rjob 0.4.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 5f95e67c5a4c2304306b39db54fbd9a38a934a5d47eb78763db9bd1485181ebc
+  data.tar.gz: e69f59a709d82b346df5dcaf3037a864a7ad509898a0ea38de70f8b41611c477
+SHA512:
+  metadata.gz: 4be5f01c00b8f5b428366d1d049c2bda14ae76aba0ccaff38567e1d89433a534d513c1195fe01aaae5562f7499d70edb8b6431e124d31fbc385f0c67671034d2
+  data.tar.gz: 97b94969dc9a440386a25e36db30f1712fff4f11d2a5eddd42809926509f7d13713f9271ded055e84f3d2390e998d51a2080a38d4903178155fbdf1fe5dbb5b2

data/bin/rjob

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require "rjob/cli"
+
+Rjob::CLI.boot

data/lib/rjob.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require 'redis'
+require 'msgpack'
+
+require 'connection_pool'
+require 'concurrent-ruby'
+require 'socket'
+require 'securerandom'
+
+require 'rjob/version'
+require 'rjob/context'
+
+require 'rjob/worker'
+
+require 'rjob/job'
+require 'rjob/job_processor'
+
+require 'rjob/scripts'
+
+module Rjob
+  def self.configure(&block)
+    ::Rjob::Context.configure(&block)
+  end
+
+  def self.enqueue(job_class, *args)
+    ::Rjob::Context.instance.enqueue_job(job_class, args)
+  end
+
+  def self.schedule_in(seconds_from_now, job_class, *args)
+    t = Time.now.to_i + seconds_from_now
+    ::Rjob::Context.instance.schedule_job_at(t, job_class, args)
+  end
+
+  def self.schedule_at(timestamp, job_class, *args)
+    ::Rjob::Context.instance.schedule_job_at(timestamp.to_i, job_class, args)
+  end
+end

data/lib/rjob/cli.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Rjob
+end
+
+class Rjob::CLI
+  def self.boot; new.boot(ARGV); end
+
+  def initialize
+    @use_rails = false
+    @run_workers = false
+  end
+
+  def boot(args)
+    STDOUT.sync = true
+    STDERR.sync = true
+
+    parse_cli_args(args)
+
+    if @use_rails
+      require File.join(Dir.pwd, "config/environment")
+    end
+
+    run_workers if @run_workers
+  end
+
+  def run_workers
+    require "rjob"
+    require "rjob/worker_process"
+
+    worker = Rjob::WorkerProcess.new(Rjob::Context.instance)
+    worker.run_forever
+  end
+
+  private
+
+  def parse_cli_args(args)
+    while args.length > 0 do
+      arg = args.shift
+      if arg == "--use-rails"
+        @use_rails = true
+      elsif arg == "--run-workers"
+        @run_workers = true
+      else
+        puts "Unrecognized argument: #{arg}"
+        puts "Exiting now"
+        exit 1
+      end
+    end
+  end
+end

data/lib/rjob/context.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+class Rjob::Context
+  attr_reader :config
+  attr_reader :prefix
+  attr_reader :bucket_count
+  attr_reader :logger
+  attr_reader :job_wrapper_proc
+  attr_reader :script_runner
+  attr_reader :recurring_jobs
+
+  def self.instance
+    return @instance if @instance
+    raise "Rjob is not configured. Please call Rjob.configure first"
+  end
+
+  def self.set_instance(instance)
+    @instance = instance
+  end
+
+  # Available options:
+  #
+  # :redis - (passed to Redis.new)
+  # :max_threads - paralallelism
+  # :bucket_count - defaults to 32
+  # :redis_pool_size - redis connection pool size. Defaults to 10
+  # :prefix - defaults to "rjob"
+  # :job_wrapper_proc - defaults to none
+  #
+  def self.configure
+    raise "Already configured!: #{@instance}" if @instance
+    config = {}
+    yield(config)
+    set_instance(new(config))
+  end
+
+  def initialize(config)
+    @config = config.dup
+    @pool_size = @config.fetch(:redis_pool_size, 10)
+
+    @bucket_count = config.fetch(:bucket_count, 32)
+    @prefix = config.fetch(:prefix, 'rjob')
+    @logger = config[:logger]
+    @job_wrapper_proc = config[:job_wrapper_proc]
+    @script_runner = Rjob::Scripts::ScriptRunner.new
+    @recurring_jobs = nil
+
+    if config.key?(:recurring_jobs)
+      require "rjob/recurring"
+
+      @recurring_jobs = config[:recurring_jobs].map do |defn|
+        Rjob::RecurringJob.from_definition(self, defn)
+      end
+    end
+
+    initialize_connection_pool
+    load_redis_scripts
+  end
+
+  def redis(&block)
+    @pool.with(&block)
+  end
+
+  def enqueue_job(job_class, args)
+    redis(&method(:enqueue_job_with_redis).curry[job_class, args])
+  end
+
+  def enqueue_job_with_redis(job_class, args, r)
+    job_data = MessagePack.pack([job_class.to_s, args])
+    @script_runner.exec(r, :enqueue_job, [], [@prefix, @bucket_count, job_data])
+  end
+
+  def schedule_job_at(timestamp, job_class, args)
+    job_data = MessagePack.pack([job_class.to_s, args])
+
+    redis do |r|
+      @script_runner.exec(r, :schedule_job_at, [], [timestamp.to_s, job_data, @prefix, @bucket_count])
+    end
+  end
+
+  def fetch_worker_class(class_name:)
+    demodularize_class(class_name)
+  end
+
+  def demodularize_class(name)
+    const = Kernel
+    name.split('::').each do |n|
+      const = const.const_get(n)
+    end
+    const
+  end
+
+  def create_redis_connection
+    redis_args = @config[:redis]
+    Redis.new(redis_args)
+  end
+
+  private
+
+  def load_redis_scripts
+    @pool.with do |redis|
+      @script_runner.load_all_scripts(redis)
+    end
+  end
+
+  def initialize_connection_pool
+    @pool = ConnectionPool.new(size: @pool_size) { create_redis_connection }
+  end
+end

data/lib/rjob/job.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+class Rjob::Job
+  DeserializationError = Class.new(StandardError)
+
+  attr_accessor :id
+  attr_accessor :retry_num
+  attr_reader :payload
+  attr_reader :context
+
+  def initialize(context)
+    @context = context
+  end
+
+  def worker_class_name
+    @deserialized_payload[0]
+  end
+
+  def worker_class
+    @context.fetch_worker_class(class_name: worker_class_name)
+  end
+
+  def worker_args
+    @deserialized_payload[1]
+  end
+
+  def payload=(str)
+    @payload = str
+    @deserialized_payload = MessagePack.unpack(str)
+  end
+
+  def serialize
+    "#{@id}!#{@retry_num}!#{@payload}".force_encoding(Encoding::ASCII_8BIT)
+  end
+
+  def self.deserialize(context, job_str)
+    first = job_str.index('!')
+    second = job_str.index('!', first + 1)
+
+    if first == nil || second == nil
+      raise DeserializationError.new("Malformed job string: '#{job_str}'")
+    end
+
+    begin
+      new(context).tap do |job|
+        job.id = job_str[0...first]
+        job.retry_num = job_str[(first + 1)...second].to_i
+        job.payload = job_str[(second + 1)..-1]
+      end
+    rescue MessagePack::MalformedFormatError => e
+      raise DeserializationError.new("Malformed job msgpack payload: #{e.message}")
+    end
+  end
+end

data/lib/rjob/job_processor.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+# Processes one single job.
+class Rjob::JobProcessor
+  attr_reader :context
+  attr_reader :error
+  attr_reader :job_str
+  attr_reader :job
+
+  def initialize(context, job_str)
+    @context = context
+    @job_str = job_str
+    @error = nil
+    @force_dont_retry = false
+    @success = false
+  end
+
+  def success?
+    @success
+  end
+
+  def stop_retry?
+    @force_dont_retry
+  end
+
+  def run
+    job = Rjob::Job.deserialize(@context, @job_str)
+    @job = job
+
+    job_args = job.worker_args
+
+    worker_class = begin
+      job.worker_class
+    rescue NameError
+      @error = { message: "No worker class '#{job.worker_class_name}'" }
+      @force_dont_retry = true
+      return
+    end
+
+    begin
+      worker_instance = worker_class.new(@context, job)
+      worker_instance.perform(*job_args)
+      @success = true
+    rescue Exception => e
+      @error = { error_class: e.class, message: e.message }
+    end
+  end
+end
+
+#     @thread_pool.post do
+#       begin
+#         klass, args = MessagePack.unpack(job_data)
+#         ::Rjob::WorkerThread.new(self, klass, args).run
+#       rescue Exception => e
+#         @failed_count.increment
+#         handle_failed_job(job, bucket, e, klass, args)
+#       ensure
+#         @processed_count.increment
+#       end
+#     end
+#   rescue Rjob::Job::DeserializationError => e
+#     @error = { exception: e }
+#   end
+# end
+
+#
+# class Rjob::WorkerThread
+#   def initialize(worker, klass, args)
+#     @worker = worker
+#     @context = @worker.context
+#     @prefix = @context.prefix
+#     @job_class, @job_args = klass, args
+#   end
+#
+#   def run
+#     klass = @context.demodularize_class(@job_class)
+#     klass.perform(@job_args)
+#   end
+# end
+#

data/lib/rjob/recurring.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+# This file just loads dependencies needed for running
+# recurring jobs
+
+require 'openssl'
+require 'time'
+require 'date'
+
+begin
+  require 'fugit'
+rescue LoadError => e
+  puts("The gem 'fugit' is required when recurring_jobs config is set for Rjob")
+  raise(e)
+end
+
+require 'rjob'
+require 'rjob/recurring_job'

data/lib/rjob/recurring_job.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+class Rjob::RecurringJob
+  attr_reader :context
+  attr_reader :cron
+  attr_reader :job_class_name
+  attr_reader :job_arguments
+  attr_reader :unique_id
+
+  def initialize(context, cron, job_class_name, job_arguments, unique_id=nil)
+    @context = context
+    @cron = cron
+    @job_class_name = job_class_name
+    @job_class = nil
+    @job_arguments = job_arguments
+
+    @unique_id = unique_id
+
+    generate_unique_id! unless @unique_id
+  end
+
+  def maybe_enqueue(redis)
+    key_name = "#{@context.prefix}:recurring:1:#{@unique_id}:lastrun"
+    current_time = Time.now
+
+    last_run_str = redis.get(key_name)
+    last_run = last_run_str ? Time.parse(last_run_str) : (current_time - 1)
+
+    next_run_on = @cron.next_time(last_run)
+    should_run = (current_time >= next_run_on.to_t)
+
+    @context.enqueue_job_with_redis(job_class, job_arguments, redis) if should_run
+
+    if should_run || last_run_str == nil
+      redis.set(key_name, current_time.utc.to_s, ex: @cron.rough_frequency * 2)
+    end
+  end
+
+  def job_class
+    @job_class ||= @context.demodularize_class(@job_class_name)
+  end
+
+  def self.from_definition(context, defn)
+    new(
+      context,
+      Fugit.parse(defn[:cron]),
+      defn[:job_class].to_s,
+      defn[:arguments],
+      defn[:unique_id]
+    )
+  end
+
+  private
+
+  def generate_unique_id!
+    digest = ::OpenSSL::Digest.new('sha256')
+    digest << @job_class_name
+    digest << @cron.original
+    @job_arguments.each { |x| digest << x.to_s }
+
+    @unique_id = digest.digest
+  end
+end

data/lib/rjob/scripts.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Rjob::Scripts
+  SCRIPTS = {
+    check_leadership: 'CheckLeadership',
+    enqueue_job: 'EnqueueJob',
+    schedule_job_at: 'ScheduleJobAt',
+    scan_buckets: 'ScanBuckets',
+    retry_job: 'RetryJob',
+    return_job_execution: 'ReturnJobExecution',
+    enqueue_scheduled_jobs: 'EnqueueScheduledJobs',
+  }.freeze
+
+  class ScriptRunner
+    def initialize
+      @scripts = {}
+    end
+
+    def load_all_scripts(redis)
+      SCRIPTS.each do |file_name, class_name|
+        klass = Rjob::Scripts.const_get(class_name)
+        script = klass.new
+        @scripts[file_name] = script
+        load_script(redis, script)
+      end
+    end
+
+    def exec(redis, name, *args)
+      script = @scripts[name]
+      redis.evalsha(script.sha1, *args)
+    end
+
+    private
+
+    def load_script(redis, script)
+      script.sha1 = redis.script(:load, script.lua_script)
+    end
+  end
+end
+
+require 'rjob/scripts/redis_script'
+
+Rjob::Scripts::SCRIPTS.each do |file_name, class_name|
+  require "rjob/scripts/#{file_name}"
+end

data/lib/rjob/scripts/check_leadership.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+class Rjob::Scripts::CheckLeadership < Rjob::Scripts::RedisScript
+  def arg_params
+    %i(worker_name time_now prefix heartbeat_timeout)
+  end
+
+  def lua_script
+    <<~LUA
+      local worker_name = ARGV[1]
+      local time_now = ARGV[2]
+      local prefix = ARGV[3]
+      local heartbeat_timeout = tonumber(ARGV[4])
+      local r = redis
+      if r.call('setnx', prefix .. ':leaderworker', worker_name) == 1 then
+        return worker_name
+      else
+        local leader = r.call('get', prefix .. ':leaderworker')
+        local last_hb = tonumber(r.call('hget', prefix .. ':worker:' .. leader, 'heartbeat'))
+        if last_hb == nil or time_now - last_hb > heartbeat_timeout then
+          r.call('set', prefix .. ':leaderworker', worker_name)
+          return worker_name
+        end
+        return leader
+      end
+    LUA
+  end
+end

data/lib/rjob/scripts/enqueue_job.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+class Rjob::Scripts::EnqueueJob < Rjob::Scripts::RedisScript
+  def arg_params
+    %i(prefix bucket_count job_data)
+  end
+
+  def lua_script
+    <<~LUA
+      local prefix = ARGV[1]
+      local bucket_count = tonumber(ARGV[2])
+      local job_data = ARGV[3]
+      local r = redis
+      local job_id = r.call('incr', prefix .. ':next')
+      local bucket = job_id % bucket_count
+      r.call('lpush', prefix .. ':jobs:' .. bucket, job_id .. '!0!' .. job_data)
+      r.call('publish', prefix .. ':jobs', bucket)
+      return job_id
+    LUA
+  end
+end

data/lib/rjob/scripts/enqueue_scheduled_jobs.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+class Rjob::Scripts::EnqueueScheduledJobs < Rjob::Scripts::RedisScript
+  def arg_params
+    %i(time_now job_limit bucket_no)
+  end
+
+  def key_params
+    %i(scheduled_key dest_key jobs_key)
+  end
+
+  def lua_script
+    <<~LUA
+      local r = redis
+      local time_now = ARGV[1]
+      local job_limit = ARGV[2]
+      local bucket_no = ARGV[3]
+
+      local scheduled_key = KEYS[1]
+      local dest_key = KEYS[2]
+      local jobs_key = KEYS[3]
+
+      local jobs = r.call('zrangebyscore', scheduled_key, 0, time_now, 'limit', 0, job_limit)
+      if #jobs == 0 then
+        return 0
+      end
+
+      local i
+      for i=1, #jobs do
+        r.call('lpush', dest_key, jobs[i])
+      end
+      r.call('zrem', scheduled_key, unpack(jobs))
+      r.call('publish', jobs_key, bucket_no)
+
+      return #jobs
+    LUA
+  end
+end

data/lib/rjob/scripts/redis_script.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class Rjob::Scripts::RedisScript
+  attr_accessor :sha1
+
+  def arg_params
+    []
+  end
+
+  def key_params
+    []
+  end
+end

data/lib/rjob/scripts/retry_job.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class Rjob::Scripts::RetryJob < Rjob::Scripts::RedisScript
+  def arg_params
+    %i(next_retry_at retry_num bucket job_id job_payload prefix)
+  end
+
+  def lua_script
+    <<~LUA
+      local timestamp = ARGV[1]
+      local retry_num = ARGV[2]
+      local bucket = ARGV[3]
+      local job_id = ARGV[4]
+      local job_payload = ARGV[5]
+      local prefix = ARGV[6]
+      local r = redis
+
+      local curr_job = job_id .. '!' .. retry_num .. '!' .. job_payload
+      local new_job = job_id .. '!' .. (retry_num + 1) .. '!' .. job_payload
+
+      r.call('lrem', prefix .. ':jobs:' .. bucket .. ':working', 1, curr_job)
+      r.call('zadd', prefix .. ':scheduled:' .. bucket, timestamp, new_job)
+
+      return job_id
+    LUA
+  end
+end

data/lib/rjob/scripts/return_job_execution.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+class Rjob::Scripts::ReturnJobExecution < Rjob::Scripts::RedisScript
+  def arg_params
+    %i(job bucket prefix)
+  end
+
+  def lua_script
+    <<~LUA
+      local job = ARGV[1]
+      local bucket = ARGV[2]
+      local prefix = ARGV[3]
+      local r = redis
+      r.call('lrem', prefix .. ':jobs:' .. bucket .. ':working', 1, job)
+      r.call('rpush', prefix .. ':jobs:' .. bucket, job)
+      return 1
+    LUA
+  end
+end

data/lib/rjob/scripts/scan_buckets.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+class Rjob::Scripts::ScanBuckets < Rjob::Scripts::RedisScript
+  def arg_params
+    %i(prefix bucket_count)
+  end
+
+  def lua_script
+    <<~LUA
+      local prefix = ARGV[1]
+      local bucket_count = ARGV[2]
+      local r = redis
+      local i
+      for i=0,bucket_count-1 do
+        local len = r.call('llen', prefix .. ':jobs:' .. i)
+        if len > 0 then
+          r.call('publish', prefix .. ':jobs', i)
+        end
+      end
+      return 1
+    LUA
+  end
+end

data/lib/rjob/scripts/schedule_job_at.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+class Rjob::Scripts::ScheduleJobAt < Rjob::Scripts::RedisScript
+  def arg_params
+    %i(timestamp job prefix bucket_count)
+  end
+
+  def lua_script
+    <<~LUA
+      local timestamp = ARGV[1]
+      local job = ARGV[2]
+      local prefix = ARGV[3]
+      local bucket_count = tonumber(ARGV[4])
+      local r = redis
+      local job_id = r.call('incr', prefix .. ':next')
+      local bucket = job_id % bucket_count
+      r.call('zadd', prefix .. ':scheduled:' .. bucket, timestamp, job_id .. '!0!' .. job)
+      return job_id
+    LUA
+  end
+end

data/lib/rjob/version.rb

@@ -0,0 +1,4 @@
+
+module Rjob
+  VERSION = "0.4.3".freeze
+end

data/lib/rjob/worker.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class Rjob::Worker
+  attr_reader :context
+  attr_reader :job
+
+  def initialize(context, job)
+    @context = context
+    @job = job
+  end
+
+  def self.retry_options
+    {
+      retry: false
+    }
+  end
+end

data/lib/rjob/worker_process.rb

@@ -0,0 +1,387 @@
+# frozen_string_literal: true
+
+# TODO: find a mechanism to recover from jobs that went to working but never returned
+
+class Rjob::WorkerProcess
+  ITERATION_TIMEOUT = 2
+  HEARTBEAT_TIMEOUT = 15
+
+  StopSubscription = Class.new(StandardError)
+
+  attr_reader :context
+  attr_reader :worker_name
+  attr_reader :state
+  attr_reader :leader
+
+  def initialize(context)
+    @context = context
+    @prefix = @context.prefix
+    @pubsub_redis = @context.create_redis_connection
+
+    init_worker_name
+
+    @iteration_no = 0
+    @max_queue_size = 20
+    max_threads = @context.config.fetch(:max_threads, 2)
+
+    @subscription_thread = nil
+    @thread_pool = Concurrent::ThreadPoolExecutor.new(
+      min_threads: [2, max_threads].min,
+      max_threads: max_threads,
+      max_queue: @max_queue_size,
+      fallback_policy: :abort # Concurrent::RejectedExecutionError
+    )
+
+    @processed_count = Concurrent::AtomicFixnum.new
+    @failed_count = Concurrent::AtomicFixnum.new
+    @returned_count = Concurrent::AtomicFixnum.new
+
+    @leader = nil
+    @state = :new
+  end
+
+  def run_forever
+    register_worker
+
+    Signal.trap("INT") do
+      if @state == :exiting
+        puts "Force exit requested. Exiting immediately"
+        exit 1
+      else
+        @state = :exiting
+        puts "Exiting..."
+      end
+    end
+
+    @state = :running
+    loop do
+      break if @state == :exited
+      run_iteration
+    end
+  ensure
+    unregister_worker
+  end
+
+  private
+
+  def disable_subscription_thread
+    return unless @subscription_thread
+    @subscription_thread.raise(StopSubscription.new)
+    @subscription_thread = nil
+  end
+
+  def enable_subscription_thread
+    return if @subscription_thread
+
+    @subscription_thread = Thread.new do
+      begin
+        @pubsub_redis.subscribe("#{@prefix}:jobs") do |on|
+          on.message do |_, bucket_no|
+            loop do
+              break unless @state == :running
+              break unless start_processing_message_from_bucket(bucket_no)
+            end
+          end
+        end
+      rescue StopSubscription => e
+        @pubsub_redis.disconnect rescue nil
+      rescue StandardError => e
+        puts "staaahp -> #{e}"
+        raise e
+        exit 1
+      end
+    end
+    @subscription_thread.run
+  end
+
+  def run_iteration
+    begin
+      stop_threshold = (@max_queue_size * 0.7).to_i
+      if @thread_pool.queue_length >= stop_threshold || @state != :running
+        disable_subscription_thread
+      elsif @state == :running
+        if !@subscription_thread
+          enable_subscription_thread
+          sleep(ITERATION_TIMEOUT)
+          scan_buckets
+        end
+      end
+
+      if @state == :exiting
+        if @thread_pool.shutdown?
+          @state = :exited
+        elsif !@thread_pool.shuttingdown?
+          @thread_pool.shutdown
+        else
+          puts "Waiting shutdown..."
+        end
+      end
+
+      report_stats
+
+      check_leadership
+
+      if leader? && @state == :running
+        exercise_leadership if @iteration_no % 2 == 0
+      end
+
+      @iteration_no += 1
+      sleep(ITERATION_TIMEOUT) unless @state == :exited
+    rescue StandardError => e
+      raise e
+    end
+  end
+
+  def check_leadership
+    @context.redis do |r|
+      if leader? && @state == :exiting
+        r.call('del', "#{@prefix}:leaderworker")
+        return
+      end
+
+      @leader = @context.script_runner.exec(r, :check_leadership,
+        [], [
+          @worker_name,
+          Time.now.to_i,
+          @prefix,
+          HEARTBEAT_TIMEOUT
+        ])
+    end
+  end
+
+  def leader?
+    @leader && @leader == @worker_name
+  end
+
+  def report_stats
+    key_prefix = "#{@prefix}:worker:#{@worker_name}"
+    state_data = {
+      heartbeat: Time.now.to_i,
+      queue_length: @thread_pool.queue_length,
+      processed: @processed_count.value,
+      failed: @failed_count.value,
+      returned: @returned_count.value,
+      state: @state
+    }
+
+    @context.redis do |r|
+      r.pipelined do
+        state_data.each do |k, v|
+          r.hset(key_prefix, k, v.to_s)
+        end
+      end
+    end
+  end
+
+  def scan_buckets
+    @context.redis do |r|
+      @context.script_runner.exec(r, :scan_buckets, [], [@prefix, @context.bucket_count])
+    end
+  end
+
+  def start_processing_message_from_bucket(bucket)
+    job_str = @context.redis do |r|
+      r.rpoplpush("#{@prefix}:jobs:#{bucket}", "#{@prefix}:jobs:#{bucket}:working")
+    end
+
+    return false if job_str == nil
+
+    # move to inside thread
+    job_processor = Rjob::JobProcessor.new(context, job_str)
+
+    begin
+      @thread_pool.post do
+
+        using_app_wrapper do
+          job_processor.run
+        end
+
+        if !job_processor.success?
+          @failed_count.increment
+          handle_job_processing_failure(bucket, job_processor)
+        else
+          remove_job_from_working(job_str, bucket)
+        end
+      end
+    rescue Concurrent::RejectedExecutionError => e
+      @returned_count.increment
+      return_job_execution(job_str, bucket)
+    ensure
+      @processed_count.increment
+    end
+  end
+
+  def remove_job_from_working(job_str, bucket)
+    @context.redis do |r|
+      r.lrem("#{@prefix}:jobs:#{bucket}:working", 1, job_str)
+    end
+  end
+
+  def retry_job(job, bucket, next_retry_at)
+    @context.redis do |r|
+      @context.script_runner.exec(r, :retry_job, [],
+        [
+          next_retry_at.to_s,
+          job.retry_num,
+          bucket,
+          job.id.to_s,
+          job.payload,
+          @prefix
+        ])
+    end
+  end
+
+  def handle_job_processing_failure(bucket, job_processor)
+    job = job_processor.job
+    error = job_processor.error
+
+    if !error
+      error = { message: "Unknown error" }
+    end
+
+    if @context.logger.respond_to?(:info)
+      @context.logger.info("Job '#{job.worker_class_name}' with args '#{job.worker_args}' failed: #{error}")
+    end
+
+    if job_processor.stop_retry?
+      move_job_to_dead(job_processor.job_str, bucket, error)
+      return
+    end
+
+    retry_options = job.worker_class.retry_options
+
+    if retry_options[:retry]
+      exceptions = retry_options.fetch(:exceptions, [StandardError])
+      should_handle = exceptions.any? { |e| e >= error[:error_class] }
+
+      retry_proc = retry_options[:next_retry_proc] || (proc { |x| 3 * x ** 4 + 15 })
+      max_retries = retry_options.fetch(:max_retries, 16) # retry for ~2 days
+
+      new_retry_num = job.retry_num + 1
+
+      if should_handle && new_retry_num <= max_retries
+        next_retry_at = Time.now.to_i + retry_proc.call(new_retry_num)
+        retry_job(job, bucket, next_retry_at)
+        return
+      end
+    end
+
+    move_job_to_dead(job_processor.job_str, bucket, error)
+  end
+
+  # TODO: this should probably be in a single redis pipelined operation
+  def move_job_to_dead(job_str, bucket, error)
+    push_job_to_dead(job_str, bucket, error)
+    remove_job_from_working(job_str, bucket)
+  end
+
+  def push_job_to_dead(job_str, bucket, error)
+    error_payload = MessagePack.pack({
+      when: Time.now.to_i,
+      error_class: error[:error_class].to_s,
+      full_message: error[:message],
+      job: job_str
+    })
+
+    @context.redis do |r|
+      r.lpush("#{@prefix}:dead", error_payload)
+    end
+  end
+
+  # When a job previously went to working state and we want to
+  # put it back (re-enqueue it).
+  #
+  # This mostly happens when we picked a job for processing but realized
+  # that we don't actually have the resources to process it at the moment.
+  def return_job_execution(job, bucket)
+    @context.redis do |r|
+      @context.script_runner.exec(r, :return_job_execution, [], [job, bucket, @prefix])
+    end
+  end
+
+  def register_worker
+    report_stats
+
+    @context.redis do |r|
+      r.lpush("#{@prefix}:workers", @worker_name)
+    end
+  end
+
+  def unregister_worker
+    @context.redis do |r|
+      r.lrem("#{@prefix}:workers", 1, @worker_name)
+      r.del("#{@prefix}:worker:#{@worker_name}")
+    end
+  end
+
+  def init_worker_name
+    host = Socket.gethostname
+    rand_factor = SecureRandom.alphanumeric(24)
+    @worker_name = [host, rand_factor].join('-')
+  end
+
+  def exercise_leadership
+    enqueue_scheduled_jobs
+
+    scan_buckets
+
+    enqueue_recurring_jobs
+  end
+
+  def enqueue_recurring_jobs
+    recurring_jobs = @context.recurring_jobs
+    return unless recurring_jobs
+
+    # Make sure all classes are loaded without error
+    recurring_jobs.each(&:job_class)
+
+    @context.redis do |redis|
+      recurring_jobs.each do |rj|
+        rj.maybe_enqueue(redis)
+      end
+    end
+  end
+
+  def enqueue_scheduled_jobs
+    time_now = Time.now.to_i
+    job_limit = 100
+
+    # Let's not be caught in an infinite loop. Thus, loop max 10 times
+    10.times do
+      re_run = false
+
+      @context.redis do |r|
+        (0...@context.bucket_count).each do |bucket|
+          num_jobs = @context.script_runner.exec(r, :enqueue_scheduled_jobs,
+            [
+              "#{@prefix}:scheduled:#{bucket}",
+              "#{@prefix}:jobs:#{bucket}",
+              "#{@prefix}:jobs"
+            ], [
+              time_now, job_limit, bucket
+            ])
+
+          re_run = true if num_jobs == job_limit
+        end
+      end
+
+      break unless re_run
+    end
+  end
+
+  def using_app_wrapper(&blk)
+    call_block = if @context.job_wrapper_proc != nil
+      proc do
+        @context.job_wrapper_proc.call(blk)
+      end
+    else
+      blk
+    end
+
+    if defined?(::Rails)
+      ::Rails.application.executor.wrap(&call_block)
+    else
+      call_block.call
+    end
+  end
+end

metadata

@@ -0,0 +1,166 @@
+--- !ruby/object:Gem::Specification
+name: rjob
+version: !ruby/object:Gem::Version
+  version: 0.4.3
+platform: ruby
+authors:
+- André D. Piske
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2021-03-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: oj
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4'
+- !ruby/object:Gem::Dependency
+  name: multi_json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.1'
+- !ruby/object:Gem::Dependency
+  name: msgpack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: connection_pool
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.2.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.2.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.6
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+description: 'RJob: asynchronous job processing'
+email: andrepiske@gmail.com
+executables:
+- rjob
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/rjob
+- lib/rjob.rb
+- lib/rjob/cli.rb
+- lib/rjob/context.rb
+- lib/rjob/job.rb
+- lib/rjob/job_processor.rb
+- lib/rjob/recurring.rb
+- lib/rjob/recurring_job.rb
+- lib/rjob/scripts.rb
+- lib/rjob/scripts/check_leadership.rb
+- lib/rjob/scripts/enqueue_job.rb
+- lib/rjob/scripts/enqueue_scheduled_jobs.rb
+- lib/rjob/scripts/redis_script.rb
+- lib/rjob/scripts/retry_job.rb
+- lib/rjob/scripts/return_job_execution.rb
+- lib/rjob/scripts/scan_buckets.rb
+- lib/rjob/scripts/schedule_job_at.rb
+- lib/rjob/version.rb
+- lib/rjob/worker.rb
+- lib/rjob/worker_process.rb
+homepage: https://gitlab.com/andrepiske/rjob
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.15
+signing_key: 
+specification_version: 4
+summary: Asynchronous job processing
+test_files: []