jiggler 0.1.0.rc4 → 0.1.0.rc5

data/lib/jiggler/launcher.rb

@@ -22,6 +22,7 @@ module Jiggler
 
       @done = true
       manager.suspend
+      logger.debug('Manager suspended')
 
       poller.terminate if config[:poller_enabled]
       monitor.terminate
@@ -30,14 +31,19 @@ module Jiggler
     def stop
       suspend
       manager.terminate
+      logger.debug('Manager terminated')
     end
 
     private
 
     def uuid
-      @uuid ||= begin
+      @uuid ||= SecureRandom.hex(6)
+    end
+
+    def identity
+      @identity ||= begin
         data_str = [
-          SecureRandom.hex(6),
+          uuid,
           config[:concurrency],
           config[:timeout],
           config[:queues].join(','),
@@ -51,7 +57,7 @@ module Jiggler
     end
 
     def collection
-      @collection ||= Stats::Collection.new(uuid)
+      @collection ||= Stats::Collection.new(uuid, identity)
     end
 
     def manager

data/lib/jiggler/manager.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'set'
+
 # This class manages the workers lifecycle
 module Jiggler
   class Manager
@@ -7,16 +9,21 @@ module Jiggler
 
     def initialize(config, collection)
       @workers = Set.new
+
       @done = false
       @config = config
       @timeout = @config[:timeout]
       @collection = collection
+      init_acknowledger_and_fetcher
+
       @config[:concurrency].times do
         @workers << init_worker
       end
     end
 
     def start
+      @acknowledger.start
+      @fetcher.start
       @workers.each(&:run)
     end
 
@@ -24,17 +31,34 @@ module Jiggler
       return if @done
 
       @done = true
-      @workers.each(&:suspend)
+      @fetcher.suspend
     end
 
     def terminate
       suspend
       schedule_shutdown
       wait_for_workers
+      wait_for_acknowledger
     end
 
     private
 
+    def init_acknowledger_and_fetcher
+      if @config.at_least_once?
+        @fetcher = AtLeastOnce::Fetcher.new(@config, @collection)
+        @acknowledger = AtLeastOnce::Acknowledger.new(@config)
+      else
+        @fetcher = AtMostOnce::Fetcher.new(@config, @collection)
+        @acknowledger = AtMostOnce::Acknowledger.new(@config)
+      end
+    end
+
+    def wait_for_acknowledger
+      logger.info('Waiting for the finished jobs to acknowledge...')
+      @acknowledger.terminate
+      @acknowledger.wait
+    end
+
     def wait_for_workers
       logger.info('Waiting for workers to finish...')
       @workers.each(&:wait)
@@ -53,7 +77,7 @@ module Jiggler
 
     def init_worker
       Jiggler::Worker.new(
-        @config, @collection, &method(:process_worker_result)
+        @config, @collection, @acknowledger, @fetcher, &method(:process_worker_result)
       )
     end
 

data/lib/jiggler/retrier.rb

@@ -30,16 +30,14 @@ module Jiggler
       
       log_error(
         err,
-        { 
-          context: '\'Job raised exception\'',
-          error_class: err.class.name,
-          name: parsed_job['name'],
-          queue: parsed_job['queue'],
-          args: parsed_job['args'],
-          attempt: parsed_job['attempt'],
-          tid: @tid,
-          jid: parsed_job['jid']
-        }
+        context: '\'Job raised exception\'',
+        error_class: err.class.name,
+        name: parsed_job['name'],
+        queue: parsed_job['queue'],
+        args: parsed_job['args'],
+        attempt: parsed_job['attempt'],
+        tid: @tid,
+        jid: parsed_job['jid']
       )
     end
 

data/lib/jiggler/scheduled/enqueuer.rb

@@ -32,7 +32,7 @@ module Jiggler
             end
           end
         rescue => err
-          log_error_short(err, { context: '\'Enqueuing jobs error\'', tid: @tid })
+          log_error_short(err, context: '\'Enqueuing jobs error\'', tid: @tid)
         end
       end
 

data/lib/jiggler/scheduled/poller.rb

@@ -12,46 +12,67 @@ module Jiggler
       def initialize(config)
         @config = config
         @enqueuer = Jiggler::Scheduled::Enqueuer.new(config)
+        @requeuer = Jiggler::Scheduled::Requeuer.new(config)
         @done = false
         @job = nil
         @count_calls = 0
-        @condition = Async::Condition.new
+        @requeuer_condition = Async::Condition.new
+        @enqueuer_condition = Async::Condition.new
       end
 
       def terminate
         @done = true
         @enqueuer.terminate
+        @requeuer.terminate
 
         Async do
-          @condition.signal
+          @requeuer_condition.signal
+          @enqueuer_condition.signal
           @job&.wait
         end
       end
 
       def start
-        @job = safe_async('Poller') do
+        @job = Async do
           @tid = tid
           initial_wait
-          until @done
-            enqueue
-            wait unless @done
+          safe_async('Poller') do
+            until @done
+              enqueue
+              wait(@enqueuer_condition) unless @done
+            end
           end
+          safe_async('Requeuer') do
+            until @done
+              handle_stale_in_process_queues
+              logger.debug('Executing requeuer')
+              wait(@requeuer_condition, in_process_interval) unless @done
+            end
+          end if @config.at_least_once?
         end
       end
 
       def enqueue
-        # logger.warn('Poller runs')
         @enqueuer.enqueue_jobs
       end
 
+      def handle_stale_in_process_queues
+        @requeuer.handle_stale
+      end
+
       private
 
-      def wait
+      def wait(condition, interval = random_poll_interval)
         Async(transient: true) do
-          sleep(random_poll_interval)
-          @condition.signal
+          sleep(interval)
+          condition.signal
         end
-        @condition.wait
+        condition.wait
+      end
+
+      def in_process_interval
+        # 60 to 120 seconds by default
+        [@config[:in_process_interval] * rand, 60].max
       end
 
       def random_poll_interval
@@ -66,31 +87,22 @@ module Jiggler
       end
 
       def fetch_count
-        @config.with_sync_redis do |conn| 
-          conn.call('SCAN', '0', 'MATCH', @config.process_scan_key).last.size
-        rescue => err
-          log_error_short(err, { context: '\'Poller getting processes error\'', tid: @tid })
-          1
-        end
+        scan_all(@config.process_scan_key).size
+      rescue => err
+        log_error_short(err, { context: '\'Poller getting processes error\'', tid: @tid })
+        1
       end
 
       def process_count
         count = fetch_count
-        count = 1 if count == 0
+        count = 1 if count.zero?
         count
       end
 
       # wait a random amount of time so in case of multiple processes 
       # their pollers won't be synchronized
       def initial_wait
-        total = INITIAL_WAIT + (12 * rand)
-
-        # in case of an early exit skip the initial wait
-        Async(transient: true) do
-          sleep(total)
-          @condition.signal
-        end
-        @condition.wait
+        wait(@enqueuer_condition, INITIAL_WAIT + (12 * rand))
       end
     end
   end

data/lib/jiggler/scheduled/requeuer.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Jiggler
+  module Scheduled
+    class Requeuer
+      include Support::Helper
+
+      def initialize(config)
+        @config = config
+        @done = false
+        @tid = tid
+      end
+
+      def handle_stale
+        requeue_data.each do |(rqueue, queue, _)|
+          @config.with_async_redis do |conn|
+            loop do
+              return if @done
+              # reprocessing is prioritised so we push at the right side
+              break if conn.call('LMOVE', rqueue, queue, 'RIGHT', 'RIGHT').nil?
+            end
+          end
+        end
+      rescue => err
+        log_error_short(err, context: '\'Requeuing jobs error\'', tid: @tid)
+      end
+
+      def terminate
+        @done = true
+      end
+
+      private
+
+      def requeue_data
+        grouped_queues = in_progress_queues.map do |queue|
+          [queue, *queue.split(":#{AtLeastOnce::Fetcher::RESERVE_QUEUE_SUFFIX}:")]
+        end.group_by(&:last)
+        # returns [[queue_in_progress, queue, uuid]] for non-running processes
+        grouped_queues.except(*running_processes_uuid).values.flatten(1)
+      end
+
+      def running_processes_uuid
+        scan_all(@config.process_scan_key).map do |process|
+          process.split(':')[2]
+        end
+      end
+
+      def in_progress_queues
+        scan_all(in_progress_wildcard)
+      end
+
+      def in_progress_wildcard
+        "*#{AtLeastOnce::Fetcher::RESERVE_QUEUE_SUFFIX}*"
+      end
+    end
+  end
+end

data/lib/jiggler/server.rb

@@ -4,6 +4,7 @@
 require 'jiggler/support/helper'
 require 'jiggler/scheduled/enqueuer'
 require 'jiggler/scheduled/poller'
+require 'jiggler/scheduled/requeuer'
 require 'jiggler/stats/collection'
 require 'jiggler/stats/monitor'
 require 'jiggler/errors'
@@ -11,4 +12,10 @@ require 'jiggler/retrier'
 require 'jiggler/launcher'
 require 'jiggler/manager'
 require 'jiggler/worker'
+require 'jiggler/base_acknowledger'
+require 'jiggler/base_fetcher'
+require 'jiggler/at_least_once/acknowledger'
+require 'jiggler/at_least_once/fetcher'
+require 'jiggler/at_most_once/acknowledger'
+require 'jiggler/at_most_once/fetcher'
 require 'jiggler/cli'

data/lib/jiggler/stats/collection.rb

@@ -3,10 +3,11 @@
 module Jiggler
   module Stats
     class Collection
-      attr_reader :uuid, :data
+      attr_reader :uuid, :identity, :data
 
-      def initialize(uuid)
+      def initialize(uuid, identity)
         @uuid = uuid
+        @identity = identity
         @data = {
           processed: 0,
           failures: 0,

data/lib/jiggler/stats/monitor.rb

@@ -52,7 +52,7 @@ module Jiggler
 
         config.with_async_redis do |conn|
           conn.pipelined do |pipeline|
-            pipeline.call('SET', collection.uuid, process_data, ex: exp)
+            pipeline.call('SET', collection.identity, process_data, ex: exp)
             pipeline.call('INCRBY', config.processed_counter, processed_jobs)
             pipeline.call('INCRBY', config.failures_counter, failed_jobs)
           end
@@ -96,7 +96,7 @@ module Jiggler
       private
 
       def cleanup_with(conn)
-        conn.call('DEL', collection.uuid)
+        conn.call('DEL', collection.identity)
       end
     end
   end

data/lib/jiggler/summary.rb

@@ -84,7 +84,14 @@ module Jiggler
     end
 
     def fetch_and_format_queues(conn)
-      lists = conn.call('SCAN', '0', 'MATCH', config.queue_scan_key).last
+      cursor = '0'
+      lists = []
+
+      loop do
+        cursor, current_lists = conn.call('SCAN', cursor, 'MATCH', config.queue_scan_key)
+        lists += current_lists.select { |list| !list.include?(':in_progress:') } # TODO: replace with constant
+        break if cursor == '0'
+      end
       lists_data = {}
 
       collected_data = conn.pipelined do |pipeline|

data/lib/jiggler/support/helper.rb

@@ -7,17 +7,17 @@ module Jiggler
         Async do
           yield
         rescue Exception => ex
-          log_error(ex, { context: name, tid: tid })        
+          log_error(ex, context: name, tid: tid)
         end
       end
 
-      def log_error(ex, ctx = {})
+      def log_error(ex, **ctx)
         err_context = ctx.compact.map { |k, v| "#{k}=#{v}" }.join(' ')
         logger.error("error_message='#{ex.message}' #{err_context}")
         logger.error(ex.backtrace.first(12).join("\n")) unless ex.backtrace.nil?
       end
 
-      def log_error_short(err, ctx = {})
+      def log_error_short(err, **ctx)
         err_context = ctx.compact.map { |k, v| "#{k}=#{v}" }.join(' ')
         logger.error("error_message='#{err.message}' #{err_context}")
       end
@@ -25,6 +25,20 @@ module Jiggler
       def logger
         @config.logger
       end
+
+      def scan_all(mask)
+        @config.with_sync_redis do |conn|
+          start = '0'
+          all_keys = []
+          loop do
+            start, keys = conn.call('SCAN', start, 'MATCH', mask)
+            break if keys.empty?
+            all_keys += keys
+            break if start == '0'
+          end
+          all_keys
+        end
+      end
   
       def tid
         return unless Async::Task.current?

data/lib/jiggler/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Jiggler
-  VERSION = '0.1.0.rc4'
+  VERSION = '0.1.0.rc5'
 end

data/lib/jiggler/web.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'erb'
-
 module Jiggler
   class Web
     WEB_PATH = File.expand_path("#{File.dirname(__FILE__)}/web")

data/lib/jiggler/worker.rb

@@ -3,18 +3,19 @@
 module Jiggler
   class Worker
     include Support::Helper
-    TIMEOUT = 2 # timeout for brpop
 
     CurrentJob = Struct.new(:queue, :args, keyword_init: true)
 
-    attr_reader :current_job, :config, :done, :collection
+    attr_reader :current_job, :config, :collection, :acknowledger, :fetcher
 
-    def initialize(config, collection, &callback)
+    def initialize(config, collection, acknowledger, fetcher, &callback)
       @done = false
       @current_job = nil
       @callback = callback
       @config = config
       @collection = collection
+      @acknowledger = acknowledger
+      @fetcher = fetcher
     end
 
     def run
@@ -43,14 +44,9 @@ module Jiggler
     end
 
     def terminate
-      @done = true
       @runner&.stop
     end
 
-    def suspend
-      @done = true
-    end
-
     def wait
       @runner&.wait
     end
@@ -59,21 +55,20 @@ module Jiggler
 
     def process_job
       @current_job = fetch_one
-      return if current_job.nil? # timed out brpop or done
+      return if current_job.nil? # done
+
       execute_job
       @current_job = nil
     end
 
     def fetch_one
-      queue, args = config.with_sync_redis { |conn| conn.blocking_call(false, 'BRPOP', *queues, TIMEOUT) }
-      return nil unless queue
-
-      if @done
-        requeue(queue, args)
-        nil
-      else
-        CurrentJob.new(queue: queue, args: args)
+      job = fetcher.fetch
+      if job == :done
+        logger.debug('Suspending the worker')
+        @done = true
+        return
       end
+      job
     rescue Async::Stop => err
       raise err
     rescue => err
@@ -86,17 +81,16 @@ module Jiggler
     def execute_job
       parsed_args = Oj.load(current_job.args, mode: :compat)
       execute(parsed_args, current_job.queue)
+      acknowledger.ack(current_job)
     rescue Async::Stop => err
       raise err
     rescue UnknownJobError => err
       collection.incr_failures
       log_error_short(
         err,
-        {
-          error_class: err.class.name,
-          job: parsed_args,
-          tid: @tid
-        }
+        error_class: err.class.name,
+        job: parsed_args,
+        tid: @tid
       )
     rescue JSON::ParserError => err
       collection.incr_failures
@@ -104,13 +98,10 @@ module Jiggler
     rescue Exception => ex
       log_error(
         ex,
-        {
-          context: '\'Internal exception\'',
-          tid: @tid,
-          jid: parsed_args['jid']
-        }
+        context: '\'Internal exception\'',
+        tid: @tid,
+        jid: parsed_args['jid']
       )
-      # raise ex
     end
 
     def execute(parsed_job, queue)
@@ -139,10 +130,8 @@ module Jiggler
     def handle_fetch_error(ex)
       log_error_short(
         ex,
-        {
-          context: '\'Fetch error\'',
-          tid: @tid
-        }
+        context: '\'Fetch error\'',
+        tid: @tid
       )
       sleep(TIMEOUT + rand(5) * config[:concurrency]) # sleep for a while before retrying
     end
@@ -159,10 +148,6 @@ module Jiggler
       collection.data[:current_jobs].delete(@tid)
     end
 
-    def queues
-      @queues ||= config.prefixed_queues
-    end
-
     def constantize(str)
       return Object.const_get(str) unless str.include?('::')