jiggler 0.1.0.rc2

data/lib/jiggler/redis_store.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require 'redis_client'
+
+module Jiggler
+  class RedisStore
+    def initialize(options = {})
+      @options = options
+    end
+
+    def pool      
+      @options[:async] ? async_pool : sync_pool
+    end
+
+    def async_pool
+      @async_pool ||= begin
+        config = RedisClient.config(url: @options[:redis_url], timeout: nil)
+        Async::Pool::Controller.wrap(limit: @options[:concurrency]) do
+          config.new_client
+        end
+      end
+    end
+
+    def sync_pool
+      @sync_pool ||= begin
+        config = RedisClient.config(url: @options[:redis_url])
+        pool = config.new_pool(size: @options[:concurrency])
+        def pool.acquire(&block)
+          with(&block)
+        end
+        pool
+      end
+    end
+  end
+end
+
+module Jiggler
+  class RedisClient < ::RedisClient
+    def concurrency
+      1
+    end
+
+    def viable?
+      connected?
+    end
+
+    def closed?
+      @raw_connection.nil?
+    end
+
+    def reusable?
+      !@raw_connection.nil?
+    end
+  end
+end

data/lib/jiggler/retrier.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+module Jiggler
+  class Retrier
+    include Support::Helper
+
+    attr_reader :config, :collection
+
+    def initialize(config, collection)
+      @config = config
+      @collection = collection
+      @tid = tid
+    end
+
+    def wrapped(instance, parsed_job, queue)
+      logger.info {
+        "Starting #{instance.class.name} queue=#{instance.class.queue} tid=#{@tid} jid=#{parsed_job['jid']}"
+      }
+      yield
+      logger.info { 
+        "Finished #{instance.class.name} queue=#{instance.class.queue} tid=#{@tid} jid=#{parsed_job['jid']}"
+      }
+    rescue Async::Stop => stop
+      raise stop
+    rescue => err
+      raise Async::Stop if exception_caused_by_shutdown?(err)
+
+      process_retry(instance, parsed_job, queue, err)
+      collection.incr_failures
+      
+      log_error(
+        err,
+        { 
+          context: '\'Job raised exception\'',
+          error_class: err.class.name,
+          name: parsed_job['name'],
+          queue: parsed_job['queue'],
+          args: parsed_job['args'],
+          attempt: parsed_job['attempt'],
+          tid: @tid,
+          jid: parsed_job['jid']
+        }
+      )
+    end
+
+    private
+
+    def process_retry(jobinst, parsed_job, queue, exception)
+      job_class = jobinst.class
+      parsed_job['started_at'] ||= Time.now.to_f
+
+      message = exception_message(exception)
+      if message.respond_to?(:scrub!)
+        message.force_encoding('utf-8')
+        message.scrub!
+      end
+
+      parsed_job['error_message'] = message
+      parsed_job['error_class'] = exception.class.name
+      
+      max_retry_attempts = parsed_job['retries'].to_i
+      count = parsed_job['attempt'].to_i + 1
+      return retries_exhausted(jobinst, parsed_job, exception) if count > max_retry_attempts
+
+      delay = count**4 + 15
+      retry_at = Time.now.to_f + delay
+      parsed_job['retry_at'] = retry_at
+      if count > 1
+        parsed_job['retried_at'] = Time.now.to_f
+      end
+      parsed_job['attempt'] = count
+      parsed_job['queue'] = job_class.retry_queue
+      payload = Oj.dump(parsed_job, mode: :compat)
+
+      config.with_async_redis do |conn|
+        conn.call('ZADD', config.retries_set, retry_at.to_s, payload)
+      end
+    end
+
+    def retries_exhausted(jobinst, parsed_job, exception)
+      logger.debug('Retrier') { 
+        "Retries exhausted for #{parsed_job['name']} jid=#{parsed_job['jid']}" 
+      }
+
+      send_to_morgue(parsed_job)
+    end
+
+    def send_to_morgue(parsed_job)
+      logger.warn('Retrier') { 
+        "#{parsed_job['name']} has been sent to dead jid=#{parsed_job['jid']}"
+      }
+      payload = Oj.dump(parsed_job, mode: :compat)
+      now = Time.now.to_f
+
+      config.with_async_redis do |conn|
+        conn.multi do |xa|
+          xa.call('ZADD', config.dead_set, now.to_s, payload)
+          xa.call('ZREMRANGEBYSCORE', config.dead_set, '-inf', now - config[:dead_timeout])
+          xa.call('ZREMRANGEBYRANK', config.dead_set, 0, - config[:max_dead_jobs])
+        end
+      end
+    end
+
+    def exception_caused_by_shutdown?(e, checked_causes = [])
+      return false unless e.cause
+
+      # Handle circular causes
+      checked_causes << e.object_id
+      return false if checked_causes.include?(e.cause.object_id)
+
+      e.cause.instance_of?(Async::Stop) ||
+        exception_caused_by_shutdown?(e.cause, checked_causes)
+    end
+
+    def exception_message(exception)
+      # Message from app code
+      exception.message.to_s[0, 10_000]
+    rescue
+      'Exception message unavailable'
+    end
+  end
+end

data/lib/jiggler/scheduled/enqueuer.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Jiggler
+  module Scheduled
+    class Enqueuer
+      include Support::Helper
+
+      LUA_ZPOPBYSCORE = <<~LUA
+        local key, now = KEYS[1], ARGV[1]
+        local jobs = redis.call('zrangebyscore', key, '-inf', now, 'limit', 0, 1)
+        if jobs[1] then
+          redis.call('zrem', key, jobs[1])
+          return jobs[1]
+        end
+      LUA
+
+      def initialize(config)
+        @config = config
+        @done = false
+        @lua_zpopbyscore_sha = nil
+        @tid = tid
+      end
+
+      def enqueue_jobs
+        @config.with_async_redis do |conn|
+          sorted_sets.each do |sorted_set|
+            # Get next item in the queue with score (time to execute) <= now
+            job_args = zpopbyscore(conn, key: sorted_set, argv: Time.now.to_f.to_s)
+            while !@done && job_args
+              push_job(conn, job_args)
+              job_args = zpopbyscore(conn, key: sorted_set, argv: Time.now.to_f.to_s)
+            end
+          end
+        rescue => err
+          log_error_short(err, { context: '\'Enqueuing jobs error\'', tid: @tid })
+        end
+      end
+
+      def terminate
+        @done = true
+      end
+
+      def push_job(conn, job_args)
+        name = Oj.load(job_args, mode: :compat)['queue'] || @config.default_queue
+        list_name = "#{@config.queue_prefix}#{name}"
+        # logger.debug('Poller Enqueuer') { "Pushing #{job_args} to #{list_name}" }
+        conn.call('LPUSH', list_name, job_args)
+      rescue => err
+        log_error_short(
+          err, { 
+            context: '\'Pushing scheduled job error\'', 
+            tid: @tid,
+            job_args: job_args,
+            queue: list_name
+          }
+        )
+      end
+
+      private
+      
+      def sorted_sets
+        @sorted_sets ||= [@config.retries_set, @config.scheduled_set].freeze
+      end
+
+      def zpopbyscore(conn, key: nil, argv: nil)
+        if @lua_zpopbyscore_sha.nil?
+          @lua_zpopbyscore_sha = conn.call('SCRIPT', 'LOAD', LUA_ZPOPBYSCORE)
+        end
+        conn.call('EVALSHA', @lua_zpopbyscore_sha, 1, key, argv)
+      rescue RedisClient::CommandError => e
+        raise unless e.message.start_with?('NOSCRIPT')
+
+        @lua_zpopbyscore_sha = nil
+        retry
+      end     
+    end
+  end
+end

data/lib/jiggler/scheduled/poller.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+# The Poller checks Redis every N seconds for jobs in the retry or scheduled
+# set have passed their timestamp and should be enqueued.
+module Jiggler
+  module Scheduled    
+    class Poller
+      include Support::Helper
+
+      INITIAL_WAIT = 5
+
+      def initialize(config)
+        @config = config
+        @enqueuer = Jiggler::Scheduled::Enqueuer.new(config)
+        @done = false
+        @job = nil
+        @count_calls = 0
+        @condition = Async::Condition.new
+      end
+
+      def terminate
+        @done = true
+        @enqueuer.terminate
+
+        Async do
+          @condition.signal
+          @job&.wait
+        end
+      end
+
+      def start
+        @job = safe_async('Poller') do
+          @tid = tid
+          initial_wait
+          until @done
+            enqueue
+            wait unless @done
+          end
+        end
+      end
+
+      def enqueue
+        # logger.warn('Poller runs')
+        @enqueuer.enqueue_jobs
+      end
+
+      private
+
+      def wait
+        Async(transient: true) do
+          sleep(random_poll_interval)
+          @condition.signal
+        end
+        @condition.wait
+      end
+
+      def random_poll_interval
+        count = process_count
+        interval = @config[:poll_interval] 
+
+        if count < 10
+          interval * rand + interval.to_f / 2
+        else
+          interval * rand
+        end
+      end
+
+      def fetch_count
+        @config.with_sync_redis do |conn| 
+          conn.call('SCAN', '0', 'MATCH', @config.process_scan_key).last.size
+        rescue => err
+          log_error_short(err, { context: '\'Poller getting processes error\'', tid: @tid })
+          1
+        end
+      end
+
+      def process_count
+        count = fetch_count
+        count = 1 if count == 0
+        count
+      end
+
+      # wait a random amount of time so in case of multiple processes 
+      # their pollers won't be synchronized
+      def initial_wait
+        total = INITIAL_WAIT + (12 * rand)
+
+        # in case of an early exit skip the initial wait
+        Async(transient: true) do
+          sleep(total)
+          @condition.signal
+        end
+        @condition.wait
+      end
+    end
+  end
+end

data/lib/jiggler/stats/collection.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Jiggler
+  module Stats
+    class Collection
+      attr_reader :uuid, :data
+
+      def initialize(uuid)
+        @uuid = uuid
+        @data = {
+          processed: 0,
+          failures: 0,
+          current_jobs: {}
+        }
+      end
+
+      def incr_processed
+        @data[:processed] += 1
+      end
+
+      def incr_failures
+        @data[:failures] += 1
+      end
+    end
+  end
+end

data/lib/jiggler/stats/monitor.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module Jiggler
+  module Stats
+    class Monitor
+      include Support::Helper
+
+      attr_reader :config, :collection, :data_key, :exp
+
+      def initialize(config, collection)
+        @config = config
+        @collection = collection
+        @done = false
+        @condition = Async::Condition.new
+        # the key expiration should be greater than the stats interval
+        # to avoid cases where the monitor is blocked
+        # by long running workers and the key is not updated in time
+        @exp = @config[:stats_interval] + 180 # interval + 3 minutes
+        @rss_path = "/proc/#{Process.pid}/status"
+      end
+
+      def start
+        @job = safe_async('Monitor') do
+          @tid = tid
+          until @done
+            load_data_into_redis
+            wait unless @done
+          end
+        end
+      end
+
+      def terminate
+        @condition.signal
+        @done = true
+        cleanup
+      end
+  
+      def process_data
+        Oj.dump({
+          heartbeat: Time.now.to_f,
+          rss: process_rss,
+          current_jobs: collection.data[:current_jobs],
+        }, mode: :compat)
+      end
+
+      def load_data_into_redis
+        # logger.warn('Monitor runs')
+        processed_jobs = collection.data[:processed]
+        failed_jobs = collection.data[:failures]
+        collection.data[:processed] -= processed_jobs
+        collection.data[:failures] -= failed_jobs
+
+        config.with_async_redis do |conn|
+          conn.pipelined do |pipeline|
+            pipeline.call('SET', collection.uuid, process_data, ex: exp)
+            pipeline.call('INCRBY', config.processed_counter, processed_jobs)
+            pipeline.call('INCRBY', config.failures_counter, failed_jobs)
+          end
+
+          cleanup_with(conn) if @done
+        rescue => err
+          log_error_short(err, { context: '\'Monitor loading stats error\'', tid: @tid })
+        end
+      end
+
+      def process_rss
+        case RUBY_PLATFORM
+        when /linux/
+          IO.readlines(@rss_path).each do |line|
+            next unless line.start_with?('VmRSS:')
+            break line.split[1].to_i
+          end
+        when /darwin|bsd/
+          `ps -o pid,rss -p #{Process.pid}`.lines.last.split.last.to_i
+        else
+          nil
+        end
+      end
+
+      def cleanup
+        config.with_async_redis do |conn| 
+          cleanup_with(conn)
+        rescue => err
+          log_error_short(err, { context: '\'Cleanup error\'', tid: @tid })
+        end
+      end
+
+      def wait
+        Async(transient: true) do
+          sleep(config[:stats_interval])
+          @condition.signal
+        end
+        @condition.wait
+      end
+
+      private
+
+      def cleanup_with(conn)
+        conn.call('DEL', collection.uuid)
+      end
+    end
+  end
+end

data/lib/jiggler/summary.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module Jiggler
+  class Summary
+    KEYS = %w[
+      retry_jobs_count
+      dead_jobs_count
+      scheduled_jobs_count
+      failures_count
+      processed_count
+      processes
+      queues
+    ].freeze
+
+    attr_reader :config
+
+    def initialize(config)
+      @config = config
+    end
+
+    def all
+      summary = {}
+      collected_data = config.client_redis_pool.acquire do |conn|
+        data = conn.pipelined do |pipeline|
+          pipeline.call('ZCARD', config.retries_set)
+          pipeline.call('ZCARD', config.dead_set)
+          pipeline.call('ZCARD', config.scheduled_set)
+          pipeline.call('GET', config.failures_counter)
+          pipeline.call('GET', config.processed_counter)
+        end
+        [*data, fetch_and_format_processes(conn), fetch_and_format_queues(conn)]
+      end
+      KEYS.each_with_index do |key, index|
+        val = collected_data[index]
+        val = val.to_i if index <= 4 # counters
+        summary[key] = val
+      end
+      summary
+    end
+
+    def last_retry_jobs(num)
+      config.client_redis_pool.acquire do |conn|
+        conn.call('ZRANGE', config.retries_set, '+inf', '-inf', 'BYSCORE', 'REV', 'LIMIT', 0, num)
+      end.map { |job| Oj.load(job, mode: :compat) }
+    end
+
+    def last_scheduled_jobs(num)
+      config.client_redis_pool.acquire do |conn|
+        conn.call('ZRANGE', config.scheduled_set, '+inf', '-inf', 'BYSCORE', 'REV', 'LIMIT', 0, num, 'WITHSCORES')
+      end.map do |(job, score)|
+        Oj.load(job).merge('scheduled_at' => score)
+      end
+    end
+
+    def last_dead_jobs(num)
+      config.client_redis_pool.acquire do |conn|
+        conn.call('ZRANGE', config.dead_set, '+inf', '-inf', 'BYSCORE', 'REV', 'LIMIT', 0, num)
+      end.map { |job| Oj.load(job, mode: :compat) }
+    end
+
+    private
+
+    def fetch_processes(conn)
+      # in case they keys were deleted/modified could return incorrect results
+      conn.call('SCAN', '0', 'MATCH', config.process_scan_key).last
+    end
+
+    def fetch_and_format_processes(conn)
+      fetch_processes(conn).reduce({}) do |acc, uuid|
+        process_data = Oj.load(conn.call('GET', uuid), mode: :compat) || {}
+        values = uuid.split(':')
+        acc[uuid] = process_data.merge({
+          'name' => values[0..2].join(':'),
+          'concurrency' => values[3],
+          'timeout' => values[4],
+          'queues' => values[5],
+          'poller_enabled' => values[6] == '1',
+          'started_at' => values[7],
+          'pid' => values[8]
+        })
+        acc[uuid]['hostname'] = values[9..-1].join(':')
+        acc
+      end
+    end
+
+    def fetch_and_format_queues(conn)
+      lists = conn.call('SCAN', '0', 'MATCH', config.queue_scan_key).last
+      lists_data = {}
+
+      collected_data = conn.pipelined do |pipeline|
+        lists.each do |list|
+          pipeline.call('LLEN', list)
+        end
+      end
+      lists.each_with_index do |list, index|
+        lists_data[list.split(':').last] = collected_data[index]
+      end
+      lists_data
+    end
+  end
+end

data/lib/jiggler/support/helper.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Jiggler
+  module Support
+    module Helper
+      def safe_async(name)
+        Async do
+          yield
+        rescue Exception => ex
+          log_error(ex, { context: name, tid: tid })        
+        end
+      end
+
+      def log_error(ex, ctx = {})
+        err_context = ctx.compact.map { |k, v| "#{k}=#{v}" }.join(' ')
+        logger.error("error_message='#{ex.message}' #{err_context}")
+        logger.error(ex.backtrace.first(12).join("\n")) unless ex.backtrace.nil?
+      end
+
+      def log_error_short(err, ctx = {})
+        err_context = ctx.compact.map { |k, v| "#{k}=#{v}" }.join(' ')
+        logger.error("error_message='#{err.message}' #{err_context}")
+      end
+  
+      def logger
+        @config.logger
+      end
+  
+      def tid
+        return unless Async::Task.current?
+        (Async::Task.current.object_id ^ ::Process.pid).to_s(36)
+      end
+    end
+  end
+end

data/lib/jiggler/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Jiggler
+  VERSION = '0.1.0.rc2'
+end

data/lib/jiggler/web/assets/stylesheets/application.css

@@ -0,0 +1,64 @@
+body {
+  background: rgb(45, 44, 49);
+  color: #d8d9da;
+  padding: 20px;
+  font-family: "Segoe UI","Roboto","Oxygen","Ubuntu","Cantarell","Fira Sans","Droid Sans","Helvetica Neue",sans-serif;
+}
+
+.main-table {
+  border-collapse: collapse;
+  margin: 1rem 0;
+  width: 100%;
+}
+
+.main-table th, td {
+  border: 1px solid #858585;
+  border-collapse: collapse;
+  padding: 0.5rem;
+}
+
+.main-table tr:nth-child(even) {
+  background-color: rgb(45, 44, 49);
+}
+
+.main-table thead {
+  background-color: rgb(54 52 64);
+}
+
+.main-table tbody tr:hover {
+  background-color: rgb(56, 55, 61);
+}
+
+.right {
+  text-align: right;
+}
+
+.stats-entry {
+  background-color: #363440;
+  border: 1px solid #858585;
+  display: inline-block;
+  padding: 10px;
+}
+
+.badge {
+  background-color: #858585;
+  color: #fff;
+  padding: 0.1rem 0.3rem;
+  text-align: center;
+  border-radius: 3px;
+  font-size: 0.7rem;
+  font-weight: 500;
+  vertical-align: top;
+}
+
+.badge-success {
+  background-color: #008B69;
+}
+
+.outdated {
+  color: #858585;
+}
+
+.outdated .badge-success {
+  background-color: #858585;
+}