jiggler 0.1.0.rc2

data/lib/jiggler/cli.rb

@@ -0,0 +1,263 @@
+# frozen_string_literal: true
+
+require 'singleton'
+require 'optparse'
+require 'yaml'
+require 'async'
+require 'async/io/trap'
+require 'async/pool'
+
+module Jiggler
+  class CLI
+    include Singleton
+    CONTEXT_SWITCHER_THRESHOLD = 0.5
+
+    attr_reader :logger, :config, :environment
+
+    SIGNAL_HANDLERS = {
+      :INT => ->(cli) {
+        cli.logger.fatal('Received INT, shutting down')
+        cli.stop 
+      },
+      :TERM => ->(cli) {
+        cli.logger.fatal('Received TERM, shutting down')
+        cli.stop 
+      },
+      :TSTP => ->(cli) {
+        cli.logger.info('Received TSTP, no longer accepting new work')
+        cli.suspend
+      }
+    }
+    UNHANDLED_SIGNAL_HANDLER = ->(cli) { cli.logger.info('No signal handler registered, ignoring') }
+    SIGNAL_HANDLERS.default = UNHANDLED_SIGNAL_HANDLER
+    SIGNAL_HANDLERS.freeze
+    
+    def parse_and_init(args = ARGV.dup)
+      @config ||= Jiggler.config
+
+      setup_options(args)
+      initialize_logger
+      validate!
+      load_app
+    end
+
+    def start
+      return unless ping_redis
+      @cond = Async::Condition.new
+      Async do
+        setup_signal_handlers
+        patch_scheduler
+        @launcher = Launcher.new(config)
+        @launcher.start
+        Async do
+          @cond.wait
+        end
+      end
+      @switcher&.exit
+    end
+
+    def stop
+      @launcher.stop
+      logger.info('Jiggler is stopped, bye!')
+      @cond.signal
+    end
+
+    def suspend
+      @launcher.suspend
+      logger.info('Jiggler is suspended')
+    end
+
+    private
+
+    # forces scheduler to switch fibers if they take more than threshold to execute
+    def patch_scheduler
+      @switcher = Thread.new(Fiber.scheduler) do |scheduler|
+        loop do
+          sleep(CONTEXT_SWITCHER_THRESHOLD)
+          switch = scheduler.context_switch
+          next if switch.nil?
+          next if Process.clock_gettime(Process::CLOCK_MONOTONIC) - switch < CONTEXT_SWITCHER_THRESHOLD
+
+          Process.kill('URG', Process.pid)
+        end
+      end
+
+      Signal.trap('URG') do
+        next Fiber.scheduler.context_switch!(nil) unless Async::Task.current?
+        Async::Task.current.yield
+      end
+
+      Fiber.scheduler.instance_eval do
+        def context_switch
+          @context_switch
+        end
+
+        def context_switch!(value = Process.clock_gettime(Process::CLOCK_MONOTONIC))
+          @context_switch = value
+        end
+
+        def block(...)
+          context_switch!(nil)
+          super
+        end
+
+        def kernel_sleep(...)
+          context_switch!(nil)
+          super
+        end
+
+        def resume(fiber, *args)
+          context_switch!
+          super
+        end
+      end
+    end
+
+    def setup_signal_handlers
+      SIGNAL_HANDLERS.each do |signal, handler|
+        trap = Async::IO::Trap.new(signal)
+        trap.install!
+        Async(transient: true) do
+          trap.wait
+          invoked_traps[signal] += 1
+          handler.call(self)
+        end
+      end
+    end
+
+    def invoked_traps
+      @invoked_traps ||= Hash.new { |h, k| h[k] = 0 }
+    end
+
+    def validate!
+      if config[:queues].any? { |q| q.include?(':') }
+        raise ArgumentError, 'Queue names cannot contain colons'
+      end
+
+      [:concurrency, :client_concurrency, :timeout].each do |opt|
+        raise ArgumentError, "#{opt}: #{config[opt]} is not a valid value" if config[opt].to_i <= 0
+      end
+    end
+
+    def parse_options(argv)
+      opts = {}
+      @parser = option_parser(opts)
+      @parser.parse!(argv)
+      opts
+    end
+
+    def option_parser(opts)
+      parser = OptionParser.new do |o|
+        o.on '-c', '--concurrency INT', 'Number of fibers to use on the server' do |arg|
+          opts[:concurrency] = Integer(arg)
+        end
+
+        o.on '-e', '--environment ENV', 'Application environment' do |arg|
+          opts[:environment] = arg
+        end
+
+        o.on '-q', '--queue QUEUE1,QUEUE2', 'Queues to process' do |arg|
+          opts[:queues] ||= []
+          arg.split(',').each do |queue|
+            opts[:queues] << queue
+          end
+        end
+
+        o.on '-r', '--require PATH', 'File to require' do |arg|
+          opts[:require] = arg
+        end
+
+        o.on '-t', '--timeout NUM', 'Shutdown timeout' do |arg|
+          opts[:timeout] = Integer(arg)
+        end
+
+        o.on '-v', '--verbose', 'Print more verbose output' do |arg|
+          opts[:verbose] = arg
+        end
+
+        o.on '-C', '--config PATH', 'Path to YAML config file' do |arg|
+          opts[:config_file] = arg
+        end
+
+        o.on '-V', '--version', 'Print version and exit' do
+          puts("Jiggler #{Jiggler::VERSION}")
+          exit(0)
+        end
+
+        o.on_tail '-h', '--help', 'Show help' do
+          puts o 
+          exit(0)
+        end
+      end
+      parser.banner = 'Jiggler [options]'
+      parser
+    end
+
+    def setup_options(args)
+      opts = parse_options(args)
+
+      set_environment(opts)
+
+      opts = parse_config(opts[:config_file]).merge(opts) if opts[:config_file]
+      opts[:queues] = [Jiggler::Config::DEFAULT_QUEUE] if opts[:queues].nil?
+      opts[:server_mode] = true # cli starts only in server mode
+      config.merge!(opts)
+    end
+
+    def set_environment(opts)
+      opts[:environment] ||= ENV['APP_ENV'] || 'development'
+      @environment = opts[:environment]
+    end
+
+    def initialize_logger
+      @logger = config.logger
+      logger.level = ::Logger::DEBUG if config[:verbose]
+    end
+
+    def symbolize_keys_deep!(hash)
+      hash.keys.each do |k|
+        symkey = k.respond_to?(:to_sym) ? k.to_sym : k
+        hash[symkey] = hash.delete k
+        symbolize_keys_deep! hash[symkey] if hash[symkey].is_a? Hash
+      end
+    end
+
+    def parse_config(path)
+      erb = ERB.new(File.read(path))
+      erb.filename = File.expand_path(path)
+      opts = YAML.safe_load(erb.result, permitted_classes: [Symbol], aliases: true) || {}
+
+      symbolize_keys_deep!(opts)
+
+      opts = opts.merge(opts.delete(environment.to_sym) || {})
+      opts.delete(:strict)
+
+      opts
+    rescue => error
+      raise ArgumentError, "Error parsing config file: #{error.message}"
+    end
+
+    def ping_redis
+      config.with_sync_redis { |conn| conn.call('PING') }
+      true
+    rescue => err
+      logger.fatal("Redis connection error: #{err.message}")
+      false
+    end
+
+    def load_app
+      if config[:require].nil? || config[:require].empty?
+        logger.warn('No require option specified. Please specify a Ruby file to require with --require')
+        # allow to start empty server
+        return
+      end
+      # the code required by this file is expected to call Jiggler.configure
+      # thus it'll be executed in the context of the current process
+      # and apply the configuration for the server
+      require config[:require]
+    rescue LoadError => e
+      logger.fatal("Could not load jobs: #{e.message}")
+      exit(1)
+    end
+  end
+end

data/lib/jiggler/config.rb

@@ -0,0 +1,165 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+require 'logger'
+
+module Jiggler
+  class Config
+    extend Forwardable
+
+    DEFAULT_QUEUE = 'default'
+    QUEUE_PREFIX = 'jiggler:list:'
+    SERVER_PREFIX = 'jiggler:svr:'
+    RETRIES_SET = 'jiggler:set:retries'
+    SCHEDULED_SET = 'jiggler:set:scheduled'
+    DEAD_SET = 'jiggler:set:dead'
+    PROCESSED_COUNTER = 'jiggler:stats:processed_counter'
+    FAILURES_COUNTER = 'jiggler:stats:failures_counter'
+
+    DEFAULTS = {
+      require: nil,
+      environment: 'development',
+      concurrency: 10,
+      timeout: 25,
+      max_dead_jobs: 10_000,
+      stats_interval: 10,
+      poller_enabled: true,
+      poll_interval: 5,
+      dead_timeout: 180 * 24 * 60 * 60, # 6 months in seconds
+      # client settings
+      client_concurrency: 10,
+      client_redis_pool: nil,
+      client_async: false,
+    }
+
+    def initialize(options = {})
+      @options = DEFAULTS.merge(options)
+      @options[:redis_url] = ENV['REDIS_URL'] if @options[:redis_url].nil? && ENV['REDIS_URL']
+      @options[:queues] ||= [DEFAULT_QUEUE]
+      @directory = {}
+    end
+
+    def queue_prefix
+      QUEUE_PREFIX
+    end
+
+    def retries_set
+      RETRIES_SET
+    end
+
+    def scheduled_set
+      SCHEDULED_SET
+    end
+
+    def dead_set
+      DEAD_SET
+    end
+
+    def default_queue
+      DEFAULT_QUEUE
+    end
+
+    # jiggler main process prefix
+    def server_prefix
+      SERVER_PREFIX
+    end
+
+    def processed_counter
+      PROCESSED_COUNTER
+    end
+
+    def failures_counter
+      FAILURES_COUNTER
+    end
+
+    def process_scan_key
+      @process_scan_key ||= "#{server_prefix}*"
+    end
+
+    def queue_scan_key
+      @queue_scan_key ||= "#{queue_prefix}*"
+    end
+
+    def prefixed_queues
+      @prefixed_queues ||= @options[:queues].map do |name| 
+        "#{QUEUE_PREFIX}#{name}" 
+      end
+    end
+
+    def with_async_redis
+      Async do
+        redis_pool.acquire do |conn|
+          yield conn
+        end
+      end
+    end
+
+    def with_sync_redis
+      Sync do
+        redis_pool.acquire do |conn|
+          yield conn
+        end
+      end
+    end
+
+    def redis_options
+      @redis_options ||= begin
+        opts = @options.slice(
+          :concurrency,
+          :redis_url
+        )
+
+        opts[:concurrency] += 2 # monitor + safety margin
+        opts[:concurrency] += 1 if @options[:poller_enabled]
+        opts[:async] = true
+
+        opts
+      end
+    end
+
+    def client_redis_options
+      @client_redis_options ||= begin
+        opts = @options.slice(
+          :redis_url,
+          :client_redis_pool
+        )
+
+        opts[:concurrency] = @options[:client_concurrency]
+        opts[:async] = @options[:client_async]
+        opts
+      end
+    end
+
+    def redis_pool
+      @redis_pool ||= Jiggler::RedisStore.new(redis_options).pool
+    end
+
+    def client_redis_pool
+      @client_redis_pool ||= begin
+        @options[:client_redis_pool] || Jiggler::RedisStore.new(client_redis_options).pool
+      end
+    end
+
+    def client_redis_pool=(new_pool)
+      @client_redis_pool = new_pool
+    end
+
+    def cleaner
+      @cleaner ||= Jiggler::Cleaner.new(self)
+    end
+
+    def summary
+      @summary ||= Jiggler::Summary.new(self)
+    end
+
+    def logger=(new_logger)
+      @logger = new_logger
+    end
+
+    def logger
+      @logger ||= ::Logger.new(STDOUT, level: :info)
+    end
+    
+    def_delegators :@options, :[], :[]=, :fetch, :key?, :has_key?, :merge!, :delete, :slice
+  end
+end

data/lib/jiggler/core.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require 'oj'
+require 'securerandom'
+
+module Jiggler  
+  def self.config
+    @config ||= Jiggler::Config.new
+  end
+
+  def self.logger
+    config.logger
+  end
+
+  def self.configure(&block)
+    block.call(config)
+  end
+
+  def self.summary
+    config.summary.all
+  end
+end

data/lib/jiggler/errors.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Jiggler
+  class UnknownJobError < StandardError; end
+end

data/lib/jiggler/job.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+module Jiggler
+  module Job
+    module ClassMethods
+      def enqueue(*args)
+        Enqueuer.new(self).enqueue(*args)
+      end
+
+      def enqueue_in(seconds, *args)
+        Enqueuer.new(self).enqueue_in(seconds, *args)
+      end
+
+      # MyJob.with_options(queue: 'custom', retries: 3).enqueue(*args)
+      def with_options(options)
+        Enqueuer.new(self, options)
+      end
+
+      def enqueue_bulk(args_arr)
+        Enqueuer.new(self).enqueue_bulk(args_arr)
+      end
+
+      def queue
+        @queue || Jiggler::Config::DEFAULT_QUEUE
+      end
+
+      def retry_queue
+        @retry_queue || Jiggler::Config::DEFAULT_QUEUE
+      end
+
+      def retries
+        @retries || 0
+      end
+
+      def job_options(queue: nil, retries: nil, retry_queue: nil)
+        @queue = queue || Jiggler::Config::DEFAULT_QUEUE
+        @retries = retries || 0
+        @retry_queue = retry_queue || queue
+      end
+    end
+
+    class Enqueuer
+      def initialize(klass, options = {})
+        @options = options
+        @klass = klass
+      end
+
+      def with_options(options)
+        @options.merge(options)
+        self
+      end
+
+      def enqueue(*args)
+        Jiggler.config.client_redis_pool.acquire do |conn|
+          conn.call('LPUSH', list_name, job_args(args))
+        end
+      end
+
+      def enqueue_bulk(args_arr)
+        Jiggler.config.client_redis_pool.acquire do |conn|
+          conn.pipelined do |pipeline|
+            args_arr.each do |args|
+              pipeline.call('LPUSH', list_name, job_args(args))
+            end
+          end
+        end
+      end
+
+      def enqueue_in(seconds, *args)
+        timestamp = Time.now.to_f + seconds
+        Jiggler.config.client_redis_pool.acquire do |conn|
+          conn.call(
+            'ZADD',
+            config.scheduled_set, 
+            timestamp, 
+            job_args(args)
+          )
+        end
+      end
+
+      def list_name
+        "#{config.queue_prefix}#{@options.fetch(:queue, @klass.queue)}"
+      end
+
+      def job_args(raw_args)
+        Oj.dump({ name: @klass.name, args: raw_args, **job_options }, mode: :compat)
+      end
+
+      def job_options
+        retries = @options.fetch(:retries, @klass.retries)
+        jid = @options.fetch(:jid, SecureRandom.hex(8))
+        { retries: retries, jid: jid }
+      end
+
+      def config
+        @config ||= Jiggler.config
+      end
+    end
+
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    def enqueue(*args)
+      Enqueuer.new(self.class).enqueue(*args)
+    end
+
+    def enqueue_in(seconds, *args)
+      Enqueuer.new(self.class).enqueue_in(seconds, *args)
+    end
+
+    def perform(**args)
+      raise "#{self.class} must implement 'perform' method"
+    end
+  end
+end

data/lib/jiggler/launcher.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Jiggler
+  class Launcher
+    include Support::Helper
+
+    attr_reader :config
+
+    def initialize(config)
+      @done = false
+      @config = config
+    end
+
+    def start
+      poller.start if config[:poller_enabled]
+      monitor.start
+      manager.start
+    end
+
+    def suspend
+      return if @done
+
+      @done = true
+      manager.suspend
+
+      poller.terminate if config[:poller_enabled]
+      monitor.terminate
+    end
+
+    def stop
+      suspend
+      manager.terminate
+    end
+
+    private
+
+    def uuid
+      @uuid ||= begin
+        data_str = [
+          SecureRandom.hex(6),
+          config[:concurrency],
+          config[:timeout],
+          config[:queues].join(','),
+          config[:poller_enabled] ? '1' : '0',
+          Time.now.to_i,
+          Process.pid,
+          ENV['DYNO'] || Socket.gethostname
+        ].join(':')
+        "#{config.server_prefix}#{data_str}"
+      end
+    end
+
+    def collection
+      @collection ||= Stats::Collection.new(uuid)
+    end
+
+    def manager
+      @manager ||= Manager.new(config, collection)
+    end
+
+    def poller
+      @poller ||= Scheduled::Poller.new(config)
+    end
+
+    def monitor
+      @monitor ||= Stats::Monitor.new(config, collection)
+    end
+  end
+end

data/lib/jiggler/manager.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Jiggler
+  class Manager
+    include Support::Helper
+
+    def initialize(config, collection)
+      @workers = Set.new
+      @done = false
+      @config = config
+      @timeout = @config[:timeout]
+      @collection = collection
+      @config[:concurrency].times do
+        @workers << init_worker
+      end
+    end
+
+    def start
+      @workers.each(&:run)
+    end
+
+    def suspend
+      return if @done
+
+      @done = true
+      @workers.each(&:suspend)
+    end
+
+    def terminate
+      suspend
+      schedule_shutdown
+      wait_for_workers
+    end
+
+    private
+
+    def wait_for_workers
+      logger.info('Waiting for workers to finish...')
+      @workers.each(&:wait)
+      @shutdown_task.stop
+    end
+
+    def schedule_shutdown
+      @shutdown_task = Async do
+        sleep(@timeout)
+
+        next if @workers.empty?
+
+        hard_shutdown
+      end
+    end
+
+    def init_worker
+      Jiggler::Worker.new(
+        @config, @collection, &method(:process_worker_result)
+      )
+    end
+
+    def process_worker_result(worker, reason = nil)
+      @workers.delete(worker)
+      unless @done
+        new_worker = init_worker
+        @workers << new_worker
+        new_worker.run
+      end
+    end
+
+    def hard_shutdown
+      logger.warn('Hard shutdown, terminating workers...')
+      @workers.each(&:terminate)
+    end
+  end
+end