quiq 0.2.0

data/bin/quiq

@@ -0,0 +1,42 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative '../lib/quiq'
+require 'optparse'
+
+options = { path: Dir.pwd, queues: %w[default], log_level: Logger::DEBUG }
+OptionParser.new do |opts|
+  opts.banner = 'Usage: quiq [options]'
+
+  opts.on('-p', '--path PATH', 'Location of the workers to load') do |path|
+    options[:path] = File.expand_path(path)
+  end
+
+  opts.on('-q', '--queues NAMES', Array,
+          'Comma-separated list of queues to poll') do |queues|
+    options[:queues] = queues
+  end
+
+  opts.on('-l', '--log-level LEVEL', %i[debug info warn error],
+          'The logging level') do |level|
+    options[:log_level] = level
+  end
+
+  opts.on '-v', '--version', 'Output version and exit' do
+    puts "Quiq #{Quiq::VERSION}"
+    exit
+  end
+
+  opts.on_tail('-h', '--help', 'Show this message') do
+    puts opts
+    exit
+  end
+end.parse!
+
+begin
+  Quiq.boot(options)
+rescue StandardError => e
+  warn e.message
+  warn e.backtrace.join("\n")
+  exit 1
+end

data/bin/quiqload

@@ -0,0 +1,46 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Usage: time RUBYOPT="-W0" bin/quiqload
+
+require_relative '../lib/quiq'
+require_relative '../testapp/config/environment.rb'
+require 'optparse'
+
+options = { number: 10_000, wait: 1 }
+OptionParser.new do |opts|
+  opts.banner = 'Usage: quiqload [options]'
+
+  opts.on('-n', '--number JOBS', Integer, 'Number of jobs to enqueue') do |number|
+    options[:number] = number.to_i
+  end
+
+  opts.on('-w', '--wait DURATION', Integer,
+          'Idle time within each job (in seconds)') do |wait|
+    options[:wait] = wait.to_i
+  end
+
+  opts.on_tail('-h', '--help', 'Show this message') do
+    puts opts
+    exit
+  end
+end.parse!
+
+Quiq.logger.info("Enqueuing #{options[:number]} jobs")
+options[:number].times { |i| TestJob.perform_later(i, options[:wait]) }
+
+Thread.new do
+  loop do
+    queue_size = Async do
+      queue = Quiq::Queue.processing_name('default')
+      Quiq.redis.llen queue
+    end.wait
+
+    if queue_size.zero?
+      Quiq.logger.info("Done processing #{options[:number]} jobs")
+      break
+    end
+
+    sleep 0.1
+  end
+end.join

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/quiq.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require_relative 'quiq/version'
+require_relative 'quiq/config'
+require_relative 'quiq/server'
+require_relative 'quiq/client'
+require 'async/redis'
+
+module Quiq
+  extend self
+
+  def configuration
+    Config.instance
+  end
+
+  def configure
+    yield(configuration) if block_given?
+  end
+
+  def redis
+    configuration.redis.client
+  end
+
+  def boot(options)
+    configuration.parse_options(**options)
+
+    # Load the workers source code
+    path = configuration.path
+    if File.directory?(path)
+      Dir.glob(File.join(path, '*.rb')).each { |file| require file }
+    else
+      require path
+    end
+
+    Server.instance.run!
+  end
+
+  def queues
+    configuration.queues
+  end
+
+  def current_task
+    Async::Task.current
+  end
+
+  def logger
+    configuration.logger
+  end
+end

data/lib/quiq/client.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'uri'
+
+module Quiq
+  class Client
+    def push(job, scheduled_at)
+      serialized_job = JSON.dump(job.serialize)
+
+      if scheduled_at
+        Async { Scheduler.enqueue_at(serialized_job, scheduled_at) }
+      else
+        Async { Queue.push(job.queue_name, serialized_job) }
+      end
+    end
+
+    def self.push(job, scheduled_at: nil)
+      new.push(job, scheduled_at)
+    end
+  end
+end

data/lib/quiq/config.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require 'logger'
+require 'singleton'
+require_relative 'redis'
+
+module Quiq
+  class Config
+    include Singleton
+
+    attr_reader :queues, :path
+    attr_writer :logger
+
+    def redis=(server)
+      @redis = Redis.new(server)
+    end
+
+    def redis
+      @redis ||= Redis.new
+    end
+
+    def logger
+      @logger ||= begin
+        level = @log_level || Logger::DEBUG
+        ::Logger.new(STDOUT, level: level)
+      end
+    end
+
+    def parse_options(path:, queues:, log_level:)
+      @path = path
+      @queues = queues
+      @log_level = log_level
+    end
+  end
+end

data/lib/quiq/job.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Quiq
+  class Job
+    def initialize(raw, queue)
+      @raw = raw
+      @queue = queue
+    end
+
+    def run
+      Async do
+        begin
+          # First parse the raw message from redis
+          payload = JSON.parse(@raw)
+
+          # Then load the definition of the job + its arguments
+          klass = Object.const_get(payload['job_class'])
+          args = payload['arguments']
+
+          # Then run the task
+          klass.new.perform(*args)
+        rescue JSON::ParserError => e
+          Quiq.logger.warn("Invalid format: #{e}")
+          send_to_dlq(@raw, e)
+        rescue StandardError => e
+          Quiq.logger.debug("Sending message to DLQ: #{e}")
+          send_to_dlq(payload, e)
+        ensure
+          # Remove the job from the processing list
+          Queue.delete(@queue.processing, @raw)
+        end
+      end
+    end
+
+    private
+
+    def send_to_dlq(payload, exception)
+      if payload.is_a?(Hash)
+        payload['error'] = exception.to_s
+        payload['backtrace'] = exception.backtrace
+        message = JSON.dump(payload)
+      else
+        message = @raw
+      end
+
+      Queue.send_to_dlq(message)
+    end
+  end
+end

data/lib/quiq/queue.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Quiq
+  class Queue
+    PREFIX = 'queue'
+    PROCESSING_SUFFIX = 'processing'
+    DEAD_LETTER_QUEUE = 'dead'
+
+    attr_reader :name, :processing
+
+    def initialize(name)
+      @name = self.class.formatted_name(name)
+      @processing = self.class.processing_name(name)
+    end
+
+    def push(job)
+      pushed = Quiq.redis.lpush(@name, job)
+      return unless pushed <= 0
+
+      Quiq.logger.error("Could not push to the queue: #{@name}")
+      false
+    end
+
+    def pop
+      Quiq.redis.brpoplpush(@name, @processing, 0)
+    end
+
+    # Insert elements that weren't fully processed at the tail of the queue to avoid loss
+    # @note that they should be enqueued at the head of the queue, but Redis lacks a LPOPRPUSH command
+    def purge_processing!
+      Async do
+        Quiq.redis.pipeline do |pipe|
+          loop do
+            job = pipe.sync.call('RPOPLPUSH', @processing, @name)
+            Quiq.logger.warn("Requeuing job #{job} in #{@name}") unless job.nil?
+            break if job.nil?
+          end
+          pipe.close
+        end
+      end.wait
+    end
+
+    def self.push(queue, job)
+      @queue = new(queue)
+      @queue.push(job)
+    end
+
+    def self.delete(queue, job)
+      Quiq.redis.lrem(queue, 0, job)
+    end
+
+    def self.formatted_name(name)
+      "#{PREFIX}:#{name}"
+    end
+
+    def self.processing_name(name)
+      "#{PREFIX}:#{name}:#{PROCESSING_SUFFIX}"
+    end
+
+    def self.send_to_dlq(job)
+      @dlq ||= Queue.new(DEAD_LETTER_QUEUE)
+      @dlq.push(job)
+    end
+  end
+end

data/lib/quiq/redis.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require 'uri'
+
+module Quiq
+  class Redis
+    DEFAULT_REDIS_URL = 'redis://localhost:6379'
+
+    attr_reader :client
+
+    def initialize(server = DEFAULT_REDIS_URL)
+      uri = URI(server)
+      endpoint = Async::IO::Endpoint.tcp(uri.host, uri.port)
+      @client = Async::Redis::Client.new(endpoint)
+    end
+  end
+end

data/lib/quiq/scheduler.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require 'singleton'
+
+module Quiq
+  class Scheduler
+    include Singleton
+
+    SCHEDULER_KEY = 'quiq:schedule'
+
+    def start
+      # Set the process name
+      Process.setproctitle('quiq scheduler')
+
+      Async do
+        loop do
+          sleep 0.2
+
+          # TODO: use ZRANGEBYSCORE instead to batch enqueuing
+          job, scheduled_at = Quiq.redis.zrange(
+            SCHEDULER_KEY, 0, 0, with_scores: true
+          )
+
+          enqueue(job) if job && scheduled_at.to_f <= Time.now.to_f
+        end
+      ensure
+        Quiq.redis.close
+      end
+    end
+
+    def self.enqueue_at(job, scheduled_at)
+      Quiq.redis.zadd(SCHEDULER_KEY, scheduled_at, job)
+    end
+
+    private
+
+    # Push the job in its queue and remove from scheduler_queue
+    def enqueue(job)
+      begin
+        payload = JSON.parse(job)
+      rescue JSON::ParserError => e
+        Quiq.logger.warn("Invalid format: #{e}")
+        Queue.send_to_dlq(job)
+      end
+
+      # TODO: wrap those 2 calls in a transaction
+      Queue.push(payload['queue_name'], job)
+      Quiq.redis.zrem(SCHEDULER_KEY, job)
+    end
+  end
+end

data/lib/quiq/server.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require 'singleton'
+require 'async/redis'
+require_relative 'worker'
+require_relative 'scheduler'
+
+module Quiq
+  class Server
+    include Singleton
+
+    def run!
+      # Launch one worker per queue
+      Quiq.queues.each do |queue|
+        fork { Worker.new(queue).start }
+      end
+
+      # Launch scheduler for jobs to be performed at certain time
+      fork { Scheduler.instance.start }
+
+      # Set the process name
+      Process.setproctitle("quiq master #{Quiq.configuration.path}")
+
+      # TODO: handle graceful shutdowns
+      Process.waitall
+    end
+  end
+end

data/lib/quiq/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Quiq
+  VERSION = '0.2.0'
+end

data/lib/quiq/worker.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require_relative 'job'
+require_relative 'queue'
+
+module Quiq
+  class Worker
+    def initialize(queue)
+      @queue = Queue.new(queue)
+    end
+
+    def start
+      # Set the process name
+      Process.setproctitle("quiq worker #{@queue.name}")
+
+      # Reschedule jobs that get terminated before completion
+      # Beware that the jobs must be idempotent!
+      @queue.purge_processing!
+
+      # Then start processing enqueued jobs
+      Async do
+        loop { Job.new(@queue.pop, @queue).run }
+      ensure
+        Quiq.redis.close
+      end
+    end
+  end
+end