anway 0.1.1

data/lib/anway/connection.rb

@@ -0,0 +1,195 @@
+# frozen_string_literal: true
+
+require 'digest'
+require 'json'
+require 'openssl'
+require 'socket'
+
+module Anway
+  # One socket. Timeouts use nonblocking calls, so that plain and TLS sockets
+  # follow the same path.
+  class Connection
+    PROTOCOL_VERSION = 2
+    READ_SIZE = 16_384
+
+    def initialize(endpoint, timeout:, tls: nil, wid: nil, labels: [])
+      @timeout = timeout
+      @buffer = String.new(encoding: Encoding::BINARY)
+      @io = open_socket(endpoint, tls)
+
+      handshake(endpoint, wid, labels)
+    rescue StandardError
+      close
+
+      raise
+    end
+
+    # The argument is a string, or an object that becomes JSON.
+    def command(verb, argument = nil, wait: 0)
+      deadline = monotonic + @timeout + wait
+      line = argument.nil? ? verb : "#{verb} #{argument.is_a?(String) ? argument : JSON.generate(argument)}"
+
+      write_all("#{line}\r\n", deadline)
+      read_reply(deadline)
+    rescue SystemCallError, IOError, SocketError, OpenSSL::SSL::SSLError => e
+      raise ConnectionError, e.message
+    end
+
+    def close
+      @io&.close
+    rescue SystemCallError, IOError, OpenSSL::SSL::SSLError
+      nil
+    ensure
+      @io = nil
+    end
+
+    private
+
+    def open_socket(endpoint, tls)
+      socket = Socket.tcp(endpoint.host, endpoint.port, connect_timeout: @timeout, resolv_timeout: @timeout)
+
+      return socket unless endpoint.tls
+
+      wrap_tls(socket, endpoint.host, tls)
+    rescue SystemCallError, SocketError, OpenSSL::SSL::SSLError => e
+      raise ConnectionError, e.message
+    end
+
+    def wrap_tls(socket, host, tls)
+      context = OpenSSL::SSL::SSLContext.new
+
+      context.set_params(tls || {})
+
+      secure = OpenSSL::SSL::SSLSocket.new(socket, context)
+
+      secure.hostname = host
+      secure.sync_close = true
+
+      connect_tls(secure)
+
+      secure
+    end
+
+    def connect_tls(secure)
+      deadline = monotonic + @timeout
+
+      loop do
+        case secure.connect_nonblock(exception: false)
+        when :wait_readable then wait(secure, deadline, :read)
+        when :wait_writable then wait(secure, deadline, :write)
+        else return
+        end
+      end
+    end
+
+    def handshake(endpoint, wid, labels)
+      deadline = monotonic + @timeout
+      greeting = JSON.parse(read_reply(deadline).delete_prefix('HI '))
+
+      raise Error, "unsupported protocol version #{greeting['v']}" unless greeting['v'] == PROTOCOL_VERSION
+
+      hello = { v: PROTOCOL_VERSION }
+
+      if greeting['s']
+        hello[:pwdhash] = password_hash(endpoint.password.to_s, greeting['s'], greeting['i'])
+      end
+
+      if wid
+        hello.merge!(wid: wid, hostname: Socket.gethostname, pid: Process.pid, labels: labels)
+      end
+
+      command('HELLO', hello)
+    end
+
+    def password_hash(password, salt, iterations)
+      digest = Digest::SHA256.digest(password + salt)
+
+      (iterations - 1).times { digest = Digest::SHA256.digest(digest) }
+
+      digest.unpack1('H*')
+    end
+
+    def write_all(data, deadline)
+      data = data.b
+
+      until data.empty?
+        written = @io.write_nonblock(data, exception: false)
+
+        case written
+        when :wait_readable then wait(@io, deadline, :read)
+        when :wait_writable then wait(@io, deadline, :write)
+        else data = data.byteslice(written..)
+        end
+      end
+    end
+
+    def read_reply(deadline)
+      line = read_line(deadline)
+
+      raise ConnectionError, 'empty reply' if line.empty?
+
+      body = line[1..]
+
+      case line[0]
+      when '+' then body.force_encoding(Encoding::UTF_8)
+      when '-' then raise server_error(body)
+      when '$' then read_bulk(Integer(body), deadline)
+      else raise ConnectionError, "unexpected reply: #{line.inspect}"
+      end
+    end
+
+    def server_error(body)
+      code, message = body.force_encoding(Encoding::UTF_8).split(' ', 2)
+
+      return NotUnique.new if code == 'NOTUNIQUE'
+
+      ServerError.new(code, message)
+    end
+
+    def read_line(deadline)
+      until (index = @buffer.index("\n"))
+        fill(deadline)
+      end
+
+      @buffer.slice!(0..index).chomp
+    end
+
+    def read_bulk(size, deadline)
+      return nil if size.negative?
+
+      fill(deadline) while @buffer.bytesize < size + 2
+
+      body = @buffer.slice!(0, size)
+
+      @buffer.slice!(0, 2)
+
+      body.force_encoding(Encoding::UTF_8)
+    end
+
+    def fill(deadline)
+      chunk = @io.read_nonblock(READ_SIZE, exception: false)
+
+      case chunk
+      when :wait_readable then wait(@io, deadline, :read)
+      when :wait_writable then wait(@io, deadline, :write)
+      when nil then raise ConnectionError, 'server closed the connection'
+      else @buffer << chunk
+      end
+    end
+
+    def wait(io, deadline, mode)
+      remaining = deadline - monotonic
+
+      raise ConnectionError, 'timeout' unless remaining.positive?
+
+      readers = mode == :read ? [io] : nil
+      writers = mode == :write ? [io] : nil
+
+      IO.select(readers, writers, nil, remaining)
+    end
+
+    def monotonic
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+  end
+end

data/lib/anway/errors.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Anway
+  class Error < StandardError; end
+
+  class NotUnique < Error
+    def initialize(message = 'job not unique')
+      super
+    end
+  end
+
+  class Closed < Error
+    def initialize(message = 'client is closed')
+      super
+    end
+  end
+
+  class ConnectionError < Error; end
+
+  class UnknownJob < Error; end
+
+  class ServerError < Error
+    attr_reader :code
+
+    def initialize(code, message)
+      @code = code
+
+      super("#{code} #{message}")
+    end
+  end
+
+  # Inherits from Exception, so that `rescue => e` in a job does not catch it.
+  class Shutdown < Exception; end # rubocop:disable Lint/InheritException
+end

data/lib/anway/executor.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Anway
+  # Runs one job through the reloader and the middleware.
+  module Executor
+    module_function
+
+    def call(job)
+      Anway.config.reloader.call do
+        klass = Worker.find(job.type)
+
+        chain(job) do
+          instance = klass.new
+
+          instance.job = job
+          instance.perform(*job.args)
+        end
+      end
+    end
+
+    def chain(job, &block)
+      runner = Anway.config.middleware.reverse.reduce(block) do |nextone, middleware|
+        -> { middleware.call(job) { nextone.call } }
+      end
+
+      runner.call
+    end
+  end
+end

data/lib/anway/job.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'securerandom'
+require 'time'
+
+module Anway
+  class Job
+    ATTRIBUTES = %w[queue reserve_for at retry backtrace created_at enqueued_at failure].freeze
+
+    attr_accessor :jid, :type, :args, :custom, *ATTRIBUTES.map(&:to_sym)
+
+    def self.from_hash(hash)
+      job = allocate
+
+      job.jid = hash['jid']
+      job.type = hash['jobtype']
+      job.args = hash['args'] || []
+      job.custom = hash['custom']
+
+      ATTRIBUTES.each { |name| job.public_send("#{name}=", hash[name]) }
+
+      job
+    end
+
+    # Options: queue, retry, reserve_for, backtrace, at, custom, jid.
+    def initialize(type, args = [], **options)
+      validate_json!(args)
+
+      @jid = options[:jid] || SecureRandom.hex(12)
+      @type = type
+      @args = args
+      @queue = options[:queue]&.to_s
+      @retry = options[:retry]
+      @reserve_for = options[:reserve_for]
+      @backtrace = options[:backtrace]
+      @custom = options[:custom]&.transform_keys(&:to_s)
+
+      run_at(options[:at]) if options[:at]
+    end
+
+    def run_at(time)
+      @at = time.to_time.utc.iso8601(9)
+
+      self
+    end
+
+    def set_custom(key, value)
+      @custom = (@custom || {}).merge(key.to_s => value)
+
+      self
+    end
+
+    def to_h
+      {
+        'jid' => jid, 'jobtype' => type, 'args' => args, 'custom' => custom,
+        **ATTRIBUTES.to_h { |name| [name, public_send(name)] }
+      }.compact
+    end
+
+    private
+
+    # Other languages read these args, so only JSON types are valid.
+    def validate_json!(value)
+      case value
+      when nil, true, false, String, Integer, Float
+        nil
+      when Array
+        value.each { |item| validate_json!(item) }
+      when Hash
+        value.each do |key, item|
+          raise ArgumentError, "job hash keys must be strings: #{key.inspect}" unless key.is_a?(String)
+
+          validate_json!(item)
+        end
+      else
+        raise ArgumentError, "job args must be JSON types: #{value.inspect}"
+      end
+    end
+  end
+end

data/lib/anway/pool.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module Anway
+  # Connections open at first use. A forked process starts with an empty pool
+  # and does not touch the sockets of its parent.
+  class Pool
+    # A wrapper, because a pop timeout also returns nil.
+    Slot = Data.define(:connection)
+
+    def initialize(size, &opener)
+      @size = size
+      @opener = opener
+      @lock = Mutex.new
+      @closed = false
+
+      reset
+    end
+
+    def with(timeout:)
+      connection = checkout(timeout)
+      healthy = false
+
+      begin
+        connection ||= @opener.call
+
+        result = yield connection
+        healthy = true
+
+        result
+      rescue ServerError, NotUnique
+        healthy = true
+
+        raise
+      ensure
+        checkin(connection, healthy)
+      end
+    end
+
+    def close
+      @lock.synchronize do
+        return if @closed
+
+        @closed = true
+      end
+
+      @slots.close
+
+      @size.times do
+        @slots.pop(timeout: 1)&.connection&.close
+      end
+    end
+
+    private
+
+    def reset
+      @pid = Process.pid
+      @slots = Thread::SizedQueue.new(@size)
+
+      @size.times { @slots.push(Slot.new(nil)) }
+    end
+
+    def checkout(timeout)
+      raise Closed if @closed
+
+      @lock.synchronize { reset unless @pid == Process.pid }
+
+      entry = @slots.pop(timeout: timeout)
+
+      raise Closed if entry.nil? && @slots.closed?
+      raise ConnectionError, 'no free connection' if entry.nil?
+
+      entry.connection
+    end
+
+    def checkin(connection, healthy)
+      connection&.close unless healthy
+
+      return connection&.close if @slots.closed?
+
+      @slots.push(Slot.new(healthy ? connection : nil))
+    rescue ClosedQueueError
+      connection&.close
+    end
+  end
+end

data/lib/anway/queues.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Anway
+  # Entries are names, or [name, weight] pairs. Without weights, the order is
+  # strict. With weights, each fetch uses a new random order.
+  class Queues
+    attr_reader :names
+
+    def initialize(entries)
+      pairs = entries.map { |entry| Array(entry) }
+
+      @names = pairs.map { |name, _| name.to_s }.uniq
+      @weighted = pairs.any? { |pair| pair.size > 1 }
+      @pool = pairs.flat_map { |name, weight| [name.to_s] * Integer(weight || 1) }
+    end
+
+    def order
+      @weighted ? @pool.shuffle.uniq : @names
+    end
+  end
+end

data/lib/anway/railtie.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Anway
+  class Railtie < Rails::Railtie
+    # Rails looks up the adapter in active_job.set_configs.
+    initializer 'anway.active_job', before: 'active_job.set_configs' do
+      require 'active_job/queue_adapters/anway_adapter' if defined?(ActiveJob)
+    end
+
+    initializer 'anway.transactions' do
+      ActiveSupport.on_load(:active_record) do
+        Anway.config.push_wrapper = ->(&block) { ActiveRecord.after_all_transactions_commit(&block) }
+      end
+    end
+
+    config.after_initialize do |app|
+      Anway.config.logger = Rails.logger if Rails.logger
+      Anway.config.reloader = ->(&block) { app.reloader.wrap(&block) }
+
+      Anway.config.on_error { |error, _job| Rails.error.report(error, handled: true, source: 'anway') }
+    end
+  end
+end