tuber 0.0.1 → 0.5.1

data/Rakefile

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require 'rake/testtask'
+require 'yard'
+require 'redcarpet'
+
+# rake test
+Rake::TestTask.new do |t|
+  t.libs.push "lib"
+  t.test_files = FileList[File.expand_path('../test/**/*_test.rb', __FILE__)] -
+    FileList[File.expand_path('../test/**/tuber_test.rb', __FILE__)]
+  t.verbose = true
+end
+
+# rake test:integration
+Rake::TestTask.new("test:integration") do |t|
+  t.libs.push "lib"
+  t.test_files = FileList[File.expand_path('../test/**/tuber_test.rb', __FILE__)]
+  t.verbose = true
+end
+
+# rake test:full
+Rake::TestTask.new("test:full") do |t|
+  t.libs.push "lib"
+  t.test_files = FileList[File.expand_path('../test/**/*_test.rb', __FILE__)]
+  t.verbose = true
+end
+
+YARD::Rake::YardocTask.new do |t|
+  t.files   = ['lib/tuber/**/*.rb']
+  t.options = []
+end
+
+task :default => 'test:full'

data/TODO

@@ -0,0 +1 @@
+- Remove connection from pool if it's not responding and be able to add more connections and reattempt later

data/examples/demo.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+require 'term/ansicolor'
+class String; include Term::ANSIColor; end
+def step(msg); "\n[STEP] #{msg}...".yellow; end
+$:.unshift("../lib")
+require 'tuber'
+
+# Establish a pool of beanstalks
+puts step("Connecting to Beanstalk")
+bc = Tuber.new('localhost')
+puts bc
+
+# Print out key stats
+puts step("Print Stats")
+p bc.stats.keys
+p [bc.stats.total_connections, bc.stats[:total_connections], bc.stats['total_connections']]
+
+# find tube
+puts step("Find tube")
+tube = bc.tubes.find('tube2')
+puts tube
+
+# Put job onto tube
+puts step("Put job")
+response = tube.put "foo bar", :pri => 1000, :ttr => 10, :delay => 0
+puts response
+
+# peek tube
+puts step("Peek tube")
+p tube.peek :ready
+
+# watch tube
+bc.tubes.watch!('tube2')
+
+# Check tube stats
+puts step("Get tube stats")
+p tube.stats.keys
+p tube.stats.name
+p tube.stats.current_jobs_ready
+
+# Reserve job from tube
+puts step("Reserve job")
+p job = bc.tubes.reserve
+jid = job.id
+
+# pause tube
+puts step("Pause tube")
+p tube.pause(1)
+
+# Register jobs
+puts step("Register jobs for tubes")
+bc.jobs.register('tube_test', :retry_on => [Timeout::Error]) do |job|
+ p 'tube_test'
+ p job
+ raise Tuber::AbortProcessingError
+end
+
+bc.jobs.register('tube_test2', :retry_on => [Timeout::Error]) do |job|
+ p 'tube_test2'
+ p job
+ raise Tuber::AbortProcessingError
+end
+
+p bc.jobs.processors
+
+response = bc.tubes.find('tube_test').put "foo register", :pri => 1000, :ttr => 10, :delay => 0
+response = bc.tubes.find('tube_test2').put "foo baz", :pri => 1000, :ttr => 10, :delay => 0
+
+# Process jobs
+puts step("Process jobs")
+2.times { bc.jobs.process! }
+
+# Get job from id (peek job)
+puts step("Get job from id")
+p bc.jobs.find(jid)
+p bc.jobs.peek(jid)
+
+# Check job stats
+puts step("Get job stats")
+p job.stats.keys
+p job.stats.tube
+p job.stats.state
+
+# bury job
+puts step("Bury job")
+p job.bury
+
+# delete job
+puts step("Delete job")
+p job.delete
+
+# list tubes
+puts step("List tubes")
+p bc.tubes.watched
+p bc.tubes.used
+p bc.tubes.all

data/lib/tuber/configuration.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+class Tuber
+  class Configuration
+    attr_accessor :default_put_delay   # default delay value to put a job
+    attr_accessor :default_put_pri     # default priority value to put a job
+    attr_accessor :default_put_ttr     # default ttr value to put a job
+    attr_accessor :job_parser          # default job_parser to parse job body
+    attr_accessor :job_serializer      # default serializer for job body
+    attr_accessor :tuber_url           # default server url
+    alias_method :beanstalkd_url,  :tuber_url   # compatibility with beaneater configs
+    alias_method :beanstalkd_url=, :tuber_url=
+    attr_accessor :connect_timeout     # TCP connect timeout in seconds
+    attr_accessor :resolv_timeout      # DNS resolve timeout in seconds
+    attr_accessor :read_timeout        # socket read timeout in seconds
+    attr_accessor :write_timeout       # socket write timeout in seconds
+
+    def initialize
+      @default_put_delay   = 0
+      @default_put_pri     = 65536
+      @default_put_ttr     = 120
+      @job_parser          = lambda { |body| body }
+      @job_serializer      = lambda { |body| body }
+      @tuber_url           = nil
+      @connect_timeout     = nil
+      @resolv_timeout      = nil
+      @read_timeout        = nil
+      @write_timeout       = nil
+    end
+  end # Configuration
+end # Tuber

data/lib/tuber/connection.rb

@@ -0,0 +1,366 @@
+# frozen_string_literal: true
+
+require 'yaml'
+require 'socket'
+
+class Tuber
+  # Represents a connection to a beanstalkd instance.
+  class Connection
+
+    # Default number of retries to send a command to a connection
+    MAX_RETRIES = 3
+
+    # Default retry interval
+    DEFAULT_RETRY_INTERVAL = 1
+
+    # @!attribute address
+    #   @return [String] returns Beanstalkd server address
+    #   @example
+    #     @conn.address # => "localhost:11300"
+    # @!attribute host
+    #   @return [String] returns Beanstalkd server host
+    #   @example
+    #     @conn.host # => "localhost"
+    # @!attribute port
+    #  @return [Integer] returns Beanstalkd server port
+    #  @example
+    #    @conn.port # => "11300"
+    # @!attribute connection
+    #   @return [Net::TCPSocket] returns connection object
+    attr_reader :address, :host, :port, :connection
+
+    # @!attribute tubes_watched
+    #   @returns [Array<String>] returns currently watched tube names
+    # @!attribute tube_used
+    #   @returns [String] returns currently used tube name
+    attr_accessor :tubes_watched, :tube_used
+
+    # Default port value for beanstalk connection
+    DEFAULT_PORT = 11300
+
+    # Commands that must not be retransmitted after a dropped connection.
+    # The socket dying between write and readline leaves the first send's
+    # fate unknown: a re-sent put can insert a duplicate job, and a re-sent
+    # delete/release/bury/touch acts on a job whose reservation died with
+    # the old socket — the server answers NOT_FOUND for work that actually
+    # succeeded. For these verbs the connection is healed but the original
+    # error is re-raised so the caller decides. Everything else (reserve,
+    # watch, stats, peek, ...) converges to the same state on a re-send and
+    # keeps the transparent retry.
+    NON_IDEMPOTENT_COMMANDS = %w[put delete delete-batch release bury touch touch-all kick kick-job].freeze
+
+    # Initializes new connection.
+    #
+    # @param [String] address beanstalkd instance address.
+    # @example
+    #   Tuber::Connection.new('127.0.0.1')
+    #   Tuber::Connection.new('127.0.0.1:11300')
+    #
+    #   ENV['TUBER_URL'] = '127.0.0.1:11300'
+    #   @b = Tuber.new
+    #   @b.connection.host # => '127.0.0.1'
+    #   @b.connection.port # => '11300'
+    #
+    def initialize(address)
+      @address = address || _host_from_env || Tuber.configuration.tuber_url
+      @mutex = Mutex.new
+      @tube_used = 'default'
+      @tubes_watched = ['default']
+
+      establish_connection
+    rescue
+      _raise_not_connected!
+    end
+
+    # Send commands to beanstalkd server via connection.
+    #
+    # @param [Hash{String => String, Number}>] options Retained for compatibility
+    # @param [String] command Beanstalkd command
+    # @return [Array<Hash{String => String, Number}>] Beanstalkd command response
+    # @example
+    #   @conn = Tuber::Connection.new
+    #   @conn.transmit('bury 123')
+    #   @conn.transmit('stats')
+    #
+    def transmit(command, **options)
+      verb = command.to_s[/\A\S+/]
+      retransmit = !NON_IDEMPOTENT_COMMANDS.include?(verb)
+      _with_retry(retransmit: retransmit, **options.slice(:retry_interval, :init)) do
+        @mutex.synchronize do
+          _raise_not_connected! unless connection
+
+          command = command.dup.force_encoding('ASCII-8BIT') if command.respond_to?(:force_encoding)
+          connection.write(command.to_s + "\r\n")
+          res = connection.readline
+          parse_response(command, res)
+        end
+      end
+    end
+
+    # Reserves a batch of jobs atomically.
+    #
+    # Without a +timeout+ (or with +timeout+ of 0) the command is non-blocking:
+    # it returns whatever is ready immediately, possibly an empty array. With a
+    # positive +timeout+ it long-polls, blocking until the first job arrives (up
+    # to +timeout+ seconds) and then draining everything ready, up to +count+.
+    #
+    # @param [Integer] count Maximum number of jobs to reserve
+    # @param [Integer] timeout Seconds to long-poll for the first job (nil = non-blocking)
+    # @return [Array<Hash>] Array of job hashes with :status, :id, :body keys
+    # @raise [Tuber::DeadlineSoonError] A reserved job's TTR is about to expire
+    #
+    def reserve_batch(count, timeout = nil)
+      _with_retry do
+        @mutex.synchronize do
+          _raise_not_connected! unless connection
+
+          cmd = timeout ? "reserve-batch #{count} #{timeout}" : "reserve-batch #{count}"
+          connection.write(cmd + "\r\n")
+
+          header = connection.readline.chomp
+          status, actual_count_str = header.split(/\s/, 2)
+
+          raise UnexpectedResponse.from_status(status, cmd) unless status == "RESERVED_BATCH"
+
+          actual_count = actual_count_str.to_i
+          jobs = []
+          actual_count.times do
+            line = connection.readline.chomp
+            _, job_id, bytes_str = line.split(/\s/)
+            bytes = bytes_str.to_i
+            body = connection.read(bytes)
+            crlf = connection.read(2)
+            raise ExpectedCrlfError.new("EXPECTED_CRLF", cmd) unless crlf == "\r\n"
+
+            body = config.job_parser.call(body)
+            jobs << { status: "RESERVED", id: job_id, body: body }
+          end
+          jobs
+        end
+      end
+    end
+
+    # Deletes a batch of jobs atomically.
+    #
+    # @param [Array<Integer, String>] ids Job IDs to delete
+    # @return [Hash] with :deleted and :not_found counts
+    #
+    def delete_batch(ids)
+      _with_retry(retransmit: false) do
+        @mutex.synchronize do
+          _raise_not_connected! unless connection
+
+          cmd = "delete-batch #{ids.join(' ')}"
+          connection.write(cmd + "\r\n")
+
+          res = connection.readline.chomp
+          status, deleted, not_found = res.split(/\s/)
+
+          raise UnexpectedResponse.from_status(status, cmd) unless status == "DELETED_BATCH"
+
+          { deleted: deleted.to_i, not_found: not_found.to_i }
+        end
+      end
+    end
+
+    # Close connection with beanstalkd server.
+    #
+    # @example
+    #  @conn.close
+    #
+    def close
+      if @connection
+        @connection.close
+        @connection = nil
+      end
+    end
+
+    # Returns string representation of job.
+    #
+    # @example
+    #  @conn.inspect
+    #
+    def to_s
+      "#<Tuber::Connection host=#{host.inspect} port=#{port.inspect}>"
+    end
+    alias :inspect :to_s
+
+    def add_to_watched(tube_name)
+      @tubes_watched << tube_name
+      @tubes_watched.uniq
+    end
+
+    def remove_from_watched(tube_name)
+      @tubes_watched.delete(tube_name)
+    end
+
+    protected
+
+    # Establish a connection based on beanstalk address.
+    #
+    # @return [Net::TCPSocket] connection for specified address.
+    # @raise [Tuber::NotConnected] Could not connect to specified beanstalkd instance.
+    # @example
+    #  establish_connection('localhost:3005')
+    #
+    def establish_connection
+      @address = address.first if address.is_a?(Array)
+      match = address.split(':')
+      @host, @port = match[0], Integer(match[1] || DEFAULT_PORT)
+
+      tcp_opts = { connect_timeout: config.connect_timeout, resolv_timeout: config.resolv_timeout }.compact
+
+      socket = if RUBY_VERSION >= "3.0" && tcp_opts.any?
+        TCPSocket.new(@host, @port, **tcp_opts)
+      else
+        TCPSocket.new(@host, @port)
+      end
+
+      begin
+        socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_RCVTIMEO, _timeval_for(config.read_timeout)) if config.read_timeout
+        socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, _timeval_for(config.write_timeout)) if config.write_timeout
+        @connection = socket
+      rescue
+        socket.close rescue nil
+        raise
+      end
+    end
+
+    # Parses the response and returns the useful beanstalk response.
+    # Will read the body if one is indicated by the status.
+    #
+    # @param [String] cmd Beanstalk command transmitted
+    # @param [String] res Telnet command response
+    # @return [Array<Hash{String => String, Number}>] Beanstalk response with `status`, `id`, `body`
+    # @raise [Tuber::UnexpectedResponse] Response from beanstalk command was an error status
+    # @example
+    #  parse_response("delete 56", "DELETED 56\nFOO")
+    #   # => { :body => "FOO", :status => "DELETED", :id => 56 }
+    #
+    def parse_response(cmd, res)
+      status = res.chomp
+      body_values = status.split(/\s/)
+      status = body_values[0]
+      if status == "DRAINING" && cmd.strip.start_with?("drain")
+        return { status: status }
+      end
+      raise UnexpectedResponse.from_status(status, cmd) if UnexpectedResponse::ERROR_STATES.include?(status)
+      body = nil
+      if status == 'FLUSHED'
+        return { status: status, id: body_values[1] }
+      end
+      if ['OK','FOUND', 'RESERVED'].include?(status)
+        bytes_size = body_values[-1].to_i
+        raw_body = connection.read(bytes_size)
+        body = if status == 'OK'
+          psych_v4_valid_body = raw_body.gsub(/^(.*?): (.*)$/) { "#{$1}: #{$2.gsub(/[\:\-\~]/, '_')}" }
+          YAML.load(psych_v4_valid_body)
+        else
+          config.job_parser.call(raw_body)
+        end
+        crlf = connection.read(2) # \r\n
+        raise ExpectedCrlfError.new('EXPECTED_CRLF', cmd) if crlf != "\r\n"
+      end
+      id = body_values[1]
+      response = { :status => status }
+      response[:id] = id if id
+      response[:body] = body if body
+      response[:state] = body_values[2] if status == 'INSERTED' && body_values[2]
+      response
+    end
+
+    # Returns configuration options for tuber
+    #
+    # @return [Tuber::Configuration] configuration object
+    def config
+      Tuber.configuration
+    end
+
+    private
+
+    def _initialize_tubes
+      if @tubes_watched != ['default']
+        tubes_watched.each do |t|
+          transmit("watch #{t}", init: false)
+        end
+
+        transmit("ignore default", init: false)
+      end
+
+      transmit("use #{tube_used}", init: false) if @tube_used != 'default'
+    end
+
+    # Wrapper method for capturing certain failures and retry the payload block
+    #
+    # @param [Proc] block The command to execute.
+    # @param [Integer] retry_interval The time to wait before the next retry
+    # @param [Boolean] retransmit Whether the command is safe to re-send after
+    #   a reconnect. When false the connection is still healed (reconnect +
+    #   tube re-init) but the original connection error is re-raised.
+    # @param [Integer] tries The maximum number of tries in draining mode
+    # @return [Object] Result of the block passed
+    #
+    def _with_retry(retry_interval: DEFAULT_RETRY_INTERVAL, init: true, tries: MAX_RETRIES, retransmit: true, &block)
+      yield
+    rescue EOFError, Errno::ECONNRESET, Errno::EPIPE,
+      Errno::ECONNREFUSED => ex
+      _reconnect(ex, retry_interval)
+      _initialize_tubes if init
+      raise ex unless retransmit
+      retry
+    rescue Tuber::DrainingError
+      tries -= 1
+      if tries.zero?
+        close
+        raise
+      end
+      sleep(retry_interval)
+      retry
+    end
+
+    # Tries to re-establish connection to the beanstalkd
+    #
+    # @param [Exception] original_exception The exception caused the retry
+    # @param [Integer] retry_interval The time to wait before the next reconnect
+    # @param [Integer] tries The maximum number of attempts to reconnect
+    def _reconnect(original_exception, retry_interval, tries=MAX_RETRIES)
+      close
+      establish_connection
+    rescue Errno::ECONNREFUSED
+      tries -= 1
+      if tries.zero?
+        _raise_not_connected!
+      end
+      sleep(retry_interval || DEFAULT_RETRY_INTERVAL)
+      retry
+    end
+
+    # The host provided by the TUBER_URL environment variable (or BEANSTALKD_URL,
+    # honoured for compatibility with beanstalkd tooling), if available.
+    #
+    # @return [String] A server host address
+    # @example
+    #  ENV['TUBER_URL'] = "localhost:1212"
+    #   # => 'localhost:1212'
+    #
+    def _host_from_env
+      url = ENV['TUBER_URL'] || ENV['BEANSTALKD_URL']
+      url.respond_to?(:length) && url.length > 0 && url.strip
+    end
+
+    # Packs a timeout value (in seconds) into a struct timeval binary string.
+    # Supports fractional seconds (e.g., 0.5 => 500000 usec).
+    def _timeval_for(timeout)
+      sec = timeout.to_i
+      usec = ((timeout.to_f - sec) * 1_000_000).to_i
+      [sec, usec].pack('l_l_')
+    end
+
+    # Raises an error to be triggered when the connection has failed
+    # @raise [Tuber::NotConnected] Beanstalkd is no longer connected
+    def _raise_not_connected!
+      raise Tuber::NotConnected, "Connection to beanstalk '#{@host}:#{@port}' is closed!"
+    end
+
+  end # Connection
+end # Tuber

data/lib/tuber/errors.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+class Tuber
+  # Raises when a beanstalkd instance is no longer accessible.
+  class NotConnected < RuntimeError; end
+  # Raises when the tube name specified is invalid.
+  class InvalidTubeName < RuntimeError; end
+  # Raises when a job has not been reserved properly.
+  class JobNotReserved < RuntimeError; end
+
+  # Abstract class for errors that occur when a command does not complete successfully.
+  class UnexpectedResponse < RuntimeError
+    # Set of status states that are considered errors
+    ERROR_STATES = %w(OUT_OF_MEMORY INTERNAL_ERROR
+      BAD_FORMAT UNKNOWN_COMMAND JOB_TOO_BIG DRAINING
+      TIMED_OUT DEADLINE_SOON NOT_FOUND NOT_IGNORED EXPECTED_CRLF)
+
+    # @!attribute status
+    #   @return [String] returns beanstalkd response status
+    #   @example @ex.status # => "NOT_FOUND"
+    # @!attribute cmd
+    #   @return [String] returns beanstalkd request command
+    #   @example @ex.cmd # => "stats-job 23"
+    attr_reader :status, :cmd
+
+    # Initialize unexpected response error
+    #
+    # @param [Tuber::UnexpectedResponse] status Unexpected response object
+    # @param [String] cmd Beanstalkd request command
+    #
+    # @example
+    #   Tuber::UnexpectedResponse.new(NotFoundError, 'bury 123')
+    #
+    def initialize(status, cmd)
+      @status, @cmd = status, cmd
+      super("Response failed with: #{status}")
+    end
+
+    # Translate beanstalkd error status to ruby Exeception
+    #
+    # @param [String] status Beanstalkd error status
+    # @param [String] cmd Beanstalkd request command
+    #
+    # @return [Tuber::UnexpectedResponse] Exception for the status provided
+    # @example
+    #   Tuber::UnexpectedResponse.new('NOT_FOUND', 'bury 123')
+    #
+    def self.from_status(status, cmd)
+      error_klazz_name = status.split('_').map { |w| w.capitalize }.join
+      error_klazz_name << "Error" unless error_klazz_name =~ /Error$/
+      error_klazz = Tuber.const_get(error_klazz_name)
+      error_klazz.new(status, cmd)
+    end
+  end
+
+  # Raises when the beanstalkd instance runs out of memory
+  class OutOfMemoryError < UnexpectedResponse; end
+  # Raises when the beanstalkd instance is draining and new jobs cannot be inserted
+  class DrainingError < UnexpectedResponse; end
+  # Raises when the job or tube cannot be found
+  class NotFoundError < UnexpectedResponse; end
+  # Raises when the job reserved is going to be released within a second.
+  class DeadlineSoonError < UnexpectedResponse; end
+  # Raises when a beanstalkd has an internal error.
+  class InternalError < UnexpectedResponse; end
+  # Raises when a command was not properly formatted.
+  class BadFormatError < UnexpectedResponse; end
+  # Raises when a command was sent that is unknown.
+  class UnknownCommandError < UnexpectedResponse; end
+  # Raises when command does not have proper CRLF suffix.
+  class ExpectedCrlfError < UnexpectedResponse; end
+  # Raises when the body of a job was too large.
+  class JobTooBigError < UnexpectedResponse; end
+  # Raises when a job was attempted to be reserved but the timeout occurred.
+  class TimedOutError < UnexpectedResponse; end
+  # Raises when a tube could not be ignored because it is the last watched tube.
+  class NotIgnoredError < UnexpectedResponse; end
+end