tuber 0.0.1 → 0.5.1

data/lib/tuber/job/collection.rb

@@ -0,0 +1,160 @@
+# frozen_string_literal: true
+
+class Tuber
+  # Exception to stop processing jobs during a `process!` loop.
+  # Simply `raise AbortProcessingError` in any job process handler to stop the processing loop.
+  class AbortProcessingError < RuntimeError; end
+
+  # Represents collection of job-related commands.
+  class Jobs
+
+    # @!attribute processors
+    #   @return [Array<Proc>] returns Collection of proc to handle beanstalkd jobs
+    # @!attribute client
+    #   @return [Tuber] returns the client instance
+    # @!attribute current_job
+    #   @return [Tuber] returns the currently processing job in the process loop
+    attr_reader :processors, :client, :current_job
+
+    # Number of retries to process a job.
+    MAX_RETRIES = 3
+
+    # Delay in seconds before to make job ready again.
+    RELEASE_DELAY = 1
+
+    # Number of seconds to wait for a job before checking a different server.
+    RESERVE_TIMEOUT = nil
+
+    # Creates new jobs instance.
+    #
+    # @param [Tuber] client The tuber client instance.
+    # @example
+    #  Tuber::Jobs.new(@client)
+    #
+    def initialize(client)
+      @client = client
+    end
+
+    # Delegates transmit to the connection object.
+    #
+    # @see Tuber::Connection#transmit
+    def transmit(command, options={})
+      # Empty **options must not be forwarded: on Ruby 2.7 it arrives as an
+      # extra positional {} at the receiver.
+      return client.connection.transmit(command) if options.empty?
+      client.connection.transmit(command, **options)
+    end
+
+    # Peek (or find) job by id from beanstalkd.
+    #
+    # @param [Integer] id Job id to find
+    # @return [Tuber::Job] Job matching given id
+    # @example
+    #   @tuber.jobs[123] # => <Tuber::Job>
+    #   @tuber.jobs.find(123) # => <Tuber::Job>
+    #   @tuber.jobs.peek(123) # => <Tuber::Job>
+    #
+    # @api public
+    def find(id)
+      res = transmit("peek #{id}")
+      Job.new(client, res)
+    rescue Tuber::NotFoundError
+      nil
+    end
+    alias_method :peek, :find
+    alias_method :[], :find
+
+    # Extends the ttr of every job currently reserved by this connection.
+    #
+    # A single heartbeat for a whole +reserve_batch+ window: the server already
+    # tracks the reserved set per connection, so no ids are sent and jobs that
+    # were already deleted, released, buried or lost to a ttr timeout are simply
+    # absent. Each job keeps its own ttr; deadlines are extended individually.
+    #
+    # The returned count is how many jobs the connection *actually* still holds.
+    # If it is lower than expected, jobs hit their ttr and went back to the queue
+    # while the worker was busy.
+    #
+    # @return [Integer] Number of held jobs whose deadline was extended
+    # @example
+    #   @tuber.jobs.touch_all # => 10
+    #
+    # @api public
+    def touch_all
+      transmit("touch-all")[:id].to_i
+    end
+
+    # Register a processor to handle beanstalkd job on particular tube.
+    #
+    # @param [String] tube_name Tube name
+    # @param [Hash{String=>RuntimeError}] options settings for processor
+    # @param [Proc] block Process beanstalkd job
+    # @option options [Integer] max_retries Number of retries to process a job
+    # @option options [Array<RuntimeError>] retry_on Collection of errors to rescue and re-run processor
+    #
+    # @example
+    #   @beanstalk.jobs.register('some-tube', :retry_on => [SomeError]) do |job|
+    #     do_something(job)
+    #   end
+    #
+    #   @beanstalk.jobs.register('other-tube') do |job|
+    #     do_something_else(job)
+    #   end
+    #
+    # @api public
+    def register(tube_name, options={}, &block)
+      @processors ||= {}
+      max_retries = options[:max_retries] || MAX_RETRIES
+      retry_on = Array(options[:retry_on])
+      @processors[tube_name.to_s] = { :block => block, :retry_on => retry_on, :max_retries => max_retries }
+    end
+
+    # Sets flag to indicate that process loop should stop after current job
+    def stop!
+      @stop = true
+    end
+
+    # Returns whether the process loop should stop
+    #
+    # @return [Boolean] if true the loop should stop after current processing
+    def stop?
+      !!@stop
+    end
+
+    # Watch, reserve, process and delete or bury or release jobs.
+    #
+    # @param [Hash{String => Integer}] options Settings for processing
+    # @option options [Integer] release_delay Delay in seconds before to make job ready again
+    # @option options [Integer] reserve_timeout Number of seconds to wait for a job before checking a different server
+    #
+    # @api public
+    def process!(options={})
+      release_delay = options.delete(:release_delay) || RELEASE_DELAY
+      reserve_timeout = options.delete(:reserve_timeout) || RESERVE_TIMEOUT
+      client.tubes.watch!(*processors.keys)
+      while !stop? do
+        begin
+          @current_job = client.tubes.reserve(reserve_timeout)
+          processor = processors[@current_job.tube]
+          begin
+            processor[:block].call(@current_job)
+            @current_job.delete
+          rescue *processor[:retry_on]
+            if @current_job.stats.releases < processor[:max_retries]
+              @current_job.release(:delay => release_delay)
+            end
+          end
+        rescue AbortProcessingError
+          break
+        rescue Tuber::JobNotReserved, Tuber::NotFoundError, Tuber::TimedOutError
+          retry
+        rescue StandardError # handles unspecified errors
+          @current_job.bury if @current_job
+        ensure # bury if still reserved
+          @current_job.bury if @current_job && @current_job.exists? && @current_job.reserved?
+          @current_job = nil
+        end
+      end
+    end # process!
+  end # Jobs
+end # Tuber

data/lib/tuber/job/record.rb

@@ -0,0 +1,226 @@
+# frozen_string_literal: true
+
+class Tuber
+  # Represents job related commands.
+  class Job
+
+    # @!attribute id
+    #   @return [Integer] id for the job.
+    # @!attribute body
+    #   @return [String] the job's body.
+    # @!attribute reserved
+    #   @return [Boolean] whether the job has been reserved.
+    # @!attribute client
+    #   @return [Tuber] returns the client instance
+    attr_reader :id, :body, :reserved, :client
+
+
+    # Initializes a new job object.
+    #
+    # @param [Hash{Symbol => String,Number}] res Result from beanstalkd response
+    #
+    def initialize(client, res)
+      @client     = client
+      @id         = res[:id]
+      @body       = res[:body]
+      @reserved   = res[:status] == 'RESERVED'
+    end
+
+    # Sends command to bury a reserved job.
+    #
+    # @param [Hash{Symbol => Integer}] options Settings to bury job
+    # @option options [Integer] pri Assign new priority to job
+    # @return [Hash{Symbol => String,Number}] Beanstalkd response for the command.
+    #
+    # @example
+    #   @job.bury({:pri => 100})
+    #     # => {:status=>"BURIED", :body=>nil}
+    #
+    # @api public
+    def bury(options={})
+      options = { :pri => stats.pri }.merge(options)
+      with_reserved("bury #{id} #{options[:pri]}") do
+        @reserved = false
+      end
+    end
+
+    # Sends command to release a job back to ready state.
+    #
+    # @param [Hash{String => Integer}] options Settings to release job
+    # @option options [Integer] pri Assign new priority to job
+    # @option options [Integer] delay Assign new delay to job
+    # @return [Hash{Symbol => String,Number}] Beanstalkd response for the command.
+    # @example
+    #   @tuber.jobs.find(123).release(:pri => 10, :delay => 5)
+    #     # => {:status=>"RELEASED", :body=>nil}
+    #
+    # @api public
+    def release(options={})
+      options = { :pri => stats.pri, :delay => stats.delay }.merge(options)
+      with_reserved("release #{id} #{options[:pri]} #{options[:delay]}") do
+        @reserved = false
+      end
+    end
+
+    # Sends command to touch job which extends the ttr.
+    #
+    # @return [Hash{Symbol => String,Number}] Beanstalkd response for the command.
+    # @example
+    #   @tuber.jobs.find(123).touch
+    #     # => {:status=>"TOUCHED", :body=>nil}
+    #
+    # @api public
+    def touch
+      with_reserved("touch #{id}")
+    end
+
+    # Sends command to delete a job.
+    #
+    # @return [Hash{Symbol => String,Number}] Beanstalkd response for the command.
+    # @example
+    #   @tuber.jobs.find(123).delete
+    #     # => {:status=>"DELETED", :body=>nil}
+    #
+    # @api public
+    def delete
+      transmit("delete #{id}") { @reserved = false }
+    end
+
+    # Sends command to kick a buried job.
+    #
+    # @return [Hash{Symbol => String,Number}] Beanstalkd response for the command.
+    # @example
+    #   @tuber.jobs.find(123).kick
+    #     # => {:status=>"KICKED", :body=>nil}
+    #
+    # @api public
+    def kick
+      transmit("kick-job #{id}")
+    end
+
+    # Sends command to get stats about job.
+    #
+    # @return [Tuber::StatStruct] struct filled with relevant job stats
+    # @example
+    #   @tuber.jobs.find(123).stats
+    #   @job.stats.tube # => "some-tube"
+    #
+    # @api public
+    def stats
+      res = transmit("stats-job #{id}")
+      StatStruct.from_hash(res[:body])
+    end
+
+    # Check if job is currently in a reserved state.
+    #
+    # @return [Boolean] Returns true if the job is in a reserved state
+    # @example
+    #   @tuber.jobs.find(123).reserved?
+    #
+    # @api public
+    def reserved?
+      @reserved || self.stats.state == "reserved"
+    end
+
+    # Check if the job still exists.
+    #
+    # @return [Boolean] Returns true if the job still exists
+    # @example
+    #   @tuber.jobs.find(123).exists?
+    #
+    # @api public
+    def exists?
+      !self.stats.nil?
+    rescue Tuber::NotFoundError
+      false
+    end
+
+    # Returns the name of the tube this job is in
+    #
+    # @return [String] The name of the tube for this job
+    # @example
+    #   @tuber.jobs.find(123).tube
+    #     # => "some-tube"
+    #
+    # @api public
+    def tube
+      @tube ||= self.stats.tube
+    end
+
+    # Returns the ttr of this job
+    #
+    # @return [Integer] The ttr of this job
+    # @example
+    #   @tuber.jobs.find(123).ttr
+    #     # => 123
+    #
+    # @api public
+    def ttr
+      @ttr ||= self.stats.ttr
+    end
+
+    # Returns the pri of this job
+    #
+    # @return [Integer] The pri of this job
+    # @example
+    #   @tuber.jobs.find(123).pri
+    #     # => 1
+    #
+    def pri
+      self.stats.pri
+    end
+
+    # Returns the delay of this job
+    #
+    # @return [Integer] The delay of this job
+    # @example
+    #   @tuber.jobs.find(123).delay
+    #     # => 5
+    #
+    def delay
+      self.stats.delay
+    end
+
+    # Returns string representation of job
+    #
+    # @return [String] string representation
+    # @example
+    #   @tuber.jobs.find(123).to_s
+    #   @tuber.jobs.find(123).inspect
+    #
+    def to_s
+      "#<Tuber::Job id=#{id} body=#{body.inspect}>"
+    end
+    alias :inspect :to_s
+
+    protected
+
+    # Transmit command to beanstalkd instance and fetch response.
+    #
+    # @param [String] cmd Beanstalkd command to send.
+    # @return [Hash{Symbol => String,Number}] Beanstalkd response for the command.
+    # @example
+    #  transmit('stats')
+    #  transmit('stats') { 'success' }
+    #
+    def transmit(cmd, &block)
+      res = client.connection.transmit(cmd)
+      yield if block_given?
+      res
+    end
+
+    # Transmits a command which requires the job to be reserved.
+    #
+    # @param [String] cmd Beanstalkd command to send.
+    # @return [Hash{Symbol => String,Number}] Beanstalkd response for the command.
+    # @raise [Tuber::JobNotReserved] Command cannot execute since job is not reserved.
+    # @example
+    #   with_reserved("bury 26") { @reserved = false }
+    #
+    def with_reserved(cmd, &block)
+      raise JobNotReserved unless reserved?
+      transmit(cmd, &block)
+    end
+
+  end # Job
+end # Tuber

data/lib/tuber/job.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require 'tuber/job/record'
+require 'tuber/job/collection'

data/lib/tuber/stats/fast_struct.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+class Tuber
+  #
+  # Borrowed from:
+  # https://github.com/dolzenko/faster_open_struct/blob/master/lib/faster_open_struct.rb
+  #
+  # Up to 40 (!) times more memory efficient version of OpenStruct
+  #
+  # Differences from Ruby MRI OpenStruct:
+  #
+  # 1. Doesn't `dup` passed initialization hash (NOTE: only reference to hash is stored)
+  #
+  # 2. Doesn't convert hash keys to symbols (by default string keys are used,
+  #    with fallback to symbol keys)
+  #
+  # 3. Creates methods on the fly on `OpenStruct` class, instead of singleton class.
+  #    Uses `module_eval` with string to avoid holding scope references for every method.
+  #
+  # 4. Refactored, crud clean, spec covered :)
+  #
+  # @private
+  class FasterOpenStruct
+    # Undefine particularly nasty interfering methods on Ruby 1.8
+    undef :type if method_defined?(:type)
+    undef :id if method_defined?(:id)
+
+    def initialize(hash = nil)
+      @hash = hash || {}
+      @initialized_empty = hash == nil
+    end
+
+    def method_missing(method_name_sym, *args)
+      if method_name_sym.to_s[-1] == ?=
+        if args.size != 1
+          raise ArgumentError, "wrong number of arguments (#{args.size} for 1)", caller(1)
+        end
+
+        if self.frozen?
+          raise TypeError, "can't modify frozen #{self.class}", caller(1)
+        end
+
+        __new_ostruct_member__(method_name_sym.to_s.chomp("="))
+        send(method_name_sym, args[0])
+      elsif args.size == 0
+        __new_ostruct_member__(method_name_sym)
+        send(method_name_sym)
+      else
+        raise NoMethodError, "undefined method `#{method_name_sym}' for #{self}", caller(1)
+      end
+    end
+
+    def __new_ostruct_member__(method_name_sym)
+      self.class.module_eval <<-END_EVAL, __FILE__, __LINE__ + 1
+      def #{ method_name_sym }
+        @hash.fetch("#{ method_name_sym }", @hash[:#{ method_name_sym }]) # read by default from string key, then try symbol
+                                                                          # if string key doesn't exist
+      end
+      END_EVAL
+
+      unless method_name_sym.to_s[-1] == ?? # can't define writer for predicate method
+        self.class.module_eval <<-END_EVAL, __FILE__, __LINE__ + 1
+        def #{ method_name_sym }=(val)
+          if @hash.key?("#{ method_name_sym }") || @initialized_empty       # write by default to string key (when it is present
+                                                                            # in initialization hash or initialization hash
+                                                                            # wasn't provided)
+            @hash["#{ method_name_sym }"] = val                             # if it doesn't exist - write to symbol key
+          else
+            @hash[:#{ method_name_sym }] = val
+          end
+        end
+        END_EVAL
+      end
+    end
+
+    def empty?
+      @hash.empty?
+    end
+
+    #
+    # Compare this object and +other+ for equality.
+    #
+    def ==(other)
+      return false unless other.is_a?(self.class)
+      @hash == other.instance_variable_get(:@hash)
+    end
+
+    #
+    # Returns a string containing a detailed summary of the keys and values.
+    #
+    def inspect
+      str = "#<#{ self.class }"
+      str << " #{ @hash.map { |k, v| "#{ k }=#{ v.inspect }" }.join(", ") }" unless @hash.empty?
+      str << ">"
+    end
+    alias :to_s :inspect
+  end # FasterOpenStruct
+end # Tuber

data/lib/tuber/stats/stat_struct.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+class Tuber
+  # Represents a stats hash with proper underscored keys
+  class StatStruct < FasterOpenStruct
+    # Convert a stats hash into a struct.
+    #
+    # @param [Hash{String => String}] hash Hash Stats hash to convert to struct
+    # @return [Tuber::StatStruct, nil] Stats struct from hash
+    # @example
+    #   s = StatStruct.from_hash(:foo => "bar")
+    #   s.foo # => 'bar'
+    #
+    def self.from_hash(hash)
+      return unless hash.is_a?(Hash)
+      underscore_hash = hash.inject({}) { |r, (k, v)| r[k.to_s.gsub(/-/, '_')] = v; r }
+      self.new(underscore_hash)
+    end
+
+    # Access value for stat with specified key.
+    #
+    # Keys are stored underscored (see {.from_hash}), so hyphenated beanstalkd
+    # names like "current-jobs-ready" are underscored before lookup. This also
+    # avoids defining an invalid, hyphenated method name via method_missing.
+    #
+    # @param [String] key Key to fetch from stats.
+    # @return [String, Integer] Value for specified stat key.
+    # @example
+    #  @stats['foo'] # => "bar"
+    #  @stats['current-jobs-ready'] # => 5
+    #
+    def [](key)
+      self.send(key.to_s.gsub(/-/, '_'))
+    end
+
+    # Returns set of keys within this struct
+    #
+    # @return [Array<String>] Value for specified stat key.
+    # @example
+    #  @stats.keys # => ['foo', 'bar', 'baz']
+    #
+    def keys
+      @hash.keys.map { |k| k.to_s }
+    end
+
+    # Returns the initialization hash
+    #
+    def to_h
+      @hash
+    end
+  end # StatStruct
+end # Tuber

data/lib/tuber/stats.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require 'tuber/stats/fast_struct'
+require 'tuber/stats/stat_struct'
+
+class Tuber
+  # Represents stats related to the beanstalkd pool.
+  class Stats
+
+    # @!attribute client
+    #   @return [Tuber] returns the client instance
+    attr_reader :client
+
+    # Creates new stats instance.
+    #
+    # @param [Tuber] client The tuber client instance.
+    # @example
+    #  Tuber::Stats.new(@client)
+    #
+    def initialize(client)
+      @client = client
+    end
+
+    # Returns keys for stats data
+    #
+    # @return [Array<String>] Set of keys for stats.
+    # @example
+    #  @bp.stats.keys # => ["version", "total_connections"]
+    #
+    # @api public
+    def keys
+      data.keys
+    end
+
+    # Returns value for specified key.
+    #
+    # @param [String,Symbol] key Name of key to retrieve
+    # @return [String,Integer] Value of specified key
+    # @example
+    #  @bp.stats['total_connections'] # => 4
+    #
+    def [](key)
+      data[key]
+    end
+
+    # Delegates inspection to the real data structure
+    #
+    # @return [String] returns a string containing a detailed stats summary
+    def inspect
+      data.to_s
+    end
+    alias :to_s :inspect
+
+    # Defines a cached method for looking up data for specified key
+    # Protects against infinite loops by checking stacktrace
+    # @api public
+    def method_missing(name, *args, &block)
+      if caller.first !~ /`(method_missing|data')/ && data.keys.include?(name.to_s)
+        self.class.class_eval <<-CODE, __FILE__, __LINE__
+          def #{name}; data[#{name.inspect}]; end
+        CODE
+        data[name.to_s]
+      else # no key matches or caught infinite loop
+        super
+      end
+    end
+
+    protected
+
+    # Returns struct based on stats data from response.
+    #
+    # @return [Tuber::StatStruct] the stats
+    # @example
+    #  self.data # => { 'version' : 1.7, 'total_connections' : 23 }
+    #  self.data.total_connections # => 23
+    #
+    def data
+      StatStruct.from_hash(client.connection.transmit('stats')[:body])
+    end
+  end # Stats
+end # Tuber