backburner-allq 1.0.0

data/lib/backburner/allq_wrapper.rb

@@ -0,0 +1,317 @@
+require 'base64'
+require 'yajl'
+require 'timeout'
+require 'lz4-ruby'
+require 'allq_client'
+
+module Backburner
+  class AllqWatcher
+    attr_accessor :tube
+
+    def initialize(tube, allq_wrapper)
+      @tube = tube
+      @allq_wrapper = allq_wrapper
+    end
+
+    def watch
+      Thread.new do
+        ran = false
+        job = @allq_wrapper.get(@tube_name)
+        if job.body
+          perform(job)
+          ran = true
+        end
+        # Wait if nothing returned
+        sleep(rand() * 3) unless ran
+      end
+    end
+  end
+
+  class AllQJob
+    attr_accessor :id
+
+    # Body
+    attr_accessor :body
+
+    # Tube name
+    attr_accessor :tube
+
+    # Expired count
+    attr_accessor :expireds
+
+    # Release count
+    attr_accessor :releases
+
+    def initialize(wrapper, job_resposne)
+      @client = wrapper
+      @id = job_resposne.id
+      @body = job_resposne.body
+      @expireds = job_resposne.expireds
+      @releases = job_resposne.releases
+      @tube = job_resposne.tube.to_s
+    end
+
+    def done
+      @client.done(self)
+    end
+
+    def delete
+      @client.delete(self)
+    end
+
+    def touch
+      @client.touch(self)
+    end
+
+    def kick
+      @client.kick(self)
+    end
+
+    def release(delay = 0)
+      @client.release(self, delay)
+    end
+
+    def bury
+      @client.bury(self)
+    end
+
+    def stats
+      { 'expireds' => expireds, 'releases' => releases }
+    end
+  end
+
+  class AllQWrapper
+    def initialize(url = Blitline::Constants::ALLQ_DEFAULT_EU_URL)
+      allq_conf = Allq::Configuration.new do |config|
+        config.host = url
+      end
+
+      raw_client = Allq::ApiClient.new(allq_conf)
+      @client = Allq::ActionsApi.new(raw_client)
+      @admin = Allq::AdminApi.new(raw_client)
+      @recent_times = []
+
+    end
+
+    def speed
+      if @recent_times.size > 0
+        return @recent_times.sum(0.0) /  @recent_times.size
+      end
+      return 0
+    end
+
+    def touch(job)
+      @client.touch_put(job.id)
+    end
+
+    def done(job)
+      @client.job_delete(job.id)
+    end
+
+    def delete(job)
+      @client.job_delete(job.id)
+    end
+
+    def release(job, _delay = 0)
+      @client.release_put(job.id)
+    end
+
+    def bury(job); end
+
+    def tube_names
+      stats_hash = stats
+      stats_hash.keys
+    end
+
+    def peek_buried(tube_name = 'default')
+      job = nil
+      job = @client.peek_get(tube_name, buried: true)
+      return nil if job.body.nil?
+
+      job.body = Base64.decode64(job.body) if job
+      job_obj = Blitline::AllQJob.new(self, job)
+      job_obj
+    end
+
+    def get(tube_name = 'default')
+      job = nil
+      delt = Blitline::Utils.get_delta do
+        job = @client.job_get(tube_name)
+      end
+      puts "Allq http get delta: #{delt} #{tube_name} #{job}" if delt.to_f > 1.5
+
+      @recent_times.push(delt.to_f)
+      @recent_times.shift if @recent_times.size > 2
+      # Inplace decode
+      job.body = Base64.decode64(job.body) if job&.body
+
+      job_obj = Blitline::AllQJob.new(self, job)
+      job_obj
+    rescue StandardError => ex
+      if ex.message == "Couldn't resolve host name"
+        BlitlineLogger.log("COUDNT RESOLVE HOST NAME------ SHOULD REBOOT")
+      else
+        BlitlineLogger.log(ex)
+      end
+    end
+
+    def close
+    rescue StandardError => ex
+      BlitlineLogger.log(ex)
+    end
+
+    def map_priority(app_priority)
+      app_priority = app_priority.to_i
+
+      # IF already using allq-like priority, stick with it
+      return app_priority if app_priority < 11 && app_priority > 0
+
+      default = 5
+
+      return default if app_priority == 32_000
+
+      return 6 if app_priority >= 33_000
+
+      return 7 if app_priority >= 35_000
+
+      return 8 if app_priority >= 37_000
+
+      default
+    end
+
+    def log_result(job_result)
+      BlitlineLogger.log("ALLQ-HTTP-JOB-ID=#{job_result.job_id}")
+    rescue StandardError => ex
+      BlitlineLogger.log(ex)
+    end
+
+    def build_new_job(body, options)
+      adjusted_priority = map_priority(options[:pri] || 5)
+
+      ttl = options[:ttl] || 600
+      tube_name = options[:tube_name] || 'default'
+      delay = options[:delay] || 0
+      parent_id = options[:parent_id]
+
+      new_job = Allq::NewJob.new(tube: tube_name,
+                                 body: Base64.strict_encode64(body),
+                                 ttl: ttl,
+                                 delay: delay,
+                                 priority: adjusted_priority,
+                                 parent_id: parent_id)
+      new_job
+    end
+
+    def build_new_parent_job(body, options)
+      adjusted_priority = map_priority(options[:pri] || 5)
+      ttl = options[:ttl] || 600
+      tube_name = options[:tube_name] || 'default'
+      delay = options[:delay] || 0
+      parent_id = options[:parent_id]
+      limit = options[:limit]
+      timeout = options[:timeout] || 3_600
+      run_on_timeout = options[:run_on_timeout] || false
+
+      new_parent_job = Allq::NewParentJob.new(tube: tube_name,
+                                              body: Base64.strict_encode64(body),
+                                              ttl: ttl,
+                                              delay: delay,
+                                              priority: adjusted_priority,
+                                              timeout: timeout,
+                                              run_on_timeout: run_on_timeout,
+                                              limit: limit)
+      new_parent_job
+    end
+
+    def put2(body, pri = 5, ttl = 600, tube_name = "default", delay = 0)
+      # Old school way
+      options = {
+        pri: pri,
+        ttl: ttl,
+        tube_name: tube_name,
+        delay: delay
+      }
+      put(body, options)
+    end
+
+    def put(body, options)
+      # New school put
+      retry_count = 0
+      is_parent = options[:is_parent] || false
+      result = nil
+
+      begin
+        Timeout.timeout(10) do
+          if body && body.to_s.include?('["default"]')
+            BlitlineLogger.log "PUTTING DEFAULT! #{caller.inspect}"
+          end
+
+          delt = Blitline::Utils.get_delta do
+            if is_parent
+              new_job = build_new_parent_job(body, options)
+              result = @client.parent_job_post(new_job)
+            else
+              new_job = build_new_job(body, options)
+              result = @client.job_post(new_job)
+            end
+            raise 'PUT returned nil' if result.nil? || result.to_s == ''
+          end
+          BlitlineLogger.log "Allq http put delta: #{delt}"
+        end
+      rescue Timeout::Error
+        BlitlineLogger.log('ALLQ_PUT_TIMEOUT')
+        sleep(5)
+        retry_count += 1
+        retry if retry_count < 4
+        raise 'Failed to put on allq, we are investigating the problem, please try again'
+      rescue StandardError => ex
+        BlitlineLogger.log('Failed to ALLQ PUT')
+        BlitlineLogger.log(ex)
+        retry_count += 1
+        sleep(5)
+        retry if retry_count < 4
+        raise 'Failed to put on allq, we are investigating the problem, please try again'
+      end
+      result
+    end
+
+    def stats
+      raw_stats = @admin.stats_get
+      final_stats = {}
+
+      raw_stats.each do |agg|
+        agg.stats.each do |tube_ref|
+          name = tube_ref.tube
+          final_stats[name] = {} unless final_stats[name]
+          final_stats[name]['ready'] = final_stats[name]['ready'].to_i + tube_ref.ready.to_i
+          final_stats[name]['reserved'] = final_stats[name]['reserved'].to_i + tube_ref.reserved.to_i
+          final_stats[name]['delayed'] = final_stats[name]['delayed'].to_i + tube_ref.delayed.to_i
+          final_stats[name]['buried'] = final_stats[name]['buried'].to_i + tube_ref.buried.to_i
+          final_stats[name]['parents'] = final_stats[name]['parents'].to_i + tube_ref.parents.to_i
+        end
+      end
+      final_stats
+    rescue StandardError => ex
+      BlitlineLogger.log(ex)
+      {}
+    end
+
+    def get_ready_by_tube(name)
+      count = -1
+      tube_stats = stats[name]
+      count = tube_stats['ready'].to_i if tube_stats && tube_stats['ready']
+      count
+    rescue StandardError => ex
+      BlitlineLogger.log(ex)
+      -1
+    end
+
+    def size
+      result = get_ready_by_tube('default')
+      result.to_i
+    rescue StandardError => ex
+      BlitlineLogger.log(ex)
+      0
+    end
+  end
+end

data/lib/backburner/async_proxy.rb

@@ -0,0 +1,25 @@
+module Backburner
+  # BasicObject for 1.8.7
+  class BasicObject
+    instance_methods.each do |m|
+      undef_method(m) if m.to_s !~ /(?:^__|^nil?$|^send$|^object_id$)/
+    end
+  end unless defined?(::BasicObject)
+
+  # Class allows async task to be proxied
+  class AsyncProxy < BasicObject
+    # Options include `pri` (priority), `delay` (delay in secs), `ttr` (time to respond)
+    #
+    # @example
+    #   AsyncProxy.new(User, 10, :pri => 1000, :ttr => 1000)
+    #
+    def initialize(klazz, id=nil, opts={})
+      @klazz, @id, @opts = klazz, id, opts
+    end
+
+    # Enqueue as job when a method is invoked
+    def method_missing(method, *args, &block)
+      ::Backburner::Worker.enqueue(@klazz, [@id, method, *args], @opts)
+    end
+  end # AsyncProxy
+end # Backburner

data/lib/backburner/cli.rb

@@ -0,0 +1,53 @@
+require 'dante'
+
+module Backburner
+  class CLI
+
+    def self.start(args)
+      runner = Dante::Runner.new('backburner')
+      runner.description = "Execute a backburner worker process"
+      runner.with_options do |opts|
+        opts.on("-r", "--require PATH", String, "The path to load as the environment.") do |req|
+          options[:require] = req
+        end
+        opts.on("-q", "--queues PATH", String, "The specific queues to work.") do |queues|
+          options[:queues] = queues
+        end
+        opts.on("-e", "--environment ENVIRONMENT", String, "The environment to run Backburner within") do |environment|
+          options[:environment] = environment
+        end
+      end
+      runner.execute do |opts|
+        queues = (opts[:queues] ? opts[:queues].split(',') : nil) rescue nil
+        load_environment(opts[:require], opts[:environment])
+        Backburner.work(queues)
+      end
+    end
+
+    protected
+
+      def self.load_environment(file = nil, environment = nil)
+        file ||= "."
+        if File.directory?(file) && File.exists?(File.expand_path("#{file}/config/environment.rb"))
+          ENV["RAILS_ENV"] = environment if environment && ENV["RAILS_ENV"].nil?
+          require "rails"
+          require File.expand_path("#{file}/config/environment.rb")
+          if defined?(::Rails) && ::Rails.respond_to?(:application)
+            # Rails 3
+            ::Rails.application.eager_load!
+          elsif defined?(::Rails::Initializer)
+            # Rails 2.3
+            $rails_rake_task = false
+            ::Rails::Initializer.run :load_application_classes
+          end
+        elsif File.directory?(file) && File.exists?(File.expand_path("#{file}/config/boot.rb"))
+          ENV["RACK_ENV"] = environment if environment && ENV["RACK_ENV"].nil?
+          ENV["PADRINO_ROOT"] = file
+          require File.expand_path("#{file}/config/boot.rb")
+        elsif File.file?(file)
+          require File.expand_path(file)
+        end
+      end
+
+  end
+end

data/lib/backburner/configuration.rb

@@ -0,0 +1,48 @@
+module Backburner
+  class Configuration
+    PRIORITY_LABELS = { :high => 0, :medium => 5, :low => 9 }
+
+    attr_accessor :allq_url       # beanstalk url connection
+    attr_accessor :tube_namespace      # namespace prefix for every queue
+    attr_reader   :namespace_separator # namespace separator
+    attr_accessor :default_priority    # default job priority
+    attr_accessor :respond_timeout     # default job timeout
+    attr_accessor :on_error            # error handler
+    attr_accessor :max_job_retries     # max job retries
+    attr_accessor :retry_delay         # (minimum) retry delay in seconds
+    attr_accessor :retry_delay_proc    # proc to calculate delay (and allow for back-off)
+    attr_accessor :default_queues      # default queues
+    attr_accessor :logger              # logger
+    attr_accessor :default_worker      # default worker class
+    attr_accessor :primary_queue       # the general queue
+    attr_accessor :priority_labels     # priority labels
+    attr_accessor :reserve_timeout     # duration to wait to reserve on a single server
+    attr_accessor :job_serializer_proc # proc to write the job body to a string
+    attr_accessor :job_parser_proc     # proc to parse a job body from a string
+
+    def initialize
+      @allq_url       = "127.0.0.1:8090"
+      @tube_namespace      = ""
+      @namespace_separator = "."
+      @default_priority    = 5
+      @respond_timeout     = 120
+      @on_error            = nil
+      @max_job_retries     = 0
+      @retry_delay         = 5
+      @retry_delay_proc    = lambda { |min_retry_delay, num_retries| min_retry_delay + (num_retries ** 3) }
+      @default_queues      = []
+      @logger              = nil
+      @default_worker      = Backburner::Workers::Simple
+      @primary_queue       = "backburner-jobs"
+      @priority_labels     = PRIORITY_LABELS
+      @reserve_timeout     = nil
+      @job_serializer_proc = lambda { |body| body.to_json }
+      @job_parser_proc     = lambda { |body| JSON.parse(body) }
+    end
+
+    def namespace_separator=(val)
+      raise 'Namespace separator cannot used reserved queue configuration separator ":"' if val == ':'
+      @namespace_separator = val
+    end
+  end # Configuration
+end # Backburner

data/lib/backburner/connection.rb

@@ -0,0 +1,157 @@
+require 'delegate'
+
+module Backburner
+  class Connection < SimpleDelegator
+    class BadURL < RuntimeError; end
+
+    attr_accessor :url, :allq_wrapper
+
+    # If a proc is provided, it will be called (and given this connection as an
+    # argument) whenever the connection is reconnected.
+    # @example
+    #   connection.on_reconnect = lambda { |conn| puts 'reconnected!' }
+    attr_accessor :on_reconnect
+
+    # Constructs a backburner connection
+    # `url` can be a string i.e '127.0.0.1:3001' or an array of
+    # addresses (however, only the first element in the array will
+    # be used)
+    def initialize(url, &on_reconnect)
+      @url = url
+      @allq_wrapper = nil
+      @on_reconnect = on_reconnect
+      connect!
+    end
+
+    # Close the connection, if it exists
+    def close
+      @allq_wrapper.close if @allq_wrapper
+      @allq_wrapper = nil
+      __setobj__(@allq_wrapper)
+    end
+
+    # Determines if the connection to allq is currently open
+    def connected?
+      begin
+        !!(@allq_wrapper && @allq_wrapper.connection && @allq_wrapper.connection.connection && !@allq_wrapper.connection.connection.closed?) # Would be nice if beaneater provided a connected? method
+      rescue
+        false
+      end
+    end
+
+    # Attempt to reconnect to allq. Note: the connection will not be watching
+    # or using the tubes it was before it was reconnected (as it's actually a
+    # completely new connection)
+    # @raise [Beaneater::NotConnected] If allq fails to connect
+    def reconnect!
+      close
+      connect!
+      @on_reconnect.call(self) if @on_reconnect.respond_to?(:call)
+    end
+
+    # Yield to a block that will be retried several times if the connection to
+    # allq goes down and is able to be re-established.
+    #
+    # @param options Hash Options. Valid options are:
+    #   :max_retries       Integer The maximum number of times the block will be yielded to.
+    #                              Defaults to 4
+    #   :on_retry          Proc    An optional proc that will be called for each retry. Will be
+    #                              called after the connection is re-established and :retry_delay
+    #                              has passed but before the block is yielded to again
+    #   :retry_delay       Float   The amount to sleep before retrying. Defaults to 1.0
+    # @raise Beaneater::NotConnected If a connection is unable to be re-established
+    def retryable(options = {}, &block)
+      options = {:max_retries => 4, :on_retry => nil, :retry_delay => 1.0}.merge!(options)
+      retry_count = options[:max_retries]
+
+      begin
+        yield
+
+      rescue Beaneater::NotConnected
+        if retry_count > 0
+          reconnect!
+          retry_count -= 1
+          sleep options[:retry_delay]
+          options[:on_retry].call if options[:on_retry].respond_to?(:call)
+          retry
+        else # stop retrying
+          raise e
+        end
+      end
+    end
+
+    protected
+
+    # Attempt to ensure we're connected to allq if the missing method is
+    # present in the delegate and we haven't shut down the connection on purpose
+    # @raise [Beaneater::NotConnected] If allq fails to connect after multiple attempts.
+    def method_missing(m, *args, &block)
+      ensure_connected! if respond_to_missing?(m, false)
+      super
+    end
+
+    # Connects to a allq queue
+    # @raise Beaneater::NotConnected if the connection cannot be established
+    def connect!
+      @allq_wrapper = Backburner::AllqWrapper.new(allq_addresses)
+      __setobj__(@allq_wrapper)
+      @allq_wrapper
+    end
+
+    def put(tube_name, data, opt)
+      pri = (opt[:pri] || 5).to_i
+      delay = opt[:delay].to_i
+      ttr = (opt[:ttr] || 600).to_i
+      @allq_wrapper.put2(data, pri, ttr, delay)
+    end
+
+    def get(tube_name)
+      @allq_wrapper.get(tube_name)
+    end
+
+    # Attempts to ensure a connection to allq is established but only if
+    # we're not connected already
+    # @param max_retries Integer The maximum number of times to attempt connecting. Defaults to 4
+    # @param retry_delay Float   The time to wait between retrying to connect. Defaults to 1.0
+    # @raise [Beaneater::NotConnected] If allq fails to connect after multiple attempts.
+    # @return Connection This Connection is returned if the connection to allq is open or was able to be reconnected
+    def ensure_connected!(max_retries = 4, retry_delay = 1.0)
+      return self if connected?
+
+      begin
+        reconnect!
+        return self
+
+      rescue Exception => e
+        if max_retries > 0
+          max_retries -= 1
+          sleep retry_delay
+          retry
+        else # stop retrying
+          raise e
+        end
+      end
+    end
+
+    # Returns the allq queue addresses
+    #
+    # @example
+    #   allq_addresses => ["127.0.0.1:11300"]
+    #
+    def allq_addresses
+      uri = self.url.is_a?(Array) ? self.url.first : self.url
+      allq_host_and_port(uri)
+    end
+
+    # Returns a host and port based on the uri_string given
+    #
+    # @example
+    #   allq_host_and_port("allq://127.0.0.1") => "127.0.0.1:11300"
+    #
+    def allq_host_and_port(uri_string)
+      uri = URI.parse(uri_string)
+      raise(BadURL, uri_string) if uri.scheme != 'allq'.freeze
+      "#{uri.host}:#{uri.port || 11300}"
+    end
+  end # Connection
+end # Backburner