siberite-client 0.8.0

data/lib/siberite/client/proxy.rb

@@ -0,0 +1,15 @@
+module Siberite
+  class Client
+    class Proxy
+      attr_reader :client
+
+      def initialize(client)
+        @client = client
+      end
+
+      def method_missing(method, *args, &block)
+        client.send(method, *args, &block)
+      end
+    end
+  end
+end

data/lib/siberite/client/stats_helper.rb

@@ -0,0 +1,122 @@
+module Siberite::Client::StatsHelper
+  STATS_TIMEOUT    = 3
+  QUEUE_STAT_NAMES = %w{items bytes total_items logsize expired_items mem_items mem_bytes age discarded waiters open_transactions}
+
+  def sizeof(queue)
+    stat_info = stat(queue)
+    stat_info ? stat_info['items'] : 0
+  end
+
+  def available_queues
+    stats['queues'].keys.sort
+  end
+
+  def stats
+    alive, dead = 0, 0
+
+    results = servers.map do |server|
+      begin
+        result = stats_for_server(server)
+        alive += 1
+        result
+      rescue Exception
+        dead += 1
+        nil
+      end
+    end.compact
+
+    stats = merge_stats(results)
+    stats['alive_servers'] = alive
+    stats['dead_servers']  = dead
+    stats
+  end
+
+  def stat(queue)
+    stats['queues'][queue]
+  end
+
+  private
+
+  def stats_for_server(server)
+    server_name, port = server.split(/:/)
+    socket = nil
+    with_timeout STATS_TIMEOUT do
+      socket = TCPSocket.new(server_name, port)
+    end
+    socket.puts "STATS"
+
+    stats = Hash.new
+    stats['queues'] = Hash.new
+    while line = socket.readline
+      if line =~ /^STAT queue_(\S+?)_(#{QUEUE_STAT_NAMES.join("|")}) (\S+)/
+        queue_name, queue_stat_name, queue_stat_value = $1, $2, deserialize_stat_value($3)
+        stats['queues'][queue_name] ||= Hash.new
+        stats['queues'][queue_name][queue_stat_name] = queue_stat_value
+      elsif line =~ /^STAT (\w+) (\S+)/
+        stat_name, stat_value = $1, deserialize_stat_value($2)
+        stats[stat_name] = stat_value
+      elsif line =~ /^END/
+        socket.close
+        break
+      elsif defined?(RAILS_DEFAULT_LOGGER)
+        RAILS_DEFAULT_LOGGER.debug("SiberiteClient#stats_for_server: Ignoring #{line}")
+      end
+    end
+
+    stats
+  ensure
+    socket.close if socket && !socket.closed?
+  end
+
+  def merge_stats(all_stats)
+    result = Hash.new
+
+    all_stats.each do |stats|
+      stats.each do |stat_name, stat_value|
+        if result.has_key?(stat_name)
+          if stat_value.kind_of?(Hash)
+            result[stat_name] = merge_stats([result[stat_name], stat_value])
+          else
+            result[stat_name] += stat_value
+          end
+        else
+          result[stat_name] = stat_value
+        end
+      end
+    end
+
+    result
+  end
+
+  def deserialize_stat_value(value)
+    case value
+    when /^\d+\.\d+$/
+        value.to_f
+    when /^\d+$/
+        value.to_i
+    else
+      value
+    end
+  end
+
+  begin
+    require "system_timer"
+
+    def with_timeout(seconds, &block)
+      SystemTimer.timeout_after(seconds, &block)
+    end
+
+  rescue LoadError
+    if ! defined?(RUBY_ENGINE)
+      # MRI 1.8, all other interpreters define RUBY_ENGINE, JRuby and
+      # Rubinius should have no issues with timeout.
+      warn "WARNING: using the built-in Timeout class which is known to have issues when used for opening connections. Install the SystemTimer gem if you want to make sure the Siberite client will not hang."
+    end
+
+    require "timeout"
+
+    def with_timeout(seconds, &block)
+      Timeout.timeout(seconds, &block)
+    end
+  end
+end

data/lib/siberite/client/transactional.rb

@@ -0,0 +1,125 @@
+class Siberite::Client::Transactional < Siberite::Client::Proxy
+
+  # Raised when a caller attempts to use this proxy across
+  # multiple queues.
+  class MultipleQueueException < StandardError; end
+
+  # Raised when a caller attempts to retry a job if
+  # there is no current open transaction
+  class NoOpenTransaction < StandardError; end
+
+  # Raised when a retry fails when max retries is exceeded
+  class RetriesExceeded < StandardError
+    def initialize(job)
+      super "Max retries of #{job.retries} exceeded for item: #{job.job.inspect}"
+    end
+  end
+
+  class RetryableJob < Struct.new(:retries, :job); end
+
+  # Number of times to retry a job before giving up
+  DEFAULT_RETRIES = 10
+
+  # Pct. of the time during 'normal' processing we check the error queue first
+  ERROR_PROCESSING_RATE = 0.1
+
+  # ==== Parameters
+  # client<Siberite::Client>:: Client
+  # max_retries<Integer>:: Number of times to retry a job before
+  #                        giving up. Defaults to DEFAULT_RETRIES
+  # error_rate<Float>:: Pct. of the time during 'normal'
+  #                     processing we check the error queue
+  #                     first. Defaults to ERROR_PROCESSING_RATE
+  # per_server<Integer>:: Number of gets to execute against a
+  #                       single server, before changing
+  #                       servers. Defaults to MAX_PER_SERVER
+  #
+  def initialize(client, max_retries = nil, error_rate = nil)
+    @max_retries = max_retries || DEFAULT_RETRIES
+    @error_rate  = error_rate || ERROR_PROCESSING_RATE
+    @counter     = 0 # Command counter
+    super(client)
+  end
+
+  attr_reader :current_queue
+
+  # Returns job from the +key+ queue 1 - ERROR_PROCESSING_RATE
+  # pct. of the time. Every so often, checks the error queue for
+  # jobs and returns a retryable job.
+  #
+  # ==== Returns
+  # Job, possibly retryable, or nil
+  #
+  # ==== Raises
+  # MultipleQueueException
+  #
+  def get(key, opts = {})
+    raise MultipleQueueException if current_queue && key != current_queue
+
+    close_last_transaction
+
+    if read_from_error_queue?
+      queue = key + "_errors"
+      job = client.get_from_last(queue, opts.merge(:open => true))
+    else
+      queue = key
+      job = client.get(queue, opts.merge(:open => true))
+    end
+
+    if job
+      @job = job.is_a?(RetryableJob) ? job : RetryableJob.new(0, job)
+      @last_read_queue = queue
+      @current_queue = key
+      @job.job
+    end
+  end
+
+  def current_try
+    @job.retries + 1
+  end
+
+  def close_last_transaction #:nodoc:
+    return unless @last_read_queue
+
+    client.get_from_last(@last_read_queue, :close => true)
+    @last_read_queue = @current_queue = @job = nil
+  end
+
+  # Enqueues the current job on the error queue for later
+  # retry. If the job has been retried DEFAULT_RETRIES times,
+  # gives up entirely.
+  #
+  # ==== Parameters
+  # item (optional):: if specified, the job set to the error
+  #                   queue with the given payload instead of what
+  #                   was originally fetched.
+  #
+  # ==== Returns
+  # Boolean:: true if the job is enqueued in the retry queue, false otherwise
+  #
+  # ==== Raises
+  # NoOpenTransaction
+  #
+  def retry(item = nil)
+    raise NoOpenTransaction unless @last_read_queue
+
+    job = item ? RetryableJob.new(@job.retries, item) : @job.dup
+
+    job.retries += 1
+
+    if job.retries < @max_retries
+      client.set(current_queue + "_errors", job)
+    else
+      raise RetriesExceeded.new(job)
+    end
+
+  ensure
+    close_last_transaction
+  end
+
+  private
+
+  def read_from_error_queue?
+    rand < @error_rate
+  end
+end

data/lib/siberite/client/unmarshal.rb

@@ -0,0 +1,32 @@
+module Siberite
+  class Client
+    class Unmarshal < Proxy
+      def get(key, opts = {})
+        response = client.get(key, opts.merge(:raw => true))
+        return response if opts[:raw]
+
+        if is_marshaled?(response)
+          Marshal.load(response)
+        else
+          response
+        end
+      end
+
+      if RUBY_VERSION.respond_to?(:getbyte)
+        def is_marshaled?(object)
+          o = object.to_s
+          o.getbyte(0) == Marshal::MAJOR_VERSION && o.getbyte(1) == Marshal::MINOR_VERSION
+        rescue Exception
+          false
+        end
+      else
+        def is_marshaled?(object)
+          o = object.to_s
+          o[0] == Marshal::MAJOR_VERSION && o[1] == Marshal::MINOR_VERSION
+        rescue Exception
+          false
+        end
+      end
+    end
+  end
+end

data/lib/siberite/client/version.rb

@@ -0,0 +1,3 @@
+module Siberite
+  VERSION = "0.8.0"
+end

data/lib/siberite/config.rb

@@ -0,0 +1,48 @@
+require 'yaml'
+
+module Siberite
+  module Config
+    class ConfigNotLoaded < StandardError; end
+
+    extend self
+
+    attr_accessor :environment, :config
+
+    def load(config_file)
+      self.config = YAML.load_file(config_file)
+    end
+
+    def environment
+      @environment ||= 'development'
+    end
+
+    def config
+      @config or raise ConfigNotLoaded
+    end
+
+    def namespace(namespace)
+      client_args_from config[namespace.to_s][environment.to_s]
+    end
+
+    def default
+      client_args_from config[environment.to_s]
+    end
+
+    def new_client(space = nil)
+      Client.new *(space ? namespace(space) : default)
+    end
+
+    alias method_missing namespace
+
+    private
+
+    def client_args_from(config)
+      sanitized = config.inject({}) do |sanitized, (key, val)|
+        sanitized[key.to_sym] = val; sanitized
+      end
+      servers = sanitized.delete(:servers)
+
+      [servers, sanitized]
+    end
+  end
+end

data/siberite-client.gemspec

@@ -0,0 +1,35 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'siberite/client/version'
+
+Gem::Specification.new do |s|
+  s.name = "siberite-client"
+  s.version = Siberite::VERSION
+
+  s.authors = ["Matt Freels", "Rael Dornfest", "Anton Bogdanovich"]
+  s.summary = "Ruby Siberite client"
+  s.description = "Ruby client for the Siberite queue server"
+  s.email = "anton@bogdanovich.co"
+  s.license = "Apache-2"
+  s.extra_rdoc_files = [
+    "README.md"
+  ]
+  s.files = `git ls-files -z`.split("\x0")
+  s.homepage = "http://github.com/bogdanovich/siberite-ruby"
+
+  s.add_dependency(%q<memcached>, [">= 0.19.6"])
+
+  s.files         = `git ls-files -z`.split("\x0")
+  s.executables   = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  s.test_files    = s.files.grep(%r{^(test|spec|features)/})
+  s.require_paths = ["lib"]
+
+  s.add_development_dependency "pry", "~> 0.10"
+  s.add_development_dependency "pry-byebug", "~> 2.0"
+  s.add_development_dependency "rspec", "~> 3.2"
+  s.add_development_dependency "rr", "~> 1.1"
+  s.add_development_dependency "activesupport", "> 3.1"
+  s.add_development_dependency "bundler", "~> 1.7"
+  s.add_development_dependency "rake", "~> 10.0"
+end

data/spec/siberite/client/blocking_spec.rb

@@ -0,0 +1,29 @@
+require 'spec_helper'
+
+describe "Siberite::Client::Blocking" do
+  describe "Instance Methods" do
+    before do
+      @raw_client = Siberite::Client.new(*Siberite::Config.default)
+      @client = Siberite::Client::Blocking.new(@raw_client)
+    end
+
+    describe "#get" do
+      before do
+        @queue = "some_queue"
+      end
+
+      it "blocks on a get until the get works" do
+        mock(@raw_client).
+          get(@queue) { nil }.times(5).then.get(@queue) { :mcguffin }
+        @client.get(@queue).should == :mcguffin
+      end
+
+      describe "#get_without_blocking" do
+        it "does not block" do
+          mock(@raw_client).get(@queue) { nil }
+          @client.get_without_blocking(@queue).should be_nil
+        end
+      end
+    end
+  end
+end

data/spec/siberite/client/envelope_spec.rb

@@ -0,0 +1,45 @@
+require 'spec_helper'
+
+class Envelope
+  class << self; attr_accessor :unwraps end
+
+  def initialize(item); @item = item end
+  def unwrap; self.class.unwraps += 1; @item end
+end
+
+describe Siberite::Client::Envelope do
+  describe "Instance Methods" do
+    before do
+      Envelope.unwraps = 0
+      @raw_client = Siberite::Client.new(*Siberite::Config.default)
+      @client = Siberite::Client::Envelope.new(Envelope, @raw_client)
+    end
+
+    describe "#get and #set" do
+      describe "envelopes" do
+        it "integrates" do
+          @client.set("a_queue", :mcguffin)
+          @client.get("a_queue").should == :mcguffin
+          Envelope.unwraps.should == 1
+        end
+
+        it "creates an envelope on a set" do
+          mock(Envelope).new(:mcguffin)
+          @client.set('a_queue', :mcguffin)
+        end
+
+        it "unwraps an envelope on a get" do
+          envelope = Envelope.new(:mcguffin)
+          mock(@raw_client).get('a_queue') { envelope }
+          mock.proxy(envelope).unwrap
+          @client.get('a_queue').should == :mcguffin
+        end
+
+        it "does not unwrap a nil get" do
+          mock(@raw_client).get('a_queue') { nil }
+          @client.get('a_queue').should be_nil
+        end
+      end
+    end
+  end
+end