siberite-client 0.8.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e7848ea76b5512913552938cfac31d21fedce5b2
+  data.tar.gz: 7cbdf9d5fbed9c3aa10ccd8aa917d3f882f2f8a6
+SHA512:
+  metadata.gz: 1762a60bc54881a542068783b82ffe10bfbee2f5aeaba80831e0542e5365552bb9f3e6e21c72c77e8976ad90cf159ef0a1f94c4bad9598b701e010d263836eb6
+  data.tar.gz: c74d681cc09007725fbe9d6996e66ea4a6820c713266be633e2997cd227d04265e7c468d146b5dfdf996b2cd01abe2e7a866691d57fa010d3847f47c3d075f28

data/.gitignore

@@ -0,0 +1,3 @@
+pkg/*
+Gemfile.lock
+*.gem

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in ngrok-tunnel.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,14 @@
+Copyright 2015 Anton Bogdanovich
+Copyright 2010-2014 Twitter, Inc.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.

data/README.md

@@ -0,0 +1,63 @@
+## siberite-client: Talk to Siberite queue server from Ruby
+
+siberite-client is a library that allows you to talk to a [Siberite](http://github.com/robey/siberite) queue server from ruby. As Siberite uses the memcache protocol, siberite-client is implemented as a wrapper around the memcached gem.
+
+
+## Installation
+
+you will need to install memcached.gem, though rubygems should do this for you. just:
+
+    sudo gem install siberite-client
+
+
+## Basic Usage
+
+`Siberite::Client.new` takes a list of servers and an options hash. See the [rdoc for Memcached](http://blog.evanweaver.com/files/doc/fauna/memcached/classes/Memcached.html) for an explanation of what the various options do.
+
+    require 'siberite'
+
+    $queue = Siberite::Client.new('localhost:22133')
+    $queue.set('a_queue', 'foo')
+    $queue.get('a_queue') # => 'foo'
+
+
+## Client Proxies
+
+siberite-client comes with a number of decorators that change the behavior of the raw client.
+
+    $queue = Siberite::Client.new('localhost:22133')
+    $queue.get('empty_queue') # => nil
+
+    $queue = Siberite::Client::Blocking.new(Siberite::Client.new('localhost:22133'))
+    $queue.get('empty_queue') # does not return until it pulls something from the queue
+
+
+## Configuration Management
+
+Siberite::Config provides some tools for pulling queue config out of a YAML config file.
+
+    Siberite::Config.load 'path/to/siberite.yml'
+    Siberite::Config.environment = 'production' # defaults to development
+
+    $queue = Siberite::Config.new_client
+
+This tells siberite-client to look for `path/to/siberite.yml`, and pull the client configuration out of
+the 'production' key in that file. Sample config:
+
+    defaults: &defaults
+      distribution: :random
+      timeout: 2
+      connect_timeout: 1
+
+    production:
+      <<: *defaults
+      servers:
+        - siberite01.example.com:22133
+        - siberite02.example.com:22133
+        - siberite03.example.com:22133
+
+    development:
+      <<: *defaults
+      servers:
+        - localhost:22133
+      show_backtraces: true

data/Rakefile

@@ -0,0 +1,18 @@
+ROOT_DIR = File.expand_path(File.dirname(__FILE__))
+
+require 'rubygems' rescue nil
+require 'rake'
+require 'rspec/core/rake_task'
+
+task :default => :spec
+
+desc "Run all specs in spec directory."
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.rspec_opts = ['--options', "\"#{ROOT_DIR}/spec/spec.opts\""]
+end
+
+desc "Run benchmarks"
+RSpec::Core::RakeTask.new(:benchmark) do |t|
+  t.rspec_opts = ['--options', "\"#{ROOT_DIR}/spec/spec.opts\""]
+  t.pattern = 'spec/*_benchmark.rb'
+end

data/lib/siberite-client.rb

@@ -0,0 +1 @@
+require 'siberite'

data/lib/siberite.rb

@@ -0,0 +1,9 @@
+require 'yaml'
+require 'socket'
+require 'memcached'
+
+require 'siberite/client'
+
+module Siberite
+  autoload :Config, 'siberite/config'
+end

data/lib/siberite/client.rb

@@ -0,0 +1,202 @@
+require 'forwardable'
+
+module Siberite
+  class Client
+    require 'siberite/client/stats_helper'
+
+    autoload :Proxy, 'siberite/client/proxy'
+    autoload :Envelope, 'siberite/client/envelope'
+    autoload :Blocking, 'siberite/client/blocking'
+    autoload :Partitioning, 'siberite/client/partitioning'
+    autoload :Unmarshal, 'siberite/client/unmarshal'
+    autoload :Namespace, 'siberite/client/namespace'
+    autoload :Json, 'siberite/client/json'
+    autoload :Transactional, "siberite/client/transactional"
+
+    SIBERITE_OPTIONS = [:gets_per_server, :exception_retry_limit, :get_timeout_ms].freeze
+
+    DEFAULT_OPTIONS = {
+      :retry_timeout => 0,
+      :exception_retry_limit => 5,
+      :timeout => 0.25,
+      :gets_per_server => 100,
+      :get_timeout_ms => 10
+    }.freeze
+
+    # Exceptions which are connection failures we retry after
+    RECOVERABLE_ERRORS = [
+      Memcached::ServerIsMarkedDead,
+      Memcached::ATimeoutOccurred,
+      Memcached::ConnectionBindFailure,
+      Memcached::ConnectionFailure,
+      Memcached::ConnectionSocketCreateFailure,
+      Memcached::Failure,
+      Memcached::MemoryAllocationFailure,
+      Memcached::ReadFailure,
+      Memcached::ServerError,
+      Memcached::SystemError,
+      Memcached::UnknownReadFailure,
+      Memcached::WriteFailure,
+      Memcached::NotFound
+    ]
+
+    extend Forwardable
+    include StatsHelper
+
+    attr_accessor :servers, :options
+    attr_reader :current_queue, :siberite_options, :current_server
+
+    def_delegators :@write_client, :add, :append, :cas, :decr, :incr, :get_orig, :prepend
+
+    def initialize(*servers)
+      opts = servers.last.is_a?(Hash) ? servers.pop : {}
+      opts = DEFAULT_OPTIONS.merge(opts)
+
+      @siberite_options = extract_siberite_options!(opts)
+      @default_get_timeout = siberite_options[:get_timeout_ms]
+      @gets_per_server = siberite_options[:gets_per_server]
+      @exception_retry_limit = siberite_options[:exception_retry_limit]
+      @counter = 0
+      @shuffle = true
+
+      # we handle our own retries so that we can apply different
+      # policies to sets and gets, so set memcached limit to 0
+      opts[:exception_retry_limit] = 0
+      opts[:distribution] = :random # force random distribution
+
+      self.servers  = Array(servers).flatten.compact
+      self.options  = opts
+
+      @server_count = self.servers.size # Minor optimization.
+      @read_client  = Memcached.new(self.servers[rand(@server_count)], opts)
+      @write_client = Memcached.new(self.servers[rand(@server_count)], opts)
+    end
+
+    def delete(key, expiry=0)
+      with_retries { @write_client.delete key }
+    rescue Memcached::NotFound, Memcached::ServerEnd
+    end
+
+    def set(key, value, ttl=0, raw=false)
+      with_retries { @write_client.set key, value, ttl, !raw }
+      true
+    rescue Memcached::NotStored
+      false
+    end
+
+    # This provides the necessary semantic to support transactionality
+    # in the Transactional client. It temporarily disables server
+    # shuffling to allow the client to close any open transactions on
+    # the current server before jumping.
+    #
+    def get_from_last(*args)
+      @shuffle = false
+      get *args
+    ensure
+      @shuffle = true
+    end
+
+    # ==== Parameters
+    # key<String>:: Queue name
+    # opts<Boolean,Hash>:: True/false toggles Marshalling. A Hash
+    #                      allows collision-avoiding options support.
+    #
+    # ==== Options (opts)
+    # :open<Boolean>:: Begins a transactional read.
+    # :close<Boolean>:: Ends a transactional read.
+    # :abort<Boolean>:: Cancels an existing transactional read
+    # :peek<Boolean>:: Return the head of the queue, without removal
+    # :timeout<Integer>:: Milliseconds to block for a new item
+    # :raw<Boolean>:: Toggles Marshalling. Equivalent to the "old
+    #                 style" second argument.
+    #
+    def get(key, opts = {})
+      raw = opts[:raw] || false
+      commands = extract_queue_commands(opts)
+
+      val =
+        begin
+          shuffle_if_necessary! key
+          @read_client.get key + commands, !raw
+        rescue *RECOVERABLE_ERRORS
+          # we can't tell the difference between a server being down
+          # and an empty queue, so just return nil. our sticky server
+          # logic should eliminate piling on down servers
+          nil
+        end
+
+      # nil result without :close and :abort, force next get to jump from
+      # current server
+      if !val && @shuffle && !opts[:close] && !opts[:abort]
+        @counter = @gets_per_server
+      end
+
+      val
+    end
+
+    def flush(queue)
+      count = 0
+      while sizeof(queue) > 0
+        count += 1 while get queue, :raw => true
+      end
+      count
+    end
+
+    def peek(queue)
+      get queue, :peek => true
+    end
+
+    private
+
+    def extract_siberite_options!(opts)
+      siberite_opts, memcache_opts = opts.inject([{}, {}]) do |(siberite, memcache), (key, opt)|
+        (SIBERITE_OPTIONS.include?(key) ? siberite : memcache)[key] = opt
+        [siberite, memcache]
+      end
+      opts.replace(memcache_opts)
+      siberite_opts
+    end
+
+    def shuffle_if_necessary!(key)
+      return unless @server_count > 1
+      # Don't reset servers on the first request:
+      # i.e. @counter == 0 && @current_queue == nil
+      if @shuffle &&
+          (@counter > 0 && key != @current_queue) ||
+          @counter >= @gets_per_server
+        @counter = 0
+        @current_queue = key
+        @read_client.reset(servers[rand(@server_count)])
+      else
+        @counter +=1
+      end
+    end
+
+    def extract_queue_commands(opts)
+      commands = [:open, :close, :abort, :peek].select do |key|
+        opts[key]
+      end
+
+      if timeout = (opts[:timeout] || @default_get_timeout)
+        commands << "t=#{timeout}"
+      end
+
+      commands.map { |c| "/#{c}" }.join('')
+    end
+
+    def with_retries #:nodoc:
+      yield
+    rescue *RECOVERABLE_ERRORS
+      tries ||= @exception_retry_limit + 1
+      tries -= 1
+      @write_client.reset(servers[rand(@server_count)])
+
+      if tries > 0
+        retry
+      else
+        raise
+      end
+    end
+
+  end
+end

data/lib/siberite/client/blocking.rb

@@ -0,0 +1,36 @@
+module Siberite
+  class Client
+    class Blocking < Proxy
+
+      # random backoff sleeping
+
+      SLEEP_TIMES = [[0] * 1, [0.01] * 2, [0.1] * 2, [0.5] * 2, [1.0] * 1].flatten
+
+      def get(*args)
+        count = 0
+
+        while count += 1
+
+          if response = client.get(*args)
+            return response
+          end
+
+          sleep_for_count(count)
+        end
+      end
+
+      def get_without_blocking(*args)
+        client.get(*args)
+      end
+
+      private
+
+      def sleep_for_count(count)
+        base = SLEEP_TIMES[count] || SLEEP_TIMES.last
+
+        time = ((rand * base) + base) / 2
+        sleep time if time > 0
+      end
+    end
+  end
+end

data/lib/siberite/client/envelope.rb

@@ -0,0 +1,25 @@
+module Siberite
+  class Client
+    class Envelope < Proxy
+      attr_accessor :envelope_class
+
+      def initialize(envelope_class, client)
+        @envelope_class = envelope_class
+        super(client)
+      end
+
+      def get(*args)
+        response = client.get(*args)
+        if response.respond_to?(:unwrap)
+          response.unwrap
+        else
+          response
+        end
+      end
+
+      def set(key, value, *args)
+        client.set(key, envelope_class.new(value), *args)
+      end
+    end
+  end
+end

data/lib/siberite/client/json.rb

@@ -0,0 +1,16 @@
+require 'json'
+
+module Siberite
+  class Client
+    class Json < Proxy
+      def get(*args)
+        response = client.get(*args)
+        if response.is_a?(String)
+          HashWithIndifferentAccess.new(JSON.parse(response)) rescue response
+        else
+          response
+        end
+      end
+    end
+  end
+end

data/lib/siberite/client/namespace.rb

@@ -0,0 +1,29 @@
+module Siberite
+  class Client
+    class Namespace < Proxy
+      def initialize(namespace, client)
+        @namespace = namespace
+        @matcher = /\A#{Regexp.escape(@namespace)}:(.+)/
+        super(client)
+      end
+
+      %w(set get delete flush stat).each do |method|
+        class_eval "def #{method}(key, *args); client.#{method}(namespace(key), *args) end", __FILE__, __LINE__
+      end
+
+      def available_queues
+        client.available_queues.map {|q| in_namespace(q) }.compact
+      end
+
+      def in_namespace(key)
+        if match = @matcher.match(key)
+          match[1]
+        end
+      end
+
+      def namespace(key)
+        "#{@namespace}:#{key}"
+      end
+    end
+  end
+end

data/lib/siberite/client/partitioning.rb

@@ -0,0 +1,35 @@
+module Siberite
+  class Client
+    class Partitioning < Proxy
+
+      def initialize(client_map)
+        @clients = client_map.inject({}) do |clients, (keys, client)|
+          Array(keys).inject(clients) do |_, key|
+            clients.update(key => client)
+          end
+        end
+      end
+
+      def clients
+        @clients.values.uniq
+      end
+
+      def default_client
+        @clients[:default]
+      end
+      alias client default_client
+
+      %w(set get delete flush stat).each do |method|
+        class_eval "def #{method}(key, *args); client_for(key).#{method}(key, *args) end", __FILE__, __LINE__
+      end
+
+      def stats
+        merge_stats(clients.map {|c| c.stats })
+      end
+
+      def client_for(key)
+        @clients[key.to_s.split('/', 2).first] || default_client
+      end
+    end
+  end
+end