redis_ha 0.0.1

data/README.md

@@ -0,0 +1,38 @@
+RedisHA
+=======
+
+Three basic CRDTs (set, hashmap and counter) for redis. Also includes
+a ConnectionPool that allows you to run concurrent redis commands on
+multiple connections w/o using eventmachine/em-hiredis.
+
+
+Installation
+------------
+
+    gem install redis_ha
+
+or in your Gemfile:
+
+    gem 'redis_ha', '~> 0.1'
+
+
+License
+-------
+
+Copyright (c) 2011 Paul Asmuth
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to use, copy and modify copies of the Software, subject 
+to the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/lib/redis_ha/base.rb

@@ -0,0 +1,11 @@
+class RedisHA::Base
+
+  attr_accessor :pool, :key, :merge_strategy
+
+  def initialize(pool, key)
+    @pool = pool
+    @pool.ensure_connected
+    @key = key
+  end
+
+end

data/lib/redis_ha/connection.rb

@@ -0,0 +1,96 @@
+class RedisHA::Connection < Thread
+
+  attr_accessor :status, :buffer
+
+  def initialize(redis_opts, opts = {})
+    self.abort_on_exception = true
+
+    @read_timeout = opts[:read_timeout]
+    @retry_timeout = opts[:retry_timeout]
+    @redis_opts = redis_opts
+
+    @queue = Queue.new
+    @buffer = Array.new
+    @lock = Mutex.new
+
+    super do
+      run
+    end
+  end
+
+  def next
+    @lock.synchronize do
+      @buffer.shift
+    end
+  end
+
+  def <<(msg)
+    @queue << msg
+  end
+
+private
+
+  def run
+    while job = @queue.pop
+      semaphore, *msg = job
+
+      @lock.synchronize do
+        @buffer << send(*msg)
+      end
+
+      semaphore.decrement
+    end
+  end
+
+  def connect
+    with_timeout_and_check do
+      @redis = Redis.new(@redis_opts)
+      @redis.ping
+    end
+  end
+
+  def method_missing(*msg)
+    with_timeout_and_check do
+      @redis.send(*msg)
+    end
+  end
+
+  def with_timeout_and_check(&block)
+    return nil unless up_or_retry?
+    with_timeout(&block)
+  end
+
+  def with_timeout
+    ret = Timeout::timeout(@read_timeout) do
+      yield
+    end
+  rescue Redis::CannotConnectError
+    mark_as_down; nil
+  rescue Timeout::Error
+    mark_as_down; nil
+  else
+    mark_as_up; ret
+  end
+
+  def up_or_retry?
+    return true if @status == :up
+    return true unless @down_since
+
+    down_diff = Time.now.to_f - @down_since
+    return true if down_diff > @retry_timeout
+    false
+  end
+
+  def mark_as_down
+    @status = :down
+    @down_since = Time.now.to_f
+  end
+
+  def mark_as_up
+    return if @status == :up
+    @status = :up
+    @down_since = nil
+  end
+
+end
+

data/lib/redis_ha/connection_pool.rb

@@ -0,0 +1,68 @@
+class RedisHA::ConnectionPool
+
+  # timeout after which a redis connection is considered down. the
+  # default is 500ms
+  DEFAULT_READ_TIMEOUT  = 0.5
+
+  # timeout after which a redis that was marked as down is retried
+  # the default is 5s
+  DEFAULT_RETRY_TIMEOUT = 5
+
+  attr_accessor :status, :connections, :read_timeout, :retry_timeout
+
+  def initialize
+    @read_timeout  = DEFAULT_READ_TIMEOUT
+    @retry_timeout = DEFAULT_RETRY_TIMEOUT
+
+    @connections = []
+    @connected = false
+  end
+
+  def connect(*conns)
+    @connected = true
+
+    conns.each do |conn|
+      @connections << setup(conn)
+    end
+
+    async(:connect)
+  end
+
+  def ensure_connected
+    unless @connected
+      raise RedisHA::Error.new("you need to call Base.connect first")
+    end
+
+    unless @connections.map(&:status).include?(:up)
+      raise RedisHA::Error.new("no servers available")
+    end
+  end
+
+  def method_missing(*msg)
+    ensure_connected
+    async(*msg)
+  end
+
+private
+
+  def async(*msg)
+    @semaphore = RedisHA::Semaphore.new(@connections.size)
+
+    @connections.each do |conn|
+      conn << [@semaphore, *msg]
+    end
+
+    @semaphore.wait
+
+    @connections.map(&:next).tap do
+      ensure_connected
+    end
+  end
+
+  def setup(redis_opts)
+    RedisHA::Connection.new(redis_opts,
+      :retry_timeout => @retry_timeout,
+      :read_timeout => @read_timeout)
+  end
+
+end

data/lib/redis_ha/counter.rb

@@ -0,0 +1,31 @@
+class RedisHA::Counter < RedisHA::Base
+
+  # this lambda defines how the individual response hashes are merged
+  # the default is to select the maximum value
+  DEFAULT_MERGE_STRATEGY = ->(v) { v.map(&:to_i).max }
+
+  def incr(n = 1)
+    pool.incrby(@key, n)
+    true
+  end
+
+  def decr(n = 1)
+    pool.decrby(@key, n)
+    true
+  end
+
+  def set(n)
+    pool.set(@key, n)
+    true
+  end
+
+  def get
+    versions = pool.get(@key).compact
+    merge_strategy[versions]
+  end
+
+  def merge_strategy
+    @merge_strategy || DEFAULT_MERGE_STRATEGY
+  end
+
+end

data/lib/redis_ha/hash_map.rb

@@ -0,0 +1,31 @@
+class RedisHA::HashMap < RedisHA::Base
+
+  # this lambda defines how the individual response hashes are merged
+  # the default is to merge in reverse-chronological order
+  DEFAULT_MERGE_STRATEGY = ->(v) { v
+    .sort{ |a,b| a[:_time] <=> b[:_time] }
+    .inject({}){ |t,c| t.merge!(c) } }
+
+
+  def set(data = {})
+    data.merge!(:_time => Time.now.to_i)
+    pool.set(@key, Marshal.dump(data))
+    true
+  end
+
+  def get
+    versions = pool.get(@key).map do |v|
+      next if v.nil? || v == ""
+      puts v.inspect
+      Marshal.load(v) rescue nil
+    end.compact
+    merge_strategy[versions].tap do |merged|
+      merged.delete(:_time)
+    end
+  end
+
+  def merge_strategy
+    @merge_strategy || DEFAULT_MERGE_STRATEGY
+  end
+
+end

data/lib/redis_ha/semaphore.rb

@@ -0,0 +1,20 @@
+class RedisHA::Semaphore
+
+  POLL_INTERVAL = 0.00001
+
+  def initialize(n)
+    @lock = Mutex.new
+    @n = n
+  end
+
+  def decrement
+    @lock.synchronize do
+      @n -= 1
+    end
+  end
+
+  def wait
+    sleep(POLL_INTERVAL) while @n != 0
+  end
+
+end

data/lib/redis_ha/set.rb

@@ -0,0 +1,26 @@
+class RedisHA::Set < RedisHA::Base
+
+  # this lambda defines how the individual response hashes are merged
+  # the default is set union
+  DEFAULT_MERGE_STRATEGY = ->(v) { v.inject(&:+).uniq }
+
+  def add(*items)
+    pool.sadd(@key, *items)
+    true
+  end
+
+  def rem(*items)
+    pool.srem(@key, *items)
+    true
+  end
+
+  def get
+    versions = pool.smembers(@key).compact
+    merge_strategy[versions]
+  end
+
+  def merge_strategy
+    @merge_strategy || DEFAULT_MERGE_STRATEGY
+  end
+
+end

data/lib/redis_ha.rb

@@ -0,0 +1,16 @@
+require "rubygems"
+require "redis"
+require "timeout"
+
+module RedisHA
+  class Error < StandardError
+  end
+end
+
+require "redis_ha/base"
+require "redis_ha/semaphore"
+require "redis_ha/connection"
+require "redis_ha/connection_pool"
+require "redis_ha/hash_map"
+require "redis_ha/set"
+require "redis_ha/counter"

data/lib/test.rb

@@ -0,0 +1,71 @@
+require "rubygems"; require "redis"; require "pp"; require "ripl"
+
+def bm(label)
+  t = Time.now.to_f
+  yield
+  d = (Time.now.to_f - t) * 1000
+  puts "#{label}: #{d.to_i}ms"
+end
+
+$: << ::File.expand_path("..", __FILE__)
+require "redis_ha"
+
+pool = RedisHA::ConnectionPool.new
+pool.retry_timeout = 0.5
+pool.read_timeout = 0.5
+pool.connect(
+  {:host => "localhost", :port => 6379},
+  {:host => "localhost", :port => 6380},
+  {:host => "localhost", :port => 6381},
+  {:host => "localhost", :port => 6385})
+
+map = RedisHA::HashMap.new(pool, "fnordmap")
+set = RedisHA::Set.new(pool, "fnordset")
+ctr = RedisHA::Counter.new(pool, "fnordctr")
+
+Ripl.start :binding => binding
+exit
+
+
+bm "1000x HashMap.set w/ retries" do
+  1000.times do |n|
+    map.set(:fu=>:bar, :fnord=>:bar)
+  end
+end
+
+pool = RedisHA::ConnectionPool.new
+pool.retry_timeout = 50
+pool.read_timeout = 0.5
+pool.connect(
+  {:host => "localhost", :port => 6379},
+  {:host => "localhost", :port => 6380},
+  {:host => "localhost", :port => 6385})
+
+map = RedisHA::HashMap.new(pool, "fnordmap")
+
+bm "1000x HashMap.set w/o retries" do
+  1000.times do |n|
+    map.set(:fu=>:bar, :fnord=>:bar)
+  end
+end
+
+
+bm "sequential connect" do
+  pool = RedisHA::ConnectionPool.new
+  pool.connect(:host => "localhost", :port => 6379)
+  pool.connect(:host => "localhost", :port => 6380)
+  pool.connect(:host => "localhost", :port => 6385)
+  pool.connect(:host => "localhost", :port => 6382)
+  pool.connect(:host => "localhost", :port => 6383)
+end
+
+bm "async connect" do
+  pool = RedisHA::ConnectionPool.new
+  pool.connect(
+    {:host => "localhost", :port => 6379},
+    {:host => "localhost", :port => 6380},
+    {:host => "localhost", :port => 6385},
+    {:host => "localhost", :port => 6382},
+    {:host => "localhost", :port => 6383}
+  )
+end

data/redis_ha.gemspec

@@ -0,0 +1,21 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = "redis_ha"
+  s.version     = "0.0.1"
+  s.date        = Date.today.to_s
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Paul Asmuth"]
+  s.email       = ["paul@paulasmuth.com"]
+  s.homepage    = "http://github.com/paulasmuth/redis_ha"
+  s.summary     = %q{basic CRDTs and a HA connection pool for redis}
+  s.description = %q{Three basic CRDTs (set, hashmap and counter) for redis. Also includes a ConnectionPool that allows you to run concurrent redis commands on multiple connections w/o using eventmachine/em-hiredis.}
+  s.licenses    = ["MIT"]
+
+  s.add_dependency "redis", ">= 2.2.2"
+
+  s.files         = `git ls-files`.split("\n") - [".gitignore", ".rspec", ".travis.yml"]
+  s.test_files    = `git ls-files -- spec/*`.split("\n")
+  s.require_paths = ["lib"]
+end

metadata

@@ -0,0 +1,70 @@
+--- !ruby/object:Gem::Specification
+name: redis_ha
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Paul Asmuth
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-11-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: &19924060 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.2.2
+  type: :runtime
+  prerelease: false
+  version_requirements: *19924060
+description: Three basic CRDTs (set, hashmap and counter) for redis. Also includes
+  a ConnectionPool that allows you to run concurrent redis commands on multiple connections
+  w/o using eventmachine/em-hiredis.
+email:
+- paul@paulasmuth.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- lib/redis_ha.rb
+- lib/redis_ha/base.rb
+- lib/redis_ha/connection.rb
+- lib/redis_ha/connection_pool.rb
+- lib/redis_ha/counter.rb
+- lib/redis_ha/hash_map.rb
+- lib/redis_ha/semaphore.rb
+- lib/redis_ha/set.rb
+- lib/test.rb
+- redis_ha.gemspec
+homepage: http://github.com/paulasmuth/redis_ha
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.17
+signing_key: 
+specification_version: 3
+summary: basic CRDTs and a HA connection pool for redis
+test_files: []