redis_migrator 0.0.1

data/spec/mock_redis/lib/mock_redis/zset.rb

@@ -0,0 +1,110 @@
+require 'forwardable'
+require 'set'
+
+class MockRedis
+  class Zset
+    include Enumerable
+    extend Forwardable
+
+    attr_reader :members, :scores
+
+    def_delegators :members, :empty?, :include?, :size
+
+    def initialize
+      @members = Set.new
+      @scores  = Hash.new
+    end
+
+    def initialize_copy(source)
+      super
+      @members = @members.clone
+      @scores  = @scores.clone
+    end
+
+    def add(score, member)
+      members.add(member)
+      if score.to_f.to_i == score.to_f
+        scores[member] = score.to_f.to_i
+      else
+        scores[member] = score.to_f
+      end
+      self
+    end
+
+    def delete?(member)
+      scores.delete(member)
+      members.delete?(member) and self
+    end
+
+    def each
+      members.each {|m| yield score(m), m}
+    end
+
+    def in_range(min, max)
+      in_from_the_left = case min
+                         when "-inf"
+                             lambda {|_| true }
+                         when "+inf"
+                             lambda {|_| false }
+                         when /\((.*)$/
+                             val = $1.to_f
+                           lambda {|x| x.to_f > val }
+                         else
+                           lambda {|x| x.to_f >= min.to_f }
+                         end
+
+      in_from_the_right = case max
+                          when "-inf"
+                              lambda {|_| false }
+                          when "+inf"
+                              lambda {|_| true }
+                          when /\((.*)$/
+                              val = $1.to_f
+                            lambda {|x| x.to_f < val }
+                          else
+                            lambda {|x| x.to_f <= max.to_f }
+                          end
+
+      sorted.find_all do |(score, member)|
+        in_from_the_left[score] && in_from_the_right[score]
+      end
+    end
+
+    def intersection(other)
+      if !block_given?
+        intersection(other, &:+)
+      else
+        self.members.intersection(other.members).reduce(self.class.new) do |acc, m|
+          new_score = yield(self.score(m), other.score(m))
+          acc.add(new_score, m)
+        end
+      end
+    end
+
+    def score(member)
+      scores[member]
+    end
+
+    def sorted
+      members.map do |m|
+        [score(m), m]
+      end.sort_by(&:first)
+    end
+
+    def sorted_members
+      sorted.map(&:last)
+    end
+
+    def union(other)
+      if !block_given?
+        union(other, &:+)
+      else
+        self.members.union(other.members).reduce(self.class.new) do |acc, m|
+          new_score = yield(self.score(m), other.score(m))
+          acc.add(new_score, m)
+        end
+      end
+    end
+
+  end
+end

data/spec/mock_redis/lib/mock_redis/zset_methods.rb

@@ -0,0 +1,210 @@
+require 'mock_redis/assertions'
+require 'mock_redis/utility_methods'
+require 'mock_redis/zset'
+
+class MockRedis
+  module ZsetMethods
+    include Assertions
+    include UtilityMethods
+
+    def zadd(key, score, member)
+      assert_scorey(score)
+
+      retval = !zscore(key, member)
+      with_zset_at(key) {|z| z.add(score, member.to_s)}
+      retval
+    end
+
+    def zcard(key)
+      with_zset_at(key, &:size)
+    end
+
+    def zcount(key, min, max)
+      assert_scorey(min, 'min or max')
+      assert_scorey(max, 'min or max')
+
+      with_zset_at(key) do |z|
+        z.count do |score, _|
+          score >= min && score <= max
+        end
+      end
+    end
+
+    def zincrby(key, increment, member)
+      assert_scorey(increment)
+      member = member.to_s
+      with_zset_at(key) do |z|
+        old_score = z.include?(member) ? z.score(member) : 0
+        new_score = old_score + increment
+        z.add(new_score, member)
+        new_score.to_s
+      end
+    end
+
+    def zinterstore(destination, keys, options={})
+      assert_has_args(keys, 'zinterstore')
+
+      data[destination] = combine_weighted_zsets(keys, options, :intersection)
+      zcard(destination)
+    end
+
+    def zrange(key, start, stop, options={})
+      with_zset_at(key) do |z|
+        to_response(z.sorted[start..stop] || [], options)
+      end
+    end
+
+    def zrangebyscore(key, min, max, options={})
+      with_zset_at(key) do |zset|
+        all_results = zset.in_range(min, max)
+        to_response(apply_limit(all_results, options[:limit]), options)
+      end
+    end
+
+    def zrank(key, member)
+      with_zset_at(key) {|z| z.sorted_members.index(member.to_s) }
+    end
+
+    def zrem(key, member)
+      with_zset_at(key) {|z| !!z.delete?(member.to_s)}
+    end
+
+    def zrevrange(key, start, stop, options={})
+      with_zset_at(key) do |z|
+        to_response(z.sorted.reverse[start..stop], options)
+      end
+    end
+
+    def zremrangebyrank(key, start, stop)
+      zrange(key, start, stop).
+        each {|member| zrem(key, member)}.
+        size
+    end
+
+    def zremrangebyscore(key, min, max)
+      zrangebyscore(key, min, max).
+        each {|member| zrem(key, member)}.
+        size
+    end
+
+    def zrevrangebyscore(key, max, min, options={})
+      with_zset_at(key) do |zset|
+        to_response(
+          apply_limit(
+            zset.in_range(min, max).reverse,
+            options[:limit]),
+          options)
+      end
+    end
+
+    def zrevrank(key, member)
+      with_zset_at(key) {|z| z.sorted_members.reverse.index(member.to_s) }
+    end
+
+    def zscore(key, member)
+      with_zset_at(key) do |z|
+        score = z.score(member.to_s)
+        score.to_s if score
+      end
+    end
+
+    def zunionstore(destination, keys, options={})
+      assert_has_args(keys, 'zunionstore')
+
+      data[destination] = combine_weighted_zsets(keys, options, :union)
+      zcard(destination)
+    end
+
+    private
+    def apply_limit(collection, limit)
+      if limit
+        if limit.is_a?(Array) && limit.length == 2
+          offset, count = limit
+          collection.drop(offset).take(count)
+        else
+          raise RuntimeError, "ERR syntax error"
+        end
+      else
+        collection
+      end
+    end
+
+    def to_response(score_member_pairs, options)
+      score_member_pairs.map do |(score,member)|
+        if options[:with_scores] || options[:withscores]
+          [member, score.to_s]
+        else
+          member
+        end
+      end.flatten
+    end
+
+    def combine_weighted_zsets(keys, options, how)
+      weights = options.fetch(:weights, keys.map { 1 })
+      if weights.length != keys.length
+        raise RuntimeError, "ERR syntax error"
+      end
+
+      aggregator = case options.fetch(:aggregate, :sum).to_s.downcase.to_sym
+                   when :sum
+                     proc {|a,b| [a,b].compact.reduce(&:+)}
+                   when :min
+                     proc {|a,b| [a,b].compact.min}
+                   when :max
+                     proc {|a,b| [a,b].compact.max}
+                   else
+                     raise RuntimeError, "ERR syntax error"
+                   end
+
+      with_zsets_at(*keys) do |*zsets|
+        zsets.zip(weights).map do |(zset, weight)|
+          zset.reduce(Zset.new) do |acc, (score, member)|
+            acc.add(score * weight, member)
+          end
+        end.reduce do |za, zb|
+          za.send(how, zb, &aggregator)
+        end
+      end
+
+    end
+
+    def with_zset_at(key, &blk)
+      with_thing_at(key, :assert_zsety, proc {Zset.new}, &blk)
+    end
+
+    def with_zsets_at(*keys, &blk)
+      if keys.length == 1
+        with_zset_at(keys.first, &blk)
+      else
+        with_zset_at(keys.first) do |set|
+          with_zsets_at(*(keys[1..-1])) do |*sets|
+            blk.call(*([set] + sets))
+          end
+        end
+      end
+    end
+
+    def zsety?(key)
+      data[key].nil? || data[key].kind_of?(Zset)
+    end
+
+    def assert_zsety(key)
+      unless zsety?(key)
+        raise RuntimeError,
+        "ERR Operation against a key holding the wrong kind of value"
+      end
+    end
+
+    def looks_like_float?(x)
+      # ugh, exceptions for flow control.
+      !!Float(x) rescue false
+    end
+
+    def assert_scorey(value, what='value')
+      unless looks_like_float?(value)
+        raise RuntimeError, "ERR #{what} is not a double"
+      end
+    end
+
+  end
+end

data/spec/mock_redis/lib/mock_redis.rb

@@ -0,0 +1,119 @@
+require 'set'
+
+require 'mock_redis/assertions'
+require 'mock_redis/database'
+require 'mock_redis/distributed'
+require 'mock_redis/expire_wrapper'
+require 'mock_redis/multi_db_wrapper'
+require 'mock_redis/transaction_wrapper'
+require 'mock_redis/undef_redis_methods'
+
+class MockRedis
+  include UndefRedisMethods
+
+  attr_reader :options
+
+  DEFAULTS = {
+     :scheme => "redis",
+     :host => "127.0.0.1",
+     :port => 6379,
+     :path => nil,
+     :timeout => 5.0,
+     :password => nil,
+     :db => 0,
+  }
+
+  def initialize(*args)
+    @options = _parse_options(args.first)    
+
+    @db = TransactionWrapper.new(
+      ExpireWrapper.new(
+        MultiDbWrapper.new(
+          Database.new(*args))))
+  end
+
+  def id
+    "redis://#{self.host}:#{self.port}/#{self.db}"
+  end
+
+  def call(command, &block)
+     self.send(*command)
+  end
+
+  def host
+    self.options[:host]
+  end
+
+  def port
+    self.options[:port]
+  end
+
+  def db
+    self.options[:db]
+  end
+
+  def client
+    self
+  end
+
+  def respond_to?(method, include_private=false)
+    super || @db.respond_to?(method, include_private)
+  end
+
+  def method_missing(method, *args, &block)
+    @db.send(method, *args, &block)
+  end
+
+  def initialize_copy(source)
+    super
+    @db = @db.clone
+  end
+
+
+  protected
+
+  def _parse_options(options)
+    return {} if options.nil?
+
+    defaults = DEFAULTS.dup
+
+    url = options[:url] || ENV["REDIS_URL"]
+
+    # Override defaults from URL if given
+    if url
+      require "uri"
+
+      uri = URI(url)
+
+      if uri.scheme == "unix"
+        defaults[:path]   = uri.path
+      else
+        # Require the URL to have at least a host
+        raise ArgumentError, "invalid url" unless uri.host
+
+        defaults[:scheme]   = uri.scheme
+        defaults[:host]     = uri.host
+        defaults[:port]     = uri.port if uri.port
+        defaults[:password] = uri.password if uri.password
+        defaults[:db]       = uri.path[1..-1].to_i if uri.path
+      end
+    end
+
+    options = defaults.merge(options)
+
+    if options[:path]
+      options[:scheme] = "unix"
+      options.delete(:host)
+      options.delete(:port)
+    else
+      options[:host] = options[:host].to_s
+      options[:port] = options[:port].to_i
+    end
+
+    options[:timeout] = options[:timeout].to_f
+    options[:db] = options[:db].to_i
+
+    options
+  end
+
+end

data/spec/redis_helper_spec.rb

@@ -0,0 +1,58 @@
+describe Redis::Helper do 
+  before do
+    Redis.should_receive(:new).any_number_of_times {|options|  
+      MockRedis.new(options) 
+    }
+
+    @migrator = Redis::Migrator.new(["redis://localhost:6379"],
+                                    ["redis://localhost:6377"])
+
+    @r1 = @migrator.old_cluster
+    @r2 = @migrator.new_cluster
+    @migrator.stub!(:redis).and_return(@r1)
+    @pipe = PipeMock.new(@r2)
+  end
+
+  it "should copy a string" do
+    @r1.set("a", "some_string")
+    @migrator.copy_string(@pipe, "a")
+
+    @r2.get("a").should == "some_string"
+  end
+
+  it "should copy a hash" do
+    @r1.hmset("myhash", 
+      "first_name", "James",
+      "last_name", "Randi",
+      "age", "83")
+
+    @migrator.copy_hash(@pipe, "myhash")
+
+    @r2.hgetall("myhash").should == {"first_name" => "James", "last_name" => "Randi", "age" => "83"}
+  end
+
+  it "should copy a list" do
+    ('a'..'z').to_a.each { |val| @r1.lpush("mylist", val) }
+
+    @migrator.copy_list(@pipe, "mylist")
+
+    @r2.lrange("mylist", 0, -1).should == ('a'..'z').to_a
+  end
+
+  it "should copy a set" do
+    ('a'..'z').to_a.each { |val| @r1.sadd("myset", val) } 
+
+    @migrator.copy_set(@pipe, "myset")
+
+    @r2.smembers("myset").should == ('a'..'z').to_a
+  end
+
+  it "should copy zset" do
+    ('a'..'z').to_a.each { |val| @r1.zadd("myzset", rand(100), val) } 
+
+    @migrator.copy_zset(@pipe, "myzset")
+
+    @r2.zrange("myzset", 0, -1, :with_scores => true).sort.should == @r1.zrange("myzset", 0, -1, :with_scores => true).sort
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,29 @@
+$LOAD_PATH.unshift File.expand_path(File.join(File.dirname(__FILE__), "mock_redis/lib"))
+
+require_relative "../lib/redis_migrator.rb"
+
+# include patched version of mock_redis
+# that works with Redis::Distributed
+require "mock_redis.rb"
+
+class PipeMock
+  def initialize(redis)
+    @redis = redis
+  end
+
+  def close; true; end
+
+  def <<(val)
+    val[0] = val[0].downcase.to_sym
+    @redis.send(*val)
+  end
+end
+
+
+class Redis
+  module Helper
+    def to_redis_proto(*cmd)
+      cmd
+    end
+  end
+end

metadata

@@ -0,0 +1,107 @@
+--- !ruby/object:Gem::Specification
+name: redis_migrator
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Artem Yankov
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-07-05 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: &70245281843860 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.2.2
+  type: :runtime
+  prerelease: false
+  version_requirements: *70245281843860
+description: Redis-migrator takes a list of nodes for your old cluster and list of
+  nodes for your new cluster and determines for which keys routes were changed. Then
+  it moves those keys to new nodes.
+email:
+- artem.yankov@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- README.md
+- lib/redis_migrator.rb
+- lib/redis_migrator/redis_helper.rb
+- lib/redis_migrator/redis_populator.rb
+- migrator_benchmark.rb
+- redis_migrator.gemspec
+- spec/migrator_spec.rb
+- spec/mock_redis/lib/mock_redis.rb
+- spec/mock_redis/lib/mock_redis/assertions.rb
+- spec/mock_redis/lib/mock_redis/database.rb
+- spec/mock_redis/lib/mock_redis/distributed.rb
+- spec/mock_redis/lib/mock_redis/exceptions.rb
+- spec/mock_redis/lib/mock_redis/expire_wrapper.rb
+- spec/mock_redis/lib/mock_redis/hash_methods.rb
+- spec/mock_redis/lib/mock_redis/list_methods.rb
+- spec/mock_redis/lib/mock_redis/multi_db_wrapper.rb
+- spec/mock_redis/lib/mock_redis/set_methods.rb
+- spec/mock_redis/lib/mock_redis/string_methods.rb
+- spec/mock_redis/lib/mock_redis/transaction_wrapper.rb
+- spec/mock_redis/lib/mock_redis/undef_redis_methods.rb
+- spec/mock_redis/lib/mock_redis/utility_methods.rb
+- spec/mock_redis/lib/mock_redis/version.rb
+- spec/mock_redis/lib/mock_redis/zset.rb
+- spec/mock_redis/lib/mock_redis/zset_methods.rb
+- spec/redis_helper_spec.rb
+- spec/spec_helper.rb
+homepage: http://rubygems.org/gems/redis_migrator
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.17
+signing_key: 
+specification_version: 3
+summary: A tool to redistribute keys in your redis cluster when its topography has
+  changed
+test_files:
+- spec/migrator_spec.rb
+- spec/mock_redis/lib/mock_redis.rb
+- spec/mock_redis/lib/mock_redis/assertions.rb
+- spec/mock_redis/lib/mock_redis/database.rb
+- spec/mock_redis/lib/mock_redis/distributed.rb
+- spec/mock_redis/lib/mock_redis/exceptions.rb
+- spec/mock_redis/lib/mock_redis/expire_wrapper.rb
+- spec/mock_redis/lib/mock_redis/hash_methods.rb
+- spec/mock_redis/lib/mock_redis/list_methods.rb
+- spec/mock_redis/lib/mock_redis/multi_db_wrapper.rb
+- spec/mock_redis/lib/mock_redis/set_methods.rb
+- spec/mock_redis/lib/mock_redis/string_methods.rb
+- spec/mock_redis/lib/mock_redis/transaction_wrapper.rb
+- spec/mock_redis/lib/mock_redis/undef_redis_methods.rb
+- spec/mock_redis/lib/mock_redis/utility_methods.rb
+- spec/mock_redis/lib/mock_redis/version.rb
+- spec/mock_redis/lib/mock_redis/zset.rb
+- spec/mock_redis/lib/mock_redis/zset_methods.rb
+- spec/redis_helper_spec.rb
+- spec/spec_helper.rb