redis_migrator 0.0.1 → 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 98a618ae2641c4bf6eadc6cae753d8219f0ab039
+  data.tar.gz: e07f091488059a7d20f7ce0bdf05fb43566632ef
+SHA512:
+  metadata.gz: 7cda9a9894f2a79c4a500eb15cf9ab6f5efe6fcf512112b9dcfe818c0f672ee4e1894d2aae5d03624db69aee3452821cabf1597c3c6f1588f93785bdd6b5711f
+  data.tar.gz: f6d7e25a19eff8d52cc6704d528eeb6c5fa03bfd5ea4d8f6ac5fec92ba675b14c55d1c3ff8b5d332511b6c69ef86f784e5e5a6d2a0fc381ca9026a1a24126bd5

data/.gitignore

@@ -17,3 +17,4 @@ tmp
 _yardoc
 doc/
 .DS_Store
+.idea/

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/README.md

@@ -22,7 +22,7 @@ and determines for which keys routes were changed. Then it moves those keys to n
     old_redis_hosts = ["redis://host1.com:6379", "redis://host2.com:6379"]
 
     # a list of redis-urls for a new cluster
-    old_redis_hosts = ["redis://host1.com:6379", "redis://host2.com:6379", "redis://host3.com:6379"]
+    new_redis_hosts = ["redis://host1.com:6379", "redis://host2.com:6379", "redis://host3.com:6379"]
 
     migrator = Redis::Migrator.new(old_redis_hosts, new_redis_hosts)
     migrator.run
@@ -31,5 +31,17 @@ and determines for which keys routes were changed. Then it moves those keys to n
 * ruby 1.9 or jruby (with --1.9 flag)
 * redis >=2.4.14 (only on machine where migrator will be running)
 
+##Contributing
+
+    # First fork the project.
+    # Then bundle
+    bundle
+
+    # and make sure tests pass
+    bundle exec rspec spec
+
+    # Add features and profit.
+    # Send a pull request back to the original repository.
+
 ##TODO
 * Error handling 

data/Rakefile

@@ -0,0 +1,6 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new
+
+task :test => :spec

data/lib/redis_migrator/redis_helper.rb

@@ -1,9 +1,8 @@
 class Redis
   module Helper
-
     def to_redis_proto(*cmd)
       cmd.inject("*#{cmd.length}\r\n") {|acc, arg|
-        acc << "$#{arg.length}\r\n#{arg}\r\n"
+        acc << "$#{arg.to_s.bytesize}\r\n#{arg}\r\n"
       }
     end
 
@@ -13,40 +12,10 @@ class Redis
       db = path[1..-1].to_i rescue 0
 
       {
-        :host => node.host,
-        :port => node.port || 6379,
-        :db   => db
+        host: node.host,
+        port: node.port || 6379,
+        db:   db
       }
     end
-
-    def copy_string(pipe, key)
-      value = redis.get(key)
-      pipe << to_redis_proto("SET", key, value)
-    end
-  
-    def copy_hash(pipe, key)
-      redis.hgetall(key).each do |field, value|
-        pipe << to_redis_proto("HSET", key, field, value) 
-      end
-    end
-  
-    def copy_list(pipe, key)
-      redis.lrange(key, 0, -1).each do |value|
-        pipe << to_redis_proto("LPUSH", key, value)
-      end
-    end
-  
-    def copy_set(pipe, key)
-      redis.smembers(key).each do |member|
-        pipe << to_redis_proto("SADD", key, member)
-      end
-    end
-  
-    def copy_zset(pipe, key)
-      redis.zrange(key, 0, -1, :with_scores => true).each_slice(2) do |member, score|
-        pipe << to_redis_proto("ZADD", key, score, member)
-      end 
-    end
-
   end
-end
+end

data/lib/redis_migrator/redis_native_migrator.rb

@@ -0,0 +1,27 @@
+class Redis
+  class NativeMigrator
+    def initialize(old_hosts)
+      Thread.current[:redis] = Redis::Distributed.new(old_hosts)
+    end
+
+    def redis
+      Thread.current[:redis]
+    end
+
+    def migrate(node_options, keys, _)
+      new_node_options = { host: node_options[:host],
+                           port: node_options[:port],
+                           db:   node_options[:db] }
+
+      grouped_by_old_nodes = keys.group_by do |key|
+        redis.node_for(key)
+      end
+
+      grouped_by_old_nodes.each do |old_node, node_keys|
+        old_node.pipelined do
+          node_keys.each { |key| old_node.migrate(key, new_node_options) }
+        end
+      end
+    end
+  end
+end

data/lib/redis_migrator/redis_pipe_migrator.rb

@@ -0,0 +1,66 @@
+class Redis
+  class PipeMigrator
+    include Redis::Helper
+
+    def initialize(old_hosts)
+      Thread.current[:redis] = Redis::Distributed.new(old_hosts)
+    end
+
+    def redis
+      Thread.current[:redis]
+    end
+
+    def migrate(node_options, keys, options)
+      host, port, db = node_options[:host], node_options[:port], node_options[:db]
+      pipe = IO.popen("redis-cli -h #{host} -p #{port} -n #{db} --pipe", IO::RDWR)
+
+      keys.each {|key|
+        copy_key(pipe, key)
+
+        #remove key from old node
+        redis.node_for(key).del(key) unless options[:do_not_remove]
+      }
+
+      pipe.close
+    end
+
+    # Copy a given Redis key to a Redis pipe
+    # @param pipe [IO] a pipe opened  redis-cli --pipe
+    # @param key [String] a Redis key that needs to be copied
+    def copy_key(pipe, key)
+      key_type = redis.type(key)
+      return false unless ['list', 'hash', 'string', 'set', 'zset'].include?(key_type)
+
+      self.send("copy_#{key_type}", pipe, key)
+    end
+
+    def copy_string(pipe, key)
+      value = redis.get(key)
+      pipe << to_redis_proto('SET', key, value)
+    end
+
+    def copy_hash(pipe, key)
+      redis.hgetall(key).each do |field, value|
+        pipe << to_redis_proto('HSET', key, field, value)
+      end
+    end
+
+    def copy_list(pipe, key)
+      redis.lrange(key, 0, -1).each do |value|
+        pipe << to_redis_proto('RPUSH', key, value)
+      end
+    end
+
+    def copy_set(pipe, key)
+      redis.smembers(key).reverse.each do |member|
+        pipe << to_redis_proto('SADD', key, member)
+      end
+    end
+
+    def copy_zset(pipe, key)
+      redis.zrange(key, 0, -1, with_scores: true).each do |member, score|
+        pipe << to_redis_proto('ZADD', key, score, member)
+      end
+    end
+  end
+end

data/lib/redis_migrator/redis_populator.rb

@@ -33,7 +33,7 @@ class Redis
 
       keys.each do |key|
         size.times.map do |x| 
-          f << to_redis_proto("SADD", key, ::Digest::MD5.hexdigest("f" + x.to_s))
+          f << to_redis_proto('SADD', key, ::Digest::MD5.hexdigest('f' + x.to_s))
         end
       end
 

data/lib/redis_migrator.rb

@@ -1,7 +1,8 @@
-require 'rubygems'
 require 'redis'
 require 'redis/distributed'
 require_relative 'redis_migrator/redis_helper'
+require_relative 'redis_migrator/redis_pipe_migrator'
+require_relative 'redis_migrator/redis_native_migrator'
 
 class Redis
   class Migrator
@@ -48,19 +49,8 @@ class Redis
     # @param options [Hash] additional options, such as :do_not_remove => true
     def migrate_keys(node, keys, options={})
       return false if keys.empty? || keys.nil?
-      
-      Thread.current[:redis] = Redis::Distributed.new(old_hosts)
 
-      pipe = IO.popen("redis-cli -h #{node[:host]} -p #{node[:port]} -n #{node[:db]} --pipe", IO::RDWR)
-
-      keys.each {|key|
-        copy_key(pipe, key)
-
-        #remove key from old node
-        redis.node_for(key).del(key) unless options[:do_not_remove]
-      }
-
-      pipe.close
+      migrator(options[:do_not_remove]).new(old_hosts).migrate(node, keys, options)
     end
 
     # Runs a migration process for a Redis cluster. 
@@ -82,15 +72,24 @@ class Redis
       threads.each{|t| t.join}
     end
 
-    # Copy a given Redis key to a Redis pipe
-    # @param pipe [IO] a pipe opened  redis-cli --pipe 
-    # @param key [String] a Redis key that needs to be copied
-    def copy_key(pipe, key)
-      key_type = old_cluster.type(key)
-      return false unless ['list', 'hash', 'string', 'set', 'zset'].include?(key_type)
+    private
+
+    def nodes
+      old_cluster.nodes + new_cluster.nodes
+    end
 
-      self.send("copy_#{key_type}", pipe, key)
+    def old_nodes
+      @old_nodes ||= nodes.select { |node| node.info['redis_version'].to_f < 2.6 }
     end
 
+    def migrator(keep_original)
+      @migrator ||= begin
+        if old_nodes.any? || keep_original
+          Redis::PipeMigrator
+        else
+          Redis::NativeMigrator
+        end
+      end
+    end
   end # class Migrator
 end # class Redis

data/migrator_benchmark.rb

@@ -18,5 +18,4 @@ migrator = Redis::Migrator.new(old_redis_hosts, new_redis_hosts)
 Benchmark.bm do |x|
   x.report("populate:") { r.populate_cluster(1000, 100) }
   x.report("migrate:")  { migrator.run }
-end
-
+end

data/redis_migrator.gemspec

@@ -3,8 +3,8 @@ $:.push File.expand_path("./lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "redis_migrator"
-  s.version     = "0.0.1"
-  s.date        = "2012-07-05"
+  s.version     = "0.1.1"
+  s.date        = "2014-04-10"
   s.platform    = Gem::Platform::RUBY
   s.authors     = ["Artem Yankov"]
   s.email       = ["artem.yankov@gmail.com"]
@@ -16,5 +16,9 @@ Gem::Specification.new do |s|
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
-  s.add_dependency('redis', '>= 2.2.2')
-end
+  s.add_dependency('redis', '>= 3.0.0')
+  s.add_development_dependency 'rspec', '~> 2.6'
+  s.add_development_dependency 'rake'
+  s.add_development_dependency 'debugger'
+  s.add_development_dependency 'mock_redis'
+end

data/spec/different_redis_type_migrator.rb

@@ -0,0 +1,67 @@
+shared_examples 'different redis type migrator' do
+  context 'key of type' do
+    let(:keys) { [key] }
+
+    subject { migrator.migrate(node, keys, {}) }
+
+    context 'string' do
+      let(:key) { 'a' }
+
+      it 'should copy' do
+        old_cluster.set(key, 'some_string')
+        subject
+        destination_cluster.get(key).should == 'some_string'
+      end
+    end
+
+
+    context 'hash' do
+      let(:key) { 'myhash' }
+
+      it 'should copy' do
+        old_cluster.hmset(key,
+                          'first_name', 'James',
+                          'last_name', 'Randi',
+                          'age', '83')
+        subject
+        destination_cluster.hgetall(key).should == {'first_name' => 'James',
+                                                    'last_name' => 'Randi',
+                                                    'age' => '83'}
+      end
+    end
+
+    context 'list' do
+      let(:key) { 'mylist' }
+
+      it 'should copy' do
+        ('a'..'z').to_a.each { |val| old_cluster.lpush(key, val) }
+        values = old_cluster.lrange(key, 0, -1)
+        subject
+        destination_cluster.lrange(key, 0, -1).should == values
+      end
+    end
+
+    context 'set' do
+      let(:key) { 'myset' }
+      it 'should copy' do
+        ('a'..'z').to_a.each { |val| old_cluster.sadd(key, val) }
+        values = old_cluster.smembers(key)
+        subject
+        destination_cluster.smembers(key).should == values
+      end
+    end
+
+    context 'zset' do
+      let(:key) { 'myzset' }
+      it 'should copy' do
+        ('a'..'z').to_a.each { |val| old_cluster.zadd(key, rand(100), val) }
+        old_range = old_cluster.zrange(key, 0, -1, with_scores: true).sort
+
+        subject
+
+        new_range = destination_cluster.zrange(key, 0, -1, with_scores: true).sort
+        new_range.should == old_range
+      end
+    end
+  end
+end

data/spec/pretested_migrator.rb

@@ -0,0 +1,47 @@
+shared_context 'common keys' do
+  let(:keys) { %w(q s j) }
+  let(:node) { { host: 'localhost', port: 6378, db: 1 } }
+  let(:options) { {} }
+
+  before do
+    prefill_cluster(old_cluster)
+    migrator.migrate(node, keys, options)
+  end
+
+  def common_keys(cluster)
+    (cluster.keys('*') & keys).sort
+  end
+
+  subject { common_keys(cluster) }
+end
+
+shared_examples 'pretested migrator' do
+  include_context 'common keys'
+
+  context do
+    let(:cluster) { destination_cluster }
+    it 'should copy given keys to a new cluster' do
+      should == %w(j q s)
+    end
+  end
+end
+
+shared_examples 'safe pretested migrator' do
+  include_context 'common keys'
+
+  context do
+    let(:cluster) { old_cluster }
+
+    it 'should remove copied keys from the old redis node' do
+      should == []
+    end
+
+    context 'when asked to not remove' do
+      let(:options) { { do_not_remove: true } }
+      it 'should keep keys on old node' do
+        should == ["j", "q", "s"]
+      end
+    end
+  end
+end
+

data/spec/redis_migrator_spec.rb

@@ -0,0 +1,41 @@
+require_relative 'shared_hosts_context'
+
+describe Redis::Migrator do
+  include_context 'shared hosts context'
+  let(:migrator) { Redis::Migrator.new(old_hosts, new_hosts) }
+
+  describe '#changed_keys' do
+    before { prefill_cluster(migrator.old_cluster) }
+
+    it 'should show keys which need migration' do
+      migrator.changed_keys.should == {'redis://localhost:6377/0' => %w(h q s y j m n o)}
+    end
+  end
+
+  describe '#migrator' do
+    subject { migrator.send(:migrator, keep_original) }
+    let(:keep_original) { false }
+
+    before do
+      allow_any_instance_of(MockRedis).to(
+          receive(:info).and_return({ 'redis_version' => version })
+      )
+    end
+
+    context 'when all instances are old' do
+      let(:version) { '2.4.1' }
+      it { should == Redis::PipeMigrator}
+    end
+
+    context 'when all instances are new' do
+      let(:version) { '2.6.14' }
+      it { should == Redis::NativeMigrator }
+
+      context 'when asking to preserve data on source' do
+        let(:keep_original) { true }
+
+        it { should == Redis::PipeMigrator }
+      end
+    end
+  end
+end

data/spec/redis_native_migrator_spec.rb

@@ -0,0 +1,44 @@
+require_relative 'shared_hosts_context'
+require_relative 'different_redis_type_migrator'
+require_relative 'pretested_migrator'
+
+describe Redis::NativeMigrator do
+  let(:migrator) { Redis::NativeMigrator.new(old_hosts) }
+  include_context 'shared hosts context'
+
+  let(:old_cluster) { Redis::Distributed.new(old_hosts) }
+  let(:new_cluster) { Redis::Distributed.new(new_hosts) }
+
+  before { allow(migrator).to receive(:redis).and_return(old_cluster) }
+
+  describe '#migrate' do
+    context do
+      let(:node) { new_cluster.node_for(key) }
+      let(:source_node) { old_cluster.node_for(key) }
+      let(:destination_cluster) { source_node.client.select(10); source_node }
+      before do
+        allow_any_instance_of(MockRedis).to receive(:migrate) do |key|
+          source_node.move(key, 10)
+        end
+      end
+
+      it_behaves_like 'different redis type migrator'
+    end
+
+    context do
+      let(:old_cluster) { Redis::Distributed.new([old_hosts.first]) }
+      let(:source_node) { old_cluster.nodes.first }
+      let(:destination_cluster) { source_node.client.select(10); source_node }
+
+      before do
+        allow_any_instance_of(MockRedis).to receive(:migrate) do |key|
+          source_node.move(key, 10)
+        end
+      end
+
+      it_behaves_like 'pretested migrator'
+      # Not supported in Redis < 3.0
+      # it_behaves_like 'safe pretested migrator'
+    end
+  end
+end

data/spec/redis_pipe_migrator_spec.rb

@@ -0,0 +1,51 @@
+require_relative 'shared_hosts_context'
+require_relative 'different_redis_type_migrator'
+require_relative 'pretested_migrator'
+
+describe Redis::PipeMigrator do
+  let(:migrator) { Redis::PipeMigrator.new(old_hosts) }
+  include_context 'shared hosts context'
+
+  let(:old_cluster) { Redis::Distributed.new(old_hosts) }
+  let(:destination_cluster) { Redis::Distributed.new(new_hosts) }
+  let(:pipe) { PipeMock.new(destination_cluster) }
+
+  before { allow(migrator).to receive(:redis).and_return(old_cluster) }
+
+  describe '#migrate' do
+    context do
+      let(:node) { {} }
+      before { expect(IO).to receive(:popen).and_return(pipe) }
+      it_behaves_like 'different redis type migrator'
+    end
+
+    context do
+      before do
+        command = 'redis-cli -h localhost -p 6378 -n 1 --pipe'
+        expect(IO).to receive(:popen).with(command, IO::RDWR).and_return(pipe)
+      end
+
+      it_behaves_like 'pretested migrator'
+      it_behaves_like 'safe pretested migrator'
+    end
+  end
+
+  describe '#copy_key' do
+    subject { migrator.copy_key(nil, key) }
+
+    context 'with unknown key' do
+      let(:key) { 'some_key' }
+      it { should == false }
+    end
+
+    context 'when known set key' do
+      let(:key) { 'a' }
+      before { old_cluster.sadd('a', 1) }
+
+      it 'calls copy_set' do
+        expect(migrator).to receive(:copy_set).with(nil, 'a')
+        subject
+      end
+    end
+  end
+end

data/spec/shared_hosts_context.rb

@@ -0,0 +1,10 @@
+shared_context 'shared hosts context' do
+  let(:old_hosts) { %w(redis://localhost:6379 redis://localhost:6378) }
+  let(:new_hosts) { old_hosts + ['redis://localhost:6377'] }
+
+  before do
+    expect(Redis).to receive(:new).at_least(1).times do |options|
+      MockRedis.new(options)
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -1,10 +1,6 @@
-$LOAD_PATH.unshift File.expand_path(File.join(File.dirname(__FILE__), "mock_redis/lib"))
-
+require 'rspec'
 require_relative "../lib/redis_migrator.rb"
-
-# include patched version of mock_redis
-# that works with Redis::Distributed
-require "mock_redis.rb"
+require 'mock_redis'
 
 class PipeMock
   def initialize(redis)
@@ -26,4 +22,10 @@ class Redis
       cmd
     end
   end
-end
+end
+
+def prefill_cluster(cluster)
+  ('a'..'z').to_a.each do |key|
+    (1..5).to_a.each {|val| cluster.sadd(key, val)}
+  end
+end