redis_migrator 0.0.1

data/.gitignore

@@ -0,0 +1,19 @@
+*.gem
+*.rbc
+.bundle
+.config
+coverage
+InstalledFiles
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+
+# YARD artifacts
+.yardoc
+_yardoc
+doc/
+.DS_Store

data/.rspec

@@ -0,0 +1,2 @@
+--colour -d --require spec_helper
+

data/README.md

@@ -0,0 +1,35 @@
+Redis-migrator
+==============
+Redis-migrator is a tool to redistribute keys in your redis cluster when its topography has
+changed. 
+
+##How it works
+
+Say you are using Redis::Distributed to distribute your writes and reads across different
+redis nodes. Redis::Distributed uses consitent hashing algorithm to determine where a command
+will go. If you changed configuration of your cluster, for example, changed a hostname or added a new node then routes for some of the keys will change too. If you try to read such a key - you won't get a data from its old node.
+
+Redis-migrator takes a list of nodes for your old cluster and list of nodes for your new cluster 
+and determines for which keys routes were changed. Then it moves those keys to new nodes.
+
+##Install  
+`gem install redis_migrator`
+
+##Usage  
+    require 'redis_migrator'
+
+    # a list of redis-urls for an old cluster
+    old_redis_hosts = ["redis://host1.com:6379", "redis://host2.com:6379"]
+
+    # a list of redis-urls for a new cluster
+    old_redis_hosts = ["redis://host1.com:6379", "redis://host2.com:6379", "redis://host3.com:6379"]
+
+    migrator = Redis::Migrator.new(old_redis_hosts, new_redis_hosts)
+    migrator.run
+
+##Requirements
+* ruby 1.9 or jruby (with --1.9 flag)
+* redis >=2.4.14 (only on machine where migrator will be running)
+
+##TODO
+* Error handling 

data/lib/redis_migrator/redis_helper.rb

@@ -0,0 +1,52 @@
+class Redis
+  module Helper
+
+    def to_redis_proto(*cmd)
+      cmd.inject("*#{cmd.length}\r\n") {|acc, arg|
+        acc << "$#{arg.length}\r\n#{arg}\r\n"
+      }
+    end
+
+    def parse_redis_url(redis_url)
+      node = URI(redis_url)
+      path = node.path
+      db = path[1..-1].to_i rescue 0
+
+      {
+        :host => node.host,
+        :port => node.port || 6379,
+        :db   => db
+      }
+    end
+
+    def copy_string(pipe, key)
+      value = redis.get(key)
+      pipe << to_redis_proto("SET", key, value)
+    end
+  
+    def copy_hash(pipe, key)
+      redis.hgetall(key).each do |field, value|
+        pipe << to_redis_proto("HSET", key, field, value) 
+      end
+    end
+  
+    def copy_list(pipe, key)
+      redis.lrange(key, 0, -1).each do |value|
+        pipe << to_redis_proto("LPUSH", key, value)
+      end
+    end
+  
+    def copy_set(pipe, key)
+      redis.smembers(key).each do |member|
+        pipe << to_redis_proto("SADD", key, member)
+      end
+    end
+  
+    def copy_zset(pipe, key)
+      redis.zrange(key, 0, -1, :with_scores => true).each_slice(2) do |member, score|
+        pipe << to_redis_proto("ZADD", key, score, member)
+      end 
+    end
+
+  end
+end

data/lib/redis_migrator/redis_populator.rb

@@ -0,0 +1,63 @@
+require 'digest'
+require_relative 'redis_helper'
+
+class Redis
+  class Populator
+    include Redis::Helper
+
+    attr_accessor :redis
+
+    def initialize(redis_hosts)
+      @redis = Redis::Distributed.new(redis_hosts)
+    end
+
+    # Generate sets' keys to populate redis cluster
+    # @param num size [Integer] the number of sets that have to be created
+    # @return hash of keys grouped by redis node
+    def generate_keys(num)
+      num.times.inject({}) do |acc, i| 
+        key = ::Digest::MD5.hexdigest(i.to_s)
+        node = redis.node_for(key).client
+        hash_key = "redis://#{node.host}:#{node.port}/#{node.db}"
+        acc[hash_key] = [] if acc[hash_key].nil?
+        acc[hash_key] << key
+        acc
+      end
+    end
+
+    # Populates sets with the given amount of members
+    # @param node [Hash] a parsed redis_url 
+    # @param keys [Array] an array of sets' keys that need to be populated 
+    def populate_keys(node, keys, size)
+      f = IO.popen("redis-cli -h #{node[:host]} -p #{node[:port]} -n #{node[:db]} --pipe", IO::RDWR)
+
+      keys.each do |key|
+        size.times.map do |x| 
+          f << to_redis_proto("SADD", key, ::Digest::MD5.hexdigest("f" + x.to_s))
+        end
+      end
+
+      f.close
+    end
+
+    # Populates redis cluster 
+    # @param keys_num [Integer] amount of redis sets that need to be populated
+    # @param num [Integer] number of members in each set
+    def populate_cluster(keys_num, num)
+      redis.flushdb
+
+      nodes = generate_keys(keys_num)
+      threads = []
+
+      nodes.keys.each do |node_url|
+        node = parse_redis_url(node_url)
+
+        threads << Thread.new(node, nodes[node_url], num) {|node, keys, size|
+          populate_keys(node, keys, size)
+        }
+      end
+
+      threads.each{|t| t.join}
+    end
+  end # class Populator
+end # class Redis

data/lib/redis_migrator.rb

@@ -0,0 +1,96 @@
+require 'rubygems'
+require 'redis'
+require 'redis/distributed'
+require_relative 'redis_migrator/redis_helper'
+
+class Redis
+  class Migrator
+    include Redis::Helper
+
+    attr_accessor :old_cluster, :new_cluster, :old_hosts, :new_hosts
+
+    def initialize(old_hosts, new_hosts)
+      @old_hosts = old_hosts
+      @new_hosts = new_hosts
+      @old_cluster = Redis::Distributed.new(old_hosts)
+      @new_cluster = Redis::Distributed.new(new_hosts)
+    end
+
+    def redis
+      Thread.current[:redis]
+    end
+
+    # Finds redis keys for which migration is needed
+    # @return a hash of keys grouped by node they need to be written to
+    # @example Returned value
+    #   { "redis://host1.com" => ['key1', 'key2', 'key3'],
+    #     "redis://host2.com => ['key4', 'key5', 'key6']" }
+    def changed_keys
+      keys = @old_cluster.keys("*")
+
+      keys.inject({}) do |acc, key|
+        old_node = @old_cluster.node_for(key).client
+        new_node = @new_cluster.node_for(key).client
+
+        if (old_node.host != new_node.host) || (old_node.port != new_node.port)
+          hash_key = "redis://#{new_node.host}:#{new_node.port}/#{new_node.db}"
+          acc[hash_key] = [] if acc[hash_key].nil?
+          acc[hash_key] << key
+        end
+
+        acc
+      end
+    end
+
+    # Migrates a given array of keys to a given redis node
+    # @param node [Hash] options for redis node keys will be migrated to
+    # @param keys [Array] array of keys that need to be migrated
+    # @param options [Hash] additional options, such as :do_not_remove => true
+    def migrate_keys(node, keys, options={})
+      return false if keys.empty? || keys.nil?
+      
+      Thread.current[:redis] = Redis::Distributed.new(old_hosts)
+
+      pipe = IO.popen("redis-cli -h #{node[:host]} -p #{node[:port]} -n #{node[:db]} --pipe", IO::RDWR)
+
+      keys.each {|key|
+        copy_key(pipe, key)
+
+        #remove key from old node
+        redis.node_for(key).del(key) unless options[:do_not_remove]
+      }
+
+      pipe.close
+    end
+
+    # Runs a migration process for a Redis cluster. 
+    # @param [Hash] additional options such as :do_not_remove => true
+    def run(options={})
+      keys_to_migrate = changed_keys
+      puts "Migrating #{keys_to_migrate.values.flatten.count} keys"
+      threads = []
+
+      keys_to_migrate.keys.each do |node_url|
+        node = parse_redis_url(node_url)
+        
+        #spawn a separate thread for each Redis pipe
+        threads << Thread.new(node, keys_to_migrate[node_url]) {|node, keys|
+          migrate_keys(node, keys, options)
+        }
+      end
+
+      threads.each{|t| t.join}
+    end
+
+    # Copy a given Redis key to a Redis pipe
+    # @param pipe [IO] a pipe opened  redis-cli --pipe 
+    # @param key [String] a Redis key that needs to be copied
+    def copy_key(pipe, key)
+      key_type = old_cluster.type(key)
+      return false unless ['list', 'hash', 'string', 'set', 'zset'].include?(key_type)
+
+      self.send("copy_#{key_type}", pipe, key)
+    end
+
+  end # class Migrator
+end # class Redis

data/migrator_benchmark.rb

@@ -0,0 +1,22 @@
+require 'redis_migrator'
+require 'uri'
+require 'benchmark'
+require 'redis_migrator/redis_populator'
+
+# a list of hosts for an old cluster
+# You either have to start 3 Redis instances on your local - each on its
+# own port: 6379, 6378, 6377. Or, better, have a real Redis nodes running
+old_redis_hosts = ["redis://localhost:6379/9", "redis://localhost:6378/9"]
+
+# a list of hosts for a new cluster
+new_redis_hosts = ["redis://localhost:6379/9", "redis://localhost:6378/9", "redis://localhost:6377/9"]
+
+r = Redis::Populator.new(old_redis_hosts)
+
+migrator = Redis::Migrator.new(old_redis_hosts, new_redis_hosts)                               
+
+Benchmark.bm do |x|
+  x.report("populate:") { r.populate_cluster(1000, 100) }
+  x.report("migrate:")  { migrator.run }
+end
+

data/redis_migrator.gemspec

@@ -0,0 +1,20 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("./lib", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = "redis_migrator"
+  s.version     = "0.0.1"
+  s.date        = "2012-07-05"
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Artem Yankov"]
+  s.email       = ["artem.yankov@gmail.com"]
+  s.homepage    = "http://rubygems.org/gems/redis_migrator"
+  s.summary     = %q{A tool to redistribute keys in your redis cluster when its topography has changed}
+  s.description = %q{Redis-migrator takes a list of nodes for your old cluster and list of nodes for your new cluster and determines for which keys routes were changed. Then it moves those keys to new nodes.}
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+  s.add_dependency('redis', '>= 2.2.2')
+end

data/spec/migrator_spec.rb

@@ -0,0 +1,63 @@
+describe Redis::Migrator do
+  
+  before do
+    Redis.should_receive(:new).any_number_of_times {|options|  
+      MockRedis.new(options) 
+    }
+  
+    @migrator = Redis::Migrator.new(["redis://localhost:6379", "redis://localhost:6378"],
+                                    ["redis://localhost:6379", "redis://localhost:6378", "redis://localhost:6377"])
+
+    #populate old cluster with some keys
+    ('a'..'z').to_a.each do |key|
+      (1..5).to_a.each {|val| @migrator.old_cluster.sadd(key, val)}
+    end
+  end
+
+  describe Redis::Migrator, "#changed_keys" do
+    it "should show keys which need migration" do
+      @migrator.changed_keys.should == {"redis://localhost:6377/0" => ["h", "q", "s", "y", "j", "m", "n", "o"]}
+    end
+  end
+
+  describe Redis::Migrator, "#migrate_keys" do
+    let(:keys) { ["q", "s", "j"] }
+    let(:node) { {:host => "localhost", :port => 6378, :db => 1} }
+
+    before do
+      Redis::Distributed.should_receive(:new).and_return(@migrator.old_cluster)
+      
+      @pipe = PipeMock.new(@migrator.new_cluster)
+      IO.should_receive(:popen).with("redis-cli -h localhost -p 6378 -n 1 --pipe", IO::RDWR).and_return(@pipe)
+    end
+
+    it "should copy given keys to a new cluster" do
+      @migrator.migrate_keys(node, keys)
+      (@migrator.new_cluster.keys("*") & keys).sort.should == ["j", "q", "s"]
+    end
+
+    it "should remove copied keys from the old redis node" do
+      @migrator.migrate_keys(node, keys)
+      (@migrator.old_cluster.keys("*") & keys).sort.should == []      
+    end
+
+    it "should keep keys on old node if asked" do
+      @migrator.migrate_keys(node, keys, :do_not_remove => true)
+      (@migrator.old_cluster.keys("*") & keys).sort.should == ["j", "q", "s"]      
+    end
+  end
+
+  describe Redis::Migrator, "#copy_key" do
+
+    it "should return FALSE for unknown key" do
+      @migrator.copy_key(nil, "some_key").should == false
+    end
+
+    it "should call copy_set if given key is set" do
+      @migrator.should_receive(:copy_set).with(nil, "a")
+      @migrator.copy_key(nil, "a")
+    end
+
+  end
+
+end

data/spec/mock_redis/lib/mock_redis/assertions.rb

@@ -0,0 +1,13 @@
+class MockRedis
+  module Assertions
+    private
+
+    def assert_has_args(args, command)
+      unless args.any?
+        raise RuntimeError,
+        "ERR wrong number of arguments for '#{command}' command"
+      end
+    end
+      
+  end
+end