redis_migrator 0.1.1 → 0.1.2

data/lib/redis_migrator.rb

@@ -27,9 +27,7 @@ class Redis
     #   { "redis://host1.com" => ['key1', 'key2', 'key3'],
     #     "redis://host2.com => ['key4', 'key5', 'key6']" }
     def changed_keys
-      keys = @old_cluster.keys("*")
-
-      keys.inject({}) do |acc, key|
+      old_cluster_keys.inject({}) do |acc, key|
         old_node = @old_cluster.node_for(key).client
         new_node = @new_cluster.node_for(key).client
 
@@ -43,6 +41,16 @@ class Redis
       end
     end
 
+    # Returns all the keys that need to be redistributed. Can be overidden to
+    # return only a subset of old keys if migrating from a single redis environment
+    # to a mixed environment with both regular and distributed redis instances
+    # @return an array of keys that should be checked for redistributing
+    # @example Returned value
+    #   ['key1', 'key2', 'key3']
+    def old_cluster_keys
+      @old_cluster_keys ||= @old_cluster.keys("*")
+    end
+
     # Migrates a given array of keys to a given redis node
     # @param node [Hash] options for redis node keys will be migrated to
     # @param keys [Array] array of keys that need to be migrated

data/lib/redis_migrator/redis_native_migrator.rb

@@ -18,10 +18,26 @@ class Redis
       end
 
       grouped_by_old_nodes.each do |old_node, node_keys|
-        old_node.pipelined do
-          node_keys.each { |key| old_node.migrate(key, new_node_options) }
+        destination_node_options = new_node_options.merge(
+          timeout: 30 # lets add a generous timeout here
+        )
+
+        node_keys.each_slice(1000) do |slice|
+          old_node.pipelined do
+            slice.each do |key|
+              migrate_key(old_node, key, destination_node_options)
+            end
+          end
         end
       end
     end
+
+    def migrate_key(node, key, options)
+      if Redis::VERSION > '3.0.4'
+        node.migrate(key, options)
+      else
+        node.migrate([options[:host], options[:port], key, options[:db], options[:timeout]])
+      end
+    end
   end
 end

data/redis_migrator.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("./lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "redis_migrator"
-  s.version     = "0.1.1"
+  s.version     = "0.1.2"
   s.date        = "2014-04-10"
   s.platform    = Gem::Platform::RUBY
   s.authors     = ["Artem Yankov"]

metadata

@@ -1,7 +1,8 @@
 --- !ruby/object:Gem::Specification
 name: redis_migrator
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
+  prerelease: 
 platform: ruby
 authors:
 - Artem Yankov
@@ -12,74 +13,59 @@ date: 2014-04-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &70148249978680 !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: 3.0.0
   type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 3.0.0
+  version_requirements: *70148249978680
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &70148249978100 !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '2.6'
   type: :development
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '2.6'
+  version_requirements: *70148249978100
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &70148249977600 !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+  version_requirements: *70148249977600
 - !ruby/object:Gem::Dependency
   name: debugger
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &70148249976980 !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+  version_requirements: *70148249976980
 - !ruby/object:Gem::Dependency
   name: mock_redis
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &70148249976460 !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+  version_requirements: *70148249976460
 description: Redis-migrator takes a list of nodes for your old cluster and list of
   nodes for your new cluster and determines for which keys routes were changed. Then
   it moves those keys to new nodes.
@@ -110,26 +96,27 @@ files:
 - spec/spec_helper.rb
 homepage: http://rubygems.org/gems/redis_migrator
 licenses: []
-metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 1.8.17
 signing_key: 
-specification_version: 4
+specification_version: 3
 summary: A tool to redistribute keys in your redis cluster when its topography has
   changed
 test_files:
@@ -140,4 +127,3 @@ test_files:
 - spec/redis_pipe_migrator_spec.rb
 - spec/shared_hosts_context.rb
 - spec/spec_helper.rb
-has_rdoc: 

checksums.yaml

@@ -1,7 +0,0 @@
----
-SHA1:
-  metadata.gz: 98a618ae2641c4bf6eadc6cae753d8219f0ab039
-  data.tar.gz: e07f091488059a7d20f7ce0bdf05fb43566632ef
-SHA512:
-  metadata.gz: 7cda9a9894f2a79c4a500eb15cf9ab6f5efe6fcf512112b9dcfe818c0f672ee4e1894d2aae5d03624db69aee3452821cabf1597c3c6f1588f93785bdd6b5711f
-  data.tar.gz: f6d7e25a19eff8d52cc6704d528eeb6c5fa03bfd5ea4d8f6ac5fec92ba675b14c55d1c3ff8b5d332511b6c69ef86f784e5e5a6d2a0fc381ca9026a1a24126bd5