redis-copy 0.0.3 → 0.0.5

data/README.md

@@ -21,7 +21,7 @@ The current options can be grabbed using the `--help` flag.
 
 ```
 $ redis-copy --help
-redis-copy v0.0.2
+redis-copy v0.0.5
 Usage: redis-copy [options] <source> <destination>
     <source> and <destination> must be redis connection uris
     like [redis://]<hostname>[:<port>][/<db>]
@@ -31,13 +31,14 @@ Specific options:
                                        auto:    uses new if available, otherwise fallback
                                        new:     use redis DUMP and RESTORE commands (faster)
                                        classic: migrates via multiple type-specific commands
+        --[no-]pipeline              Use redis pipeline where available (default true)
+    -d, --[no-]debug                 Write debug output (default false)
+    -t, --[no-]trace                 Enable backtrace on failure (default false)
+    -f, --[no-]fail-fast             Abort on first failure (default false)
+        --[no-]verify [PERCENT]      Verify percentage of transfers -- VERY SLOW (default 0)
+        --[no-]prompt                Prompt for confirmation (default true)
+        --[no-]allow-nonempty        Allow non-empty destination (default false)
         --[no-]dry-run               Output configuration and exit
-    -d, --[no-]debug                 Write debug output
-    -t, --[no-]trace                 Enable backtrace on failure
-    -f, --[no-]fail-fast             Abort on first failure
-    -y, --yes                        Automatically accept any prompts
-        --[no-]allow-nonempty        Allow non-empty destination
-
 ```
 
 ## Example:

data/lib/redis-copy.rb

@@ -45,6 +45,17 @@ module RedisCopy
           ui.notify("FAIL: #{key.dump}")
           ui.abort if options[:fail_fast]
         end
+
+        probably(options[:verify]) do
+          if strategem.verify?(key)
+            stats[:verified] += 1
+          else
+            stats[:borked] += 1
+            ui.notify("BORK: #{key.dump}")
+            ui.abort if options[:fail_fast]
+          end
+        end
+
         ui.notify("PROGRESS: #{stats.inspect}") if (stats[:attempt] % 1000).zero?
       end.tap do |stats|
         ui.notify("DONE: #{stats.inspect}")
@@ -53,6 +64,17 @@ module RedisCopy
 
     private
 
+    # yields the block, probably.
+    # @param probability [Integer] 0-100
+    # @return [void]
+    # used by ::copy in determining whether to verify.
+    def probably(probability)
+      return if probability.zero?
+      if probability >= 100 || (Random::rand(100) % 100) < probability
+        yield
+      end
+    end
+
     def same_redis?(redis_a, redis_b)
       # Redis::Client#id returns the connection uri
       # e.g. 'redis://localhost:6379/0'

data/lib/redis-copy/cli.rb

@@ -10,10 +10,12 @@ module RedisCopy
       ui:             :command_line,
       key_emitter:    :default,
       strategy:       :auto,
+      verify:         0,
       pipeline:       :true,
       fail_fast:      false,
       prompt:         true,
       trace:          false,
+      debug:          false,
       allow_nonempty: false,
     }.freeze unless defined?(DEFAULTS)
 
@@ -64,8 +66,21 @@ module RedisCopy
           options[:fail_fast] = ff
         end
 
-        opts.on('--[no-]prompt', "Prompt for confirmation (default #{DEFAULTS[:prompt]})") do
-          options[:prompt] = true
+        opts.on('--[no-]verify [PERCENT]',
+          "Verify percentage of transfers -- VERY SLOW (default #{DEFAULTS[:verify]})"
+        ) do |verify|
+          options[:verify] = case verify
+                             when /\A1?[0-9]{2}\z/
+                               verify.to_i
+                             when false, 'false', 'none'
+                               0
+                             else
+                               100
+                             end
+        end
+
+        opts.on('--[no-]prompt', "Prompt for confirmation (default #{DEFAULTS[:prompt]})") do |prompt|
+          options[:prompt] = prompt
         end
 
         opts.on('--[no-]allow-nonempty', "Allow non-empty destination (default #{DEFAULTS[:allow_nonempty]})") do |allow_nonempty|

data/lib/redis-copy/key-emitter.rb

@@ -28,6 +28,10 @@ module RedisCopy
       raise NotImplementedError
     end
 
+    def dbsize
+      @redis.dbsize
+    end
+
     def to_s
       self.class.name.demodulize.humanize
     end
@@ -36,10 +40,16 @@ module RedisCopy
     class Default
       include KeyEmitter
 
-      def initialize(redis, ui, options = {})
-        ui.abort unless ui.confirm? <<-EOWARNING.strip_heredoc
+      def keys
+        dbsize = self.dbsize
+
+        # HT: http://stackoverflow.com/a/11466770
+        dbsize_str = dbsize.to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse
+
+        @ui.abort unless (dbsize < 10_000) || (@ui.confirm? <<-EOWARNING.strip_heredoc)
           WARNING: #{self} key emitter uses redis.keys('*') to
-          get its list of keys.
+          get its list of keys, and you have #{dbsize_str} keys in
+          your source DB.
 
           The redis keys command [reference](http://redis.io/commands/keys)
           says this:
@@ -52,10 +62,7 @@ module RedisCopy
           > regular application code. If you're looking for a way to find
           > keys in a subset of your keyspace, consider using sets.
         EOWARNING
-        super
-      end
 
-      def keys
         @ui.debug "REDIS: #{@redis.client.id} KEYS *"
         @redis.keys('*').to_enum
       end

data/lib/redis-copy/strategy.rb

@@ -21,6 +21,10 @@ module RedisCopy
       copierklass.new(source, destination, ui, options)
     end
 
+    def self.included(base)
+      base.send(:include, Verifier)
+    end
+
     # @param source [Redis]
     # @param destination [Redis]
     def initialize(source, destination, ui, options = {})
@@ -40,5 +44,46 @@ module RedisCopy
       return super if defined? super
       raise NotImplementedError
     end
+
+    def verify?(key)
+      @ui.debug("VERIFY: #{key.dump}")
+      type = @src.type(key)
+      proc = case type
+             when 'string' then proc { |r| r.get key }
+             when 'hash'   then proc { |r| r.hgetall key }
+             when 'zset'   then proc { |r| r.zrange(key, 0, -1, :with_scores => true) }
+             when 'set'    then proc { |r| r.smembers(key).sort }
+             when 'list'   then proc { |r| r.lrange(key, 0, -1) }
+             else
+               @ui.debug("BORK: #{key.dump} has unknown type #{type.dump}!")
+               return false
+             end
+
+      return false unless same_response?(&proc)
+      return false unless same_response? { |r| r.ttl key }
+
+      true
+    end
+
+    private
+
+    def same_response?(&blk)
+      responses = {
+        source:      capture_result(@src, &blk),
+        destination: capture_result(@dst, &blk)
+      }
+      if (responses[:source] == responses[:destination])
+        return true
+      else
+        @ui.debug("MISMATCH: #{keys.dump} #{responses.inspect}")
+        return false
+      end
+    end
+
+    def capture_result(redis, &block)
+      return [:returned, block.call(redis)]
+      rescue Object => exception
+      return [:raised, exception]
+    end
   end
 end

data/lib/redis-copy/strategy/classic.rb

@@ -6,6 +6,7 @@ module RedisCopy
       include Strategy
 
       def copy(key)
+        @ui.debug("COPY: #{key.dump}")
         vtype = @src.type(key)
         ttl = @src.ttl(key)
 

data/lib/redis-copy/strategy/new.rb

@@ -6,6 +6,8 @@ module RedisCopy
       include Strategy
 
       def copy(key)
+        @ui.debug("COPY: #{key.dump}")
+
         ttl = @src.ttl(key)
         # TTL returns seconds, -1 means none set
         # RESTORE ttl is in miliseconds, 0 means none set

data/lib/redis-copy/ui.rb

@@ -1,5 +1,8 @@
 # encoding: utf-8
 
+require_relative 'ui/auto_run'
+require_relative 'ui/command_line'
+
 module RedisCopy
   module UI
     def self.load(options = {})

data/lib/redis-copy/version.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
 
 module RedisCopy
-  VERSION = '0.0.3'
+  VERSION = '0.0.5'
 end

data/spec/redis-copy/key-emitter_spec.rb

@@ -6,9 +6,11 @@ describe RedisCopy::KeyEmitter::Default do
   let(:ui) { double.as_null_object }
   let(:instance) { RedisCopy::KeyEmitter::Default.new(redis, ui)}
   let(:connection_uri) { 'redis://12.34.56.78:9000/15' }
+  let(:key_count) { 100_000 }
 
   before(:each) do
     redis.stub_chain('client.id').and_return(connection_uri)
+    redis.stub(:dbsize) { key_count }
     ui.stub(:debug).with(anything)
   end
 
@@ -28,6 +30,22 @@ describe RedisCopy::KeyEmitter::Default do
           exactly(:once)
         instance.keys
       end
+      context 'when source has > 10,000 keys' do
+        let(:key_count) { 100_000 }
+        it 'should ask for confirmation' do
+          ui.should_receive(:confirm?) do |confirmation|
+            confirmation.should match /\b100,000\b/
+          end
+          instance.keys
+        end
+      end
+      context 'when source has <= 10,000 keys' do
+        let(:key_count) { 1_000 }
+        it 'should not ask for confirmation' do
+          ui.should_not_receive(:confirm?)
+          instance.keys
+        end
+      end
     end
   end
 end

data/spec/redis-copy/strategy_spec.rb

@@ -31,60 +31,76 @@ class RedisMultiplex < Struct.new(:source, :destination)
   end
 end
 
+shared_examples_for(:no_ttl) do
+  # key, redis,
+  subject { redis.ttl(key) }
+  it { should eq -1 }
+end
+
+shared_examples_for(:ttl_set) do
+  # key, redis, ttl
+  subject { redis.ttl(key) }
+  it { should eq ttl }
+end
+
+shared_examples_for '#verify?' do
+  before(:each) do
+    ui.stub(:debug).and_call_original
+    ui.stub(:notify) do |message|
+      puts message
+    end
+  end
+  it 'should verify successfully' do
+    strategy.verify?(key).should be_true
+  end
+end
+
 shared_examples_for(RedisCopy::Strategy) do
   let(:key) { rand(16**128).to_s(16) }
   after(:each) { multiplex.both { |redis| redis.del(key) } }
+  let(:ttl) { 100 }
 
   context '#copy' do
+    before(:each) { populate.call }
     context 'string' do
       let(:source_string) { rand(16**256).to_s(16) }
-      before(:each) { source.set(key, source_string) }
+      let(:populate) { proc {source.set(key, source_string)} }
       [true,false].each do |with_expiry|
         context "with_expiry(#{with_expiry})" do
-          before(:each) { source.expire(key, 100) } if with_expiry
+          before(:each) { source.expire(key, ttl) } if with_expiry
           context 'before' do
             context 'source' do
+              let(:redis) { source }
               subject { source.get(key) }
               it { should_not be_nil }
               it { should eq source_string }
-              context 'ttl' do
-                subject { source.ttl(key) }
-                it { should eq 100 } if with_expiry
-                it { should eq -1 } unless with_expiry
-              end
+              it_should_behave_like (with_expiry ? :ttl_set : :no_ttl)
             end
             context 'destination' do
+              let(:redis) { destination }
               subject { destination.get(key) }
               it { should be_nil }
-              context 'ttl' do
-                subject { destination.ttl(key) }
-                it { should eq -1 }
-              end
+              it_should_behave_like :no_ttl
             end
           end
 
           context 'after' do
             before(:each) { strategy.copy(key) }
             context 'source' do
+              let(:redis) { source }
               subject { source.get(key) }
               it { should_not be_nil }
               it { should eq source_string }
-              context 'ttl' do
-                subject { source.ttl(key) }
-                it { should eq 100 } if with_expiry
-                it { should eq -1 } unless with_expiry
-              end
+              it_should_behave_like (with_expiry ? :ttl_set : :no_ttl)
             end
             context 'destination' do
+              let(:redis) { destination }
               subject { destination.get(key) }
               it { should_not be_nil }
               it { should eq source_string }
-              context 'ttl' do
-                subject { destination.ttl(key) }
-                it { should eq 100 } if with_expiry
-                it { should eq -1 } unless with_expiry
-              end
+              it_should_behave_like (with_expiry ? :ttl_set : :no_ttl)
             end
+            it_should_behave_like '#verify?'
           end
         end
       end
@@ -94,53 +110,43 @@ shared_examples_for(RedisCopy::Strategy) do
       let(:source_list) do
         %w(foo bar baz buz bingo jango)
       end
-      before(:each) { source_list.each{|x| source.rpush(key, x)} }
+      let(:populate) { proc { source_list.each{|x| source.rpush(key, x)} } }
       [true,false].each do |with_expiry|
         context "with_expiry(#{with_expiry})" do
           before(:each) { source.expire(key, 100) } if with_expiry
           context 'before' do
             context 'source' do
+              let(:redis) { source }
               subject { source.lrange(key, 0, -1) }
               it { should_not be_empty }
               it { should eq source_list }
-              context 'ttl' do
-                subject { source.ttl(key) }
-                it { should eq 100 } if with_expiry
-                it { should eq -1 } unless with_expiry
-              end
+              it_should_behave_like (with_expiry ? :ttl_set : :no_ttl)
             end
             context 'destination' do
+              let(:redis) { destination }
               subject { destination.lrange(key, 0, -1) }
               it { should be_empty }
-              context 'ttl' do
-                subject { destination.ttl(key) }
-                it { should eq -1 }
-              end
+              it_should_behave_like :no_ttl
             end
           end
 
           context 'after' do
             before(:each) { strategy.copy(key) }
             context 'source' do
+              let(:redis) { source }
               subject { source.lrange(key, 0, -1) }
               it { should_not be_empty }
               it { should eq source_list }
-              context 'ttl' do
-                subject { source.ttl(key) }
-                it { should eq 100 } if with_expiry
-                it { should eq -1 } unless with_expiry
-              end
+              it_should_behave_like (with_expiry ? :ttl_set : :no_ttl)
             end
             context 'destination' do
+              let(:redis) { destination }
               subject { destination.lrange(key, 0, -1) }
               it { should_not be_empty }
               it { should eq source_list }
-              context 'ttl' do
-                subject { destination.ttl(key) }
-                it { should eq 100 } if with_expiry
-                it { should eq -1 } unless with_expiry
-              end
+              it_should_behave_like (with_expiry ? :ttl_set : :no_ttl)
             end
+            it_should_behave_like '#verify?'
           end
         end
       end
@@ -150,52 +156,41 @@ shared_examples_for(RedisCopy::Strategy) do
       let(:source_list) do
         %w(foo bar baz buz bingo jango)
       end
-      before(:each) { source_list.each{|x| source.sadd(key, x)} }
+      let(:populate) { proc { source_list.each{|x| source.sadd(key, x)} } }
       [true,false].each do |with_expiry|
         context "with_expiry(#{with_expiry})" do
           before(:each) { source.expire(key, 100) } if with_expiry
           context 'before' do
             context 'source' do
+              let(:redis) { source }
               subject { source.smembers(key) }
               it { should_not be_empty }
               it { should =~ source_list }
-              context 'ttl' do
-                subject { source.ttl(key) }
-                it { should eq 100 } if with_expiry
-                it { should eq -1 } unless with_expiry
-              end
+              it_should_behave_like (with_expiry ? :ttl_set : :no_ttl)
             end
             context 'destination' do
+              let(:redis) { destination }
               subject { destination.smembers(key) }
               it { should be_empty }
-              context 'ttl' do
-                subject { destination.ttl(key) }
-                it { should eq -1 }
-              end
+              it_should_behave_like :no_ttl
             end
           end
 
           context 'after' do
             before(:each) { strategy.copy(key) }
             context 'source' do
+              let(:redis) { source }
               subject { source.smembers(key) }
               it { should_not be_empty }
               it { should =~ source_list }
-              context 'ttl' do
-                subject { source.ttl(key) }
-                it { should eq 100 } if with_expiry
-                it { should eq -1 } unless with_expiry
-              end
+              it_should_behave_like (with_expiry ? :ttl_set : :no_ttl)
             end
             context 'destination' do
+              let(:redis) { destination }
               subject { destination.smembers(key) }
               it { should_not be_empty }
               it { should =~ source_list }
-              context 'ttl' do
-                subject { destination.ttl(key) }
-                it { should eq 100 } if with_expiry
-                it { should eq -1 } unless with_expiry
-              end
+              it_should_behave_like (with_expiry ? :ttl_set : :no_ttl)
             end
           end
         end
@@ -209,53 +204,43 @@ shared_examples_for(RedisCopy::Strategy) do
           'baz' => 'buz'
         }
       end
-      before(:each) { source.mapped_hmset(key, source_hash) }
+      let(:populate) { proc { source.mapped_hmset(key, source_hash) } }
       [true,false].each do |with_expiry|
         context "with_expiry(#{with_expiry})" do
           before(:each) { source.expire(key, 100) } if with_expiry
           context 'before' do
             context 'source' do
+              let(:redis) { source }
               subject { source.hgetall(key) }
               it { should_not be_empty }
               it { should eq source_hash }
-              context 'ttl' do
-                subject { source.ttl(key) }
-                it { should eq 100 } if with_expiry
-                it { should eq -1 } unless with_expiry
-              end
+              it_should_behave_like (with_expiry ? :ttl_set : :no_ttl)
             end
             context 'destination' do
+              let(:redis) { destination }
               subject { destination.hgetall(key) }
               it { should be_empty }
-              context 'ttl' do
-                subject { destination.ttl(key) }
-                it { should eq -1 }
-              end
+              it_should_behave_like :no_ttl
             end
           end
 
           context 'after' do
             before(:each) { strategy.copy(key) }
             context 'source' do
+              let(:redis) { source }
               subject { source.hgetall(key) }
               it { should_not be_empty }
               it { should eq source_hash }
-              context 'ttl' do
-                subject { source.ttl(key) }
-                it { should eq 100 } if with_expiry
-                it { should eq -1 } unless with_expiry
-              end
+              it_should_behave_like (with_expiry ? :ttl_set : :no_ttl)
             end
             context 'destination' do
+              let(:redis) { destination }
               subject { destination.hgetall(key) }
               it { should_not be_empty }
               it { should eq source_hash }
-              context 'ttl' do
-                subject { destination.ttl(key) }
-                it { should eq 100 } if with_expiry
-                it { should eq -1 } unless with_expiry
-              end
+              it_should_behave_like (with_expiry ? :ttl_set : :no_ttl)
             end
+            it_should_behave_like '#verify?'
           end
         end
       end
@@ -272,53 +257,43 @@ shared_examples_for(RedisCopy::Strategy) do
       end
       let(:vs_source_zset) { source_zset.to_a }
       let(:sv_source_zset) { vs_source_zset.map(&:reverse) }
-      before(:each) { source.zadd(key, sv_source_zset) }
+      let(:populate) { proc { source.zadd(key, sv_source_zset) } }
       [true,false].each do |with_expiry|
         context "with_expiry(#{with_expiry})" do
           before(:each) { source.expire(key, 100) } if with_expiry
           context 'before' do
             context 'source' do
+              let(:redis) { source }
               subject { source.zrange(key, 0, -1, :with_scores => true) }
               it { should_not be_empty }
               it { should =~ vs_source_zset }
-              context 'ttl' do
-                subject { source.ttl(key) }
-                it { should eq 100 } if with_expiry
-                it { should eq -1 } unless with_expiry
-              end
+              it_should_behave_like (with_expiry ? :ttl_set : :no_ttl)
             end
             context 'destination' do
+              let(:redis) { destination }
               subject { destination.zrange(key, 0, -1, :with_scores => true) }
               it { should be_empty }
-              context 'ttl' do
-                subject { destination.ttl(key) }
-                it { should eq -1 }
-              end
+              it_should_behave_like :no_ttl
             end
           end
 
           context 'after' do
             before(:each) { strategy.copy(key) }
             context 'source' do
+              let(:redis) { source }
               subject { source.zrange(key, 0, -1, :with_scores => true) }
               it { should_not be_empty }
               it { should =~ vs_source_zset }
-              context 'ttl' do
-                subject { source.ttl(key) }
-                it { should eq 100 } if with_expiry
-                it { should eq -1 } unless with_expiry
-              end
+              it_should_behave_like (with_expiry ? :ttl_set : :no_ttl)
             end
             context 'destination' do
+              let(:redis) { destination }
               subject { destination.zrange(key, 0, -1, :with_scores => true) }
               it { should_not be_empty }
               it { should =~ vs_source_zset }
-              context 'ttl' do
-                subject { destination.ttl(key) }
-                it { should eq 100 } if with_expiry
-                it { should eq -1 } unless with_expiry
-              end
+              it_should_behave_like (with_expiry ? :ttl_set : :no_ttl)
             end
+            it_should_behave_like '#verify?'
           end
         end
       end
@@ -328,7 +303,8 @@ end
 
 describe RedisCopy::Strategy do
   let(:options) { Hash.new } # append using before(:each) { options.update(foo: true) }
-  let(:ui) { double.as_null_object }
+  # let(:ui) { double.as_null_object }
+  let(:ui) { RedisCopy::UI::CommandLine.new(options) }
   let(:strategy) { strategy_class.new(source, destination, ui, options)}
   let(:multiplex) { RedisMultiplex.new(source, destination) }
   let(:source) { Redis.new(db: 14) }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: redis-copy
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.5
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-11 00:00:00.000000000 Z
+date: 2013-11-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler