redis-copy 0.0.6 → 1.0.0.rc.0

data/.travis.yml

@@ -3,6 +3,7 @@ script: "bundle exec rake run"
 
 rvm:
   - 1.9.3
+  - 2.0.0
 gemfile:
   - .travis/Gemfile.redis-gem-3.0
   - .travis/Gemfile.redis-gem-master

data/README.md

@@ -2,12 +2,11 @@
 
 This utility provides a way to move the contents of one redis DB to another
 redis DB. It is inspired by the [redis-copy.rb script][original] included in
-the redis source, but supports the following additional features:
+the redis source, but aims to always support all object types and to use the
+most-efficient methods and commands available to your redis versions:
 
- - all known data types (original supported `set`, `list`, and `string`,
-   dropping the others without warning)
  - if available on both dbs, will use `DUMP`/`RESTORE` commands (redis v2.6+)
- - support for more than just db0
+ - if available on source db, will use `SCAN` instead of `KEYS` (redis v2.8+)
 
 [original]: https://github.com/antirez/redis/commits/unstable/utils/redis-copy.rb
 
@@ -21,17 +20,18 @@ The current options can be grabbed using the `--help` flag.
 
 ```
 $ redis-copy --help
-redis-copy v0.0.5
+redis-copy v1.0.0.rc.0 (with redis-rb 3.0.6)
 Usage: redis-copy [options] <source> <destination>
     <source> and <destination> must be redis connection uris
     like [redis://][<username>:<password>@]<hostname>[:<port>][/<db>]
 
 Specific options:
-        --strategy STRATEGY          Select strategy (auto, new, classic) (default auto)
-                                       auto:    uses new if available, otherwise fallback
-                                       new:     use redis DUMP and RESTORE commands (faster)
-                                       classic: migrates via multiple type-specific commands
+        --pattern PATTERN            Only transfer matching keys (default *)
+                                     See http://redis.io/commands/keys for more info.
         --[no-]pipeline              Use redis pipeline where available (default true)
+    -r, --require FILENAME           Require a script; useful for loading third-party
+                                     implementations of key-emitter or copy strategies.
+                                     Relative paths *must* begin with `../' or `./'.
     -d, --[no-]debug                 Write debug output (default false)
     -t, --[no-]trace                 Enable backtrace on failure (default false)
     -f, --[no-]fail-fast             Abort on first failure (default false)
@@ -44,11 +44,11 @@ Specific options:
 ## Example:
 
 ```
-$ redis-copy --fail-fast --yes old.redis.host/9 new.redis.host:6380/3
+$ redis-copy --no-prompt old.redis.host/9 new.redis.host:6380/3
 Source:      redis://old.redis.host:6379/9
 Destination: redis://new.redis.host:6380/3 (empty)
-Key Emitter: Default
-Strategy:    New
+Key Emitter: Scan
+Strategy:    DumpRestore
 PROGRESS {:success=>1000, :attempt=>1000}
 PROGRESS {:success=>2000, :attempt=>2000}
 PROGRESS {:success=>3000, :attempt=>3000}
@@ -56,6 +56,22 @@ PROGRESS {:success=>4000, :attempt=>4000}
 DONE: {:success=>4246, :attempt=>4246}
 ```
 
+## Extensibility:
+
+`RedisCopy` uses the [implements][] gem to define interfaces for key-emitter
+and copy strategies, so implementations can be supplied by third-parties,
+secondary gems, or even a local script; the interface shared examples are even
+available on your load-path so you can ensure your implementation adheres to
+the interface.
+
+See the existing implementations and their specs for examples, and use the
+`--require` command-line flag to load up your own. Since `implements` treats
+last-loaded implementations as inherently better, `RedisCopy` will automatically
+pick up your implementation and attempt to use it before the bundled
+implementations.
+
+[implements]: https://rubygems.org/gems/implements
+
 ## Contributing
 
 1. Fork it

data/lib/redis-copy.rb

@@ -3,6 +3,7 @@
 require 'redis'
 require 'active_support/inflector'
 require 'active_support/core_ext/string/strip' # String#strip_heredoc
+require 'implements/global'
 
 require 'redis-copy/version'
 require 'redis-copy/ui'
@@ -15,21 +16,22 @@ module RedisCopy
     # @param destination [String]
     # @options options [Hash<Symbol,Object>]
     def copy(source, destination, options = {})
-      ui = UI.load(options)
+      ui = UI.new(options)
 
       source = redis_from(source)
       destination = redis_from(destination)
 
       ui.abort('source cannot equal destination!') if same_redis?(source, destination)
 
-      key_emitter = KeyEmitter.load(source, ui, options)
-      strategem = Strategy.load(source, destination, ui, options)
+      key_emitter = KeyEmitter.new(source, ui, options)
+      strategem = Strategy.new(source, destination, ui, options)
 
       dest_empty = !(destination.randomkey) # randomkey returns string unless db empty.
 
       return false unless ui.confirm? <<-EODESC.strip_heredoc
         Source:      #{source.client.id}
         Destination: #{destination.client.id} (#{dest_empty ? '' : 'NOT '}empty)
+        Pattern:     #{options[:pattern]}
         Key Emitter: #{key_emitter}
         Strategy:    #{strategem}
       EODESC
@@ -60,6 +62,9 @@ module RedisCopy
       end.tap do |stats|
         ui.notify("DONE: #{stats.inspect}")
       end
+    rescue Implements::Implementation::NotFound => e
+      ui.notify(e.to_s)
+      ui.abort
     end
 
     private

data/lib/redis-copy/cli.rb

@@ -8,8 +8,6 @@ module RedisCopy
     REDIS_URI = (/\A(?:redis:\/\/)?(\w*:\w+@)?([a-z0-9\-.]+)(:[0-9]{1,5})?(\/(?:(?:1[0-5])|[0-9]))?\z/i).freeze
     DEFAULTS = {
       ui:             :command_line,
-      key_emitter:    :auto,
-      strategy:       :auto,
       verify:         0,
       pipeline:       :true,
       fail_fast:      false,
@@ -17,6 +15,7 @@ module RedisCopy
       trace:          false,
       debug:          false,
       allow_nonempty: false,
+      pattern:        '*',
     }.freeze unless defined?(DEFAULTS)
 
     def initialize(argv = ARGV)
@@ -37,26 +36,11 @@ module RedisCopy
         opts.separator ''
         opts.separator "Specific options:"
 
-        opts.on('--strategy STRATEGY', [:auto, :new, :classic],
-          indent_desc.(
-            "Select strategy (auto, new, classic) (default #{DEFAULTS[:strategy]})\n" +
-            "  auto:    uses new if available, otherwise fallback\n" +
-            "  new:     use redis DUMP and RESTORE commands (faster)\n" +
-            "  classic: migrates via multiple type-specific commands"
-          )
-        ) do |strategy|
-          options[:strategy] = strategy
-        end
-
-        opts.on('--emitter EMITTER', [:auto, :scan, :keys],
-          indent_desc.(
-            "Select key emitter (auto, keys, scan) (default #{DEFAULTS[:strategy]})\n" +
-            "  auto:    uses scan if available, otherwise fallback\n" +
-            "  scan:    use redis SCAN command (faster, less blocking)\n" +
-            "  keys:    uses redis KEYS command (dangerous, esp. on large datasets)"
-          )
-        ) do |emitter|
-          options[:key_emitter] = emitter
+        opts.on('--pattern PATTERN', indent_desc[
+          "Only transfer matching keys (default #{DEFAULTS[:pattern]})\n" +
+          "See http://redis.io/commands/keys for more info."
+        ]) do |pattern|
+          options[:pattern] = pattern
         end
 
         opts.on('--[no-]pipeline',
@@ -65,6 +49,20 @@ module RedisCopy
           options[:pipeline] = pipeline
         end
 
+        opts.on('-r', '--require FILENAME', indent_desc.(
+          "Require a script; useful for loading third-party\n" +
+          "implementations of key-emitter or copy strategies.\n" +
+          "Relative paths *must* begin with `../' or `./'.")
+        ) do |script|
+          begin
+            script = File.expand_path(script) if script[/\A..?\//]
+            require script
+          rescue LoadError => e
+            $stderr.puts e.message
+            exit 1
+          end
+        end
+
         opts.on('-d', '--[no-]debug', "Write debug output (default #{DEFAULTS[:debug]})") do |debug|
           options[:debug] = debug
         end
@@ -102,16 +100,21 @@ module RedisCopy
           options[:dry_run] = true
         end
 
-        opts.parse!(argv)
-        unless argv.size == 2
-          opts.abort "Source and Destination must be specified\n\n" +
-                            opts.help
+        begin
+          opts.parse!(argv)
+          unless argv.size == 2
+            opts.abort "Source and Destination must be specified\n\n" +
+                              opts.help
+          end
+          @source = argv.shift
+          @destination = argv.shift
+
+          opts.abort "source is not valid URI" unless @source =~ REDIS_URI
+          opts.abort "destination is not valid URI" unless @destination =~ REDIS_URI
+        rescue OptionParser::ParseError => error
+          $stderr.puts error
+          exit 1
         end
-        @source = argv.shift
-        @destination = argv.shift
-
-        opts.abort "source is not valid URI" unless @source =~ REDIS_URI
-        opts.abort "destination is not valid URI" unless @destination =~ REDIS_URI
       end
 
       @config = DEFAULTS.merge(options)

data/lib/redis-copy/key-emitter.rb

@@ -7,21 +7,11 @@ module RedisCopy
   # but should allow smarter implementations to be built that can handle
   # billion-key dbs without blocking on IO.
   module KeyEmitter
-    def self.load(redis, ui, options = {})
-      key_emitter = options.fetch(:key_emitter, :default)
-      scan_compatible = Scan::compatible?(redis)
-      emitklass = case key_emitter
-                  when :keys then Keys
-                  when :scan
-                    raise ArgumentError unless scan_compatible
-                    Scan
-                  when :auto then scan_compatible ? Scan : Keys
-                  end
-      emitklass.new(redis, ui, options)
-    end
+    extend Implements::Interface
 
     # @param redis [Redis]
     # @param options [Hash<Symbol:String>]
+    # @option options [String] :pattern ('*')
     def initialize(redis, ui, options = {})
       @redis    = redis
       @ui       = ui
@@ -34,65 +24,20 @@ module RedisCopy
       raise NotImplementedError
     end
 
+    def pattern
+      @pattern ||= @options.fetch(:pattern) { '*' }
+    end
+
     def dbsize
       @redis.dbsize
     end
 
     def to_s
-      self.class.name.demodulize.humanize
-    end
-
-    # The default strategy blindly uses `redis.keys('*')`
-    class Keys
-      include KeyEmitter
-
-      def keys
-        dbsize = self.dbsize
-
-        # HT: http://stackoverflow.com/a/11466770
-        dbsize_str = dbsize.to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse
-
-        @ui.abort unless (dbsize < 10_000) || (@ui.confirm? <<-EOWARNING.strip_heredoc)
-          WARNING: #{self} key emitter uses redis.keys('*') to
-          get its list of keys, and you have #{dbsize_str} keys in
-          your source DB.
-
-          The redis keys command [reference](http://redis.io/commands/keys)
-          says this:
-
-          > Warning: consider KEYS as a command that should only be used
-          > in production environments with extreme care. It may ruin
-          > performance when it is executed against large databases.
-          > This command is intended for debugging and special operations,
-          > such as changing your keyspace layout. Don't use KEYS in your
-          > regular application code. If you're looking for a way to find
-          > keys in a subset of your keyspace, consider using sets.
-        EOWARNING
-
-        @ui.debug "REDIS: #{@redis.client.id} KEYS *"
-        @redis.keys('*').to_enum
-      end
-
-      def self.compatible?(redis)
-        true
-      end
-    end
-
-    class Scan
-      include KeyEmitter
-
-      def keys
-        @redis.scan_each(count: 1000)
-      end
-
-      def self.compatible?(redis)
-        bin_version = Gem::Version.new(redis.info['redis_version'])
-        bin_requirement = Gem::Requirement.new('>= 2.7.105')
-
-        return false unless bin_requirement.satisfied_by?(bin_version)
-
-        redis.respond_to?(:scan_each)
-      end
+      self.class.name.demodulize
     end
   end
 end
+
+# Load the bundled key-emitters:
+require_relative 'key-emitter/keys'
+require_relative 'key-emitter/scan'

data/lib/redis-copy/key-emitter/interface.spec.rb

@@ -0,0 +1,79 @@
+# encoding: utf-8
+
+# The shared examples for RedisCopy::KeyEmitter are available to require
+# into consuming libraries so they can verify their implementation of the
+# RedisCopy::KeyEmitter interface. See the bundled specs for the bundled
+# key-emitters for example usage.
+if defined?(::RSpec)
+  shared_examples_for RedisCopy::KeyEmitter do
+    let(:resolved_implementation) do
+      begin
+        instance
+        true
+      rescue Implements::Implementation::NotFound
+        false
+      end
+    end
+    let(:emitter_klass) { described_class }
+    let(:redis) { Redis.new(REDIS_OPTIONS).tap(&:ping) }
+    let(:ui) { double.as_null_object }
+    let(:selector) { emitter_klass.name.underscore.dasherize } # see implements gem
+    let(:instance) { RedisCopy::KeyEmitter.implementation(selector).new(redis, ui, options) }
+    let(:options) { Hash.new }
+    let(:key_count) { 1 }
+    let(:keys) { key_count.times.map{|i| i.to_s(16) } }
+    let(:glob_matcher) do
+      lambda do |rglob|
+        fglob = rglob.gsub('*','**')
+        lambda do |key|
+          File::fnmatch(fglob,key)
+        end
+      end
+    end
+
+    before(:each) do
+      unless resolved_implementation
+        pending "#{emitter_klass} not supported in your environment"
+      end
+      key_count.times.each_slice(50) do |keys|
+        kv = keys.map{|x| x.to_s(16)}.zip(keys)
+        redis.mset(*kv.flatten)
+      end
+      ui.stub(:debug).with(anything)
+    end
+    after(:each) { redis.flushdb }
+
+    context '#keys' do
+      let(:key_count) { 64 }
+      context 'the result' do
+        subject { instance.keys }
+        its(:to_a) { should =~ keys }
+      end
+      context 'with pattern "[139]*"' do
+        let(:options) { {pattern: '[139]*'} }
+        context 'the result' do
+          let(:key_count) { 256 }
+          subject { instance.keys }
+          its(:to_a) { should =~ keys.select(&glob_matcher['[139]*']) }
+        end
+      end
+      context 'with pattern "?[2468ace]"' do
+        let(:options) { {pattern: '?[2468ace]'} }
+        context 'the result' do
+          let(:key_count) { 256 }
+          subject { instance.keys }
+          its(:to_a) { should =~ keys.select(&glob_matcher['?[2468ace]']) }
+        end
+      end
+    end
+
+    context 'implementation resolution' do
+      subject { instance }
+      its(:class) { should eq described_class }
+    end
+  end
+else
+  fail(LoadError,
+       "#{__FILE__} contains shared examples for RedisCopy::KeyEmitter. " +
+       "Require it in your specs, not your code.")
+end

data/lib/redis-copy/key-emitter/keys.rb

@@ -0,0 +1,39 @@
+# encoding: utf-8
+
+module RedisCopy
+  # Keys uses the KEYS command, which has always been available in Redis,
+  # but comes with a rather staunch warning to *never* use it in production.
+  # This is the first-required implementation of KeyEmitter, and is thus the
+  # fallback for no other emitters are available. A Warning Prompt will appear
+  # if your dbsize is greater than 10,000 keys.
+  class KeyEmitter::Keys
+    implements KeyEmitter
+
+    def keys
+      dbsize = self.dbsize
+
+      # HT: http://stackoverflow.com/a/11466770
+      dbsize_str = dbsize.to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse
+
+      @ui.abort unless (dbsize < 10_000) || (@ui.confirm? <<-EOWARNING.strip_heredoc)
+        WARNING: #{self} key emitter uses redis.keys('*') to
+        get its list of keys, and you have #{dbsize_str} keys in
+        your source DB.
+
+        The redis keys command [reference](http://redis.io/commands/keys)
+        says this:
+
+        > Warning: consider KEYS as a command that should only be used
+        > in production environments with extreme care. It may ruin
+        > performance when it is executed against large databases.
+        > This command is intended for debugging and special operations,
+        > such as changing your keyspace layout. Don't use KEYS in your
+        > regular application code. If you're looking for a way to find
+        > keys in a subset of your keyspace, consider using sets.
+      EOWARNING
+
+      @ui.debug "REDIS: #{@redis.client.id} KEYS #{pattern}"
+      @redis.keys(pattern).to_enum
+    end
+  end
+end