redis-dump 0.3.3 → 0.3.4

data/CHANGES.txt

@@ -1,8 +1,14 @@
 REDIS-DUMP, CHANGES
 
+#### 0.3.4 (2012-12-19) ###############################
+
+* ADDED: Allow loading of redis dumps with binary data (-n flag) [meekmichael]
+* FIXED: Rescue exceptions when setting values
+
+
 #### 0.3.3 (2012-11-22) ###############################
 
-* Fixed issue with return value from zrange in newer versions of redis lib (3.0+). 
+* FIXED: issue with return value from zrange in newer versions of redis lib (3.0+).
 
 #### 0.3.2 (2012-01-05) ###############################
 
@@ -47,4 +53,4 @@ REDIS-DUMP, CHANGES
 
 #### 0.1.0 (2010-11-15) ###############################
 
-Initial public release.
+Initial public release.

data/README.rdoc

@@ -68,6 +68,11 @@ For most sensitive data, you should consider encrypting the data directly withou
     
     # Decrypt the file (automated)
     $  3</path/2/passphrase.txt gpg --passphrase-fd 3 path/2/backup-db15.json.gpg
+
+
+== Loading data with binary strings
+
+If you have binary or serialized data in your Redis database, the YAJL parser may not load your dump file because it sees some of the binary data as 'invalid bytes in UTF8 string'.  If you are certain that your data is binary and not malformed UTF8, you can use the -n flag to redis-load to tell YAJL to not check the input for UTF8 validity.  Use with caution!
     
 == Installation
 

data/VERSION.yml

@@ -1,4 +1,4 @@
---- 
+---
 :MAJOR: 0
 :MINOR: 3
-:PATCH: 3
+:PATCH: 4

data/bin/redis-load

@@ -28,6 +28,9 @@ class Redis::Dump::CLI
   global :u, :uri, String, "Redis URI (e.g. redis://hostname[:port])"
   global :d, :database, Integer, "Redis database (e.g. -d 15)"
   global :s, :sleep, Integer, "Sleep for S seconds after dumping (for debugging)"
+  global :n, :no_check_utf8 do
+    Redis::Dump.check_utf8 = false
+  end
   global :V, :version, "Display version" do 
     puts "redis-dump v#{Redis::Dump::VERSION.to_s}"
     exit 0
@@ -126,4 +129,4 @@ rescue Interrupt
 rescue => ex
   STDERR.puts "ERROR (#{ex.class.to_s}): #{ex.message}"
   STDERR.puts ex.backtrace if Redis::Dump.debug
-end
+end

data/lib/redis/dump.rb

@@ -16,17 +16,24 @@ class Redis
     @debug = false
     @encoder = Yajl::Encoder.new
     @parser = Yajl::Parser.new
-    @safe = true
     @chunk_size = 10000
     @with_optimizations = true
     class << self
       attr_accessor :debug, :encoder, :parser, :safe, :host, :port, :chunk_size, :with_optimizations
+      def le(msg)
+        STDERR.puts "#%.4f: %s" % [Time.now.utc.to_f, msg]
+      end
       def ld(msg)
         STDERR.puts "#%.4f: %s" % [Time.now.utc.to_f, msg] if debug
       end
       def memory_usage
         `ps -o rss= -p #{Process.pid}`.to_i # in kb
       end
+      def check_utf8=(check)
+        if check == false
+          @parser = Yajl::Parser.new(:check_utf8 => false)
+        end
+      end
     end
     attr_accessor :dbs, :uri
     attr_reader :redis_connections
@@ -46,13 +53,13 @@ class Redis
       #self.class.ld 'CONNECT: ' << this_uri
       Redis.connect :url => this_uri
     end
-    
+
     def each_database
       @redis_connections.keys.sort.each do |redis_uri|
         yield redis_connections[redis_uri]
       end
     end
-    
+
     # See each_key
     def dump filter=nil
       filter ||= '*'
@@ -71,9 +78,9 @@ class Redis
             process_chunk idx, dump_keys_size do |count|
               Redis::Dump.ld " dumping #{chunk_entries.size} (#{count}) from #{redis.client.id}"
               output_buffer = []
-              chunk_entries.select! do |key| 
+              chunk_entries.select! do |key|
                 type = Redis::Dump.type(redis, key)
-                if self.class.with_optimizations && type == 'string' 
+                if self.class.with_optimizations && type == 'string'
                   true
                 else
                   output_buffer.push self.class.encoder.encode(Redis::Dump.dump(redis, key, type))
@@ -81,10 +88,10 @@ class Redis
                 end
               end
               unless output_buffer.empty?
-                yield output_buffer 
+                yield output_buffer
               end
               unless chunk_entries.empty?
-                yield Redis::Dump.dump_strings(redis, chunk_entries) { |obj| self.class.encoder.encode(obj) } 
+                yield Redis::Dump.dump_strings(redis, chunk_entries) { |obj| self.class.encoder.encode(obj) }
               end
               output_buffer.clear
               chunk_entries.clear
@@ -96,13 +103,13 @@ class Redis
       end
       entries
     end
-    
+
     def process_chunk idx, total_size
       idxplus = idx+1
       yield idxplus if (idxplus % self.class.chunk_size).zero? || idxplus >= total_size
     end
     private :process_chunk
-    
+
     def report filter='*'
       values = []
       total_size, dbs = 0, {}
@@ -129,7 +136,7 @@ class Redis
       puts "total: #{total_size.to_bytes}"
       values
     end
-    
+
     def load(string_or_stream, &each_record)
       count = 0
       Redis::Dump.ld " LOAD SOURCE: #{string_or_stream}"
@@ -140,12 +147,16 @@ class Redis
         end
         this_redis = redis(obj["db"])
         #Redis::Dump.ld "load[#{this_redis.hash}, #{obj}]"
-        if each_record.nil? 
+        if each_record.nil?
           if Redis::Dump.safe && this_redis.exists(obj['key'])
             #Redis::Dump.ld " record exists (no change)"
             next
           end
-          Redis::Dump.set_value this_redis, obj['key'], obj['type'], obj['value'], obj['ttl']
+          begin
+            Redis::Dump.set_value this_redis, obj['key'], obj['type'], obj['value'], obj['ttl']
+          rescue => ex
+            Redis::Dump.le '(key: %s) %s' % [obj['key'], ex.message]
+          end
         else
           each_record.call obj
         end
@@ -182,7 +193,7 @@ class Redis
         idx = -1
         keys.collect { |key|
           idx += 1
-          info = { 
+          info = {
             'db' => this_redis.client.db, 'key' => key,
             'ttl' => this_redis.ttl(key), 'type' => 'string',
             'value' => vals[idx].to_s, 'size' => vals[idx].to_s.size
@@ -201,7 +212,7 @@ class Redis
       def stringify(this_redis, key, t=nil, v=nil)
         send("stringify_#{t}", this_redis, key, v)
       end
-      
+
       def set_value_hash(this_redis, key, hash)
         hash.keys.each { |k|  this_redis.hset key, k, hash[k] }
       end
@@ -220,7 +231,7 @@ class Redis
       def set_value_none(this_redis, key, str)
         # ignore
       end
-      
+
       def value_string(this_redis, key)  this_redis.get key                                                       end
       def value_list  (this_redis, key)  this_redis.lrange key, 0, -1                                             end
       def value_set   (this_redis, key)  this_redis.smembers key                                                  end
@@ -235,7 +246,7 @@ class Redis
       def stringify_none  (this_redis, key, v=nil)  (v || '')                                                     end
     end
     extend Redis::Dump::ClassMethods
-    
+
     module VERSION
       def self.stamp
         @info[:STAMP].to_i

data/redis-dump.gemspec

@@ -5,7 +5,7 @@
 
 Gem::Specification.new do |s|
   s.name = "redis-dump"
-  s.version = "0.3.2"
+  s.version = "0.3.4"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Delano Mandelbaum"]

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: redis-dump
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.3.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-11-22 00:00:00.000000000 Z
+date: 2012-12-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
-  requirement: &70228764012240 !ruby/object:Gem::Requirement
+  requirement: &70204696859080 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0.1'
   type: :runtime
   prerelease: false
-  version_requirements: *70228764012240
+  version_requirements: *70204696859080
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70228764011740 !ruby/object:Gem::Requirement
+  requirement: &70204696858600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '2.0'
   type: :runtime
   prerelease: false
-  version_requirements: *70228764011740
+  version_requirements: *70204696858600
 - !ruby/object:Gem::Dependency
   name: uri-redis
-  requirement: &70228764011260 !ruby/object:Gem::Requirement
+  requirement: &70204696858120 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: 0.4.0
   type: :runtime
   prerelease: false
-  version_requirements: *70228764011260
+  version_requirements: *70204696858120
 - !ruby/object:Gem::Dependency
   name: drydock
-  requirement: &70228764010760 !ruby/object:Gem::Requirement
+  requirement: &70204696857640 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,7 +54,7 @@ dependencies:
         version: 0.6.9
   type: :runtime
   prerelease: false
-  version_requirements: *70228764010760
+  version_requirements: *70204696857640
 description: Backup and restore your Redis data to and from JSON.
 email: delano@solutious.com
 executables: