redis-dump 0.1.0

data/CHANGES.txt

@@ -0,0 +1,5 @@
+REDIS-DUMP, CHANGES
+
+#### 0.1.0 (2010-11-15) ###############################
+
+Initial public release.

data/LICENSE.txt

@@ -0,0 +1,19 @@
+Copyright (c) 2010 Solutious Inc, Delano Mandelbaum
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,29 @@
+= Redis-Dump v0.1 ALPHA
+
+<i>Backup and restore your Redis data to and from JSON.</i>
+
+<b>NOTE: This is alpha software. DO NOT RELY ON IT!!</b>
+
+
+== Installation
+
+One of:
+
+    $ gem install redis0dump
+    
+    
+== More Info
+
+* Codes[http://github.com/delano/redis-dump]
+
+
+== Credits
+
+* delano[http://github.com/delano]
+
+
+== Thanks
+
+* antirez and the funky redis bunch!
+
+

data/Rakefile

@@ -0,0 +1,64 @@
+require "rubygems"
+require "rake"
+require "rake/clean"
+require 'yaml'
+
+begin
+  require 'hanna/rdoctask'
+rescue LoadError
+  require 'rake/rdoctask'
+end
+ 
+config = YAML.load_file("VERSION.yml")
+task :default => ["build"]
+CLEAN.include [ 'pkg', 'doc' ]
+name = "redis-dump"
+
+begin
+  require "jeweler"
+  Jeweler::Tasks.new do |gem|
+    gem.version = "#{config[:MAJOR]}.#{config[:MINOR]}.#{config[:PATCH]}"
+    gem.name = "redis-dump"
+    gem.rubyforge_project = gem.name
+    gem.summary = "Backup and restore your Redis data to and from JSON."
+    gem.description = gem.summary
+    gem.email = "delano@solutious.com"
+    gem.homepage = "http://github.com/delano/redis-dump"
+    gem.authors = ["Delano Mandelbaum"]
+    gem.add_dependency("yajl-ruby",    ">= 0.1")
+    gem.add_dependency("redis",        ">= 2.0")
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
+end
+
+
+Rake::RDocTask.new do |rdoc|
+  version = "#{config[:MAJOR]}.#{config[:MINOR]}.#{config[:PATCH]}.#{config[:BUILD]}"
+  rdoc.rdoc_dir = "doc"
+  rdoc.title = "redis-dump #{version}"
+  rdoc.rdoc_files.include("README*")
+  rdoc.rdoc_files.include("LICENSE.txt")
+  #rdoc.rdoc_files.include("bin/*.rb")
+  rdoc.rdoc_files.include("lib/**/*.rb")
+end
+
+
+# Rubyforge Release / Publish Tasks ==================================
+
+#about 'Publish website to rubyforge'
+task 'publish:rdoc' => 'doc/index.html' do
+  sh "scp -rp doc/* rubyforge.org:/var/www/gforge-projects/#{name}/"
+end
+
+#about 'Public release to rubyforge'
+task 'publish:gem' => [:package] do |t|
+  sh <<-end
+    rubyforge add_release -o Any -a CHANGES.txt -f -n README.md #{name} #{name} #{@spec.version} pkg/#{name}-#{@spec.version}.gem &&
+    rubyforge add_file -o Any -a CHANGES.txt -f -n README.md #{name} #{name} #{@spec.version} pkg/#{name}-#{@spec.version}.tgz 
+  end
+end
+
+
+

data/VERSION.yml

@@ -0,0 +1,5 @@
+--- 
+:MAJOR: 0
+:MINOR: 1
+:PATCH: 0
+:BUILD: '001'

data/lib/redis/dump.rb

@@ -0,0 +1,201 @@
+require 'redis'
+require 'yajl'
+
+class Redis
+  class Dump
+    unless defined?(Redis::Dump::VALID_TYPES)
+      VALID_TYPES = ['string', 'set', 'list', 'zset', 'hash', 'none'].freeze
+    end
+    @debug = false
+    @encoder = Yajl::Encoder.new
+    @parser = Yajl::Parser.new
+    @safe = true
+    class << self
+      attr_accessor :debug, :encoder, :parser, :safe
+      def ld(msg)
+        STDERR.puts "#{'%.4f' % Time.now.utc.to_f}: #{msg}" if @debug
+      end
+    end
+    attr_accessor :dbs, :uri
+    attr_reader :redis_connections
+    def initialize(dbs=nil,uri="redis://127.0.0.1:6379")
+      @redis_connections = {}
+      @uri = uri
+      unless dbs.nil?
+        @dbs = Range === dbs ? dbs : (dbs..dbs)
+        @dbs = (@dbs.first.to_i..@dbs.last.to_i) # enforce integers
+        open_all_connections
+      end
+    end
+    def open_all_connections
+      dbs.to_a.each { |db| redis(db) } unless dbs.nil?
+    end
+    def redis(db)
+      redis_connections["#{uri}/#{db}"] ||= connect("#{uri}/#{db}")
+    end
+    def connect(this_uri)
+      self.class.ld 'CONNECT: ' << this_uri
+      Redis.connect :url => this_uri
+    end
+    def each_key &blk
+      @redis_connections.keys.sort.each do |redis_uri|
+        self.class.ld ['---', "DB: #{redis_connections[redis_uri].client.db}", '---'].join($/)
+        keys = redis_connections[redis_uri].keys
+        keys.each do |key|
+          blk.call redis_connections[redis_uri], key
+        end
+      end
+    end
+    def dump(&each_record)
+      values = []
+      each_key do |this_redis,key|
+        info = Redis::Dump.dump this_redis, key
+        #self.class.ld " #{key} (#{info[:type]}): #{info[:size].to_bytes}"
+        encoded = self.class.encoder.encode info
+        each_record.nil? ? (values << encoded) : each_record.call(encoded)
+      end
+      values
+    end
+    def report(&each_record)
+      values = []
+      each_key do |this_redis,key|
+        info = Redis::Dump.report this_redis, key
+        #self.class.ld " #{key} (#{info[:type]}): #{info[:size].to_bytes}"
+        each_record.nil? ? (values << info) : each_record.call(info)
+      end
+      values
+    end
+    def load(string_or_stream, &each_record)
+      count = 0
+      Redis::Dump.ld " LOAD SOURCE: #{string_or_stream}"
+      Redis::Dump.parser.parse string_or_stream do |obj|
+        unless @dbs.member?(obj["db"].to_i)
+          Redis::Dump.ld "db out of range: #{obj["db"]}"
+          next
+        end
+        this_redis = redis(obj["db"])
+        Redis::Dump.ld "load[#{this_redis.hash}, #{obj}]"
+        if each_record.nil? 
+          if Redis::Dump.safe && this_redis.exists(obj['key'])
+            Redis::Dump.ld " record exists (no change)"
+            next
+          end
+          Redis::Dump.set_value this_redis, obj['key'], obj['type'], obj['value'], obj['ttl']
+        else
+          each_record.call obj
+        end
+        count += 1
+      end
+      count
+    end
+    module ClassMethods
+      def type(this_redis, key)
+        type = this_redis.type key
+        raise TypeError, "Unknown type: #{type}" if !VALID_TYPES.member?(type)
+        type
+      end
+      def report(this_redis, key)
+        info = { 'db' => this_redis.client.db, 'key' => key }
+        info['type'] = type(this_redis, key)
+        info['size'] = stringify(this_redis, key, info['type'], info['value']).size
+        info['bytes'] = info['size'].to_bytes
+        ld "report[#{this_redis.hash}, #{info}]"
+        info
+      end
+      def dump(this_redis, key)
+        info = { 'db' => this_redis.client.db, 'key' => key }
+        info['ttl'] = this_redis.ttl key
+        info['type'] = type(this_redis, key)
+        info['value'] = value(this_redis, key, info['type'])
+        info['size'] = stringify(this_redis, key, info['type'], info['value']).size
+        ld "dump[#{this_redis.hash}, #{info}]"
+        info
+      end
+      def set_value(this_redis, key, type, value, expire=nil)
+        t ||= type
+        send("set_value_#{t}", this_redis, key, value)
+        this_redis.expire key, expire if expire.to_s.to_i > 0
+      end
+      def value(this_redis, key, t=nil)
+        t ||= type
+        send("value_#{t}", this_redis, key)
+      end
+      def stringify(this_redis, key, t=nil, v=nil)
+        t ||= type
+        send("stringify_#{t}", this_redis, key, v)
+      end
+      
+      def set_value_hash(this_redis, key, hash)
+        hash.keys.each { |k|  this_redis.hset key, k, hash[k] }
+      end
+      def set_value_list(this_redis, key, list)
+        list.each { |value|  this_redis.rpush key, value }
+      end
+      def set_value_set(this_redis, key, set)
+        set.each { |value|  this_redis.sadd key, value }
+      end
+      def set_value_zset(this_redis, key, zset)
+        zset.each { |pair|  this_redis.zadd key, pair[1].to_f, pair[0] }
+      end
+      def set_value_string(this_redis, key, str)
+        this_redis.set key, str
+      end
+      
+      def value_string(this_redis, key)  this_redis.get key                                    end
+      def value_list  (this_redis, key)  this_redis.lrange key, 0, -1                          end
+      def value_set   (this_redis, key)  this_redis.smembers key                               end
+      def value_zset  (this_redis, key)  
+        zset = this_redis.zrange(key, 0, -1, :with_scores => true).tuple
+      end
+      def value_hash  (this_redis, key)  this_redis.hgetall(key)                               end
+      def value_none  (this_redis, key)  ''                                                end
+      def stringify_string(this_redis, key, v=nil)  (v || value_string(this_redis, key))                     end
+      def stringify_list  (this_redis, key, v=nil)  (v || value_list(this_redis, key)).join                  end
+      def stringify_set   (this_redis, key, v=nil)  (v || value_set(this_redis, key)).join                   end
+      def stringify_zset  (this_redis, key, v=nil)  (v || value_zset(this_redis, key)).flatten.compact.join  end
+      def stringify_hash  (this_redis, key, v=nil)  (v || value_hash(this_redis, key)).to_a.flatten.compact.join  end
+      def stringify_none  (this_redis, key, v=nil)  (v || '')                                            end
+    end
+    extend Redis::Dump::ClassMethods
+  end
+end
+
+class Array
+  def chunk(number_of_chunks)
+    chunks = (1..number_of_chunks).collect { [] }
+    chunks.each do |a_chunk|
+      a_chunk << self.shift if self.any?
+    end
+    chunks
+  end
+  alias / chunk
+  def tuple(tuple_size=2)
+    tuples = (1..(size/tuple_size)).collect { [] }
+    tuples.each_with_index do |a_tuple,idx|
+      tuple_size.times { a_tuple << self.shift } if self.any?
+    end
+    tuples
+  end
+end
+
+class Numeric
+  def to_ms
+    (self*1000).to_i
+  end
+
+  # TODO: Use 1024?
+  def to_bytes
+    args = case self.abs.to_i
+    when (1000)..(1000**2)
+      '%3.2f%s' % [(self / 1000.to_f).to_s, 'KB']
+    when (1000**2)..(1000**3)
+      '%3.2f%s' % [(self / (1000**2).to_f).to_s, 'MB']
+    when (1000**3)..(1000**4)
+      '%3.2f%s' % [(self / (1000**3).to_f).to_s, 'GB']
+    when (1000**4)..(1000**6)
+      '%3.2f%s' % [(self / (1000**4).to_f).to_s, 'TB']
+    else
+      [self, 'B'].join
+    end
+  end
+end

data/try/10_redis_dump_try.rb

@@ -0,0 +1,57 @@
+require 'lib/redis/dump'
+
+@uri_base = "redis://127.0.0.1:6371"
+
+Redis::Dump.debug = false
+Redis::Dump.safe = true
+
+## Connect to DB
+@rdump = Redis::Dump.new 0, @uri_base
+@rdump.redis_connections.size
+#=> 1
+
+## Populate
+@rdump.redis(0).set 'stringkey', 'stringvalue'
+@rdump.redis(0).expire 'stringkey', 100
+@rdump.redis(0).hset 'hashkey', 'field_a', 'value_a'
+@rdump.redis(0).hset 'hashkey', 'field_b', 'value_b'
+@rdump.redis(0).hset 'hashkey', 'field_c', 'value_c'
+3.times { |idx| @rdump.redis(0).rpush 'listkey', "value_#{idx}" }
+4.times { |idx| @rdump.redis(0).sadd 'setkey', "value_#{idx}" }
+5.times { |idx| @rdump.redis(0).zadd 'zsetkey', idx.zero? ? 100 : 100*idx, "value_#{idx}" }
+@rdump.redis(0).keys.size
+#=> 5
+
+## Can dump
+@values = @rdump.dump
+@values.size
+#=> 5
+
+# Clear DB 0
+db0 = Redis::Dump.new 0, @uri_base
+db0.redis(0).flushdb
+db0.redis(0).keys.size
+#=> 0
+
+## Can load data
+@rdump.load @values.join
+@rdump.redis(0).keys.size
+#=> 5
+
+## DB 0 content matches previous dump content
+values = @rdump.dump
+values.sort
+#=> @values.sort
+
+## Won't load data in safe mode if records exist
+@rdump.load @values.join
+#=> 0
+
+## Will load data if records exist and safe mode is disabled
+Redis::Dump.safe = false
+@rdump.load @values.join
+#=> 5
+
+Redis::Dump.safe = true
+db0 = Redis::Dump.new 0, @uri_base
+db0.redis(0).flushdb

data/try/redis-server.conf

@@ -0,0 +1,197 @@
+# REDIS-DUMP TEST CONFIG -- 2010-11-15
+
+# Usage: 
+#
+#    $ redis-server try/redis-server.conf
+
+dir /tmp
+
+pidfile redisdump-test.pid
+logfile redisdump-test.log
+dbfilename redisdump-test.rdb
+
+port 6371
+bind 127.0.0.1
+daemonize yes
+
+timeout 300
+
+#loglevel debug
+#loglevel verbose
+loglevel warning
+
+databases 16
+
+save 900 100
+save 300 5000
+
+
+rdbcompression yes
+
+# requirepass foobared
+# maxclients 0
+
+appendonly no
+appendfilename redisdump-test.aof
+
+# TODO: Consider having separate configs when the usecase for Redis
+# changes. For example, one for production, another for batch processing. 
+#
+# Nothing is changed from here on out:
+
+################################## INCLUDES ###################################
+
+# Include one or more other config files here.  This is useful if you
+# have a standard template that goes to all redis server but also need
+# to customize a few per-server settings.  Include files can include
+# other files, so use this wisely.
+#
+# include /path/to/local.conf
+# include /path/to/other.conf
+
+################################# REPLICATION #################################
+
+# Master-Slave replication. Use slaveof to make a Redis instance a copy of
+# another Redis server. Note that the configuration is local to the slave
+# so for example it is possible to configure the slave to save the DB with a
+# different interval, or to listen to another port, and so on.
+#
+# slaveof <masterip> <masterport>
+
+# If the master is password protected (using the "requirepass" configuration
+# directive below) it is possible to tell the slave to authenticate before
+# starting the replication synchronization process, otherwise the master will
+# refuse the slave request.
+#
+# masterauth <master-password>
+
+
+# The fsync() call tells the Operating System to actually write data on disk
+# instead to wait for more data in the output buffer. Some OS will really flush 
+# data on disk, some other OS will just try to do it ASAP.
+#
+# Redis supports three different modes:
+#
+# no: don't fsync, just let the OS flush the data when it wants. Faster.
+# always: fsync after every write to the append only log . Slow, Safest.
+# everysec: fsync only if one second passed since the last fsync. Compromise.
+#
+# The default is "everysec" that's usually the right compromise between
+# speed and data safety. It's up to you to understand if you can relax this to
+# "no" that will will let the operating system flush the output buffer when
+# it wants, for better performances (but if you can live with the idea of
+# some data loss consider the default persistence mode that's snapshotting),
+# or on the contrary, use "always" that's very slow but a bit safer than
+# everysec.
+#
+# If unsure, use "everysec".
+
+# appendfsync always
+appendfsync everysec
+# appendfsync no
+
+################################ VIRTUAL MEMORY ###############################
+
+# Virtual Memory allows Redis to work with datasets bigger than the actual
+# amount of RAM needed to hold the whole dataset in memory.
+# In order to do so very used keys are taken in memory while the other keys
+# are swapped into a swap file, similarly to what operating systems do
+# with memory pages.
+#
+# To enable VM just set 'vm-enabled' to yes, and set the following three
+# VM parameters accordingly to your needs.
+
+vm-enabled no
+# vm-enabled yes
+
+# This is the path of the Redis swap file. As you can guess, swap files
+# can't be shared by different Redis instances, so make sure to use a swap
+# file for every redis process you are running. Redis will complain if the
+# swap file is already in use.
+#
+# The best kind of storage for the Redis swap file (that's accessed at random) 
+# is a Solid State Disk (SSD).
+#
+# *** WARNING *** if you are using a shared hosting the default of putting
+# the swap file under /tmp is not secure. Create a dir with access granted
+# only to Redis user and configure Redis to create the swap file there.
+vm-swap-file /tmp/redis.swap
+
+# vm-max-memory configures the VM to use at max the specified amount of
+# RAM. Everything that deos not fit will be swapped on disk *if* possible, that
+# is, if there is still enough contiguous space in the swap file.
+#
+# With vm-max-memory 0 the system will swap everything it can. Not a good
+# default, just specify the max amount of RAM you can in bytes, but it's
+# better to leave some margin. For instance specify an amount of RAM
+# that's more or less between 60 and 80% of your free RAM.
+vm-max-memory 0
+
+# Redis swap files is split into pages. An object can be saved using multiple
+# contiguous pages, but pages can't be shared between different objects.
+# So if your page is too big, small objects swapped out on disk will waste
+# a lot of space. If you page is too small, there is less space in the swap
+# file (assuming you configured the same number of total swap file pages).
+#
+# If you use a lot of small objects, use a page size of 64 or 32 bytes.
+# If you use a lot of big objects, use a bigger page size.
+# If unsure, use the default :)
+vm-page-size 32
+
+# Number of total memory pages in the swap file.
+# Given that the page table (a bitmap of free/used pages) is taken in memory,
+# every 8 pages on disk will consume 1 byte of RAM.
+#
+# The total swap size is vm-page-size * vm-pages
+#
+# With the default of 32-bytes memory pages and 134217728 pages Redis will
+# use a 4 GB swap file, that will use 16 MB of RAM for the page table.
+#
+# It's better to use the smallest acceptable value for your application,
+# but the default is large in order to work in most conditions.
+vm-pages 134217728
+
+# Max number of VM I/O threads running at the same time.
+# This threads are used to read/write data from/to swap file, since they
+# also encode and decode objects from disk to memory or the reverse, a bigger
+# number of threads can help with big objects even if they can't help with
+# I/O itself as the physical device may not be able to couple with many
+# reads/writes operations at the same time.
+#
+# The special value of 0 turn off threaded I/O and enables the blocking
+# Virtual Memory implementation.
+vm-max-threads 4
+
+############################### ADVANCED CONFIG ###############################
+
+# Glue small output buffers together in order to send small replies in a
+# single TCP packet. Uses a bit more CPU but most of the times it is a win
+# in terms of number of queries per second. Use 'yes' if unsure.
+glueoutputbuf yes
+
+# Hashes are encoded in a special way (much more memory efficient) when they
+# have at max a given numer of elements, and the biggest element does not
+# exceed a given threshold. You can configure this limits with the following
+# configuration directives.
+hash-max-zipmap-entries 64
+hash-max-zipmap-value 512
+
+# Active rehashing uses 1 millisecond every 100 milliseconds of CPU time in
+# order to help rehashing the main Redis hash table (the one mapping top-level
+# keys to values). The hash table implementation redis uses (see dict.c)
+# performs a lazy rehashing: the more operation you run into an hash table
+# that is rhashing, the more rehashing "steps" are performed, so if the
+# server is idle the rehashing is never complete and some more memory is used
+# by the hash table.
+# 
+# The default is to use this millisecond 10 times every second in order to
+# active rehashing the main dictionaries, freeing memory when possible.
+#
+# If unsure:
+# use "activerehashing no" if you have hard latency requirements and it is
+# not a good thing in your environment that Redis can reply form time to time
+# to queries with 2 milliseconds delay.
+#
+# use "activerehashing yes" if you don't have such hard requirements but
+# want to free memory asap when possible.
+activerehashing yes

metadata

@@ -0,0 +1,104 @@
+--- !ruby/object:Gem::Specification 
+name: redis-dump
+version: !ruby/object:Gem::Version 
+  hash: 27
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- Delano Mandelbaum
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-11-15 00:00:00 -05:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: yajl-ruby
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 9
+        segments: 
+        - 0
+        - 1
+        version: "0.1"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: redis
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 2
+        - 0
+        version: "2.0"
+  type: :runtime
+  version_requirements: *id002
+description: Backup and restore your Redis data to and from JSON.
+email: delano@solutious.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE.txt
+- README.rdoc
+files: 
+- CHANGES.txt
+- LICENSE.txt
+- README.rdoc
+- Rakefile
+- VERSION.yml
+- lib/redis/dump.rb
+- try/10_redis_dump_try.rb
+- try/redis-server.conf
+has_rdoc: true
+homepage: http://github.com/delano/redis-dump
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: redis-dump
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Backup and restore your Redis data to and from JSON.
+test_files: []
+