redis-load 0.1 → 0.2

data/CHANGES

@@ -1,3 +1,6 @@
+== Release 0.2
+ Added support for dumping out JSON to file and via API
+ 
 == Release 0.1
  First release to github
  Added initial support for loading JSON into Redis. Path support not yet fully tested

data/README.md

@@ -0,0 +1,73 @@
+redis-load
+==========
+
+Utility for managing data structures in Redis, allowing lists, sets, etc to be populated by loading data from flat files and JSON. Data structures can also be saved out to local files.
+
+The rationale here was to provide an easy way to persist the results of data manipulations from Redis in common formats. Similarly, I also wanted to load data structures from files for use in Redis, but where the original data has been created separately, e.g. by a map-reduce job.
+
+Author
+______
+
+Leigh Dodds (leigh.dodds@talis.com)
+
+Download
+--------
+
+[http://github.com/ldodds/redis-load][0]
+
+Installation
+------------
+
+redis-load is available as a gem:
+
+sudo gem install redis-load
+
+It has dependencies on the ruby redis library, json/pure and siren
+
+If you grab a copy from github then there's an install target in the Rakefile that will build and install the gem
+
+Usage
+-----
+
+The command-line app is called redis-load. You need to provide it with a filename and generally a key to be populated.
+
+Load the contents of a json file as entries into a server:
+
+    redis-load load-json --file myfile.json
+
+Load the contents of a file as key-value pairs. (Must be key,value):
+
+    redis-load load-keys --file mykeys.txt
+
+Load the contents of a file into a list. Each line is a list entry:
+
+    redis-load load-list --key mylist --file mylist.txt
+
+Load the contents of a file into a set. Each line is a set entry:
+
+    redis-load load-set --key myset --file myset.txt
+
+Load the contents of a file into a sort set. Each line is a key-value pair of entry,score:
+
+    redis-load load-zset --key myzset --file myzset.txt
+
+Save a list into a file:
+
+    redis-load save-list --key mylist --file mylist.txt
+
+Save a set into a file:
+
+    redis-load save-set --key myset --file myset.txt
+
+Save a sorted set into a file:
+
+    redis-load save-zset --key myzset --file myzset.txt
+
+Note that when saving a file any existing file will automatically be over-written.
+
+License
+-------
+
+redis-load is free and unencumbered public domain software. For more information, see http://unlicense.org/ or the accompanying UNLICENSE file.
+
+[0]: http://github.com/ldodds/redis-load

data/Rakefile

@@ -5,11 +5,11 @@ require 'rake/testtask'
 require 'rake/clean'
 
 NAME = "redis-load"
-VER = "0.1"
+VER = "0.2"
 
-RDOC_OPTS = ['--quiet', '--title', 'redis-load Reference', '--main', 'README']
+RDOC_OPTS = ['--quiet', '--title', 'redis-load Reference', '--main', 'README.md']
 
-PKG_FILES = %w( README Rakefile CHANGES ) + 
+PKG_FILES = %w( README.md Rakefile CHANGES ) + 
   Dir.glob("{bin,doc,tests,examples,lib}/**/*")
 
 CLEAN.include ['*.gem', 'pkg']  
@@ -20,7 +20,7 @@ SPEC =
     s.platform = Gem::Platform::RUBY
     s.required_ruby_version = ">= 1.8.5"    
     s.has_rdoc = true
-    s.extra_rdoc_files = ["README", "CHANGES"]
+    s.extra_rdoc_files = ["README.md", "CHANGES"]
     s.rdoc_options = RDOC_OPTS
     s.summary = "Utility of loading/saving data structures from Redis"
     s.description = s.summary
@@ -45,8 +45,8 @@ end
 Rake::RDocTask.new do |rdoc|
     rdoc.rdoc_dir = 'doc/rdoc'
     rdoc.options += RDOC_OPTS
-    rdoc.rdoc_files.include("README", "CHANGES", "lib/**/*.rb")
-    rdoc.main = "README"
+    rdoc.rdoc_files.include("README.md", "CHANGES", "lib/**/*.rb")
+    rdoc.main = "README.md"
     
 end
 

data/bin/redis-load

@@ -32,6 +32,9 @@ MODE
   load-zset
   	Load the contents of a file into a sort set. Each line is a key-value pair of entry,score  
 
+  save-json
+    Save all or some keys as JSON
+    
   save-list
     Save a list into a file
   
@@ -71,7 +74,8 @@ opts = GetoptLong::new(
          [ "--database"        , "-d" , GetoptLong::REQUIRED_ARGUMENT ],
          [ "--key"        , "-k" , GetoptLong::REQUIRED_ARGUMENT ],                  
          [ "--file"        , "-f" , GetoptLong::REQUIRED_ARGUMENT ],
-         [ "--jsonpath"        , "-j" , GetoptLong::REQUIRED_ARGUMENT ]                   
+         [ "--jsonpath"        , "-j" , GetoptLong::REQUIRED_ARGUMENT ],
+         [ "--keys"        , "-e" , GetoptLong::REQUIRED_ARGUMENT ]                            
        ).enum_for.inject({}) { |h, (k, v)| h.update k.delete('-') => v }
 
 mode = "help" if opts["--help"]
@@ -84,7 +88,7 @@ else
   	abort USAGE
   end
   
-  if (mode != "load-json" && mode != "load-keys") && opts["key"] == nil
+  if (mode != "load-json" && mode != "load-keys" && mode != "save-json") && opts["key"] == nil
     abort USAGE
   end
   
@@ -101,6 +105,8 @@ else
   		cmdline.load_set()
   	when "load-zset"
   		cmdline.load_zset()
+  	when "save-json"
+  		cmdline.save_json()  		  		
   	when "save-list"
   		cmdline.save_list()  		
   	when "save-set"

data/lib/redis-load/command_line.rb

@@ -49,6 +49,13 @@ module RedisLoad
       puts "#{counter} lines loaded."
     end
 
+    def save_json()      
+      puts "Saving json into #{@opts["file"]}"
+      keys = @opts["keys"] || "*"      
+      counter = @loader.save_json( @opts["file"], keys )
+      puts "#{counter} keys saved."      
+    end
+    
     def save_set()
       puts "Saving set #{@opts["key"]} into #{@opts["file"]}"
       size = @loader.save_set( @opts["key"], @opts["file"] )

data/lib/redis-load/loader.rb

@@ -10,6 +10,28 @@ module RedisLoad
       @redis = redis      
     end
 
+    def to_json( pattern="*", limit=-1, opts=nil )
+      keys = @redis.keys(pattern)
+      h = Hash.new
+      counter = 0
+      keys.each do |key|
+        counter = counter + 1
+        type = @redis.type(key)
+        if type == "hash"
+          h[key] = @redis.hgetall(key)
+        elsif type == "list"
+          h[key] = @redis.lrange(key, 0, limit)
+        elsif type == "set"
+          h[key] = @redis.smembers(key)
+        elsif type == "zset"
+          h[key] = @redis.zrange(key, 0, limit)
+        else
+          h[key] = @redis.get(key)
+        end  
+      end     
+      return JSON.fast_generate(h, opts), counter
+    end
+        
     def load_keys(file)
       f = File.new( file )
       counter = process_lines(f) do |redis,line|
@@ -36,7 +58,7 @@ module RedisLoad
         json = Siren.query(query, json)
       end
       counter = 0
-      #TODO could support proper results, not just path?
+      #TODO could support proper results, not just hash?
       if json != nil
         json.each_pair do |key,value|
           if value.class() == Array
@@ -88,6 +110,16 @@ module RedisLoad
       return counter
     end
     
+    def save_json(file, pattern="*", limit=-1, opts=nil)
+      counter = 0
+      File.open( file, "w" ) do |file|
+        json, count = to_json(pattern, limit, opts)
+        file.write( json )
+        counter = count
+      end
+      return counter
+    end
+      
     def save_list(key, file)
       size = @redis.llen( key )
       count = 0

metadata

@@ -1,12 +1,12 @@
 --- !ruby/object:Gem::Specification 
 name: redis-load
 version: !ruby/object:Gem::Version 
-  hash: 9
+  hash: 15
   prerelease: false
   segments: 
   - 0
-  - 1
-  version: "0.1"
+  - 2
+  version: "0.2"
 platform: ruby
 authors: 
 - Leigh Dodds
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-07-22 00:00:00 +01:00
+date: 2010-07-24 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -66,10 +66,10 @@ executables:
 extensions: []
 
 extra_rdoc_files: 
-- README
+- README.md
 - CHANGES
 files: 
-- README
+- README.md
 - Rakefile
 - CHANGES
 - bin/redis-load
@@ -87,7 +87,7 @@ rdoc_options:
 - --title
 - redis-load Reference
 - --main
-- README
+- README.md
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 

data/README

@@ -1,64 +0,0 @@
-Utility for managing data structures in Redis, allowing lists, sets, etc to be populated by loading 
-data from flat files and JSON. Data structures can also be saved out to local files.
-
-The rationale here was to provide some easy ways to persistent out the results of data manipulations from 
-Redis in easy to process formats. Similarly I also wanted to load data structures from files for use in Redis, 
-but where the original data has been created separately, e.g. by a map-reduce job.
-
-== Author
- 
-Leigh Dodds (leigh.dodds@talis.com)
-
-== Download
-
-http://github.com/ldodds/redis-load
-
-== Installation
-
-redis-load is available as a gem:
-
-sudo gem install redis-load
-
-It has dependencies on the ruby redis library, json/pure and siren
-
-If you grab a copy from github then there's an install target in the Rakefile that will build and install the gem
-
-== Usage
-
-The command-line app is called redis-load. You need to provide it with a file name and generally 
-a key to be populated:
-
-Load the contents of a json file as entries into a server
-
-  redis-load load-json --file myfile.json
-
-Load the contents of a file as key-value pairs. Must be: key,value
-  
-  redis-load load-keys --file mykeys.txt
-
-Load the contents of a file into a list. Each line is a list entry
-  	
-  redis-load load-list --key mylist --file mylist.txt
-
-Load the contents of a file into a set. Each line is a set entry
-
-  redis-load load-set --key myset --file myset.txt
-  
-Load the contents of a file into a sort set. Each line is a key-value pair of entry,score  
-  
-  redis-load load-zset --key myzset --file myzset.txt
-
-Save a list into a file
-
-  redis-load save-list --key mylist --file mylist.txt
-  
-Save a set into a file
-  
-  redis-load save-set --key myset --file myset.txt
-    
-Save a sorted set into a file
-  
-  redis-load save-zset --key myzset --file myzset.txt
-
-Note that when saving a file any existing file will automatically be over-written.
-