redis-load 0.1

data/CHANGES

@@ -0,0 +1,4 @@
+== Release 0.1
+ First release to github
+ Added initial support for loading JSON into Redis. Path support not yet fully tested
+ Added basic command line tools for loading/saving

data/README

@@ -0,0 +1,64 @@
+Utility for managing data structures in Redis, allowing lists, sets, etc to be populated by loading 
+data from flat files and JSON. Data structures can also be saved out to local files.
+
+The rationale here was to provide some easy ways to persistent out the results of data manipulations from 
+Redis in easy to process formats. Similarly I also wanted to load data structures from files for use in Redis, 
+but where the original data has been created separately, e.g. by a map-reduce job.
+
+== Author
+ 
+Leigh Dodds (leigh.dodds@talis.com)
+
+== Download
+
+http://github.com/ldodds/redis-load
+
+== Installation
+
+redis-load is available as a gem:
+
+sudo gem install redis-load
+
+It has dependencies on the ruby redis library, json/pure and siren
+
+If you grab a copy from github then there's an install target in the Rakefile that will build and install the gem
+
+== Usage
+
+The command-line app is called redis-load. You need to provide it with a file name and generally 
+a key to be populated:
+
+Load the contents of a json file as entries into a server
+
+  redis-load load-json --file myfile.json
+
+Load the contents of a file as key-value pairs. Must be: key,value
+  
+  redis-load load-keys --file mykeys.txt
+
+Load the contents of a file into a list. Each line is a list entry
+  	
+  redis-load load-list --key mylist --file mylist.txt
+
+Load the contents of a file into a set. Each line is a set entry
+
+  redis-load load-set --key myset --file myset.txt
+  
+Load the contents of a file into a sort set. Each line is a key-value pair of entry,score  
+  
+  redis-load load-zset --key myzset --file myzset.txt
+
+Save a list into a file
+
+  redis-load save-list --key mylist --file mylist.txt
+  
+Save a set into a file
+  
+  redis-load save-set --key myset --file myset.txt
+    
+Save a sorted set into a file
+  
+  redis-load save-zset --key myzset --file myzset.txt
+
+Note that when saving a file any existing file will automatically be over-written.
+ 

data/Rakefile

@@ -0,0 +1,66 @@
+require 'rake'
+require 'rake/gempackagetask'
+require 'rake/rdoctask'
+require 'rake/testtask'
+require 'rake/clean'
+
+NAME = "redis-load"
+VER = "0.1"
+
+RDOC_OPTS = ['--quiet', '--title', 'redis-load Reference', '--main', 'README']
+
+PKG_FILES = %w( README Rakefile CHANGES ) + 
+  Dir.glob("{bin,doc,tests,examples,lib}/**/*")
+
+CLEAN.include ['*.gem', 'pkg']  
+SPEC =
+  Gem::Specification.new do |s|
+    s.name = NAME
+    s.version = VER
+    s.platform = Gem::Platform::RUBY
+    s.required_ruby_version = ">= 1.8.5"    
+    s.has_rdoc = true
+    s.extra_rdoc_files = ["README", "CHANGES"]
+    s.rdoc_options = RDOC_OPTS
+    s.summary = "Utility of loading/saving data structures from Redis"
+    s.description = s.summary
+    s.author = "Leigh Dodds"
+    s.email = 'leigh.dodds@talis.com'
+    s.homepage = 'http://github.org/ldodds/redis-load'
+    #s.rubyforge_project = ''
+    s.files = PKG_FILES
+    s.require_path = "lib" 
+    s.bindir = "bin"
+    s.executables = ["redis-load"]
+    s.test_file = "tests/ts_redis-load.rb"
+    s.add_dependency("redis")
+    s.add_dependency("json_pure")
+    s.add_dependency("siren")        
+  end
+      
+Rake::GemPackageTask.new(SPEC) do |pkg|
+    pkg.need_tar = true
+end
+
+Rake::RDocTask.new do |rdoc|
+    rdoc.rdoc_dir = 'doc/rdoc'
+    rdoc.options += RDOC_OPTS
+    rdoc.rdoc_files.include("README", "CHANGES", "lib/**/*.rb")
+    rdoc.main = "README"
+    
+end
+
+Rake::TestTask.new do |test|
+  test.test_files = FileList['tests/tc_*.rb']
+end
+
+desc "Install from a locally built copy of the gem"
+task :install do
+  sh %{rake package}
+  sh %{sudo gem install pkg/#{NAME}-#{VER}}
+end
+
+desc "Uninstall the gem"
+task :uninstall => [:clean] do
+  sh %{sudo gem uninstall #{NAME}}
+end

data/bin/redis-load

@@ -0,0 +1,114 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'redis-load'
+
+require 'rubygems'
+require 'redis'
+require 'getoptlong'
+
+PROGRAM     = File::basename $0
+
+USAGE = <<-EOL 
+SYNOPSIS
+
+  #{ PROGRAM } mode [args]*
+
+DESCRIPTION
+
+MODE
+
+  load-json
+  	Load the contents of a json file as entries into a server
+
+  load-keys
+  	Load the contents of a file as key-value pairs. Must be: key,value
+  	
+  load-list
+    Load the contents of a file into a list. Each line is a list entry
+
+  load-set
+  	Load the contents of a file into a set. Each line is a set entry  
+  
+  load-zset
+  	Load the contents of a file into a sort set. Each line is a key-value pair of entry,score  
+
+  save-list
+    Save a list into a file
+  
+  save-set
+    Save a set into a file
+  
+  save-zset
+    Save a sorted set into a file
+  
+
+OPTIONS
+
+  --database            , -d
+    specify database
+  --help            , -?
+    show this message
+  --host            , -h
+    specify host on which server is running
+  --file            , -f
+    path to file containing the data to load    
+  --password            , -x
+    specify password to access server    
+  --port            , -p
+    specify port on which server is running    
+  --key				, -k
+  	specify a key into which data is loaded
+  	
+EOL
+
+mode = ARGV.shift
+
+opts = GetoptLong::new(
+         [ "--help"            , "-?" , GetoptLong::NO_ARGUMENT ],
+         [ "--host"         , "-h" , GetoptLong::REQUIRED_ARGUMENT ],
+         [ "--port"         , "-p" , GetoptLong::REQUIRED_ARGUMENT ],
+         [ "--password"        , "-x" , GetoptLong::REQUIRED_ARGUMENT ],
+         [ "--database"        , "-d" , GetoptLong::REQUIRED_ARGUMENT ],
+         [ "--key"        , "-k" , GetoptLong::REQUIRED_ARGUMENT ],                  
+         [ "--file"        , "-f" , GetoptLong::REQUIRED_ARGUMENT ],
+         [ "--jsonpath"        , "-j" , GetoptLong::REQUIRED_ARGUMENT ]                   
+       ).enum_for.inject({}) { |h, (k, v)| h.update k.delete('-') => v }
+
+mode = "help" if opts["--help"]
+
+if mode == "help"
+  USAGE.display  
+else
+  
+  if opts["file"] == nil
+  	abort USAGE
+  end
+  
+  if (mode != "load-json" && mode != "load-keys") && opts["key"] == nil
+    abort USAGE
+  end
+  
+  cmdline = RedisLoad::CommandLine.new(opts, ENV)
+  
+  case mode
+  	when "load-json"
+  		cmdline.load_json()  		    
+  	when "load-keys"
+  		cmdline.load_keys()  		  
+  	when "load-list"
+  		cmdline.load_list()
+  	when "load-set"
+  		cmdline.load_set()
+  	when "load-zset"
+  		cmdline.load_zset()
+  	when "save-list"
+  		cmdline.save_list()  		
+  	when "save-set"
+  		cmdline.save_set()  		
+  	when "save-zset"
+  		cmdline.save_zset()  		
+  	else
+  		puts "Unknown command"
+  end
+end
+         

data/lib/redis-load.rb

@@ -0,0 +1,7 @@
+require 'rubygems'
+require 'redis'
+require 'json/pure'
+require 'siren'
+
+require 'redis-load/command_line.rb'
+require 'redis-load/loader.rb'

data/lib/redis-load/command_line.rb

@@ -0,0 +1,71 @@
+module RedisLoad
+  
+  #This class acts as an adapter, taking parameters and environment variables and 
+  #using those to drive an instance of RedisLoad::Loader  
+  class CommandLine
+    
+    def initialize(opts, env)
+      @opts = opts
+      @env = env
+      
+      opts = Hash.new
+      opts[:host] = @opts["host"] if @opts["host"]
+      opts[:port] = @opts["port"] if @opts["port"]
+      opts[:db] = @opts["database"] if @opts["database"]
+      opts[:password] = @opts["password"] if @opts["password"]              
+        
+      @redis = Redis.new(opts)
+      @loader = RedisLoad::Loader.new(@redis)
+                
+    end
+    
+    def load_json()
+      puts "Loading data from JSON file #{@opts["file"]} into server"
+      counter = @loader.load_json( @opts["file"], @opts["jsonpath"] )
+      puts "#{counter} keys loaded."      
+    end
+    
+    def load_list()
+      puts "Loading #{@opts["file"]} into list #{@opts["key"]} into server"
+      counter = @loader.load_list( @opts["key"] , @opts["file"] )
+      puts "#{counter} lines loaded."
+    end
+    
+    def save_list()
+      puts "Saving list #{@opts["key"]} into #{@opts["file"]}"
+      size = @loader.save_list( @opts["key"], @opts["file"] )
+      puts "#{size} lines saved."      
+    end
+    
+    def load_keys()
+      puts "Loading keys from #{@opts["file"]} into server"
+      counter = @loader.load_keys( @opts["file"] )
+      puts "#{counter} lines loaded."
+    end
+
+    def load_set()
+      puts "Loading #{@opts["file"]} into set #{@opts["key"]} on server"
+      counter @loader.load_set( @opts["key"], @opts["file"] )
+      puts "#{counter} lines loaded."
+    end
+
+    def save_set()
+      puts "Saving set #{@opts["key"]} into #{@opts["file"]}"
+      size = @loader.save_set( @opts["key"], @opts["file"] )
+      puts "#{size} items saved."      
+    end
+  
+    def load_zset()
+      puts "Loading #{@opts["file"]} into sorted set #{@opts["key"]} on server"
+      counter = @loader.load_zset( @opts["key"], @opts["file"] )
+      puts "#{counter} items loaded."
+    end
+
+    def save_zset()
+      puts "Saving set #{@opts["key"]} into #{@opts["file"]}"
+      size = @loader.save_zset( @opts["key"], @opts["file"] )
+      puts "#{size} items saved."  
+    end                      
+
+  end
+end

data/lib/redis-load/loader.rb

@@ -0,0 +1,136 @@
+module RedisLoad
+  
+  #Exposes support for loading and saving different types of redis data 
+  #structures to and from files  
+  class Loader
+    
+    attr_reader :redis
+    
+    def initialize(redis)
+      @redis = redis      
+    end
+
+    def load_keys(file)
+      f = File.new( file )
+      counter = process_lines(f) do |redis,line|
+        line = line.chomp
+        parts = line.split(",")
+        redis.set( parts[0], parts[1] )
+      end
+      return counter
+    end
+    
+    #Load the contents of a JSON file into Redis
+    #
+    #Each key in the JSON file becomes a redis key. If the value is a literal then this is added
+    #as a redis key-value pair. If an array, then a List. If a hash, then a Hash. 
+    #
+    #Nested objects will be flattened.
+    #
+    #  file:: json file
+    #  path:: optional, path in the json file indicating the subsection to load
+    def load_json(file, path=nil)
+      f = File.new( file )
+      json = JSON.parse( f.read )
+      if path != nil
+        json = Siren.query(query, json)
+      end
+      counter = 0
+      #TODO could support proper results, not just path?
+      if json != nil
+        json.each_pair do |key,value|
+          if value.class() == Array
+            value.each do |v|
+              @redis.lpush( key, v )
+            end          
+          elsif value.class == Hash
+            value.each_pair do |field, v|
+              #TODO better handling of nesting?
+              #E.g. convert hash or array back into JSON?
+              @redis.hset(key, field, v)
+            end
+          else
+            @redis.set(key, value)
+          end
+          counter = counter + 1      
+        end        
+      end
+      return counter
+    end
+        
+    #Load the contents of a file into a Redis list
+    #
+    #  key:: name of the list
+    #  file:: name of file (string)
+    def load_list(key, file)
+      f = File.new( file )
+      counter = process_lines(f) do |redis,line|
+        redis.lpush( key , line.chomp)
+      end
+      return counter            
+    end
+    
+    def load_set(key, file)
+      f = File.new( file )
+      counter = process_lines(f) do |redis,line|
+        redis.sadd( key , line.chomp)
+      end
+      return counter
+    end
+    
+    def load_zset(key, file)
+      f = File.new( file )
+      counter = process_lines(f) do |redis,line|
+        line = line.chomp
+        parts = line.split(",")
+        redis.set( key , parts[1], parts[0] )
+      end
+      return counter
+    end
+    
+    def save_list(key, file)
+      size = @redis.llen( key )
+      count = 0
+      File.open( file , "w") do |file|
+        while (count < size)
+          file.puts( @redis.lindex(key , count) )
+          count = count + 1
+        end
+      end
+      return size      
+    end
+                    
+    def save_set(key, file)
+      members = @redis.smembers( key )
+      File.open( file , "w") do |file|
+        members.each do |m|
+          file.puts(m)
+        end
+      end
+      return members.length      
+    end
+        
+    def save_zset(key, file)
+      members = @redis.zrange( key , 0, -1, :with_scores => true)
+      File.open( file , "w") do |file|
+        members.each_slice(2) do |member,value|
+          file.puts("#{member},#{value}")
+        end
+      end
+      return members.length/2  
+    end
+    
+    #Process lines
+    #
+    #  io:: IO Object
+    def process_lines(io)
+      counter = 0
+      io.each_line do |line|
+        yield @redis, line
+        counter = counter + 1
+      end
+      return counter      
+    end
+        
+  end
+end

data/tests/ts_redis-load.rb

@@ -0,0 +1,2 @@
+$:.unshift File.dirname(__FILE__)
+require 'test/unit'

metadata

@@ -0,0 +1,121 @@
+--- !ruby/object:Gem::Specification 
+name: redis-load
+version: !ruby/object:Gem::Version 
+  hash: 9
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  version: "0.1"
+platform: ruby
+authors: 
+- Leigh Dodds
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-07-22 00:00:00 +01:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: redis
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: json_pure
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: siren
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id003
+description: Utility of loading/saving data structures from Redis
+email: leigh.dodds@talis.com
+executables: 
+- redis-load
+extensions: []
+
+extra_rdoc_files: 
+- README
+- CHANGES
+files: 
+- README
+- Rakefile
+- CHANGES
+- bin/redis-load
+- tests/ts_redis-load.rb
+- lib/redis-load.rb
+- lib/redis-load/command_line.rb
+- lib/redis-load/loader.rb
+has_rdoc: true
+homepage: http://github.org/ldodds/redis-load
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --quiet
+- --title
+- redis-load Reference
+- --main
+- README
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 61
+      segments: 
+      - 1
+      - 8
+      - 5
+      version: 1.8.5
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Utility of loading/saving data structures from Redis
+test_files: 
+- tests/ts_redis-load.rb