redis_importer 0.0.5 → 0.0.6

data/README.md

@@ -1,6 +1,12 @@
 # RedisImporter
 
-TODO: Write a gem description
+Takes csv files stored on Amazon S3, converts them to objects and then imports those objects into redis, serialized as redis hashes. This gem is the glue between several other gems:
+
+* `csv_to_object` (handles the conversion of a csv file to objects)
+* `redis_backed_model` (Gives an object a to_redis command that serializes an object as a series of redis commands)
+* `redis_pipeline` (Imports large sets of commands into a redis instance)
+
+If you have those other gems set up correctly, then RedisImporter is straightforward. Tell it where your csv files are stored and you're good to go.
 
 ## Installation
 
@@ -15,10 +21,22 @@ And then execute:
 Or install it yourself as:
 
     $ gem install redis_importer
+    
+## Configuration
+
+    rails g redis_importer:install
+    
+1. Edit config/redis_importer.yml to set your remote file-storage method and what directory should store local copies of the files during import.
+2. Edit config/s3_config.yml to set your access key, secret access key, bucket name, etc.
+
+**At this time only Amazon S3 is supported as a remote file-storage method.**
 
 ## Usage
 
-TODO: Write usage instructions here
+```ruby
+  @ri = RedisImporter::RedisImporter.new
+  @ri.import => true
+```
 
 ## Contributing
 

data/lib/redis_importer/redis_importer.rb

@@ -2,8 +2,7 @@ module RedisImporter
   class RedisImporter
     include GemConfigurator
 
-    attr_reader :files
-
+    attr_reader :files, :commands, :errors
     attr_accessor :collection
 
     def initialize
@@ -11,14 +10,16 @@ module RedisImporter
       self.collection = Object::const_get("#{@settings[:storage_method].camelcase}Collection").new()
       
       self.files = self.collection.files
+      self.commands = []
     end
     
     def import
       files.each do |file|
-        if class_exists?(file.to_class_name)
-          self.local_path = "tmp/#{file.name}"
-          file.save_to(local_path)
-          get_redis_commands
+        begin
+          convert_to_redis_commands(file) if class_exists?(file.to_class_name)
+        rescue NameError
+          @errors ||= []
+          @errors << "#{file.name} is not matched by a class #{file.to_class_name} in the system."
         end
       end
       pipeline
@@ -26,33 +27,43 @@ module RedisImporter
     
     private
 
-    attr_writer :files
-    attr_accessor :local_path
+    attr_writer :files, :commands
 
     def class_exists?(c)
-      Object::const_defined?(c)
+      Module.const_get(c)
     end
-    
+
+    def convert_to_redis_commands(file)
+      local_path = local_storage_path(file)
+      file.save_to(local_path)
+      convert_objects_to_redis_commands(get_objects(local_path))
+    end
+
     def default_settings
-      {:storage_method => 's3'}
+      {:storage_method => 's3', :local_storage_directory => 'tmp'}
     end
 
-    def get_objects
+    def get_objects(local_path)
       CsvToObject::CsvToObject.new(local_path).to_objects
     end
     
-    def get_redis_commands
-      @commands ||= []
-      get_objects.each do |obj|
-        @commands << obj.to_redis
+    def convert_objects_to_redis_commands(objects)
+      objects.each do |obj|
+        self.commands << obj.to_redis
       end
     end
     
+    def local_storage_path(file)
+      "#{@settings[:local_storage_directory]}/#{file.name}"
+    end
+
     def pipeline
-      if @commands && !@commands.empty?
+      if !self.commands.empty?
         pipeline = RedisPipeline::RedisPipeline.new
-        pipeline.add_commands(@commands.flatten)
+        pipeline.add_commands(self.commands.flatten)
         pipeline.execute_commands
+      else
+        self.commands
       end
     end
   end

data/lib/redis_importer/version.rb

@@ -1,3 +1,3 @@
 module RedisImporter
-  VERSION = "0.0.5"
+  VERSION = "0.0.6"
 end

data/spec/redis_importer_spec.rb

@@ -3,12 +3,14 @@ require_relative '../test/lib/person.rb'
 
 describe RedisImporter do
   before(:all) do
-    credentials = YAML.load_file(File.open('config/s3_config.yml'))['development']
-    connection = AWS::S3::Base.establish_connection! credentials['connection']
+    @configuration = YAML.load_file(File.open('config/redis_importer.yml'))['development']
+
+    collection = YAML.load_file(File.open('config/s3_config.yml'))['development']
+    connection = AWS::S3::Base.establish_connection! collection['connection']
   
-    bucket_name = credentials['bucket']
+    bucket_name = collection['bucket']
     Bucket.create(bucket_name)
-    bucket = Bucket.find(bucket_name)
+    @bucket = Bucket.find(bucket_name)
   
     test_csv_files = Dir.glob("test/csv/*.csv")
     test_class_names = test_csv_files.map {|f| File.basename(f).gsub('.csv','').capitalize}
@@ -25,21 +27,21 @@ describe RedisImporter do
   end
 
   it "determines its file storage type from a configuration file" do
-    @ri.settings[:storage_method].should == 's3'
+    @ri.settings[:storage_method].should == @configuration['storage_method']
   end
   
   it "uses its storage method to get a file collection module" do
-    @ri.collection.class.to_s.should == 'S3Collection'
+    @ri.collection.class.to_s.downcase.should == "#{@configuration['storage_method']}Collection".downcase
   end
   
   it "checks to see if each file is matched by a class in the system" do 
-    @ri.should_receive(:class_exists?).twice
+    @ri.should_receive(:class_exists?).exactly(@bucket.objects.count).times
     @ri.import
   end
   
   it "only passes files to csv_to_object that are matched by a class in the system" do
-    @ri.should_receive(:get_redis_commands).once
     @ri.import
+    @ri.errors.should == ["foo.csv is not matched by a class Foo in the system."]
   end
   
   it "passes the redis commands to the redis pipeline" do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: redis_importer
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-08-24 00:00:00.000000000Z
+date: 2012-08-27 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &2153402260 !ruby/object:Gem::Requirement
+  requirement: &2169532440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2153402260
+  version_requirements: *2169532440
 - !ruby/object:Gem::Dependency
   name: gem_configurator
-  requirement: &2153401840 !ruby/object:Gem::Requirement
+  requirement: &2169532020 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2153401840
+  version_requirements: *2169532020
 - !ruby/object:Gem::Dependency
   name: csv_to_object
-  requirement: &2153401420 !ruby/object:Gem::Requirement
+  requirement: &2155980220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2153401420
+  version_requirements: *2155980220
 - !ruby/object:Gem::Dependency
   name: redis_pipeline
-  requirement: &2153401000 !ruby/object:Gem::Requirement
+  requirement: &2155979800 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2153401000
+  version_requirements: *2155979800
 - !ruby/object:Gem::Dependency
   name: aws-s3
-  requirement: &2160498080 !ruby/object:Gem::Requirement
+  requirement: &2155979300 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,7 +65,7 @@ dependencies:
         version: 0.6.3
   type: :runtime
   prerelease: false
-  version_requirements: *2160498080
+  version_requirements: *2155979300
 description: Creates objects, converts them to Redis commands and executes the redis
   commands.
 email: