redis_importer 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +17 -0
- data/.rspec +2 -0
- data/Gemfile +7 -0
- data/LICENSE +22 -0
- data/README.md +29 -0
- data/Rakefile +8 -0
- data/config/.gitignore +1 -0
- data/lib/collections/S3_collection.rb +14 -0
- data/lib/extensions/s3.rb +22 -0
- data/lib/redis_importer/redis_importer.rb +59 -0
- data/lib/redis_importer/version.rb +3 -0
- data/lib/redis_importer.rb +15 -0
- data/redis_importer.gemspec +24 -0
- data/spec/redis_importer_spec.rb +53 -0
- data/spec/spec_helper.rb +19 -0
- data/test/csv/foo.csv +11 -0
- data/test/csv/person.csv +11 -0
- data/test/lib/person.rb +13 -0
- data/tmp/.gitignore +4 -0
- metadata +136 -0
data/.gitignore
ADDED
data/.rspec
ADDED
data/Gemfile
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
source 'https://rubygems.org'
|
2
|
+
|
3
|
+
# Specify your gem's dependencies in redis_importer.gemspec
|
4
|
+
gemspec
|
5
|
+
gem 'gem_configurator', :git => 'git://github.com/SeniorServiceAmerica/gem_configurator'
|
6
|
+
gem 'csv_to_object', :git => 'git://github.com/SeniorServiceAmerica/csv_to_object'
|
7
|
+
gem 'redis_pipeline', :git => 'git://github.com/SeniorServiceAmerica/redis_pipeline'
|
data/LICENSE
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2012 Ian Whitney
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
# RedisImporter
|
2
|
+
|
3
|
+
TODO: Write a gem description
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Add this line to your application's Gemfile:
|
8
|
+
|
9
|
+
gem 'redis_importer'
|
10
|
+
|
11
|
+
And then execute:
|
12
|
+
|
13
|
+
$ bundle
|
14
|
+
|
15
|
+
Or install it yourself as:
|
16
|
+
|
17
|
+
$ gem install redis_importer
|
18
|
+
|
19
|
+
## Usage
|
20
|
+
|
21
|
+
TODO: Write usage instructions here
|
22
|
+
|
23
|
+
## Contributing
|
24
|
+
|
25
|
+
1. Fork it
|
26
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
27
|
+
3. Commit your changes (`git commit -am 'Added some feature'`)
|
28
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
29
|
+
5. Create new Pull Request
|
data/Rakefile
ADDED
data/config/.gitignore
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
*yml
|
@@ -0,0 +1,14 @@
|
|
1
|
+
class S3Collection
|
2
|
+
def initialize
|
3
|
+
credentials = YAML.load_file(File.open('config/s3_credentials.yml'))['development']
|
4
|
+
connection = AWS::S3::Base.establish_connection! credentials['connection']
|
5
|
+
|
6
|
+
bucket_name = credentials['bucket']
|
7
|
+
Bucket.create(bucket_name)
|
8
|
+
@bucket = Bucket.find(bucket_name)
|
9
|
+
end
|
10
|
+
|
11
|
+
def files
|
12
|
+
@bucket
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
module AWS
|
2
|
+
module S3
|
3
|
+
class S3Object
|
4
|
+
|
5
|
+
def name
|
6
|
+
self.key
|
7
|
+
end
|
8
|
+
|
9
|
+
def to_class_name
|
10
|
+
self.key.gsub('.csv','').capitalize
|
11
|
+
end
|
12
|
+
|
13
|
+
def save_to(path)
|
14
|
+
File.open(path,'w') do |file|
|
15
|
+
S3Object.stream(self.key,self.bucket.name) do |chunk|
|
16
|
+
file.write chunk
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
module RedisImporter
|
2
|
+
class RedisImporter
|
3
|
+
include GemConfigurator
|
4
|
+
|
5
|
+
attr_reader :files
|
6
|
+
|
7
|
+
attr_accessor :collection
|
8
|
+
|
9
|
+
def initialize
|
10
|
+
configure
|
11
|
+
self.collection = Object::const_get("#{@settings[:storage_method].camelcase}Collection").new()
|
12
|
+
|
13
|
+
self.files = self.collection.files
|
14
|
+
end
|
15
|
+
|
16
|
+
def import
|
17
|
+
files.each do |file|
|
18
|
+
if class_exists?(file.to_class_name)
|
19
|
+
self.local_path = "tmp/#{file.name}"
|
20
|
+
file.save_to(local_path)
|
21
|
+
get_redis_commands
|
22
|
+
end
|
23
|
+
end
|
24
|
+
pipeline
|
25
|
+
end
|
26
|
+
|
27
|
+
private
|
28
|
+
|
29
|
+
attr_writer :files
|
30
|
+
attr_accessor :local_path
|
31
|
+
|
32
|
+
def class_exists?(c)
|
33
|
+
Object::const_defined?(c)
|
34
|
+
end
|
35
|
+
|
36
|
+
def default_settings
|
37
|
+
{:storage_method => 's3'}
|
38
|
+
end
|
39
|
+
|
40
|
+
def get_objects
|
41
|
+
CsvToObject::CsvToObject.new(local_path).to_objects
|
42
|
+
end
|
43
|
+
|
44
|
+
def get_redis_commands
|
45
|
+
@commands ||= []
|
46
|
+
get_objects.each do |obj|
|
47
|
+
@commands << obj.to_redis
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def pipeline
|
52
|
+
if @commands && !@commands.empty?
|
53
|
+
pipeline = RedisPipeline::RedisPipeline.new
|
54
|
+
pipeline.add_commands(@commands.flatten)
|
55
|
+
pipeline.execute_commands
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
require 'redis_importer/version'
|
2
|
+
|
3
|
+
# Requiring released gems defined
|
4
|
+
require 'aws/s3'
|
5
|
+
require 'active_support/inflector'
|
6
|
+
include AWS::S3
|
7
|
+
|
8
|
+
# Requiring unreleased gems
|
9
|
+
require 'bundler'
|
10
|
+
Bundler.require
|
11
|
+
|
12
|
+
# Requiring local files
|
13
|
+
require 'redis_importer/redis_importer'
|
14
|
+
require 'collections/S3_collection'
|
15
|
+
require 'extensions/s3'
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
require File.expand_path('../lib/redis_importer/version', __FILE__)
|
3
|
+
|
4
|
+
Gem::Specification.new do |gem|
|
5
|
+
gem.authors = ["Ian Whitney"]
|
6
|
+
gem.email = ["ian@ianwhitney.com"]
|
7
|
+
gem.description = %q{Creates objects, converts them to Redis commands and executes the redis commands.}
|
8
|
+
gem.summary = %q{Creates objects, converts them to Redis commands and executes the redis commands.}
|
9
|
+
gem.homepage = "https://github.com/SeniorServiceAmerica/redis_importer"
|
10
|
+
|
11
|
+
gem.files = `git ls-files`.split($\)
|
12
|
+
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
13
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
14
|
+
gem.name = "redis_importer"
|
15
|
+
gem.require_paths = ["lib"]
|
16
|
+
gem.version = RedisImporter::VERSION
|
17
|
+
|
18
|
+
gem.add_development_dependency "rspec"
|
19
|
+
gem.add_dependency('gem_configurator')
|
20
|
+
gem.add_dependency('csv_to_object')
|
21
|
+
gem.add_dependency('redis_pipeline')
|
22
|
+
gem.add_dependency('aws-s3')
|
23
|
+
gem.add_dependency('active_support')
|
24
|
+
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
require_relative '../test/lib/person.rb'
|
3
|
+
|
4
|
+
describe RedisImporter do
|
5
|
+
before(:all) do
|
6
|
+
credentials = YAML.load_file(File.open('config/s3_credentials.yml'))['development']
|
7
|
+
connection = AWS::S3::Base.establish_connection! credentials['connection']
|
8
|
+
|
9
|
+
bucket_name = credentials['bucket']
|
10
|
+
Bucket.create(bucket_name)
|
11
|
+
bucket = Bucket.find(bucket_name)
|
12
|
+
|
13
|
+
test_csv_files = Dir.glob("test/csv/*.csv")
|
14
|
+
test_class_names = test_csv_files.map {|f| File.basename(f).gsub('.csv','').capitalize}
|
15
|
+
|
16
|
+
test_csv_files.each do |file_path|
|
17
|
+
filename = File.basename(file_path)
|
18
|
+
file = File.open(file_path)
|
19
|
+
S3Object.store(filename,file,bucket_name)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
before(:each) do
|
24
|
+
@ri = RedisImporter::RedisImporter.new
|
25
|
+
end
|
26
|
+
|
27
|
+
it "determines its file storage type from a configuration file" do
|
28
|
+
@ri.settings[:storage_method].should == 's3'
|
29
|
+
end
|
30
|
+
|
31
|
+
it "uses its storage method to get a file collection module" do
|
32
|
+
@ri.collection.class.to_s.should == 'S3Collection'
|
33
|
+
end
|
34
|
+
|
35
|
+
it "checks to see if each file is matched by a class in the system" do
|
36
|
+
@ri.should_receive(:class_exists?).twice
|
37
|
+
@ri.import
|
38
|
+
end
|
39
|
+
|
40
|
+
it "only passes files to csv_to_object that are matched by a class in the system" do
|
41
|
+
@ri.should_receive(:get_redis_commands).once
|
42
|
+
@ri.import
|
43
|
+
end
|
44
|
+
|
45
|
+
it "passes the redis commands to the redis pipeline" do
|
46
|
+
@ri.should_receive(:pipeline)
|
47
|
+
@ri.import
|
48
|
+
end
|
49
|
+
|
50
|
+
it "reports the success of import" do
|
51
|
+
@ri.import.should == true
|
52
|
+
end
|
53
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
require 'redis_importer'
|
2
|
+
|
3
|
+
# This file was generated by the `rspec --init` command. Conventionally, all
|
4
|
+
# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
|
5
|
+
# Require this file using `require "spec_helper"` to ensure that it is only
|
6
|
+
# loaded once.
|
7
|
+
#
|
8
|
+
# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
|
9
|
+
RSpec.configure do |config|
|
10
|
+
config.treat_symbols_as_metadata_keys_with_true_values = true
|
11
|
+
config.run_all_when_everything_filtered = true
|
12
|
+
config.filter_run :focus
|
13
|
+
|
14
|
+
# Run specs in random order to surface order dependencies. If you find an
|
15
|
+
# order dependency and want to debug it, you can fix the order by providing
|
16
|
+
# the seed, which is printed after each run.
|
17
|
+
# --seed 1234
|
18
|
+
config.order = 'random'
|
19
|
+
end
|
data/test/csv/foo.csv
ADDED
data/test/csv/person.csv
ADDED
data/test/lib/person.rb
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
class Person
|
2
|
+
attr_reader :id, :first_name, :last_name
|
3
|
+
|
4
|
+
def initialize(args={})
|
5
|
+
@id = args[:id]
|
6
|
+
@first_name = args[:first_name]
|
7
|
+
@last_name = args[:last_name]
|
8
|
+
end
|
9
|
+
|
10
|
+
def to_redis()
|
11
|
+
["hset person:#{self.id} first_name #{self.first_name}", "hset person:#{self.id} last_name #{self.last_name}"]
|
12
|
+
end
|
13
|
+
end
|
data/tmp/.gitignore
ADDED
metadata
ADDED
@@ -0,0 +1,136 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: redis_importer
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Ian Whitney
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2012-08-23 00:00:00.000000000Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: rspec
|
16
|
+
requirement: &2157676740 !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
22
|
+
type: :development
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: *2157676740
|
25
|
+
- !ruby/object:Gem::Dependency
|
26
|
+
name: gem_configurator
|
27
|
+
requirement: &2157676320 !ruby/object:Gem::Requirement
|
28
|
+
none: false
|
29
|
+
requirements:
|
30
|
+
- - ! '>='
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '0'
|
33
|
+
type: :runtime
|
34
|
+
prerelease: false
|
35
|
+
version_requirements: *2157676320
|
36
|
+
- !ruby/object:Gem::Dependency
|
37
|
+
name: csv_to_object
|
38
|
+
requirement: &2157675900 !ruby/object:Gem::Requirement
|
39
|
+
none: false
|
40
|
+
requirements:
|
41
|
+
- - ! '>='
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
version: '0'
|
44
|
+
type: :runtime
|
45
|
+
prerelease: false
|
46
|
+
version_requirements: *2157675900
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: redis_pipeline
|
49
|
+
requirement: &2157675480 !ruby/object:Gem::Requirement
|
50
|
+
none: false
|
51
|
+
requirements:
|
52
|
+
- - ! '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
type: :runtime
|
56
|
+
prerelease: false
|
57
|
+
version_requirements: *2157675480
|
58
|
+
- !ruby/object:Gem::Dependency
|
59
|
+
name: aws-s3
|
60
|
+
requirement: &2157675060 !ruby/object:Gem::Requirement
|
61
|
+
none: false
|
62
|
+
requirements:
|
63
|
+
- - ! '>='
|
64
|
+
- !ruby/object:Gem::Version
|
65
|
+
version: '0'
|
66
|
+
type: :runtime
|
67
|
+
prerelease: false
|
68
|
+
version_requirements: *2157675060
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: active_support
|
71
|
+
requirement: &2157674640 !ruby/object:Gem::Requirement
|
72
|
+
none: false
|
73
|
+
requirements:
|
74
|
+
- - ! '>='
|
75
|
+
- !ruby/object:Gem::Version
|
76
|
+
version: '0'
|
77
|
+
type: :runtime
|
78
|
+
prerelease: false
|
79
|
+
version_requirements: *2157674640
|
80
|
+
description: Creates objects, converts them to Redis commands and executes the redis
|
81
|
+
commands.
|
82
|
+
email:
|
83
|
+
- ian@ianwhitney.com
|
84
|
+
executables: []
|
85
|
+
extensions: []
|
86
|
+
extra_rdoc_files: []
|
87
|
+
files:
|
88
|
+
- .gitignore
|
89
|
+
- .rspec
|
90
|
+
- Gemfile
|
91
|
+
- LICENSE
|
92
|
+
- README.md
|
93
|
+
- Rakefile
|
94
|
+
- config/.gitignore
|
95
|
+
- lib/collections/S3_collection.rb
|
96
|
+
- lib/extensions/s3.rb
|
97
|
+
- lib/redis_importer.rb
|
98
|
+
- lib/redis_importer/redis_importer.rb
|
99
|
+
- lib/redis_importer/version.rb
|
100
|
+
- redis_importer.gemspec
|
101
|
+
- spec/redis_importer_spec.rb
|
102
|
+
- spec/spec_helper.rb
|
103
|
+
- test/csv/foo.csv
|
104
|
+
- test/csv/person.csv
|
105
|
+
- test/lib/person.rb
|
106
|
+
- tmp/.gitignore
|
107
|
+
homepage: https://github.com/SeniorServiceAmerica/redis_importer
|
108
|
+
licenses: []
|
109
|
+
post_install_message:
|
110
|
+
rdoc_options: []
|
111
|
+
require_paths:
|
112
|
+
- lib
|
113
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
114
|
+
none: false
|
115
|
+
requirements:
|
116
|
+
- - ! '>='
|
117
|
+
- !ruby/object:Gem::Version
|
118
|
+
version: '0'
|
119
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
120
|
+
none: false
|
121
|
+
requirements:
|
122
|
+
- - ! '>='
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
125
|
+
requirements: []
|
126
|
+
rubyforge_project:
|
127
|
+
rubygems_version: 1.8.16
|
128
|
+
signing_key:
|
129
|
+
specification_version: 3
|
130
|
+
summary: Creates objects, converts them to Redis commands and executes the redis commands.
|
131
|
+
test_files:
|
132
|
+
- spec/redis_importer_spec.rb
|
133
|
+
- spec/spec_helper.rb
|
134
|
+
- test/csv/foo.csv
|
135
|
+
- test/csv/person.csv
|
136
|
+
- test/lib/person.rb
|