s3mpi 0.0.8.1 → 0.1.0.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +1 -2
- data/README.md +1 -3
- data/lib/s3mpi.rb +0 -1
- data/lib/s3mpi/converters.rb +16 -1
- data/lib/s3mpi/converters/csv.rb +36 -14
- data/lib/s3mpi/converters/identity.rb +16 -0
- data/lib/s3mpi/converters/json.rb +19 -0
- data/lib/s3mpi/interface.rb +36 -63
- data/lib/s3mpi/s3.rb +15 -3
- data/lib/s3mpi/version.rb +3 -3
- data/s3mpi.gemspec +1 -1
- data/spec/s3mpi/converters/csv_spec.rb +34 -13
- data/spec/s3mpi/converters/identity_spec.rb +21 -0
- data/spec/s3mpi/converters/json_spec.rb +57 -0
- data/spec/s3mpi/interface_spec.rb +171 -26
- data/spec/spec_helper.rb +0 -3
- data/spec/support/test.csv +1 -1
- metadata +23 -8
- data/lib/s3mpi/format.rb +0 -14
- data/spec/s3mpi/converters/parsed_csv_shared_examples.rb +0 -21
- data/spec/s3mpi/s3_spec.rb +0 -19
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 731d6db40c690f6acfbccc15328e39c16367d7d3
|
4
|
+
data.tar.gz: 02bfc423b1e53b87af1675bccfa6cc2141ac283a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 76e4335229d0b998df64da0c80fe006b76c18bfdd841079d8897baae10202f5ee8cf4f12684a673c42da174c222d507bd43227727810d2f93ca9b13d08ad3a9f
|
7
|
+
data.tar.gz: 3eda26cf66c847cd1264cc5f969af053514ac0ddb26963924d3806ae43bbc69bbefb7e0e85ce755efa0b601bc1c2f181cbec1178f2a24b6039e1f610f0014a3e
|
data/.travis.yml
CHANGED
data/README.md
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
S3 uploads of Ruby objects [![Build Status](https://travis-ci.org/robertzk/s3mpi.svg?branch=master)](https://travis-ci.org/robertzk/s3mpi)
|
1
|
+
S3 uploads of Ruby objects [![Build Status](https://travis-ci.org/robertzk/s3mpi-ruby.svg?branch=master)](https://travis-ci.org/robertzk/s3mpi-ruby)
|
2
2
|
===========
|
3
3
|
|
4
4
|
Upload and download Ruby objects to S3 using a very convenient API.
|
@@ -22,5 +22,3 @@ we can store and read Ruby objects:
|
|
22
22
|
MPI.store({ some: "ruby", object: 5 }, "some_object")
|
23
23
|
MPI.read('some_object')
|
24
24
|
```
|
25
|
-
|
26
|
-
|
data/lib/s3mpi.rb
CHANGED
data/lib/s3mpi/converters.rb
CHANGED
@@ -1,6 +1,21 @@
|
|
1
1
|
require 's3mpi/converters/csv'
|
2
|
+
require 's3mpi/converters/json'
|
3
|
+
require 's3mpi/converters/identity'
|
2
4
|
|
3
5
|
module S3MPI
|
4
6
|
module Converters
|
7
|
+
|
8
|
+
class UnknownConverterError < StandardError; end
|
9
|
+
|
10
|
+
def converter(as)
|
11
|
+
case as
|
12
|
+
when :json then Converters::JSON
|
13
|
+
when :csv then Converters::CSV
|
14
|
+
when :string, :identity then Converters::Identity
|
15
|
+
else
|
16
|
+
raise UnknownConverterError, "#{as.inspect} is not a known converter!"
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
5
20
|
end
|
6
|
-
end
|
21
|
+
end
|
data/lib/s3mpi/converters/csv.rb
CHANGED
@@ -5,17 +5,7 @@ module S3MPI
|
|
5
5
|
module CSV
|
6
6
|
extend self
|
7
7
|
|
8
|
-
|
9
|
-
#
|
10
|
-
# @param [String] csv_file_path
|
11
|
-
# Path to the CSV file.
|
12
|
-
#
|
13
|
-
# @param [Hash] options
|
14
|
-
# Passed to CSV.parse
|
15
|
-
def file_to_obj(csv_file_path, options = Hash.new)
|
16
|
-
csv_data = File.read(csv_file_path)
|
17
|
-
string_to_obj(csv_data, options)
|
18
|
-
end
|
8
|
+
class HeaderError < StandardError; end
|
19
9
|
|
20
10
|
# Convert CSV string data to an array of hashes
|
21
11
|
#
|
@@ -24,14 +14,46 @@ module S3MPI
|
|
24
14
|
#
|
25
15
|
# @param [Hash] options
|
26
16
|
# Passed to CSV.parse
|
27
|
-
|
17
|
+
#
|
18
|
+
# @return [Array]
|
19
|
+
def parse(csv_data, options = Hash.new)
|
28
20
|
options = options.merge({
|
29
21
|
headers: true,
|
30
22
|
converters: :all
|
31
|
-
|
32
23
|
})
|
33
24
|
::CSV.parse(csv_data, options).map(&:to_hash)
|
34
25
|
end
|
26
|
+
|
27
|
+
# Convert an array of hashes to CSV string data
|
28
|
+
#
|
29
|
+
# @param [Array] array_of_hashes
|
30
|
+
# An Array of Hashes
|
31
|
+
#
|
32
|
+
# @param [Hash] options
|
33
|
+
# Passed to CSV.generate
|
34
|
+
#
|
35
|
+
# @return [String]
|
36
|
+
def generate(array_of_hashes, options = Hash.new)
|
37
|
+
return "" if array_of_hashes.empty?
|
38
|
+
headers = inspect_headers(array_of_hashes)
|
39
|
+
::CSV.generate(options) do |csv|
|
40
|
+
csv << headers
|
41
|
+
array_of_hashes.each do |hash|
|
42
|
+
csv << hash.values_at(*headers)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
private
|
48
|
+
|
49
|
+
def inspect_headers(data)
|
50
|
+
data.first.keys.tap do |headers|
|
51
|
+
sorted = headers.sort
|
52
|
+
error = data.any?{ |hash| hash.keys.sort != sorted }
|
53
|
+
raise HeaderError, "the rows have inconsistent headers!" if error
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
35
57
|
end
|
36
58
|
end
|
37
|
-
end
|
59
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
require 'json'
|
2
|
+
|
3
|
+
module S3MPI
|
4
|
+
module Converters
|
5
|
+
module JSON
|
6
|
+
extend self
|
7
|
+
|
8
|
+
def parse(json_string, options = Hash.new)
|
9
|
+
opts = { quirks_mode: true }.merge(options)
|
10
|
+
::JSON.parse(json_string || 'null', **opts)
|
11
|
+
end
|
12
|
+
|
13
|
+
def generate(object, options = Hash.new)
|
14
|
+
object.to_json(options)
|
15
|
+
end
|
16
|
+
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
data/lib/s3mpi/interface.rb
CHANGED
@@ -1,12 +1,12 @@
|
|
1
|
-
require 'json'
|
2
|
-
require 's3mpi/format'
|
3
1
|
require 's3mpi/s3'
|
4
2
|
require 's3mpi/converters'
|
5
3
|
|
6
4
|
module S3MPI
|
7
5
|
class Interface
|
8
|
-
include Format
|
9
6
|
include S3
|
7
|
+
include Converters
|
8
|
+
|
9
|
+
UUID = Object.new.freeze
|
10
10
|
|
11
11
|
# Return S3 bucket under use.
|
12
12
|
#
|
@@ -18,96 +18,69 @@ module S3MPI
|
|
18
18
|
# @return [String]
|
19
19
|
attr_reader :path
|
20
20
|
|
21
|
+
# Return the default converter for store & read.
|
22
|
+
#
|
23
|
+
# @return [String]
|
24
|
+
attr_reader :default_converter
|
25
|
+
|
21
26
|
# Create a new S3MPI object that responds to #read and #store.
|
22
27
|
#
|
23
28
|
# @return [S3MPI::Interface]
|
24
29
|
#
|
25
30
|
# @api public
|
26
|
-
def initialize
|
27
|
-
@bucket =
|
28
|
-
@path = path
|
31
|
+
def initialize bucket, path = '', default_converter = :json
|
32
|
+
@bucket = parse_bucket(bucket)
|
33
|
+
@path = path.freeze
|
34
|
+
converter(default_converter) # verify it is valid
|
35
|
+
@default_converter = default_converter
|
29
36
|
end
|
30
37
|
|
31
38
|
# Store a Ruby object in an S3 bucket.
|
32
|
-
#
|
39
|
+
#
|
33
40
|
# @param [Object] obj
|
34
|
-
# Any
|
41
|
+
# Any convertable Ruby object (usually a hash or array).
|
35
42
|
# @param [String] key
|
36
43
|
# The key under which to save the object in the S3 bucket.
|
37
|
-
# @param [
|
44
|
+
# @param [Symbol] :as
|
45
|
+
# Which converter to use e.g. :json, :csv, :string
|
46
|
+
# @param [Integer] tries
|
38
47
|
# The number of times to attempt to store the object.
|
39
|
-
def store(obj, key =
|
40
|
-
|
41
|
-
|
42
|
-
|
48
|
+
def store(obj, key = UUID, as: default_converter, tries: 1)
|
49
|
+
key = SecureRandom.uuid if key.equal?(UUID)
|
50
|
+
s3_object(key).write(converter(as).generate(obj))
|
51
|
+
rescue AWS::Errors::Base
|
52
|
+
(tries -= 1) > 0 ? retry : raise
|
43
53
|
end
|
44
54
|
|
55
|
+
def store_csv(obj, key = UUID); store(obj, key, as: :csv); end
|
56
|
+
def store_json(obj, key = UUID); store(obj, key, as: :json); end
|
57
|
+
def store_string(obj, key = UUID); store(obj, key, as: :string); end
|
45
58
|
|
46
|
-
#
|
47
|
-
# Proxies to store
|
48
|
-
#
|
49
|
-
# @param [String] csv_file_path
|
50
|
-
# Path to the CSV file.
|
59
|
+
# Read and de-serialize an object from an S3 bucket.
|
51
60
|
#
|
52
|
-
# @param [Hash] options Options hash.
|
53
|
-
# Passed to CSV.parse
|
54
|
-
def store_csv(csv_file_path, options = Hash.new)
|
55
|
-
store(Converters::CSV.file_to_obj(csv_file_path, options))
|
56
|
-
end
|
57
|
-
|
58
|
-
# Store a raw object
|
59
|
-
# @param [Object] obj
|
60
|
-
# The object to store.
|
61
|
-
# @param [String] key
|
62
|
-
# The key under which the object is saved in the S3 bucket.
|
63
|
-
def store_raw(obj, key)
|
64
|
-
s3_object(key).write(obj)
|
65
|
-
end
|
66
|
-
|
67
|
-
# Read a JSON-serialized Ruby object from an S3 bucket.
|
68
|
-
#
|
69
61
|
# @param [String] key
|
70
62
|
# The key under which to save the object in the S3 bucket.
|
71
|
-
|
72
|
-
|
63
|
+
# @param [Symbol] :as
|
64
|
+
# Which converter to use e.g. :json, :csv, :string
|
65
|
+
def read(key = nil, as: default_converter)
|
66
|
+
converter(as).parse(s3_object(key).read)
|
73
67
|
rescue AWS::S3::Errors::NoSuchKey
|
74
68
|
nil
|
75
69
|
end
|
76
70
|
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
# The key under which the file is saved in the S3 bucket.
|
81
|
-
def read_csv(key=nil)
|
82
|
-
Converters::CSV.string_to_obj(s3_object(key).read)
|
83
|
-
rescue AWS::S3::Errors::NoSuchKey
|
84
|
-
nil
|
85
|
-
end
|
71
|
+
def read_csv(key = nil); read(key, as: :csv); end
|
72
|
+
def read_json(key = nil); read(key, as: :json); end
|
73
|
+
def read_string(key = nil); read(key, as: :string); end
|
86
74
|
|
87
75
|
# Check whether a key exists for this MPI interface.
|
88
|
-
#
|
76
|
+
#
|
89
77
|
# @param [String] key
|
90
78
|
# The key under which to save the object in the S3 bucket.
|
91
|
-
#
|
79
|
+
#
|
92
80
|
# @return [TrueClass,FalseClass]
|
93
81
|
def exists?(key)
|
94
82
|
s3_object(key).exists?
|
95
83
|
end
|
96
84
|
|
97
|
-
# Fetch the S3 object as an AWS::S3::S3Object.
|
98
|
-
#
|
99
|
-
# @param [String] name
|
100
|
-
# The key under which to save the object in the S3 bucket.
|
101
|
-
#
|
102
|
-
# @return [AWS::S3::S3Object]
|
103
|
-
def object(key)
|
104
|
-
AWS::S3::S3Object.new(bucket, "#{@path}#{key}")
|
105
|
-
end
|
106
|
-
|
107
|
-
def bucket
|
108
|
-
@_bucket ||= parse_bucket @bucket
|
109
|
-
end
|
110
85
|
end
|
111
86
|
end
|
112
|
-
|
113
|
-
|
data/lib/s3mpi/s3.rb
CHANGED
@@ -3,12 +3,24 @@ require 'aws-sdk-v1'
|
|
3
3
|
module S3MPI
|
4
4
|
module S3
|
5
5
|
|
6
|
-
def
|
6
|
+
def s3_object(name)
|
7
|
+
bucket.objects[full_object_key(name)]
|
8
|
+
end
|
9
|
+
|
10
|
+
private
|
11
|
+
|
12
|
+
def parse_bucket(bucket)
|
7
13
|
AWS::S3.new.buckets[bucket]
|
8
14
|
end
|
9
15
|
|
10
|
-
def
|
11
|
-
|
16
|
+
def full_object_key(name)
|
17
|
+
[path, name].flatten.reject{ |x| blank?(x) }.join('/')
|
18
|
+
end
|
19
|
+
|
20
|
+
def blank?(x)
|
21
|
+
return true if !x
|
22
|
+
return true if x.is_a?(String) && x.strip == ""
|
23
|
+
false
|
12
24
|
end
|
13
25
|
|
14
26
|
end
|
data/lib/s3mpi/version.rb
CHANGED
data/s3mpi.gemspec
CHANGED
@@ -1,25 +1,46 @@
|
|
1
1
|
require 'spec_helper'
|
2
|
-
require_relative 'parsed_csv_shared_examples'
|
3
2
|
|
4
|
-
|
5
|
-
|
6
|
-
|
3
|
+
describe S3MPI::Converters::CSV do
|
4
|
+
let(:csv_file_path) { 'spec/support/test.csv' }
|
5
|
+
let(:csv_data_string) { File.read csv_file_path }
|
7
6
|
|
8
|
-
|
9
|
-
|
7
|
+
let(:csv_as_array_of_hashes) {
|
8
|
+
[
|
9
|
+
{'integer' => 1, 'string' => 'user1@test.com', 'float' => 1.11},
|
10
|
+
{'integer' => 2, 'string' => 'user2@test.com', 'float' => 2.22},
|
11
|
+
{'integer' => 3, 'string' => 'user3@test.com', 'float' => 3.33},
|
12
|
+
]
|
13
|
+
}
|
10
14
|
|
11
|
-
|
12
|
-
|
15
|
+
describe '#parse' do
|
16
|
+
subject { described_class.parse csv_data_string }
|
13
17
|
|
14
|
-
|
15
|
-
|
18
|
+
it 'converts CSV data to an array of hashes' do
|
19
|
+
expect(subject).to be_kind_of Array
|
16
20
|
|
17
|
-
|
18
|
-
|
21
|
+
subject.each do |row|
|
22
|
+
expect(row).to be_kind_of Hash
|
23
|
+
end
|
24
|
+
end
|
19
25
|
|
20
|
-
|
26
|
+
it 'preserves integers' do
|
27
|
+
subject.each do |row|
|
28
|
+
expect(row['integer']).to eq(row['integer'].to_s.to_i)
|
29
|
+
end
|
30
|
+
end
|
21
31
|
|
32
|
+
it 'preserves floats' do
|
33
|
+
subject.each do |row|
|
34
|
+
expect(row['float']).to eq(row['float'].to_s.to_f)
|
22
35
|
end
|
23
36
|
end
|
37
|
+
|
38
|
+
it { is_expected.to eql csv_as_array_of_hashes }
|
39
|
+
end
|
40
|
+
|
41
|
+
describe '#generate' do
|
42
|
+
subject { described_class.generate csv_as_array_of_hashes }
|
43
|
+
|
44
|
+
it { is_expected.to eql csv_data_string }
|
24
45
|
end
|
25
46
|
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe S3MPI::Converters::Identity do
|
4
|
+
|
5
|
+
describe '#parse' do
|
6
|
+
let(:data){ "any\nrandom\nstring" }
|
7
|
+
|
8
|
+
it 'returns the input data' do
|
9
|
+
expect(described_class.parse(data)).to equal data
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
describe '#generate' do
|
14
|
+
let(:data){ "any\nrandom\nstring" }
|
15
|
+
|
16
|
+
it 'returns the input data' do
|
17
|
+
expect(described_class.generate(data)).to equal data
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe S3MPI::Converters::JSON do
|
4
|
+
|
5
|
+
describe '#parse' do
|
6
|
+
it 'loads a JSON hash' do
|
7
|
+
expect(described_class.parse('{ "a": 1, "b": "test" }')
|
8
|
+
).to eql({ "a" => 1, "b" => "test"})
|
9
|
+
end
|
10
|
+
|
11
|
+
it 'loads a JSON array' do
|
12
|
+
expect(described_class.parse('["a", 1, "b", 2]')
|
13
|
+
).to eql(["a", 1, "b", 2])
|
14
|
+
end
|
15
|
+
|
16
|
+
it 'loads a JSON array of hashes' do
|
17
|
+
expect(described_class.parse('[{ "a": "one" }, {"b": "two" }]')
|
18
|
+
).to eql([{ "a" => "one" }, {"b" => "two"}])
|
19
|
+
end
|
20
|
+
|
21
|
+
it 'loads a single JSON element' do
|
22
|
+
expect(described_class.parse('"hello world"')
|
23
|
+
).to eql("hello world")
|
24
|
+
end
|
25
|
+
|
26
|
+
it 'loads the JSON null as nil' do
|
27
|
+
expect(described_class.parse('null')).to be_nil
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
describe '#generate' do
|
32
|
+
it 'dumps a Hash' do
|
33
|
+
expect(described_class.generate({ "a" => 1, "b" => "test"})
|
34
|
+
).to eql('{"a":1,"b":"test"}')
|
35
|
+
end
|
36
|
+
|
37
|
+
it 'dumps an Array' do
|
38
|
+
expect(described_class.generate(["a", 1, "b", 2])
|
39
|
+
).to eql('["a",1,"b",2]')
|
40
|
+
end
|
41
|
+
|
42
|
+
it 'dumps an Array of Hashes' do
|
43
|
+
expect(described_class.generate([{ "a" => "one" }, {"b" => "two"}])
|
44
|
+
).to eql('[{"a":"one"},{"b":"two"}]')
|
45
|
+
end
|
46
|
+
|
47
|
+
it 'dumps a String' do
|
48
|
+
expect(described_class.generate("hello world")
|
49
|
+
).to eql('"hello world"')
|
50
|
+
end
|
51
|
+
|
52
|
+
it 'dumps nil as the JSON null' do
|
53
|
+
expect(described_class.generate(nil)).to eql 'null'
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
end
|
@@ -1,41 +1,186 @@
|
|
1
1
|
require 'spec_helper'
|
2
|
-
require 's3mpi/interface'
|
3
|
-
require 's3mpi/converters/csv'
|
4
|
-
require_relative '../s3mpi/converters/parsed_csv_shared_examples'
|
5
2
|
|
6
|
-
|
3
|
+
describe S3MPI::Interface do
|
4
|
+
let(:bucket) { 'some_bucket' }
|
5
|
+
let(:path) { 'some_folder' }
|
7
6
|
|
8
|
-
|
7
|
+
let(:interface) { described_class.new(bucket, path) }
|
9
8
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
9
|
+
describe '#bucket' do
|
10
|
+
subject{ interface.bucket }
|
11
|
+
it { is_expected.to be_a AWS::S3::Bucket }
|
12
|
+
it { is_expected.to have_attributes(name: bucket) }
|
13
|
+
end
|
14
|
+
|
15
|
+
describe '#path' do
|
16
|
+
subject{ interface.path }
|
17
|
+
it { is_expected.to eql path }
|
18
|
+
end
|
19
|
+
|
20
|
+
describe '#default_converter' do
|
21
|
+
subject{ interface.default_converter }
|
22
|
+
|
23
|
+
context 'default' do
|
24
|
+
it { is_expected.to eql :json }
|
25
|
+
end
|
26
|
+
|
27
|
+
context 'set on initialization to :csv' do
|
28
|
+
let(:interface) { described_class.new(bucket, path, :csv) }
|
29
|
+
it { is_expected.to eql :csv }
|
30
|
+
end
|
31
|
+
|
32
|
+
it "initialization fails if it is invalid" do
|
33
|
+
expect{ described_class.new(bucket, path, :foo) }.to raise_error \
|
34
|
+
S3MPI::Converters::UnknownConverterError
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
describe '#converter' do
|
39
|
+
def expect_converter(key, klass)
|
40
|
+
expect(interface.converter(key)).to eql klass
|
41
|
+
end
|
42
|
+
|
43
|
+
it('json'){ expect_converter :json, S3MPI::Converters::JSON }
|
44
|
+
it('csv') { expect_converter :csv, S3MPI::Converters::CSV }
|
45
|
+
it('string'){ expect_converter :string, S3MPI::Converters::Identity }
|
46
|
+
it('identity'){ expect_converter :identity, S3MPI::Converters::Identity }
|
47
|
+
|
48
|
+
it "raises for unknown keys" do
|
49
|
+
err = S3MPI::Converters::UnknownConverterError
|
50
|
+
msg = ":foo is not a known converter!"
|
51
|
+
expect{ interface.converter(:foo) }.to raise_error(err, msg)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
describe '#s3_object' do
|
56
|
+
subject { interface.s3_object(name) }
|
57
|
+
|
58
|
+
let(:name) { 'foo/bar/baz' }
|
59
|
+
|
60
|
+
it { is_expected.to be_a AWS::S3::S3Object }
|
61
|
+
it { is_expected.to have_attributes(bucket: interface.bucket) }
|
62
|
+
it { is_expected.to have_attributes(key: "#{path}/#{name}") }
|
63
|
+
|
64
|
+
{ 'empty' => '', 'whitespace' => ' ', 'nil' => nil }.each do |desc, value|
|
65
|
+
context "#{desc} path" do
|
66
|
+
let(:path) { value }
|
67
|
+
it { is_expected.to have_attributes(key: name) }
|
68
|
+
end
|
69
|
+
|
70
|
+
context "#{desc} name" do
|
71
|
+
let(:name) { value }
|
72
|
+
it { is_expected.to have_attributes(key: path) }
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
context "with a pinned s3_object" do
|
78
|
+
def pin_s3_objects!
|
79
|
+
s3_objects_to_pin.each do |key, s3_obj|
|
80
|
+
allow(interface).to receive(:s3_object).with(key).and_return(s3_obj)
|
81
|
+
end
|
82
|
+
end
|
21
83
|
|
22
|
-
|
23
|
-
|
84
|
+
def s3_objects_to_pin(*keys)
|
85
|
+
(@to_pin ||= {}).tap do |h|
|
86
|
+
keys.each { |k| h[k] = interface.s3_object(k) }
|
87
|
+
end
|
88
|
+
end
|
24
89
|
|
25
|
-
|
26
|
-
it 'sends the CSV as a row of hashes to .store' do
|
27
|
-
expect(subject).to receive(:store).with(csv_as_array_of_hashes)
|
90
|
+
let(:name) { 'name' }
|
28
91
|
|
29
|
-
|
92
|
+
before { s3_objects_to_pin(name) && AWS.stub! }
|
93
|
+
subject { pin_s3_objects! && s3_objects_to_pin[name] }
|
94
|
+
|
95
|
+
describe '#exists?' do
|
96
|
+
it 'calls .exists? on the s3 object' do
|
97
|
+
expect(subject).to receive(:exists?).and_return(retval = double)
|
98
|
+
expect(interface.exists?(name)).to equal retval
|
30
99
|
end
|
31
100
|
end
|
32
101
|
|
33
|
-
|
102
|
+
let(:as_json) { '{"a":1,"b":"two","c":[3,3,3]}' }
|
103
|
+
let(:as_hash) { { "a" => 1, "b" => "two", "c" => [3,3,3] } }
|
104
|
+
|
105
|
+
let(:as_csv) { "x,y,z\n1,2,3\n4,5,6\n7,8,9\n" }
|
106
|
+
let(:as_list) { [{"x" => 1, "y" => 2, "z" => 3},
|
107
|
+
{"x" => 4, "y" => 5, "z" => 6},
|
108
|
+
{"x" => 7, "y" => 8, "z" => 9}] }
|
109
|
+
|
110
|
+
let(:random_str) { 10.times.map{ SecureRandom.uuid}.join("\n") }
|
34
111
|
|
35
|
-
|
36
|
-
|
37
|
-
|
112
|
+
describe '#read' do
|
113
|
+
it 'can parse the raw string with the json converter' do
|
114
|
+
expect(subject).to receive(:read).and_return(as_json).twice
|
115
|
+
expect(interface.read(name, as: :json)).to eql(as_hash)
|
116
|
+
expect(interface.read_json(name)).to eql(as_hash)
|
117
|
+
end
|
118
|
+
|
119
|
+
it 'can parse the raw string with the csv converter' do
|
120
|
+
expect(subject).to receive(:read).and_return(as_csv).twice
|
121
|
+
expect(interface.read(name, as: :csv)).to eql(as_list)
|
122
|
+
expect(interface.read_csv(name)).to eql(as_list)
|
123
|
+
end
|
38
124
|
|
125
|
+
it 'can parse the raw string with the identity converter' do
|
126
|
+
expect(subject).to receive(:read).and_return(random_str).twice
|
127
|
+
expect(interface.read(name, as: :identity)).to eql(random_str)
|
128
|
+
expect(interface.read_string(name)).to eql(random_str)
|
129
|
+
end
|
130
|
+
|
131
|
+
it 'defaults to the default_converter' do
|
132
|
+
allow(interface).to receive(:default_converter).and_return(:foo)
|
133
|
+
expect(subject).to receive(:read).and_return(as_json)
|
134
|
+
expect(interface).to receive(:converter
|
135
|
+
).with(:foo).and_return(S3MPI::Converters::Identity)
|
136
|
+
expect(interface.read(name)).to eql as_json
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
140
|
+
describe '#store' do
|
141
|
+
it 'can generate json and write it to s3' do
|
142
|
+
expect(subject).to receive(:write).with(as_json).twice
|
143
|
+
interface.store(as_hash, name, as: :json)
|
144
|
+
interface.store_json(as_hash, name)
|
145
|
+
end
|
146
|
+
|
147
|
+
it 'can generate csv and write it to s3' do
|
148
|
+
expect(subject).to receive(:write).with(as_csv).twice
|
149
|
+
interface.store(as_list, name, as: :csv)
|
150
|
+
interface.store_csv(as_list, name)
|
151
|
+
end
|
152
|
+
|
153
|
+
it 'can write the raw string to s3' do
|
154
|
+
expect(subject).to receive(:write).with(random_str).twice
|
155
|
+
interface.store(random_str, name, as: :identity)
|
156
|
+
interface.store_string(random_str, name)
|
157
|
+
end
|
158
|
+
|
159
|
+
it 'uses a UUID if the key is not explicitly passed' do
|
160
|
+
s3_objects_to_pin('some_uuid')
|
161
|
+
expect(subject).not_to receive(:write)
|
162
|
+
expect(SecureRandom).to receive(:uuid).and_return('some_uuid')
|
163
|
+
expect(s3_objects_to_pin['some_uuid']).to receive(:write).with(as_json)
|
164
|
+
interface.store(as_hash)
|
165
|
+
end
|
166
|
+
|
167
|
+
it 'defaults to the default_converter' do
|
168
|
+
expect(subject).to receive(:write).with(random_str)
|
169
|
+
allow(interface).to receive(:default_converter).and_return(:foo)
|
170
|
+
expect(interface).to receive(:converter
|
171
|
+
).with(:foo).and_return(S3MPI::Converters::Identity)
|
172
|
+
interface.store(random_str, name)
|
173
|
+
end
|
174
|
+
|
175
|
+
it 'retries on AWS::Errors::ServerError' do
|
176
|
+
error = AWS::S3::Errors::RequestTimeout
|
177
|
+
expect(subject).to receive(:write).and_raise(error).twice
|
178
|
+
expect(subject).to receive(:write).and_return("SUCCESS")
|
179
|
+
generator = S3MPI::Converters::JSON
|
180
|
+
expect(generator).to receive(:generate).with(as_hash).exactly(3).times
|
181
|
+
expect(interface.store(as_hash, name, tries: 3)).to eql "SUCCESS"
|
182
|
+
end
|
39
183
|
end
|
40
184
|
end
|
185
|
+
|
41
186
|
end
|
data/spec/spec_helper.rb
CHANGED
data/spec/support/test.csv
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: s3mpi
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.1.0.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Robert Krzyzanowski
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-02-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: aws-sdk-v1
|
@@ -52,6 +52,20 @@ dependencies:
|
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '3.4'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: pry
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 0.10.4
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: 0.10.4
|
55
69
|
description: Passing objects between Ruby consoles can be cumbersome if the user has
|
56
70
|
performed some serialization and deserialization procedure. S3MPI aims to enable
|
57
71
|
simple reading and writing to S3 buckets without the typical overhead of the AWS
|
@@ -72,15 +86,16 @@ files:
|
|
72
86
|
- lib/s3mpi.rb
|
73
87
|
- lib/s3mpi/converters.rb
|
74
88
|
- lib/s3mpi/converters/csv.rb
|
75
|
-
- lib/s3mpi/
|
89
|
+
- lib/s3mpi/converters/identity.rb
|
90
|
+
- lib/s3mpi/converters/json.rb
|
76
91
|
- lib/s3mpi/interface.rb
|
77
92
|
- lib/s3mpi/s3.rb
|
78
93
|
- lib/s3mpi/version.rb
|
79
94
|
- s3mpi.gemspec
|
80
95
|
- spec/s3mpi/converters/csv_spec.rb
|
81
|
-
- spec/s3mpi/converters/
|
96
|
+
- spec/s3mpi/converters/identity_spec.rb
|
97
|
+
- spec/s3mpi/converters/json_spec.rb
|
82
98
|
- spec/s3mpi/interface_spec.rb
|
83
|
-
- spec/s3mpi/s3_spec.rb
|
84
99
|
- spec/spec_helper.rb
|
85
100
|
- spec/support/test.csv
|
86
101
|
homepage: https://github.com/robertzk/s3mpi-ruby
|
@@ -103,14 +118,14 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
103
118
|
version: '0'
|
104
119
|
requirements: []
|
105
120
|
rubyforge_project:
|
106
|
-
rubygems_version: 2.
|
121
|
+
rubygems_version: 2.4.8
|
107
122
|
signing_key:
|
108
123
|
specification_version: 4
|
109
124
|
summary: Upload and download files to S3 using a very convenient API.
|
110
125
|
test_files:
|
111
126
|
- spec/s3mpi/converters/csv_spec.rb
|
112
|
-
- spec/s3mpi/converters/
|
127
|
+
- spec/s3mpi/converters/identity_spec.rb
|
128
|
+
- spec/s3mpi/converters/json_spec.rb
|
113
129
|
- spec/s3mpi/interface_spec.rb
|
114
|
-
- spec/s3mpi/s3_spec.rb
|
115
130
|
- spec/spec_helper.rb
|
116
131
|
- spec/support/test.csv
|
data/lib/s3mpi/format.rb
DELETED
@@ -1,21 +0,0 @@
|
|
1
|
-
RSpec.shared_examples 'a parsed CSV' do
|
2
|
-
it 'converts CSV data to an array of hashes' do
|
3
|
-
expect(subject).to be_kind_of Array
|
4
|
-
|
5
|
-
subject.each do |row|
|
6
|
-
expect(row).to be_kind_of Hash
|
7
|
-
end
|
8
|
-
end
|
9
|
-
|
10
|
-
it 'preserves integers' do
|
11
|
-
subject.each do |row|
|
12
|
-
expect(row['integer']).to eq(row['integer'].to_s.to_i)
|
13
|
-
end
|
14
|
-
end
|
15
|
-
|
16
|
-
it 'preserves floats' do
|
17
|
-
subject.each do |row|
|
18
|
-
expect(row['float']).to eq(row['float'].to_s.to_f)
|
19
|
-
end
|
20
|
-
end
|
21
|
-
end
|
data/spec/s3mpi/s3_spec.rb
DELETED
@@ -1,19 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
module S3MPI
|
4
|
-
|
5
|
-
describe Format do
|
6
|
-
let(:format_mixin) {
|
7
|
-
klass = Class.new
|
8
|
-
klass.send :include, Format
|
9
|
-
klass.new
|
10
|
-
}
|
11
|
-
|
12
|
-
it 'expects #parse_json_allowing_quirks_mode to parse JSON' do
|
13
|
-
obj = {"a" => 1, "b" => "test"}
|
14
|
-
expect(format_mixin.parse_json_allowing_quirks_mode(obj.to_json)).to eql(obj)
|
15
|
-
end
|
16
|
-
|
17
|
-
end
|
18
|
-
|
19
|
-
end
|