s3mpi 0.0.7.1 → 0.0.8.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +2 -2
- data/lib/s3mpi/converters/csv.rb +37 -0
- data/lib/s3mpi/converters.rb +6 -0
- data/lib/s3mpi/interface.rb +16 -16
- data/lib/s3mpi/version.rb +1 -1
- data/lib/s3mpi.rb +1 -0
- data/s3mpi.gemspec +1 -1
- data/spec/s3mpi/converters/csv_spec.rb +25 -0
- data/spec/s3mpi/converters/parsed_csv_shared_examples.rb +21 -0
- data/spec/s3mpi/interface_spec.rb +17 -6
- data/spec/support/test.csv +4 -4
- metadata +9 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 15963407068406352b0c91e2305bf5ef1c97d487
|
4
|
+
data.tar.gz: b1107fa62e9bc82a5278585c76427fb395cdda50
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 548cd924ed23bb7e9dd0a4723182c0b0706bc195ae6b66316965d79990219acc92050a28e92557c44af3e56b8820dd2240e1737c24536993c7561bbd86c7851e
|
7
|
+
data.tar.gz: 00206b142dd05916ee9b138250d87ff416e68ad343142e19f7d595457f8e951d9872aa9b8a14ef8acc4c45d2c8e973ab2f0d4f8bc3ccc7404baa61267d9d8f61
|
data/.gitignore
CHANGED
@@ -0,0 +1,37 @@
|
|
1
|
+
require 'csv'
|
2
|
+
|
3
|
+
module S3MPI
|
4
|
+
module Converters
|
5
|
+
module CSV
|
6
|
+
extend self
|
7
|
+
|
8
|
+
# Read a CSV file and convert it to an array of hashes
|
9
|
+
#
|
10
|
+
# @param [String] csv_file_path
|
11
|
+
# Path to the CSV file.
|
12
|
+
#
|
13
|
+
# @param [Hash] options
|
14
|
+
# Passed to CSV.parse
|
15
|
+
def file_to_obj(csv_file_path, options = Hash.new)
|
16
|
+
csv_data = File.read(csv_file_path)
|
17
|
+
string_to_obj(csv_data, options)
|
18
|
+
end
|
19
|
+
|
20
|
+
# Convert CSV string data to an array of hashes
|
21
|
+
#
|
22
|
+
# @param [String] csv_data
|
23
|
+
# String of CSV data
|
24
|
+
#
|
25
|
+
# @param [Hash] options
|
26
|
+
# Passed to CSV.parse
|
27
|
+
def string_to_obj(csv_data, options = Hash.new)
|
28
|
+
options = options.merge({
|
29
|
+
headers: true,
|
30
|
+
converters: :all
|
31
|
+
|
32
|
+
})
|
33
|
+
::CSV.parse(csv_data, options).map(&:to_hash)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
data/lib/s3mpi/interface.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
require 'json'
|
2
2
|
require 's3mpi/format'
|
3
3
|
require 's3mpi/s3'
|
4
|
-
require '
|
4
|
+
require 's3mpi/converters'
|
5
5
|
|
6
6
|
module S3MPI
|
7
7
|
class Interface
|
@@ -46,20 +46,20 @@ module S3MPI
|
|
46
46
|
# Load a CSV file, and store the contents in S3
|
47
47
|
# Proxies to store
|
48
48
|
#
|
49
|
-
# @param [String] csv_file_path
|
50
|
-
#
|
49
|
+
# @param [String] csv_file_path
|
50
|
+
# Path to the CSV file.
|
51
51
|
#
|
52
52
|
# @param [Hash] options Options hash.
|
53
|
-
# Passed to CSV.
|
53
|
+
# Passed to CSV.parse
|
54
54
|
def store_csv(csv_file_path, options = Hash.new)
|
55
|
-
store
|
55
|
+
store(Converters::CSV.file_to_obj(csv_file_path, options))
|
56
56
|
end
|
57
57
|
|
58
58
|
# Store a raw object
|
59
59
|
# @param [Object] obj
|
60
60
|
# The object to store.
|
61
61
|
# @param [String] key
|
62
|
-
# The key under which
|
62
|
+
# The key under which the object is saved in the S3 bucket.
|
63
63
|
def store_raw(obj, key)
|
64
64
|
s3_object(key).write(obj)
|
65
65
|
end
|
@@ -74,6 +74,16 @@ module S3MPI
|
|
74
74
|
nil
|
75
75
|
end
|
76
76
|
|
77
|
+
# Read a CSV file from an S3 bucket. Return as array of hashes.
|
78
|
+
#
|
79
|
+
# @param [String] key
|
80
|
+
# The key under which the file is saved in the S3 bucket.
|
81
|
+
def read_csv(key=nil)
|
82
|
+
Converters::CSV.string_to_obj(s3_object(key).read)
|
83
|
+
rescue AWS::S3::Errors::NoSuchKey
|
84
|
+
nil
|
85
|
+
end
|
86
|
+
|
77
87
|
# Check whether a key exists for this MPI interface.
|
78
88
|
#
|
79
89
|
# @param [String] key
|
@@ -97,16 +107,6 @@ module S3MPI
|
|
97
107
|
def bucket
|
98
108
|
@_bucket ||= parse_bucket @bucket
|
99
109
|
end
|
100
|
-
|
101
|
-
private
|
102
|
-
def _load_csv(csv_file_path, options)
|
103
|
-
options = options.merge({
|
104
|
-
headers: true,
|
105
|
-
converters: :all
|
106
|
-
|
107
|
-
})
|
108
|
-
CSV.read(csv_file_path, options).map(&:to_hash)
|
109
|
-
end
|
110
110
|
end
|
111
111
|
end
|
112
112
|
|
data/lib/s3mpi/version.rb
CHANGED
data/lib/s3mpi.rb
CHANGED
data/s3mpi.gemspec
CHANGED
@@ -25,7 +25,7 @@ Gem::Specification.new do |s|
|
|
25
25
|
s.add_dependency 'aws-sdk-v1'
|
26
26
|
|
27
27
|
s.add_development_dependency 'rake', '~> 10.1.0'
|
28
|
-
s.add_development_dependency 'rspec', '~>
|
28
|
+
s.add_development_dependency 'rspec', '~> 3.4'
|
29
29
|
|
30
30
|
s.extra_rdoc_files = ['README.md', 'LICENSE']
|
31
31
|
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require_relative 'parsed_csv_shared_examples'
|
3
|
+
|
4
|
+
module S3MPI
|
5
|
+
module Converters
|
6
|
+
describe CSV do
|
7
|
+
|
8
|
+
let(:csv_file_path) { 'spec/support/test.csv' }
|
9
|
+
let(:csv_data_string) { File.read csv_file_path }
|
10
|
+
|
11
|
+
describe '#convert_to_obj' do
|
12
|
+
subject { described_class.string_to_obj csv_data_string }
|
13
|
+
|
14
|
+
it_behaves_like 'a parsed CSV'
|
15
|
+
end
|
16
|
+
|
17
|
+
describe '#file_to_obj' do
|
18
|
+
subject { described_class.file_to_obj csv_file_path }
|
19
|
+
|
20
|
+
it_behaves_like 'a parsed CSV'
|
21
|
+
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
RSpec.shared_examples 'a parsed CSV' do
|
2
|
+
it 'converts CSV data to an array of hashes' do
|
3
|
+
expect(subject).to be_kind_of Array
|
4
|
+
|
5
|
+
subject.each do |row|
|
6
|
+
expect(row).to be_kind_of Hash
|
7
|
+
end
|
8
|
+
end
|
9
|
+
|
10
|
+
it 'preserves integers' do
|
11
|
+
subject.each do |row|
|
12
|
+
expect(row['integer']).to eq(row['integer'].to_s.to_i)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
it 'preserves floats' do
|
17
|
+
subject.each do |row|
|
18
|
+
expect(row['float']).to eq(row['float'].to_s.to_f)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -1,4 +1,7 @@
|
|
1
1
|
require 'spec_helper'
|
2
|
+
require 's3mpi/interface'
|
3
|
+
require 's3mpi/converters/csv'
|
4
|
+
require_relative '../s3mpi/converters/parsed_csv_shared_examples'
|
2
5
|
|
3
6
|
module S3MPI
|
4
7
|
|
@@ -6,16 +9,18 @@ module S3MPI
|
|
6
9
|
|
7
10
|
let(:bucket) { 'some_bucket' }
|
8
11
|
let(:path) { 'some_folder' }
|
9
|
-
let(:csv_file_path) {'spec/support/test.csv'}
|
12
|
+
let(:csv_file_path) { 'spec/support/test.csv' }
|
13
|
+
let(:csv_data_string) { File.read csv_file_path }
|
10
14
|
let(:csv_as_array_of_hashes) {
|
11
15
|
[
|
12
|
-
{'
|
13
|
-
{'
|
14
|
-
{'
|
16
|
+
{'integer' => 1, 'string' => 'user1@test.com', 'float' => 1.11},
|
17
|
+
{'integer' => 2, 'string' => 'user2@test.com', 'float' => 2.22},
|
18
|
+
{'integer' => 3, 'string' => 'user3@test.com', 'float' => 3.33},
|
15
19
|
]
|
16
20
|
}
|
17
21
|
|
18
|
-
|
22
|
+
let(:interface) { described_class.new(bucket: bucket, path: path) }
|
23
|
+
subject { interface }
|
19
24
|
|
20
25
|
describe '.store_csv' do
|
21
26
|
it 'sends the CSV as a row of hashes to .store' do
|
@@ -25,6 +30,12 @@ module S3MPI
|
|
25
30
|
end
|
26
31
|
end
|
27
32
|
|
28
|
-
|
33
|
+
describe '.read_csv' do
|
34
|
+
|
35
|
+
before { allow(interface).to receive(:s3_object) { double(read: csv_data_string) } }
|
36
|
+
subject { interface.read_csv csv_file_path }
|
37
|
+
it_behaves_like 'a parsed CSV'
|
29
38
|
|
39
|
+
end
|
40
|
+
end
|
30
41
|
end
|
data/spec/support/test.csv
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
|
2
|
-
1,user1@test.com
|
3
|
-
2,user2@test.com
|
4
|
-
3,user3@test.com
|
1
|
+
integer,string,float
|
2
|
+
1,user1@test.com,1.11
|
3
|
+
2,user2@test.com,2.22
|
4
|
+
3,user3@test.com,3.33
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: s3mpi
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.8.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Robert Krzyzanowski
|
@@ -44,14 +44,14 @@ dependencies:
|
|
44
44
|
requirements:
|
45
45
|
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version:
|
47
|
+
version: '3.4'
|
48
48
|
type: :development
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version:
|
54
|
+
version: '3.4'
|
55
55
|
description: Passing objects between Ruby consoles can be cumbersome if the user has
|
56
56
|
performed some serialization and deserialization procedure. S3MPI aims to enable
|
57
57
|
simple reading and writing to S3 buckets without the typical overhead of the AWS
|
@@ -70,11 +70,15 @@ files:
|
|
70
70
|
- README.md
|
71
71
|
- Rakefile
|
72
72
|
- lib/s3mpi.rb
|
73
|
+
- lib/s3mpi/converters.rb
|
74
|
+
- lib/s3mpi/converters/csv.rb
|
73
75
|
- lib/s3mpi/format.rb
|
74
76
|
- lib/s3mpi/interface.rb
|
75
77
|
- lib/s3mpi/s3.rb
|
76
78
|
- lib/s3mpi/version.rb
|
77
79
|
- s3mpi.gemspec
|
80
|
+
- spec/s3mpi/converters/csv_spec.rb
|
81
|
+
- spec/s3mpi/converters/parsed_csv_shared_examples.rb
|
78
82
|
- spec/s3mpi/interface_spec.rb
|
79
83
|
- spec/s3mpi/s3_spec.rb
|
80
84
|
- spec/spec_helper.rb
|
@@ -104,6 +108,8 @@ signing_key:
|
|
104
108
|
specification_version: 4
|
105
109
|
summary: Upload and download files to S3 using a very convenient API.
|
106
110
|
test_files:
|
111
|
+
- spec/s3mpi/converters/csv_spec.rb
|
112
|
+
- spec/s3mpi/converters/parsed_csv_shared_examples.rb
|
107
113
|
- spec/s3mpi/interface_spec.rb
|
108
114
|
- spec/s3mpi/s3_spec.rb
|
109
115
|
- spec/spec_helper.rb
|