blobby-s3 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +20 -0
- data/.rspec +1 -0
- data/.rubocop.yml +47 -0
- data/Gemfile +8 -0
- data/README.md +15 -0
- data/Rakefile +9 -0
- data/blobby-s3.gemspec +21 -0
- data/lib/blobby/s3_store.rb +113 -0
- data/spec/blobby/s3_store_spec.rb +96 -0
- metadata +81 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: bb92e09ebb4fb6aa87581485a6fb49e926f97746
|
4
|
+
data.tar.gz: 426467d4d7cde0580260e569132c728fff71eea0
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 0c2519c24da67688afb17718f8c9b1b49a4d3bd8be9d50592fab3e79e0f1e9a1e46561bbd3ba9d1c413a60a279b43abe843a8a18a5360453281ba86ae64cfc91
|
7
|
+
data.tar.gz: abaf0b989d2a9ffa8e143fe3b6ac3a9c2898a8a74a5104d50d0bd602f670bddde6fcc7fa9254a33b4028b8f586dd2dc3b7becc5e8bb9daa9d1f973d1ad2a46a3
|
data/.gitignore
ADDED
data/.rspec
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
--color
|
data/.rubocop.yml
ADDED
@@ -0,0 +1,47 @@
|
|
1
|
+
Eval:
|
2
|
+
Exclude:
|
3
|
+
- "Rakefile"
|
4
|
+
|
5
|
+
Metrics/AbcSize:
|
6
|
+
Enabled: false
|
7
|
+
|
8
|
+
Metrics/LineLength:
|
9
|
+
Max: 120
|
10
|
+
|
11
|
+
Metrics/MethodLength:
|
12
|
+
Max: 30
|
13
|
+
|
14
|
+
Style/ClassAndModuleChildren:
|
15
|
+
EnforcedStyle: nested
|
16
|
+
Exclude:
|
17
|
+
- "spec/**/*"
|
18
|
+
|
19
|
+
Style/Documentation:
|
20
|
+
Exclude:
|
21
|
+
- "spec/**/*"
|
22
|
+
|
23
|
+
Style/EmptyLinesAroundBlockBody:
|
24
|
+
Enabled: false
|
25
|
+
|
26
|
+
Style/EmptyLinesAroundClassBody:
|
27
|
+
EnforcedStyle: empty_lines
|
28
|
+
|
29
|
+
Style/EmptyLinesAroundModuleBody:
|
30
|
+
Enabled: false
|
31
|
+
|
32
|
+
Style/Encoding:
|
33
|
+
EnforcedStyle: when_needed
|
34
|
+
Enabled: true
|
35
|
+
|
36
|
+
Style/FileName:
|
37
|
+
Exclude:
|
38
|
+
- "bin/*"
|
39
|
+
|
40
|
+
Style/HashSyntax:
|
41
|
+
EnforcedStyle: hash_rockets
|
42
|
+
|
43
|
+
Style/StringLiterals:
|
44
|
+
EnforcedStyle: double_quotes
|
45
|
+
|
46
|
+
Style/WordArray:
|
47
|
+
Enabled: false
|
data/Gemfile
ADDED
data/README.md
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
# Blobby::S3Store
|
2
|
+
|
3
|
+
This gem provides an S3-based implementation of the "store" interface defined by the ["blobby"](https://github.com/realestate-com-au/blobby) gem. It's been packaged separately, to avoid adding dependencies to the core gem.
|
4
|
+
|
5
|
+
The simplest use-case is writing to a single bucket:
|
6
|
+
|
7
|
+
s3_store = Blobby::S3Store.new("mybucket")
|
8
|
+
s3_store["key"].write("something big")
|
9
|
+
|
10
|
+
Credentials can be provided, if required:
|
11
|
+
|
12
|
+
credentials = { :access_key_id => "KEY, :secret_access_key => "SECRET" }
|
13
|
+
s3_store = Blobby::S3Store.new("mybucket", credentials)
|
14
|
+
|
15
|
+
If none are specified, we'll look for them in [the normal places](https://blogs.aws.amazon.com/security/post/Tx3D6U6WSFGOK2H/A-New-and-Standardized-Way-to-Manage-Credentials-in-the-AWS-SDKs).
|
data/Rakefile
ADDED
data/blobby-s3.gemspec
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
|
3
|
+
Gem::Specification.new do |gem|
|
4
|
+
|
5
|
+
gem.authors = ["Mike Williams"]
|
6
|
+
gem.email = ["mdub@dogbiscuit.org"]
|
7
|
+
gem.summary = "Store BLOBs in S3"
|
8
|
+
gem.homepage = "https://github.com/realestate-com.au/blobby-s3"
|
9
|
+
|
10
|
+
gem.name = "blobby-s3"
|
11
|
+
gem.version = "1.0.0"
|
12
|
+
|
13
|
+
gem.files = `git ls-files`.split($OUTPUT_RECORD_SEPARATOR)
|
14
|
+
gem.executables = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) }
|
15
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
16
|
+
gem.require_paths = ["lib"]
|
17
|
+
|
18
|
+
gem.add_runtime_dependency("aws-sdk-resources", "~> 2.1")
|
19
|
+
gem.add_runtime_dependency("blobby")
|
20
|
+
|
21
|
+
end
|
@@ -0,0 +1,113 @@
|
|
1
|
+
require "aws-sdk-resources"
|
2
|
+
require "blobby/key_constraint"
|
3
|
+
|
4
|
+
module Blobby
|
5
|
+
|
6
|
+
# A BLOB store backed by an S3 bucket.
|
7
|
+
#
|
8
|
+
class S3Store
|
9
|
+
|
10
|
+
# Create a new instance.
|
11
|
+
#
|
12
|
+
# bucket_name - name of the bucket to store things in
|
13
|
+
# s3_options - options passed to AWS::S3.new
|
14
|
+
#
|
15
|
+
def initialize(bucket_name, s3_options = {})
|
16
|
+
@bucket_name = bucket_name.to_str
|
17
|
+
@s3_options = s3_options.dup
|
18
|
+
@s3_options[:endpoint] = s3_endpoint_for_bucket
|
19
|
+
end
|
20
|
+
|
21
|
+
attr_reader :bucket_name
|
22
|
+
attr_reader :s3_options
|
23
|
+
|
24
|
+
def available?
|
25
|
+
bucket.objects.first
|
26
|
+
true
|
27
|
+
rescue ::Aws::Errors::ServiceError
|
28
|
+
false
|
29
|
+
end
|
30
|
+
|
31
|
+
def [](key)
|
32
|
+
KeyConstraint.must_allow!(key)
|
33
|
+
StoredObject.new(bucket.object(key))
|
34
|
+
end
|
35
|
+
|
36
|
+
class StoredObject
|
37
|
+
|
38
|
+
def initialize(s3_object)
|
39
|
+
@s3_object = s3_object
|
40
|
+
end
|
41
|
+
|
42
|
+
def exists?
|
43
|
+
s3_object.exists?
|
44
|
+
end
|
45
|
+
|
46
|
+
def read
|
47
|
+
body = s3_object.get.body
|
48
|
+
if block_given?
|
49
|
+
body.each_line do |line|
|
50
|
+
yield force_binary(line)
|
51
|
+
end
|
52
|
+
nil
|
53
|
+
else
|
54
|
+
force_binary(body.read)
|
55
|
+
end
|
56
|
+
rescue Aws::S3::Errors::NoSuchKey
|
57
|
+
nil
|
58
|
+
end
|
59
|
+
|
60
|
+
def write(payload)
|
61
|
+
s3_object.put(:body => force_binary(payload))
|
62
|
+
nil
|
63
|
+
end
|
64
|
+
|
65
|
+
def delete
|
66
|
+
return false unless s3_object.exists?
|
67
|
+
s3_object.delete
|
68
|
+
true
|
69
|
+
end
|
70
|
+
|
71
|
+
private
|
72
|
+
|
73
|
+
attr_reader :s3_object
|
74
|
+
|
75
|
+
def force_binary(s)
|
76
|
+
return s unless s.respond_to?(:encoding)
|
77
|
+
return s if s.encoding.name == "ASCII-8BIT"
|
78
|
+
s.dup.force_encoding("ASCII-8BIT")
|
79
|
+
end
|
80
|
+
|
81
|
+
end
|
82
|
+
|
83
|
+
private
|
84
|
+
|
85
|
+
def s3_client
|
86
|
+
::Aws::S3::Client.new(s3_options)
|
87
|
+
end
|
88
|
+
|
89
|
+
def s3_endpoint_for_bucket
|
90
|
+
location = s3_client.get_bucket_location(:bucket => bucket_name).location_constraint
|
91
|
+
case location
|
92
|
+
when ""
|
93
|
+
"https://s3.amazonaws.com"
|
94
|
+
when "EU"
|
95
|
+
"https://s3-eu-west-1.amazonaws.com"
|
96
|
+
else
|
97
|
+
"https://s3-#{location}.amazonaws.com"
|
98
|
+
end
|
99
|
+
rescue ::Aws::Errors::ServiceError
|
100
|
+
"https://s3.amazonaws.com"
|
101
|
+
end
|
102
|
+
|
103
|
+
def s3_resource
|
104
|
+
::Aws::S3::Resource.new(s3_options)
|
105
|
+
end
|
106
|
+
|
107
|
+
def bucket
|
108
|
+
s3_resource.bucket(bucket_name)
|
109
|
+
end
|
110
|
+
|
111
|
+
end
|
112
|
+
|
113
|
+
end
|
@@ -0,0 +1,96 @@
|
|
1
|
+
require "aws-sdk-resources"
|
2
|
+
require "blobby/s3_store"
|
3
|
+
|
4
|
+
# Load the abstract "Store" tests from "blobby".
|
5
|
+
# This depends on the gem being packaged with "spec" dir intact.
|
6
|
+
$LOAD_PATH << Gem.loaded_specs["blobby"].full_gem_path + "/spec"
|
7
|
+
|
8
|
+
require "blobby/store_behaviour"
|
9
|
+
|
10
|
+
describe Blobby::S3Store, :integration => true do
|
11
|
+
|
12
|
+
before(:all) do
|
13
|
+
unless ENV.key?("AWS_ACCESS_KEY_ID")
|
14
|
+
fail "No AWS credentials provided"
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
context "with a writable bucket" do
|
19
|
+
|
20
|
+
EXISTING_BUCKET_NAME = "fake-aws-sdk-s3-test"
|
21
|
+
|
22
|
+
let(:s3_resource) { Aws::S3::Resource.new(:region => "us-east-1")}
|
23
|
+
let(:bucket) { s3_resource.bucket(EXISTING_BUCKET_NAME) }
|
24
|
+
|
25
|
+
before do
|
26
|
+
bucket.clear!
|
27
|
+
end
|
28
|
+
|
29
|
+
subject do
|
30
|
+
described_class.new(EXISTING_BUCKET_NAME)
|
31
|
+
end
|
32
|
+
|
33
|
+
it_behaves_like Blobby::Store
|
34
|
+
|
35
|
+
describe "#write" do
|
36
|
+
|
37
|
+
let(:key) { "data/file" }
|
38
|
+
let(:content) { "CONTENT" }
|
39
|
+
|
40
|
+
before do
|
41
|
+
subject[key].write(content)
|
42
|
+
end
|
43
|
+
|
44
|
+
it "stores stuff in S3" do
|
45
|
+
expect(bucket.object(key).get.body.read).to eq(content)
|
46
|
+
end
|
47
|
+
|
48
|
+
end
|
49
|
+
|
50
|
+
describe "#delete" do
|
51
|
+
|
52
|
+
let(:key) { "my_key" }
|
53
|
+
|
54
|
+
before do
|
55
|
+
subject[key].write("content")
|
56
|
+
subject[key].delete
|
57
|
+
end
|
58
|
+
|
59
|
+
it "removes stuff from S3" do
|
60
|
+
expect(bucket.object(key)).to_not exist
|
61
|
+
end
|
62
|
+
|
63
|
+
end
|
64
|
+
|
65
|
+
end
|
66
|
+
|
67
|
+
context "when we can't talk to S3" do
|
68
|
+
|
69
|
+
let(:bogus_credentials) do
|
70
|
+
{
|
71
|
+
:access_key_id => "bogus",
|
72
|
+
:secret_access_key => "bogus"
|
73
|
+
}
|
74
|
+
end
|
75
|
+
|
76
|
+
subject do
|
77
|
+
described_class.new(EXISTING_BUCKET_NAME, bogus_credentials)
|
78
|
+
end
|
79
|
+
|
80
|
+
it { is_expected.not_to be_available }
|
81
|
+
|
82
|
+
end
|
83
|
+
|
84
|
+
context "when the bucket does not exist" do
|
85
|
+
|
86
|
+
BOGUS_BUCKET_NAME = "bogusmcbogusness"
|
87
|
+
|
88
|
+
subject do
|
89
|
+
described_class.new(BOGUS_BUCKET_NAME)
|
90
|
+
end
|
91
|
+
|
92
|
+
it { is_expected.not_to be_available }
|
93
|
+
|
94
|
+
end
|
95
|
+
|
96
|
+
end
|
metadata
ADDED
@@ -0,0 +1,81 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: blobby-s3
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 1.0.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Mike Williams
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2015-10-02 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: aws-sdk-resources
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '2.1'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '2.1'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: blobby
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
description:
|
42
|
+
email:
|
43
|
+
- mdub@dogbiscuit.org
|
44
|
+
executables: []
|
45
|
+
extensions: []
|
46
|
+
extra_rdoc_files: []
|
47
|
+
files:
|
48
|
+
- ".gitignore"
|
49
|
+
- ".rspec"
|
50
|
+
- ".rubocop.yml"
|
51
|
+
- Gemfile
|
52
|
+
- README.md
|
53
|
+
- Rakefile
|
54
|
+
- blobby-s3.gemspec
|
55
|
+
- lib/blobby/s3_store.rb
|
56
|
+
- spec/blobby/s3_store_spec.rb
|
57
|
+
homepage: https://github.com/realestate-com.au/blobby-s3
|
58
|
+
licenses: []
|
59
|
+
metadata: {}
|
60
|
+
post_install_message:
|
61
|
+
rdoc_options: []
|
62
|
+
require_paths:
|
63
|
+
- lib
|
64
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
70
|
+
requirements:
|
71
|
+
- - ">="
|
72
|
+
- !ruby/object:Gem::Version
|
73
|
+
version: '0'
|
74
|
+
requirements: []
|
75
|
+
rubyforge_project:
|
76
|
+
rubygems_version: 2.4.8
|
77
|
+
signing_key:
|
78
|
+
specification_version: 4
|
79
|
+
summary: Store BLOBs in S3
|
80
|
+
test_files:
|
81
|
+
- spec/blobby/s3_store_spec.rb
|