bfs-s3 0.3.3 → 0.3.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/bfs/bucket/s3.rb +41 -24
- data/spec/bfs/bucket/s3_spec.rb +28 -44
- metadata +5 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 006a47eb5bb98ed25ce659ef731f72ea815f629382906125dff18d90866b06c9
|
4
|
+
data.tar.gz: '09e8fb5e9c3a7496f2d6004300f20e03e62af725a9e238abd4d67fa0d66b8ed2'
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0aa16aadd6093b213199286f61a4ac965b54386e96e441b74c567d277fb5c5fc552977fb44f580ce254ce565dc79c12db23941dbd3e490e52e24288911218e17
|
7
|
+
data.tar.gz: a68af4df46647ccb88bdf6d2869b068aa26b83f75fb267a093d443146a30fc092bc73879f45b958d4e7112ceb9d769608e18fa452802e43b5674b12ebb9060d2
|
data/lib/bfs/bucket/s3.rb
CHANGED
@@ -6,16 +6,20 @@ module BFS
|
|
6
6
|
module Bucket
|
7
7
|
# S3 buckets are operating on s3
|
8
8
|
class S3 < Abstract
|
9
|
+
NF_ERRORS = [Aws::S3::Errors::NoSuchKey, Aws::S3::Errors::NoSuchBucket, Aws::S3::Errors::NotFound].freeze
|
10
|
+
|
9
11
|
attr_reader :name, :sse, :acl, :storage_class
|
10
12
|
|
11
13
|
# Initializes a new S3 bucket
|
12
14
|
# @param [String] name the bucket name
|
13
15
|
# @param [Hash] opts options
|
14
16
|
# @option opts [String] :region default region
|
17
|
+
# @option opts [String] :prefix custom namespace within the bucket
|
15
18
|
# @option opts [String] :sse default server-side-encryption setting
|
16
19
|
# @option opts [Aws::Credentials] :credentials credentials object
|
17
20
|
# @option opts [String] :access_key_id custom AWS access key ID
|
18
21
|
# @option opts [String] :secret_access_key custom AWS secret access key
|
22
|
+
# @option opts [String] :profile_name custom AWS profile name (for shared credentials)
|
19
23
|
# @option opts [Symbol] :acl canned ACL
|
20
24
|
# @option opts [String] :storage_class storage class
|
21
25
|
# @option opts [Aws::S3::Client] :client custom client, uses default_client by default
|
@@ -23,26 +27,32 @@ module BFS
|
|
23
27
|
opts = opts.dup
|
24
28
|
opts.keys.each do |key|
|
25
29
|
val = opts.delete(key)
|
26
|
-
opts[key.
|
30
|
+
opts[key.to_sym] = val unless val.nil?
|
27
31
|
end
|
28
32
|
|
29
33
|
@name = name
|
30
|
-
@sse = opts[
|
31
|
-
@
|
32
|
-
@
|
33
|
-
@
|
34
|
-
@
|
35
|
-
@client = opts['client'] || Aws::S3::Client.new(region: opts['region'])
|
34
|
+
@sse = opts[:sse] || opts[:server_side_encryption]
|
35
|
+
@prefix = opts[:prefix]
|
36
|
+
@acl = opts[:acl].to_sym if opts[:acl]
|
37
|
+
@storage_class = opts[:storage_class]
|
38
|
+
@client = opts[:client] || init_client(opts)
|
36
39
|
end
|
37
40
|
|
38
41
|
# Lists the contents of a bucket using a glob pattern
|
39
42
|
def ls(pattern='**/*', opts={})
|
43
|
+
prefix = pattern[%r{^[^\*\?\{\}\[\]]+/}]
|
44
|
+
prefix = File.join(*[@prefix, prefix].compact) if @prefix
|
45
|
+
|
46
|
+
opts = opts.merge(bucket: name, prefix: @prefix)
|
47
|
+
opts[:prefix] = prefix if prefix
|
48
|
+
|
40
49
|
next_token = nil
|
41
50
|
Enumerator.new do |y|
|
42
51
|
loop do
|
43
|
-
resp = @client.list_objects_v2 opts.merge(
|
52
|
+
resp = @client.list_objects_v2 opts.merge(continuation_token: next_token)
|
44
53
|
resp.contents.each do |obj|
|
45
|
-
|
54
|
+
name = trim_prefix(obj.key)
|
55
|
+
y << name if File.fnmatch?(pattern, name, File::FNM_PATHNAME)
|
46
56
|
end
|
47
57
|
next_token = resp.next_continuation_token.to_s
|
48
58
|
break if next_token.empty?
|
@@ -53,21 +63,18 @@ module BFS
|
|
53
63
|
# Info returns the object info
|
54
64
|
def info(path, opts={})
|
55
65
|
path = norm_path(path)
|
56
|
-
opts = opts.merge(
|
57
|
-
bucket: name,
|
58
|
-
key: opts[:prefix] ? File.join(opts[:prefix], path) : path,
|
59
|
-
)
|
66
|
+
opts = opts.merge(bucket: name, key: full_path(path))
|
60
67
|
info = @client.head_object(opts)
|
61
68
|
raise BFS::FileNotFound, path unless info
|
62
69
|
|
63
70
|
BFS::FileInfo.new(path, info.content_length, info.last_modified, info.content_type, info.metadata)
|
64
|
-
rescue
|
71
|
+
rescue *NF_ERRORS
|
65
72
|
raise BFS::FileNotFound, path
|
66
73
|
end
|
67
74
|
|
68
75
|
# Creates a new file and opens it for writing
|
69
76
|
def create(path, opts={}, &block)
|
70
|
-
path =
|
77
|
+
path = full_path(path)
|
71
78
|
opts = opts.merge(
|
72
79
|
bucket: name,
|
73
80
|
key: path,
|
@@ -92,7 +99,7 @@ module BFS
|
|
92
99
|
|
93
100
|
# Opens an existing file for reading
|
94
101
|
def open(path, opts={}, &block)
|
95
|
-
path =
|
102
|
+
path = full_path(path)
|
96
103
|
temp = Tempfile.new(File.basename(path), binmode: true)
|
97
104
|
temp.close
|
98
105
|
|
@@ -104,33 +111,43 @@ module BFS
|
|
104
111
|
@client.get_object(opts)
|
105
112
|
|
106
113
|
File.open(temp.path, binmode: true, &block)
|
107
|
-
rescue
|
108
|
-
raise BFS::FileNotFound, path
|
114
|
+
rescue *NF_ERRORS
|
115
|
+
raise BFS::FileNotFound, trim_prefix(path)
|
109
116
|
end
|
110
117
|
|
111
118
|
# Deletes a file.
|
112
119
|
def rm(path, opts={})
|
113
|
-
path =
|
120
|
+
path = full_path(path)
|
114
121
|
opts = opts.merge(
|
115
122
|
bucket: name,
|
116
123
|
key: path,
|
117
124
|
)
|
118
125
|
@client.delete_object(opts)
|
119
|
-
rescue
|
126
|
+
rescue *NF_ERRORS # rubocop:disable Lint/HandleExceptions
|
120
127
|
end
|
121
128
|
|
122
129
|
# Copies a file.
|
123
130
|
def cp(src, dst, opts={})
|
124
|
-
src =
|
125
|
-
dst =
|
131
|
+
src = full_path(src)
|
132
|
+
dst = full_path(dst)
|
126
133
|
opts = opts.merge(
|
127
134
|
bucket: name,
|
128
135
|
copy_source: "/#{name}/#{src}",
|
129
136
|
key: dst,
|
130
137
|
)
|
131
138
|
@client.copy_object(opts)
|
132
|
-
rescue
|
133
|
-
raise BFS::FileNotFound, src
|
139
|
+
rescue *NF_ERRORS
|
140
|
+
raise BFS::FileNotFound, trim_prefix(src)
|
141
|
+
end
|
142
|
+
|
143
|
+
private
|
144
|
+
|
145
|
+
def init_client(opts)
|
146
|
+
credentials = opts[:credentials]
|
147
|
+
credentials ||= Aws::Credentials.new(opts[:access_key_id].to_s, opts[:secret_access_key].to_s) if opts[:access_key_id]
|
148
|
+
credentials ||= Aws::SharedCredentials.new(profile_name: opts[:profile_name])
|
149
|
+
|
150
|
+
Aws::S3::Client.new region: opts[:region], credentials: credentials
|
134
151
|
end
|
135
152
|
end
|
136
153
|
end
|
data/spec/bfs/bucket/s3_spec.rb
CHANGED
@@ -1,58 +1,42 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
allow(client).to receive(:get_object).with(hash_including(bucket: 'mock-bucket')) do |opts|
|
14
|
-
data = files[opts[:key]]
|
15
|
-
raise Aws::S3::Errors::NoSuchKey.new(nil, nil) unless data
|
16
|
-
|
17
|
-
File.open(opts[:response_target], 'wb') {|f| f.write(data) }
|
18
|
-
nil
|
19
|
-
end
|
20
|
-
|
21
|
-
allow(client).to receive(:delete_object).with(hash_including(bucket: 'mock-bucket')) do |opts|
|
22
|
-
raise Aws::S3::Errors::NoSuchKey.new(nil, nil) unless files.key?(opts[:key])
|
23
|
-
|
24
|
-
files.delete(opts[:key])
|
25
|
-
nil
|
26
|
-
end
|
27
|
-
|
28
|
-
allow(client).to receive(:list_objects_v2).with(bucket: 'mock-bucket', continuation_token: nil) do |*|
|
29
|
-
contents = files.keys.map {|key| Aws::S3::Types::Object.new(key: key) }
|
30
|
-
Aws::S3::Types::ListObjectsV2Output.new contents: contents, next_continuation_token: ''
|
31
|
-
end
|
32
|
-
allow(client).to receive(:head_object).with(bucket: 'mock-bucket', key: 'a/b/c.txt') do |*|
|
33
|
-
Aws::S3::Types::HeadObjectOutput.new content_length: 10, last_modified: Time.now, content_type: 'text/plain', metadata: { 'key' => 'val' }
|
3
|
+
sandbox = { region: 'us-east-1', bucket: 'bsm-bfs-unittest' }.freeze
|
4
|
+
run_spec = \
|
5
|
+
begin
|
6
|
+
c = Aws::SharedCredentials.new
|
7
|
+
if c.loadable?
|
8
|
+
s = Aws::S3::Client.new(region: sandbox[:region], credentials: c)
|
9
|
+
s.head_bucket(bucket: sandbox[:bucket])
|
10
|
+
true
|
11
|
+
else
|
12
|
+
false
|
34
13
|
end
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
allow(client).to receive(:copy_object).with(hash_including(bucket: 'mock-bucket')) do |opts|
|
39
|
-
src = opts[:copy_source].sub('/mock-bucket/', '')
|
40
|
-
raise Aws::S3::Errors::NoSuchKey.new(nil, nil) unless files.key?(src)
|
14
|
+
rescue Aws::Errors::ServiceError
|
15
|
+
false
|
16
|
+
end
|
41
17
|
|
42
|
-
|
43
|
-
|
44
|
-
end
|
18
|
+
RSpec.describe BFS::Bucket::S3, if: run_spec do
|
19
|
+
let(:prefix) { "x/#{SecureRandom.uuid}/" }
|
45
20
|
|
46
|
-
|
21
|
+
subject do
|
22
|
+
described_class.new sandbox[:bucket], region: sandbox[:region], prefix: prefix
|
23
|
+
end
|
24
|
+
after :all do
|
25
|
+
bucket = described_class.new sandbox[:bucket], region: sandbox[:region], prefix: 'x/'
|
26
|
+
bucket.ls.each {|name| bucket.rm(name) }
|
47
27
|
end
|
48
28
|
|
49
|
-
subject { described_class.new('mock-bucket', client: mock_client) }
|
50
29
|
it_behaves_like 'a bucket'
|
51
30
|
|
52
31
|
it 'should resolve from URL' do
|
53
|
-
bucket = BFS.resolve(
|
32
|
+
bucket = BFS.resolve("s3://#{sandbox[:bucket]}?acl=private®ion=#{sandbox[:region]}")
|
54
33
|
expect(bucket).to be_instance_of(described_class)
|
55
|
-
expect(bucket.name).to eq(
|
34
|
+
expect(bucket.name).to eq(sandbox[:bucket])
|
56
35
|
expect(bucket.acl).to eq(:private)
|
57
36
|
end
|
37
|
+
|
38
|
+
it 'should enumerate over a large number of files' do
|
39
|
+
bucket = described_class.new sandbox[:bucket], region: sandbox[:region], prefix: 'm/'
|
40
|
+
expect(bucket.ls('**/*').count).to eq(2121)
|
41
|
+
end
|
58
42
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: bfs-s3
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Dimitrij Denissenko
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-11-
|
11
|
+
date: 2018-11-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: aws-sdk-s3
|
@@ -30,14 +30,14 @@ dependencies:
|
|
30
30
|
requirements:
|
31
31
|
- - '='
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: 0.3.
|
33
|
+
version: 0.3.4
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - '='
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: 0.3.
|
40
|
+
version: 0.3.4
|
41
41
|
description: https://github.com/bsm/bfs.rb
|
42
42
|
email: dimitrij@blacksquaremedia.com
|
43
43
|
executables: []
|
@@ -68,7 +68,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
68
68
|
version: '0'
|
69
69
|
requirements: []
|
70
70
|
rubyforge_project:
|
71
|
-
rubygems_version: 2.7.
|
71
|
+
rubygems_version: 2.7.6
|
72
72
|
signing_key:
|
73
73
|
specification_version: 4
|
74
74
|
summary: S3 bucket adapter for bfs
|