dpl-s3 1.9.2.travis.2748.5 → 1.9.2.travis.2749.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/dpl/provider/s3.rb +17 -5
- data/spec/provider/s3_spec.rb +16 -1
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b0de8b4eab5b91a8d7004a264ece8c7d37f317ce5e7de387a841634b38105f2b
|
4
|
+
data.tar.gz: bedfec95ca08bef40899f63d1bf0c7d3502dca692246d4dd95f07035022202e2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3722c131153da77fcfa1d0f9587e2f30c1e959060bcead4f3991fd99878002978f2e03415463546227ff91ac604157b7143a4f8c84f182ece5f895585cae5cb0
|
7
|
+
data.tar.gz: d6e8164b130f540a5eb03728d82105a248bec6c6dd59573b678c26704e460b57a8a01c7736120f60cb055a5d4f611eb6e30df7d1f71a9cdb87d0e3c595c3260c
|
data/lib/dpl/provider/s3.rb
CHANGED
@@ -5,10 +5,22 @@ require 'mime-types'
|
|
5
5
|
module DPL
|
6
6
|
class Provider
|
7
7
|
class S3 < Provider
|
8
|
+
DEFAULT_MAX_THREADS = 5
|
9
|
+
MAX_THREADS = 15
|
10
|
+
|
8
11
|
def api
|
9
12
|
@api ||= ::Aws::S3::Resource.new(s3_options)
|
10
13
|
end
|
11
14
|
|
15
|
+
def max_threads
|
16
|
+
return @max_threads if @max_threads
|
17
|
+
if (@max_threads = threads_wanted = options.fetch(:max_threads, DEFAULT_MAX_THREADS)) >= MAX_THREADS
|
18
|
+
log "Desired thread count #{threads_wanted} is too large. Using #{MAX_THREADS}."
|
19
|
+
@max_threads = MAX_THREADS
|
20
|
+
end
|
21
|
+
@max_threads
|
22
|
+
end
|
23
|
+
|
12
24
|
def needs_key?
|
13
25
|
false
|
14
26
|
end
|
@@ -44,8 +56,8 @@ module DPL
|
|
44
56
|
cwd = options.fetch(:local_dir, Dir.pwd)
|
45
57
|
glob_args = [cwd + "/**/*"]
|
46
58
|
glob_args << File::FNM_DOTMATCH if options[:dot_match]
|
47
|
-
files = Dir.glob(*glob_args)
|
48
|
-
upload_multithreaded(files
|
59
|
+
files = Dir.glob(*glob_args).reject {|f| File.directory?(f)}
|
60
|
+
upload_multithreaded(files)
|
49
61
|
|
50
62
|
if suffix = options[:index_document_suffix]
|
51
63
|
api.bucket(option(:bucket)).website.put(
|
@@ -58,13 +70,13 @@ module DPL
|
|
58
70
|
end
|
59
71
|
end
|
60
72
|
|
61
|
-
def upload_multithreaded(files
|
73
|
+
def upload_multithreaded(files)
|
62
74
|
file_number = 0
|
63
75
|
mutex = Mutex.new
|
64
76
|
threads = []
|
65
|
-
log "Beginning upload of #{files.length} files with #{
|
77
|
+
log "Beginning upload of #{files.length} files with #{max_threads} threads."
|
66
78
|
|
67
|
-
|
79
|
+
max_threads.times do |i|
|
68
80
|
threads[i] = Thread.new {
|
69
81
|
until files.empty?
|
70
82
|
mutex.synchronize do
|
data/spec/provider/s3_spec.rb
CHANGED
@@ -48,7 +48,7 @@ describe DPL::Provider::S3 do
|
|
48
48
|
end
|
49
49
|
|
50
50
|
before :each do
|
51
|
-
provider.
|
51
|
+
allow(provider).to receive(:s3_options).and_return(client_options)
|
52
52
|
allow_any_instance_of(::Aws::S3::Object).to receive(:upload_file).and_return(true)
|
53
53
|
allow(provider).to receive(:log).with(anything).and_return(true)
|
54
54
|
end
|
@@ -159,6 +159,21 @@ describe DPL::Provider::S3 do
|
|
159
159
|
expect(Dir).to receive(:glob).with(Dir.pwd + "/**/*", File::FNM_DOTMATCH).and_return([__FILE__])
|
160
160
|
provider.push_app
|
161
161
|
end
|
162
|
+
|
163
|
+
example "when max_threads is set" do
|
164
|
+
provider.options.update(:max_threads => 10)
|
165
|
+
expect(Dir).to receive(:glob).with(Dir.pwd + "/**/*").and_return([__FILE__])
|
166
|
+
expect(provider).to receive(:log).with("Beginning upload of 1 files with 10 threads.")
|
167
|
+
provider.push_app
|
168
|
+
end
|
169
|
+
|
170
|
+
example "when max_threads is too large" do
|
171
|
+
provider.options.update(:max_threads => 100)
|
172
|
+
expect(Dir).to receive(:glob).with(Dir.pwd + "/**/*").and_return([__FILE__])
|
173
|
+
expect(provider).to receive(:log).with("Beginning upload of 1 files with 15 threads.")
|
174
|
+
expect(provider).to receive(:log).with("Desired thread count 100 is too large. Using 15.")
|
175
|
+
provider.push_app
|
176
|
+
end
|
162
177
|
end
|
163
178
|
|
164
179
|
describe "#check_app" do
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: dpl-s3
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.9.2.travis.
|
4
|
+
version: 1.9.2.travis.2749.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Konstantin Haase
|
@@ -16,14 +16,14 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - '='
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: 1.9.2.travis.
|
19
|
+
version: 1.9.2.travis.2749.5
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - '='
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: 1.9.2.travis.
|
26
|
+
version: 1.9.2.travis.2749.5
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: aws-sdk
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|