dpl-s3 1.9.2.travis.2748.5 → 1.9.2.travis.2749.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: f853bd59ec5a942fe9b384d44d7a2d6003c889cd29c9e715566b4489fbd5f258
4
- data.tar.gz: c8566920273400763dc38e11eb1cb7d057f989af54f8449a9a411b9a829a07a2
3
+ metadata.gz: b0de8b4eab5b91a8d7004a264ece8c7d37f317ce5e7de387a841634b38105f2b
4
+ data.tar.gz: bedfec95ca08bef40899f63d1bf0c7d3502dca692246d4dd95f07035022202e2
5
5
  SHA512:
6
- metadata.gz: d5be8bae73fe0751548ea06f3f4c81a30230a8b996e22bc8c6a17748f1946e92d6dff11d5aa908c89cf13efcd2c8ceac52cb786cb53a36c4cc6fae15550b8cb5
7
- data.tar.gz: 691b680708dd4cab9cc0ad2c070442746b220c9dd8861558f2569c061c6439a8deecfd2e0ee03edf45b749127b584d24a8aa120b1be30bbf645f5cf5d6cf58bd
6
+ metadata.gz: 3722c131153da77fcfa1d0f9587e2f30c1e959060bcead4f3991fd99878002978f2e03415463546227ff91ac604157b7143a4f8c84f182ece5f895585cae5cb0
7
+ data.tar.gz: d6e8164b130f540a5eb03728d82105a248bec6c6dd59573b678c26704e460b57a8a01c7736120f60cb055a5d4f611eb6e30df7d1f71a9cdb87d0e3c595c3260c
@@ -5,10 +5,22 @@ require 'mime-types'
5
5
  module DPL
6
6
  class Provider
7
7
  class S3 < Provider
8
+ DEFAULT_MAX_THREADS = 5
9
+ MAX_THREADS = 15
10
+
8
11
  def api
9
12
  @api ||= ::Aws::S3::Resource.new(s3_options)
10
13
  end
11
14
 
15
+ def max_threads
16
+ return @max_threads if @max_threads
17
+ if (@max_threads = threads_wanted = options.fetch(:max_threads, DEFAULT_MAX_THREADS)) >= MAX_THREADS
18
+ log "Desired thread count #{threads_wanted} is too large. Using #{MAX_THREADS}."
19
+ @max_threads = MAX_THREADS
20
+ end
21
+ @max_threads
22
+ end
23
+
12
24
  def needs_key?
13
25
  false
14
26
  end
@@ -44,8 +56,8 @@ module DPL
44
56
  cwd = options.fetch(:local_dir, Dir.pwd)
45
57
  glob_args = [cwd + "/**/*"]
46
58
  glob_args << File::FNM_DOTMATCH if options[:dot_match]
47
- files = Dir.glob(*glob_args)
48
- upload_multithreaded(files.reject {|f| File.directory?(f)})
59
+ files = Dir.glob(*glob_args).reject {|f| File.directory?(f)}
60
+ upload_multithreaded(files)
49
61
 
50
62
  if suffix = options[:index_document_suffix]
51
63
  api.bucket(option(:bucket)).website.put(
@@ -58,13 +70,13 @@ module DPL
58
70
  end
59
71
  end
60
72
 
61
- def upload_multithreaded(files, thread_count = 5)
73
+ def upload_multithreaded(files)
62
74
  file_number = 0
63
75
  mutex = Mutex.new
64
76
  threads = []
65
- log "Beginning upload of #{files.length} files with #{thread_count} threads."
77
+ log "Beginning upload of #{files.length} files with #{max_threads} threads."
66
78
 
67
- thread_count.times do |i|
79
+ max_threads.times do |i|
68
80
  threads[i] = Thread.new {
69
81
  until files.empty?
70
82
  mutex.synchronize do
@@ -48,7 +48,7 @@ describe DPL::Provider::S3 do
48
48
  end
49
49
 
50
50
  before :each do
51
- provider.stub(:s3_options).and_return(client_options)
51
+ allow(provider).to receive(:s3_options).and_return(client_options)
52
52
  allow_any_instance_of(::Aws::S3::Object).to receive(:upload_file).and_return(true)
53
53
  allow(provider).to receive(:log).with(anything).and_return(true)
54
54
  end
@@ -159,6 +159,21 @@ describe DPL::Provider::S3 do
159
159
  expect(Dir).to receive(:glob).with(Dir.pwd + "/**/*", File::FNM_DOTMATCH).and_return([__FILE__])
160
160
  provider.push_app
161
161
  end
162
+
163
+ example "when max_threads is set" do
164
+ provider.options.update(:max_threads => 10)
165
+ expect(Dir).to receive(:glob).with(Dir.pwd + "/**/*").and_return([__FILE__])
166
+ expect(provider).to receive(:log).with("Beginning upload of 1 files with 10 threads.")
167
+ provider.push_app
168
+ end
169
+
170
+ example "when max_threads is too large" do
171
+ provider.options.update(:max_threads => 100)
172
+ expect(Dir).to receive(:glob).with(Dir.pwd + "/**/*").and_return([__FILE__])
173
+ expect(provider).to receive(:log).with("Beginning upload of 1 files with 15 threads.")
174
+ expect(provider).to receive(:log).with("Desired thread count 100 is too large. Using 15.")
175
+ provider.push_app
176
+ end
162
177
  end
163
178
 
164
179
  describe "#check_app" do
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: dpl-s3
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.9.2.travis.2748.5
4
+ version: 1.9.2.travis.2749.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Konstantin Haase
@@ -16,14 +16,14 @@ dependencies:
16
16
  requirements:
17
17
  - - '='
18
18
  - !ruby/object:Gem::Version
19
- version: 1.9.2.travis.2748.5
19
+ version: 1.9.2.travis.2749.5
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - '='
25
25
  - !ruby/object:Gem::Version
26
- version: 1.9.2.travis.2748.5
26
+ version: 1.9.2.travis.2749.5
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: aws-sdk
29
29
  requirement: !ruby/object:Gem::Requirement