dpl-s3 1.9.2.travis.2748.5 → 1.9.2.travis.2749.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f853bd59ec5a942fe9b384d44d7a2d6003c889cd29c9e715566b4489fbd5f258
-  data.tar.gz: c8566920273400763dc38e11eb1cb7d057f989af54f8449a9a411b9a829a07a2
+  metadata.gz: b0de8b4eab5b91a8d7004a264ece8c7d37f317ce5e7de387a841634b38105f2b
+  data.tar.gz: bedfec95ca08bef40899f63d1bf0c7d3502dca692246d4dd95f07035022202e2
 SHA512:
-  metadata.gz: d5be8bae73fe0751548ea06f3f4c81a30230a8b996e22bc8c6a17748f1946e92d6dff11d5aa908c89cf13efcd2c8ceac52cb786cb53a36c4cc6fae15550b8cb5
-  data.tar.gz: 691b680708dd4cab9cc0ad2c070442746b220c9dd8861558f2569c061c6439a8deecfd2e0ee03edf45b749127b584d24a8aa120b1be30bbf645f5cf5d6cf58bd
+  metadata.gz: 3722c131153da77fcfa1d0f9587e2f30c1e959060bcead4f3991fd99878002978f2e03415463546227ff91ac604157b7143a4f8c84f182ece5f895585cae5cb0
+  data.tar.gz: d6e8164b130f540a5eb03728d82105a248bec6c6dd59573b678c26704e460b57a8a01c7736120f60cb055a5d4f611eb6e30df7d1f71a9cdb87d0e3c595c3260c

data/lib/dpl/provider/s3.rb

@@ -5,10 +5,22 @@ require 'mime-types'
 module DPL
   class Provider
     class S3 < Provider
+      DEFAULT_MAX_THREADS = 5
+      MAX_THREADS = 15
+
       def api
         @api ||= ::Aws::S3::Resource.new(s3_options)
       end
 
+      def max_threads
+        return @max_threads if @max_threads
+        if (@max_threads = threads_wanted = options.fetch(:max_threads, DEFAULT_MAX_THREADS)) >= MAX_THREADS
+          log "Desired thread count #{threads_wanted} is too large. Using #{MAX_THREADS}."
+          @max_threads = MAX_THREADS
+        end
+        @max_threads
+      end
+
       def needs_key?
         false
       end
@@ -44,8 +56,8 @@ module DPL
         cwd = options.fetch(:local_dir, Dir.pwd)
         glob_args = [cwd + "/**/*"]
         glob_args << File::FNM_DOTMATCH if options[:dot_match]
-        files = Dir.glob(*glob_args)
-        upload_multithreaded(files.reject {|f| File.directory?(f)})
+        files = Dir.glob(*glob_args).reject {|f| File.directory?(f)}
+        upload_multithreaded(files)
 
         if suffix = options[:index_document_suffix]
           api.bucket(option(:bucket)).website.put(
@@ -58,13 +70,13 @@ module DPL
         end
       end
 
-      def upload_multithreaded(files, thread_count = 5)
+      def upload_multithreaded(files)
         file_number = 0
         mutex = Mutex.new
         threads = []
-        log "Beginning upload of #{files.length} files with #{thread_count} threads."
+        log "Beginning upload of #{files.length} files with #{max_threads} threads."
 
-        thread_count.times do |i|
+        max_threads.times do |i|
           threads[i] = Thread.new {
             until files.empty?
               mutex.synchronize do

data/spec/provider/s3_spec.rb

@@ -48,7 +48,7 @@ describe DPL::Provider::S3 do
   end
 
   before :each do
-    provider.stub(:s3_options).and_return(client_options)
+    allow(provider).to receive(:s3_options).and_return(client_options)
     allow_any_instance_of(::Aws::S3::Object).to receive(:upload_file).and_return(true)
     allow(provider).to receive(:log).with(anything).and_return(true)
   end
@@ -159,6 +159,21 @@ describe DPL::Provider::S3 do
       expect(Dir).to receive(:glob).with(Dir.pwd + "/**/*", File::FNM_DOTMATCH).and_return([__FILE__])
       provider.push_app
     end
+
+    example "when max_threads is set" do
+      provider.options.update(:max_threads => 10)
+      expect(Dir).to receive(:glob).with(Dir.pwd + "/**/*").and_return([__FILE__])
+      expect(provider).to receive(:log).with("Beginning upload of 1 files with 10 threads.")
+      provider.push_app
+    end
+
+    example "when max_threads is too large" do
+      provider.options.update(:max_threads => 100)
+      expect(Dir).to receive(:glob).with(Dir.pwd + "/**/*").and_return([__FILE__])
+      expect(provider).to receive(:log).with("Beginning upload of 1 files with 15 threads.")
+      expect(provider).to receive(:log).with("Desired thread count 100 is too large. Using 15.")
+      provider.push_app
+    end
   end
 
   describe "#check_app" do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dpl-s3
 version: !ruby/object:Gem::Version
-  version: 1.9.2.travis.2748.5
+  version: 1.9.2.travis.2749.5
 platform: ruby
 authors:
 - Konstantin Haase
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.9.2.travis.2748.5
+        version: 1.9.2.travis.2749.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.9.2.travis.2748.5
+        version: 1.9.2.travis.2749.5
 - !ruby/object:Gem::Dependency
   name: aws-sdk
   requirement: !ruby/object:Gem::Requirement