dpl-s3 1.9.2.travis.2737.5 → 1.9.2.travis.2748.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 23fadd09d309e76b532cd10bfc63c901aa0669947e021f8f0c0321aba9d726a5
-  data.tar.gz: 5983c85c786e8b57cacd40abda409637df60a0967ae5a07636acda2eefd9cbae
+  metadata.gz: f853bd59ec5a942fe9b384d44d7a2d6003c889cd29c9e715566b4489fbd5f258
+  data.tar.gz: c8566920273400763dc38e11eb1cb7d057f989af54f8449a9a411b9a829a07a2
 SHA512:
-  metadata.gz: e9568711ec6646c061fa9fce271521cef481d0a8392f1ab19b538b7b46a3e368f0a515dea4b51c0d9107204026151d3960bd178c3408b9435914e9c495023ad6
-  data.tar.gz: c7941cb04ea0ca867e1f4f658d498fc6eacd1ca68bcc1fd5623fa418c602ca2fab7a04de13e70cb5e0c80fe5e958dedc8a086c33ffe9ac0c4e92d34a236180eb
+  metadata.gz: d5be8bae73fe0751548ea06f3f4c81a30230a8b996e22bc8c6a17748f1946e92d6dff11d5aa908c89cf13efcd2c8ceac52cb786cb53a36c4cc6fae15550b8cb5
+  data.tar.gz: 691b680708dd4cab9cc0ad2c070442746b220c9dd8861558f2569c061c6439a8deecfd2e0ee03edf45b749127b584d24a8aa120b1be30bbf645f5cf5d6cf58bd

data/lib/dpl/provider/s3.rb

@@ -41,23 +41,11 @@ module DPL
       end
 
       def push_app
-        glob_args = ["**/*"]
+        cwd = options.fetch(:local_dir, Dir.pwd)
+        glob_args = [cwd + "/**/*"]
         glob_args << File::FNM_DOTMATCH if options[:dot_match]
-        Dir.chdir(options.fetch(:local_dir, Dir.pwd)) do
-          Dir.glob(*glob_args) do |filename|
-            opts                          = content_data_for(filename)
-            opts[:cache_control]          = get_option_value_by_filename(options[:cache_control], filename) if options[:cache_control]
-            opts[:acl]                    = options[:acl].gsub(/_/, '-') if options[:acl]
-            opts[:expires]                = get_option_value_by_filename(options[:expires], filename) if options[:expires]
-            opts[:storage_class]          = options[:storage_class] if options[:storage_class]
-            opts[:server_side_encryption] = "AES256" if options[:server_side_encryption]
-            unless File.directory?(filename)
-              log "uploading #{filename.inspect} with #{opts.inspect}"
-              result = api.bucket(option(:bucket)).object(upload_path(filename)).upload_file(filename, opts)
-              warn "error while uploading #{filename.inspect}" unless result
-            end
-          end
-        end
+        files = Dir.glob(*glob_args)
+        upload_multithreaded(files.reject {|f| File.directory?(f)})
 
         if suffix = options[:index_document_suffix]
           api.bucket(option(:bucket)).website.put(
@@ -70,6 +58,39 @@ module DPL
         end
       end
 
+      def upload_multithreaded(files, thread_count = 5)
+        file_number = 0
+        mutex = Mutex.new
+        threads = []
+        log "Beginning upload of #{files.length} files with #{thread_count} threads."
+
+        thread_count.times do |i|
+          threads[i] = Thread.new {
+            until files.empty?
+              mutex.synchronize do
+                file_number += 1
+                Thread.current["file_number"] = file_number
+              end
+              filename = files.pop rescue nil
+              next unless filename
+
+              opts  = content_data_for(filename)
+              opts[:cache_control]          = get_option_value_by_filename(options[:cache_control], filename) if options[:cache_control]
+              opts[:acl]                    = options[:acl].gsub(/_/, '-') if options[:acl]
+              opts[:expires]                = get_option_value_by_filename(options[:expires], filename) if options[:expires]
+              opts[:storage_class]          = options[:storage_class] if options[:storage_class]
+              opts[:server_side_encryption] = "AES256" if options[:server_side_encryption]
+              unless File.directory?(filename)
+                log "uploading #{filename.inspect} with #{opts.inspect}"
+                result = api.bucket(option(:bucket)).object(upload_path(filename)).upload_file(filename, opts)
+                warn "error while uploading #{filename.inspect}" unless result
+              end
+            end
+          }
+        end
+        threads.each { |t| t.join }
+      end
+
       def deploy
         super
       rescue ::Aws::S3::Errors::InvalidAccessKeyId

data/spec/provider/s3_spec.rb

@@ -49,6 +49,8 @@ describe DPL::Provider::S3 do
 
   before :each do
     provider.stub(:s3_options).and_return(client_options)
+    allow_any_instance_of(::Aws::S3::Object).to receive(:upload_file).and_return(true)
+    allow(provider).to receive(:log).with(anything).and_return(true)
   end
 
   describe "#check_auth" do
@@ -80,27 +82,22 @@ describe DPL::Provider::S3 do
   end
 
   describe "#push_app" do
-    example "Without local_dir" do
-      expect(Dir).to receive(:chdir).with(Dir.pwd)
-      provider.push_app
-    end
-
     example "With local_dir" do
-      provider.options.update(:local_dir => 'BUILD')
-
-      expect(Dir).to receive(:chdir).with('BUILD')
+      someDir = "/some/dir/"
+      provider.options.update(:local_dir => someDir)
+      expect(Dir).to receive(:glob).with(someDir + "/**/*").and_return([__FILE__])
       provider.push_app
     end
 
     example "Sends MIME type" do
-      expect(Dir).to receive(:glob).and_yield(__FILE__)
+      expect(Dir).to receive(:glob).and_return([__FILE__])
       expect_any_instance_of(Aws::S3::Object).to receive(:upload_file).with(anything(), hash_including(:content_type => 'application/x-ruby'))
       provider.push_app
     end
 
     example "Sets Cache and Expiration" do
       provider.options.update(:cache_control => "max-age=99999999", :expires => "2012-12-21 00:00:00 -0000")
-      expect(Dir).to receive(:glob).and_yield(__FILE__)
+      expect(Dir).to receive(:glob).and_return([__FILE__])
       expect_any_instance_of(Aws::S3::Object).to receive(:upload_file).with(anything(), hash_including(:cache_control => 'max-age=99999999', :expires => '2012-12-21 00:00:00 -0000'))
       provider.push_app
     end
@@ -108,7 +105,7 @@ describe DPL::Provider::S3 do
     example "Sets different Cache and Expiration" do
       option_list = []
       provider.options.update(:cache_control => ["max-age=99999999", "no-cache" => ["foo.html", "bar.txt"], "max-age=9999" => "*.txt"], :expires => ["2012-12-21 00:00:00 -0000", "1970-01-01 00:00:00 -0000" => "*.html"])
-      expect(Dir).to receive(:glob).and_yield("foo.html").and_yield("bar.txt").and_yield("baz.js")
+      expect(Dir).to receive(:glob).and_return(%w(foo.html bar.txt baz.js))
       allow_any_instance_of(Aws::S3::Object).to receive(:upload_file) do |obj, _data, options|
         option_list << { key: obj.key, options: options }
       end
@@ -122,28 +119,28 @@ describe DPL::Provider::S3 do
 
     example "Sets ACL" do
       provider.options.update(:acl => "public_read")
-      expect(Dir).to receive(:glob).and_yield(__FILE__)
+      expect(Dir).to receive(:glob).and_return([__FILE__])
       expect_any_instance_of(Aws::S3::Object).to receive(:upload_file).with(anything(), hash_including(:acl => "public-read"))
       provider.push_app
     end
 
     example "Sets Storage Class" do
       provider.options.update(:storage_class => "STANDARD_AI")
-      expect(Dir).to receive(:glob).and_yield(__FILE__)
+      expect(Dir).to receive(:glob).and_return([__FILE__])
       expect_any_instance_of(Aws::S3::Object).to receive(:upload_file).with(anything(), hash_including(:storage_class => "STANDARD_AI"))
       provider.push_app
     end
 
     example "Sets SSE" do
       provider.options.update(:server_side_encryption => true)
-      expect(Dir).to receive(:glob).and_yield(__FILE__)
+      expect(Dir).to receive(:glob).and_return([__FILE__])
       expect_any_instance_of(Aws::S3::Object).to receive(:upload_file).with(anything(), hash_including(:server_side_encryption => "AES256"))
       provider.push_app
     end
 
     example "Sets Website Index Document" do
       provider.options.update(:index_document_suffix => "test/index.html")
-      expect(Dir).to receive(:glob).and_yield(__FILE__)
+      expect(Dir).to receive(:glob).and_return([__FILE__])
       expect_any_instance_of(Aws::S3::BucketWebsite).to receive(:put).with(:website_configuration => { :index_document => { :suffix => "test/index.html" } })
       provider.push_app
     end
@@ -151,7 +148,7 @@ describe DPL::Provider::S3 do
     example "when detect_encoding is set" do
       path = 'foo.js'
       provider.options.update(:detect_encoding => true)
-      expect(Dir).to receive(:glob).and_yield(path)
+      expect(Dir).to receive(:glob).and_return([path])
       expect(provider).to receive(:`).at_least(1).times.with("file '#{path}'").and_return('gzip compressed')
       expect_any_instance_of(Aws::S3::Object).to receive(:upload_file).with(anything(), hash_including(:content_encoding => 'gzip'))
       provider.push_app
@@ -159,7 +156,7 @@ describe DPL::Provider::S3 do
 
     example "when dot_match is set" do
       provider.options.update(:dot_match => true)
-      expect(Dir).to receive(:glob).with("**/*", File::FNM_DOTMATCH)
+      expect(Dir).to receive(:glob).with(Dir.pwd + "/**/*", File::FNM_DOTMATCH).and_return([__FILE__])
       provider.push_app
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dpl-s3
 version: !ruby/object:Gem::Version
-  version: 1.9.2.travis.2737.5
+  version: 1.9.2.travis.2748.5
 platform: ruby
 authors:
 - Konstantin Haase
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-03-15 00:00:00.000000000 Z
+date: 2018-03-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dpl
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.9.2.travis.2737.5
+        version: 1.9.2.travis.2748.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.9.2.travis.2737.5
+        version: 1.9.2.travis.2748.5
 - !ruby/object:Gem::Dependency
   name: aws-sdk
   requirement: !ruby/object:Gem::Requirement