asset_sync 2.9.1 → 2.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 454e84d789424886800c6c11f0576701eba664d42cfbf929b3b223d6d60383d0
-  data.tar.gz: 349569046fa6e19ff1ac6a27527f6e7de300b5953b996ec04c8f2e9ecf272e13
+  metadata.gz: 62e44f8d9bde08301b849aa0747f87400c01daadbcbe5a052e9d9862a8e3e32f
+  data.tar.gz: f23f8c1f4cf737a15c0f79d7d4ca94f4800d801c8443ae81d6eb25e0605ad866
 SHA512:
-  metadata.gz: e8392af4e3c757f407b831b12ed76dc200da5a7aa5ddc9601c573be0fd965336fc9cfd4affb49d8420eb7edc95dcf1dba697e5ae59dfa5de7d445dd51efbf236
-  data.tar.gz: 4a4a1de27d7e42edc656e6ca792eaa62afe55484bfa76c1d565148af685698ad32265eb7c5752fa3ac1fe2f0216bfefb1c25e9ea3fd97fcb4796deef4cf3c689
+  metadata.gz: 8aa520c8a0e507543986db116f5a3631161d85056e049acbf045fa4f3af6c9b71f7131146c5aa63c5aef5573d0efb25c451c51078d6b439ba476e157577cedb2
+  data.tar.gz: c507c91d2d928fa8fcd30d3413fa2f6b8b04faeb6dd21ac6831bddf36a5905823f446f507ad08fc182092ff4983cb0403221854051e7b8ba3c3a70f517231f84

data/CHANGELOG.md

@@ -18,6 +18,14 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 - Nothing
 
 
+## [2.10.0] - 2020-02-26
+
+### Added
+
+- Add option `concurrent_uploads_max_threads` to limit number of threads for uploading files
+  (https://github.com/AssetSync/asset_sync/pull/398)
+
+
 ## [2.9.1] - 2020-02-20
 
 ### Fixed
@@ -946,7 +954,9 @@ Changes:
     * Merge branch 'sinatra'
 
 
-[Unreleased]: https://github.com/AssetSync/asset_sync/compare/v2.9.0...HEAD
+[Unreleased]: https://github.com/AssetSync/asset_sync/compare/v2.10.0...HEAD
+[2.10.0]: https://github.com/AssetSync/asset_sync/compare/v2.9.1...v2.10.0
+[2.9.1]: https://github.com/AssetSync/asset_sync/compare/v2.9.0...v2.9.1
 [2.9.0]: https://github.com/AssetSync/asset_sync/compare/v2.8.2...v2.9.0
 [2.8.2]: https://github.com/AssetSync/asset_sync/compare/v2.8.1...v2.8.2
 [2.8.1]: https://github.com/AssetSync/asset_sync/compare/v2.8.0...v2.8.1

data/README.md

@@ -244,6 +244,9 @@ AssetSync.configure do |config|
   #
   # Upload files concurrently
   # config.concurrent_uploads = false
+  # 
+  # Number of threads when concurrent_uploads is enabled
+  # config.concurrent_uploads_max_threads = 10
   #
   # Fail silently.  Useful for environments such as Heroku
   # config.fail_silently = true
@@ -342,6 +345,7 @@ AssetSync.config.gzip_compression == ENV['ASSET_SYNC_GZIP_COMPRESSION']
 * **manifest**: (`true, false`) when enabled, will use the `manifest.yml` generated by Rails to get the list of local files to upload. **experimental**. **default:** `'false'`
 * **include_manifest**: (`true, false`) when enabled, will upload the `manifest.yml` generated by Rails. **default:** `'false'`
 * **concurrent_uploads**: (`true, false`) when enabled, will upload the files in different Threads, this greatly improves the upload speed. **default:** `'false'`
+* **concurrent_uploads_max_threads**: when concurrent_uploads is enabled, this determines the number of threads that will be created. **default:** `10`
 * **enabled**: (`true, false`) when false, will disable asset sync. **default:** `'true'` (enabled)
 * **ignored\_files**: an array of files to ignore e.g. `['ignore_me.js', %r(ignore_some/\d{32}\.css)]` Useful if there are some files that are created dynamically on the server and you don't want to upload on deploy **default**: `[]`
 * **cache\_asset\_regexps**: an array of files to add cache headers e.g. `['cache_me.js', %r(cache_some\.\d{8}\.css)]` Useful if there are some files that are added to sprockets assets list and need to be set as 'Cacheable' on uploaded server.  Only rails compiled regexp is matched internally **default**: `[]`

data/lib/asset_sync/config.rb

@@ -27,6 +27,7 @@ module AssetSync
     attr_accessor :cache_asset_regexps
     attr_accessor :include_manifest
     attr_accessor :concurrent_uploads
+    attr_accessor :concurrent_uploads_max_threads
 
     # FOG configuration
     attr_accessor :fog_provider          # Currently Supported ['AWS', 'Rackspace']
@@ -86,6 +87,7 @@ module AssetSync
       self.cache_asset_regexps = []
       self.include_manifest = false
       self.concurrent_uploads = false
+      self.concurrent_uploads_max_threads = 10
       @additional_local_file_paths_procs = []
 
       load_yml! if defined?(::Rails) && yml_exists?
@@ -222,6 +224,7 @@ module AssetSync
       self.cache_asset_regexps    = yml['cache_asset_regexps'] if yml.has_key?("cache_asset_regexps")
       self.include_manifest       = yml['include_manifest'] if yml.has_key?("include_manifest")
       self.concurrent_uploads     = yml['concurrent_uploads'] if yml.has_key?('concurrent_uploads')
+      self.concurrent_uploads_max_threads = yml['concurrent_uploads_max_threads'] if yml.has_key?('concurrent_uploads_max_threads')
 
       self.azure_storage_account_name = yml['azure_storage_account_name'] if yml.has_key?("azure_storage_account_name")
       self.azure_storage_access_key   = yml['azure_storage_access_key'] if yml.has_key?("azure_storage_access_key")

data/lib/asset_sync/storage.rb

@@ -248,19 +248,27 @@ module AssetSync
       # fixes: https://github.com/rumblelabs/asset_sync/issues/19
       local_files_to_upload = local_files - ignored_files - remote_files + always_upload_files
       local_files_to_upload = (local_files_to_upload + get_non_fingerprinted(local_files_to_upload)).uniq
+      # Only files.
+      local_files_to_upload = local_files_to_upload.select { |f| File.file? "#{path}/#{f}" }
 
       if self.config.concurrent_uploads
-        threads = ThreadGroup.new
+        jobs = Queue.new
+        local_files_to_upload.each { |f| jobs.push(f) }
+        jobs.close
+
+        num_threads = [self.config.concurrent_uploads_max_threads, local_files_to_upload.length].min
         # Upload new files
-        local_files_to_upload.each do |f|
-          next unless File.file? "#{path}/#{f}" # Only files.
-          threads.add(Thread.new { upload_file f })
+        workers = Array.new(num_threads) do
+          Thread.new do
+            while f = jobs.pop
+              upload_file(f)
+            end
+          end
         end
-        sleep 1 while threads.list.any? # wait for threads to finish uploading
+        workers.map(&:join)
       else
         # Upload new files
         local_files_to_upload.each do |f|
-          next unless File.file? "#{path}/#{f}" # Only files.
           upload_file f
         end
       end

data/lib/asset_sync/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module AssetSync
-  VERSION = "2.9.1"
+  VERSION = "2.10.0"
 end

data/spec/unit/storage_spec.rb

@@ -70,6 +70,23 @@ describe AssetSync::Storage do
       storage.upload_files
     end
 
+    it 'should allow custom number of threads' do
+      @config.concurrent_uploads = true
+      @config.concurrent_uploads_max_threads = 2
+      storage = AssetSync::Storage.new(@config)
+
+      allow(storage).to receive(:get_local_files).and_return(@local_files)
+      allow(storage).to receive(:get_remote_files).and_return(@remote_files)
+      allow(File).to receive(:file?).and_return(true) # Pretend they all exist
+
+      expect(Thread).to receive(:new).exactly(2).times.and_call_original
+      (@local_files - @remote_files + storage.always_upload_files).each do |file|
+        expect(storage).to receive(:upload_file).with(file)
+      end
+
+      storage.upload_files
+    end
+
     it 'should upload updated non-fingerprinted files' do
       @local_files = [
         'public/image.png',

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: asset_sync
 version: !ruby/object:Gem::Version
-  version: 2.9.1
+  version: 2.10.0
 platform: ruby
 authors:
 - Simon Hamilton
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-02-20 00:00:00.000000000 Z
+date: 2020-02-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fog-core