asset_sync 2.9.1 → 2.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 454e84d789424886800c6c11f0576701eba664d42cfbf929b3b223d6d60383d0
4
- data.tar.gz: 349569046fa6e19ff1ac6a27527f6e7de300b5953b996ec04c8f2e9ecf272e13
3
+ metadata.gz: 62e44f8d9bde08301b849aa0747f87400c01daadbcbe5a052e9d9862a8e3e32f
4
+ data.tar.gz: f23f8c1f4cf737a15c0f79d7d4ca94f4800d801c8443ae81d6eb25e0605ad866
5
5
  SHA512:
6
- metadata.gz: e8392af4e3c757f407b831b12ed76dc200da5a7aa5ddc9601c573be0fd965336fc9cfd4affb49d8420eb7edc95dcf1dba697e5ae59dfa5de7d445dd51efbf236
7
- data.tar.gz: 4a4a1de27d7e42edc656e6ca792eaa62afe55484bfa76c1d565148af685698ad32265eb7c5752fa3ac1fe2f0216bfefb1c25e9ea3fd97fcb4796deef4cf3c689
6
+ metadata.gz: 8aa520c8a0e507543986db116f5a3631161d85056e049acbf045fa4f3af6c9b71f7131146c5aa63c5aef5573d0efb25c451c51078d6b439ba476e157577cedb2
7
+ data.tar.gz: c507c91d2d928fa8fcd30d3413fa2f6b8b04faeb6dd21ac6831bddf36a5905823f446f507ad08fc182092ff4983cb0403221854051e7b8ba3c3a70f517231f84
@@ -18,6 +18,14 @@ This project adheres to [Semantic Versioning](http://semver.org/).
18
18
  - Nothing
19
19
 
20
20
 
21
+ ## [2.10.0] - 2020-02-26
22
+
23
+ ### Added
24
+
25
+ - Add option `concurrent_uploads_max_threads` to limit number of threads for uploading files
26
+ (https://github.com/AssetSync/asset_sync/pull/398)
27
+
28
+
21
29
  ## [2.9.1] - 2020-02-20
22
30
 
23
31
  ### Fixed
@@ -946,7 +954,9 @@ Changes:
946
954
  * Merge branch 'sinatra'
947
955
 
948
956
 
949
- [Unreleased]: https://github.com/AssetSync/asset_sync/compare/v2.9.0...HEAD
957
+ [Unreleased]: https://github.com/AssetSync/asset_sync/compare/v2.10.0...HEAD
958
+ [2.10.0]: https://github.com/AssetSync/asset_sync/compare/v2.9.1...v2.10.0
959
+ [2.9.1]: https://github.com/AssetSync/asset_sync/compare/v2.9.0...v2.9.1
950
960
  [2.9.0]: https://github.com/AssetSync/asset_sync/compare/v2.8.2...v2.9.0
951
961
  [2.8.2]: https://github.com/AssetSync/asset_sync/compare/v2.8.1...v2.8.2
952
962
  [2.8.1]: https://github.com/AssetSync/asset_sync/compare/v2.8.0...v2.8.1
data/README.md CHANGED
@@ -244,6 +244,9 @@ AssetSync.configure do |config|
244
244
  #
245
245
  # Upload files concurrently
246
246
  # config.concurrent_uploads = false
247
+ #
248
+ # Number of threads when concurrent_uploads is enabled
249
+ # config.concurrent_uploads_max_threads = 10
247
250
  #
248
251
  # Fail silently. Useful for environments such as Heroku
249
252
  # config.fail_silently = true
@@ -342,6 +345,7 @@ AssetSync.config.gzip_compression == ENV['ASSET_SYNC_GZIP_COMPRESSION']
342
345
  * **manifest**: (`true, false`) when enabled, will use the `manifest.yml` generated by Rails to get the list of local files to upload. **experimental**. **default:** `'false'`
343
346
  * **include_manifest**: (`true, false`) when enabled, will upload the `manifest.yml` generated by Rails. **default:** `'false'`
344
347
  * **concurrent_uploads**: (`true, false`) when enabled, will upload the files in different Threads, this greatly improves the upload speed. **default:** `'false'`
348
+ * **concurrent_uploads_max_threads**: when concurrent_uploads is enabled, this determines the number of threads that will be created. **default:** `10`
345
349
  * **enabled**: (`true, false`) when false, will disable asset sync. **default:** `'true'` (enabled)
346
350
  * **ignored\_files**: an array of files to ignore e.g. `['ignore_me.js', %r(ignore_some/\d{32}\.css)]` Useful if there are some files that are created dynamically on the server and you don't want to upload on deploy **default**: `[]`
347
351
  * **cache\_asset\_regexps**: an array of files to add cache headers e.g. `['cache_me.js', %r(cache_some\.\d{8}\.css)]` Useful if there are some files that are added to sprockets assets list and need to be set as 'Cacheable' on uploaded server. Only rails compiled regexp is matched internally **default**: `[]`
@@ -27,6 +27,7 @@ module AssetSync
27
27
  attr_accessor :cache_asset_regexps
28
28
  attr_accessor :include_manifest
29
29
  attr_accessor :concurrent_uploads
30
+ attr_accessor :concurrent_uploads_max_threads
30
31
 
31
32
  # FOG configuration
32
33
  attr_accessor :fog_provider # Currently Supported ['AWS', 'Rackspace']
@@ -86,6 +87,7 @@ module AssetSync
86
87
  self.cache_asset_regexps = []
87
88
  self.include_manifest = false
88
89
  self.concurrent_uploads = false
90
+ self.concurrent_uploads_max_threads = 10
89
91
  @additional_local_file_paths_procs = []
90
92
 
91
93
  load_yml! if defined?(::Rails) && yml_exists?
@@ -222,6 +224,7 @@ module AssetSync
222
224
  self.cache_asset_regexps = yml['cache_asset_regexps'] if yml.has_key?("cache_asset_regexps")
223
225
  self.include_manifest = yml['include_manifest'] if yml.has_key?("include_manifest")
224
226
  self.concurrent_uploads = yml['concurrent_uploads'] if yml.has_key?('concurrent_uploads')
227
+ self.concurrent_uploads_max_threads = yml['concurrent_uploads_max_threads'] if yml.has_key?('concurrent_uploads_max_threads')
225
228
 
226
229
  self.azure_storage_account_name = yml['azure_storage_account_name'] if yml.has_key?("azure_storage_account_name")
227
230
  self.azure_storage_access_key = yml['azure_storage_access_key'] if yml.has_key?("azure_storage_access_key")
@@ -248,19 +248,27 @@ module AssetSync
248
248
  # fixes: https://github.com/rumblelabs/asset_sync/issues/19
249
249
  local_files_to_upload = local_files - ignored_files - remote_files + always_upload_files
250
250
  local_files_to_upload = (local_files_to_upload + get_non_fingerprinted(local_files_to_upload)).uniq
251
+ # Only files.
252
+ local_files_to_upload = local_files_to_upload.select { |f| File.file? "#{path}/#{f}" }
251
253
 
252
254
  if self.config.concurrent_uploads
253
- threads = ThreadGroup.new
255
+ jobs = Queue.new
256
+ local_files_to_upload.each { |f| jobs.push(f) }
257
+ jobs.close
258
+
259
+ num_threads = [self.config.concurrent_uploads_max_threads, local_files_to_upload.length].min
254
260
  # Upload new files
255
- local_files_to_upload.each do |f|
256
- next unless File.file? "#{path}/#{f}" # Only files.
257
- threads.add(Thread.new { upload_file f })
261
+ workers = Array.new(num_threads) do
262
+ Thread.new do
263
+ while f = jobs.pop
264
+ upload_file(f)
265
+ end
266
+ end
258
267
  end
259
- sleep 1 while threads.list.any? # wait for threads to finish uploading
268
+ workers.map(&:join)
260
269
  else
261
270
  # Upload new files
262
271
  local_files_to_upload.each do |f|
263
- next unless File.file? "#{path}/#{f}" # Only files.
264
272
  upload_file f
265
273
  end
266
274
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module AssetSync
4
- VERSION = "2.9.1"
4
+ VERSION = "2.10.0"
5
5
  end
@@ -70,6 +70,23 @@ describe AssetSync::Storage do
70
70
  storage.upload_files
71
71
  end
72
72
 
73
+ it 'should allow custom number of threads' do
74
+ @config.concurrent_uploads = true
75
+ @config.concurrent_uploads_max_threads = 2
76
+ storage = AssetSync::Storage.new(@config)
77
+
78
+ allow(storage).to receive(:get_local_files).and_return(@local_files)
79
+ allow(storage).to receive(:get_remote_files).and_return(@remote_files)
80
+ allow(File).to receive(:file?).and_return(true) # Pretend they all exist
81
+
82
+ expect(Thread).to receive(:new).exactly(2).times.and_call_original
83
+ (@local_files - @remote_files + storage.always_upload_files).each do |file|
84
+ expect(storage).to receive(:upload_file).with(file)
85
+ end
86
+
87
+ storage.upload_files
88
+ end
89
+
73
90
  it 'should upload updated non-fingerprinted files' do
74
91
  @local_files = [
75
92
  'public/image.png',
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: asset_sync
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.9.1
4
+ version: 2.10.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Simon Hamilton
@@ -11,7 +11,7 @@ authors:
11
11
  autorequire:
12
12
  bindir: bin
13
13
  cert_chain: []
14
- date: 2020-02-20 00:00:00.000000000 Z
14
+ date: 2020-02-26 00:00:00.000000000 Z
15
15
  dependencies:
16
16
  - !ruby/object:Gem::Dependency
17
17
  name: fog-core