asset_sync 2.9.1 → 2.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +11 -1
- data/README.md +4 -0
- data/lib/asset_sync/config.rb +3 -0
- data/lib/asset_sync/storage.rb +14 -6
- data/lib/asset_sync/version.rb +1 -1
- data/spec/unit/storage_spec.rb +17 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 62e44f8d9bde08301b849aa0747f87400c01daadbcbe5a052e9d9862a8e3e32f
|
4
|
+
data.tar.gz: f23f8c1f4cf737a15c0f79d7d4ca94f4800d801c8443ae81d6eb25e0605ad866
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8aa520c8a0e507543986db116f5a3631161d85056e049acbf045fa4f3af6c9b71f7131146c5aa63c5aef5573d0efb25c451c51078d6b439ba476e157577cedb2
|
7
|
+
data.tar.gz: c507c91d2d928fa8fcd30d3413fa2f6b8b04faeb6dd21ac6831bddf36a5905823f446f507ad08fc182092ff4983cb0403221854051e7b8ba3c3a70f517231f84
|
data/CHANGELOG.md
CHANGED
@@ -18,6 +18,14 @@ This project adheres to [Semantic Versioning](http://semver.org/).
|
|
18
18
|
- Nothing
|
19
19
|
|
20
20
|
|
21
|
+
## [2.10.0] - 2020-02-26
|
22
|
+
|
23
|
+
### Added
|
24
|
+
|
25
|
+
- Add option `concurrent_uploads_max_threads` to limit number of threads for uploading files
|
26
|
+
(https://github.com/AssetSync/asset_sync/pull/398)
|
27
|
+
|
28
|
+
|
21
29
|
## [2.9.1] - 2020-02-20
|
22
30
|
|
23
31
|
### Fixed
|
@@ -946,7 +954,9 @@ Changes:
|
|
946
954
|
* Merge branch 'sinatra'
|
947
955
|
|
948
956
|
|
949
|
-
[Unreleased]: https://github.com/AssetSync/asset_sync/compare/v2.
|
957
|
+
[Unreleased]: https://github.com/AssetSync/asset_sync/compare/v2.10.0...HEAD
|
958
|
+
[2.10.0]: https://github.com/AssetSync/asset_sync/compare/v2.9.1...v2.10.0
|
959
|
+
[2.9.1]: https://github.com/AssetSync/asset_sync/compare/v2.9.0...v2.9.1
|
950
960
|
[2.9.0]: https://github.com/AssetSync/asset_sync/compare/v2.8.2...v2.9.0
|
951
961
|
[2.8.2]: https://github.com/AssetSync/asset_sync/compare/v2.8.1...v2.8.2
|
952
962
|
[2.8.1]: https://github.com/AssetSync/asset_sync/compare/v2.8.0...v2.8.1
|
data/README.md
CHANGED
@@ -244,6 +244,9 @@ AssetSync.configure do |config|
|
|
244
244
|
#
|
245
245
|
# Upload files concurrently
|
246
246
|
# config.concurrent_uploads = false
|
247
|
+
#
|
248
|
+
# Number of threads when concurrent_uploads is enabled
|
249
|
+
# config.concurrent_uploads_max_threads = 10
|
247
250
|
#
|
248
251
|
# Fail silently. Useful for environments such as Heroku
|
249
252
|
# config.fail_silently = true
|
@@ -342,6 +345,7 @@ AssetSync.config.gzip_compression == ENV['ASSET_SYNC_GZIP_COMPRESSION']
|
|
342
345
|
* **manifest**: (`true, false`) when enabled, will use the `manifest.yml` generated by Rails to get the list of local files to upload. **experimental**. **default:** `'false'`
|
343
346
|
* **include_manifest**: (`true, false`) when enabled, will upload the `manifest.yml` generated by Rails. **default:** `'false'`
|
344
347
|
* **concurrent_uploads**: (`true, false`) when enabled, will upload the files in different Threads, this greatly improves the upload speed. **default:** `'false'`
|
348
|
+
* **concurrent_uploads_max_threads**: when concurrent_uploads is enabled, this determines the number of threads that will be created. **default:** `10`
|
345
349
|
* **enabled**: (`true, false`) when false, will disable asset sync. **default:** `'true'` (enabled)
|
346
350
|
* **ignored\_files**: an array of files to ignore e.g. `['ignore_me.js', %r(ignore_some/\d{32}\.css)]` Useful if there are some files that are created dynamically on the server and you don't want to upload on deploy **default**: `[]`
|
347
351
|
* **cache\_asset\_regexps**: an array of files to add cache headers e.g. `['cache_me.js', %r(cache_some\.\d{8}\.css)]` Useful if there are some files that are added to sprockets assets list and need to be set as 'Cacheable' on uploaded server. Only rails compiled regexp is matched internally **default**: `[]`
|
data/lib/asset_sync/config.rb
CHANGED
@@ -27,6 +27,7 @@ module AssetSync
|
|
27
27
|
attr_accessor :cache_asset_regexps
|
28
28
|
attr_accessor :include_manifest
|
29
29
|
attr_accessor :concurrent_uploads
|
30
|
+
attr_accessor :concurrent_uploads_max_threads
|
30
31
|
|
31
32
|
# FOG configuration
|
32
33
|
attr_accessor :fog_provider # Currently Supported ['AWS', 'Rackspace']
|
@@ -86,6 +87,7 @@ module AssetSync
|
|
86
87
|
self.cache_asset_regexps = []
|
87
88
|
self.include_manifest = false
|
88
89
|
self.concurrent_uploads = false
|
90
|
+
self.concurrent_uploads_max_threads = 10
|
89
91
|
@additional_local_file_paths_procs = []
|
90
92
|
|
91
93
|
load_yml! if defined?(::Rails) && yml_exists?
|
@@ -222,6 +224,7 @@ module AssetSync
|
|
222
224
|
self.cache_asset_regexps = yml['cache_asset_regexps'] if yml.has_key?("cache_asset_regexps")
|
223
225
|
self.include_manifest = yml['include_manifest'] if yml.has_key?("include_manifest")
|
224
226
|
self.concurrent_uploads = yml['concurrent_uploads'] if yml.has_key?('concurrent_uploads')
|
227
|
+
self.concurrent_uploads_max_threads = yml['concurrent_uploads_max_threads'] if yml.has_key?('concurrent_uploads_max_threads')
|
225
228
|
|
226
229
|
self.azure_storage_account_name = yml['azure_storage_account_name'] if yml.has_key?("azure_storage_account_name")
|
227
230
|
self.azure_storage_access_key = yml['azure_storage_access_key'] if yml.has_key?("azure_storage_access_key")
|
data/lib/asset_sync/storage.rb
CHANGED
@@ -248,19 +248,27 @@ module AssetSync
|
|
248
248
|
# fixes: https://github.com/rumblelabs/asset_sync/issues/19
|
249
249
|
local_files_to_upload = local_files - ignored_files - remote_files + always_upload_files
|
250
250
|
local_files_to_upload = (local_files_to_upload + get_non_fingerprinted(local_files_to_upload)).uniq
|
251
|
+
# Only files.
|
252
|
+
local_files_to_upload = local_files_to_upload.select { |f| File.file? "#{path}/#{f}" }
|
251
253
|
|
252
254
|
if self.config.concurrent_uploads
|
253
|
-
|
255
|
+
jobs = Queue.new
|
256
|
+
local_files_to_upload.each { |f| jobs.push(f) }
|
257
|
+
jobs.close
|
258
|
+
|
259
|
+
num_threads = [self.config.concurrent_uploads_max_threads, local_files_to_upload.length].min
|
254
260
|
# Upload new files
|
255
|
-
|
256
|
-
|
257
|
-
|
261
|
+
workers = Array.new(num_threads) do
|
262
|
+
Thread.new do
|
263
|
+
while f = jobs.pop
|
264
|
+
upload_file(f)
|
265
|
+
end
|
266
|
+
end
|
258
267
|
end
|
259
|
-
|
268
|
+
workers.map(&:join)
|
260
269
|
else
|
261
270
|
# Upload new files
|
262
271
|
local_files_to_upload.each do |f|
|
263
|
-
next unless File.file? "#{path}/#{f}" # Only files.
|
264
272
|
upload_file f
|
265
273
|
end
|
266
274
|
end
|
data/lib/asset_sync/version.rb
CHANGED
data/spec/unit/storage_spec.rb
CHANGED
@@ -70,6 +70,23 @@ describe AssetSync::Storage do
|
|
70
70
|
storage.upload_files
|
71
71
|
end
|
72
72
|
|
73
|
+
it 'should allow custom number of threads' do
|
74
|
+
@config.concurrent_uploads = true
|
75
|
+
@config.concurrent_uploads_max_threads = 2
|
76
|
+
storage = AssetSync::Storage.new(@config)
|
77
|
+
|
78
|
+
allow(storage).to receive(:get_local_files).and_return(@local_files)
|
79
|
+
allow(storage).to receive(:get_remote_files).and_return(@remote_files)
|
80
|
+
allow(File).to receive(:file?).and_return(true) # Pretend they all exist
|
81
|
+
|
82
|
+
expect(Thread).to receive(:new).exactly(2).times.and_call_original
|
83
|
+
(@local_files - @remote_files + storage.always_upload_files).each do |file|
|
84
|
+
expect(storage).to receive(:upload_file).with(file)
|
85
|
+
end
|
86
|
+
|
87
|
+
storage.upload_files
|
88
|
+
end
|
89
|
+
|
73
90
|
it 'should upload updated non-fingerprinted files' do
|
74
91
|
@local_files = [
|
75
92
|
'public/image.png',
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: asset_sync
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.10.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Simon Hamilton
|
@@ -11,7 +11,7 @@ authors:
|
|
11
11
|
autorequire:
|
12
12
|
bindir: bin
|
13
13
|
cert_chain: []
|
14
|
-
date: 2020-02-
|
14
|
+
date: 2020-02-26 00:00:00.000000000 Z
|
15
15
|
dependencies:
|
16
16
|
- !ruby/object:Gem::Dependency
|
17
17
|
name: fog-core
|