asset_sync 2.10.0 → 2.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 62e44f8d9bde08301b849aa0747f87400c01daadbcbe5a052e9d9862a8e3e32f
-  data.tar.gz: f23f8c1f4cf737a15c0f79d7d4ca94f4800d801c8443ae81d6eb25e0605ad866
+  metadata.gz: e7e222d9419b7c2039f3db9a20676ec264eaaa233c9d954fd62a28a56bdf2968
+  data.tar.gz: 3fc2471434cd60071d283a59cef97f0ff845c4066161ce8c99b93d08435abe66
 SHA512:
-  metadata.gz: 8aa520c8a0e507543986db116f5a3631161d85056e049acbf045fa4f3af6c9b71f7131146c5aa63c5aef5573d0efb25c451c51078d6b439ba476e157577cedb2
-  data.tar.gz: c507c91d2d928fa8fcd30d3413fa2f6b8b04faeb6dd21ac6831bddf36a5905823f446f507ad08fc182092ff4983cb0403221854051e7b8ba3c3a70f517231f84
+  metadata.gz: fd001d4c15476c0b72b10bf8630a08a52910189b583b000adb0b3ceea925cd66f6e0bf94e72a2774213e5133be03c4dfc6e8d9d58ba8d4b8f0b5df393a52b853
+  data.tar.gz: 8115434ee5a0d37f09350fa3e4c1e4402b47635c7b04eff6f638df65c08edca9a5ba76f05c7993ea09cc57ecd2447b16b002d85b5cb6fc0cc6d24199caddbef6

data/CHANGELOG.md

@@ -18,6 +18,14 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 - Nothing
 
 
+## [2.11.0] - 2020-03-13
+
+### Added
+
+- Add option `remote_file_list_cache_file_path` to skip scanning remote
+  (https://github.com/AssetSync/asset_sync/pull/400)
+
+
 ## [2.10.0] - 2020-02-26
 
 ### Added
@@ -954,7 +962,8 @@ Changes:
     * Merge branch 'sinatra'
 
 
-[Unreleased]: https://github.com/AssetSync/asset_sync/compare/v2.10.0...HEAD
+[Unreleased]: https://github.com/AssetSync/asset_sync/compare/v2.11.0...HEAD
+[2.11.0]: https://github.com/AssetSync/asset_sync/compare/v2.10.0...v2.11.0
 [2.10.0]: https://github.com/AssetSync/asset_sync/compare/v2.9.1...v2.10.0
 [2.9.1]: https://github.com/AssetSync/asset_sync/compare/v2.9.0...v2.9.1
 [2.9.0]: https://github.com/AssetSync/asset_sync/compare/v2.8.2...v2.9.0

data/README.md

@@ -248,6 +248,9 @@ AssetSync.configure do |config|
   # Number of threads when concurrent_uploads is enabled
   # config.concurrent_uploads_max_threads = 10
   #
+  # Path to cache file to skip scanning remote
+  # config.remote_file_list_cache_file_path = './.asset_sync_remote_file_list_cache.json'
+  #
   # Fail silently.  Useful for environments such as Heroku
   # config.fail_silently = true
   #
@@ -346,6 +349,7 @@ AssetSync.config.gzip_compression == ENV['ASSET_SYNC_GZIP_COMPRESSION']
 * **include_manifest**: (`true, false`) when enabled, will upload the `manifest.yml` generated by Rails. **default:** `'false'`
 * **concurrent_uploads**: (`true, false`) when enabled, will upload the files in different Threads, this greatly improves the upload speed. **default:** `'false'`
 * **concurrent_uploads_max_threads**: when concurrent_uploads is enabled, this determines the number of threads that will be created. **default:** `10`
+* **remote_file_list_cache_file_path**: if present, use this path to cache remote file list to skip scanning remote **default:** `nil`
 * **enabled**: (`true, false`) when false, will disable asset sync. **default:** `'true'` (enabled)
 * **ignored\_files**: an array of files to ignore e.g. `['ignore_me.js', %r(ignore_some/\d{32}\.css)]` Useful if there are some files that are created dynamically on the server and you don't want to upload on deploy **default**: `[]`
 * **cache\_asset\_regexps**: an array of files to add cache headers e.g. `['cache_me.js', %r(cache_some\.\d{8}\.css)]` Useful if there are some files that are added to sprockets assets list and need to be set as 'Cacheable' on uploaded server.  Only rails compiled regexp is matched internally **default**: `[]`

data/lib/asset_sync/config.rb

@@ -28,6 +28,7 @@ module AssetSync
     attr_accessor :include_manifest
     attr_accessor :concurrent_uploads
     attr_accessor :concurrent_uploads_max_threads
+    attr_accessor :remote_file_list_cache_file_path
 
     # FOG configuration
     attr_accessor :fog_provider          # Currently Supported ['AWS', 'Rackspace']
@@ -88,6 +89,7 @@ module AssetSync
       self.include_manifest = false
       self.concurrent_uploads = false
       self.concurrent_uploads_max_threads = 10
+      self.remote_file_list_cache_file_path = nil
       @additional_local_file_paths_procs = []
 
       load_yml! if defined?(::Rails) && yml_exists?
@@ -225,6 +227,7 @@ module AssetSync
       self.include_manifest       = yml['include_manifest'] if yml.has_key?("include_manifest")
       self.concurrent_uploads     = yml['concurrent_uploads'] if yml.has_key?('concurrent_uploads')
       self.concurrent_uploads_max_threads = yml['concurrent_uploads_max_threads'] if yml.has_key?('concurrent_uploads_max_threads')
+      self.remote_file_list_cache_file_path = yml['remote_file_list_cache_file_path'] if yml.has_key?('remote_file_list_cache_file_path')
 
       self.azure_storage_account_name = yml['azure_storage_account_name'] if yml.has_key?("azure_storage_account_name")
       self.azure_storage_access_key   = yml['azure_storage_access_key'] if yml.has_key?("azure_storage_access_key")

data/lib/asset_sync/engine.rb

@@ -43,6 +43,7 @@ module AssetSync
           config.manifest = (ENV['ASSET_SYNC_MANIFEST'] == 'true') if ENV.has_key?('ASSET_SYNC_MANIFEST')
           config.include_manifest = (ENV['ASSET_SYNC_INCLUDE_MANIFEST'] == 'true') if ENV.has_key?('ASSET_SYNC_INCLUDE_MANIFEST')
           config.concurrent_uploads = (ENV['ASSET_SYNC_CONCURRENT_UPLOADS'] == 'true') if ENV.has_key?('ASSET_SYNC_CONCURRENT_UPLOADS')
+          config.remote_file_list_cache_file_path = ENV['ASSET_SYNC_REMOTE_FILE_LIST_CACHE_FILE_PATH'] if ENV.has_key?('ASSET_SYNC_REMOTE_FILE_LIST_CACHE_FILE_PATH')
         end
 
         config.prefix = ENV['ASSET_SYNC_PREFIX'] if ENV.has_key?('ASSET_SYNC_PREFIX')

data/lib/asset_sync/storage.rb

@@ -37,6 +37,10 @@ module AssetSync
       self.config.public_path
     end
 
+    def remote_file_list_cache_file_path
+      self.config.remote_file_list_cache_file_path
+    end
+
     def ignored_files
       expand_file_names(self.config.ignored_files)
     end
@@ -58,6 +62,32 @@ module AssetSync
         (get_local_files + config.additional_local_file_paths).uniq
     end
 
+    def remote_files
+      return [] if ignore_existing_remote_files?
+      return @remote_files if @remote_files
+
+      if remote_file_list_cache_file_path && File.file?(remote_file_list_cache_file_path)
+        begin
+          content = File.read(remote_file_list_cache_file_path)
+          return @remote_files = JSON.parse(content)
+        rescue JSON::ParserError
+          warn "Failed to parse #{remote_file_list_cache_file_path} as json"
+        end
+      end
+
+      @remote_files = get_remote_files
+    end
+
+    def update_remote_file_list_cache(local_files_to_upload)
+      return unless remote_file_list_cache_file_path
+      return if ignore_existing_remote_files?
+
+      File.open(self.remote_file_list_cache_file_path, 'w') do |file|
+        uploaded = local_files_to_upload + remote_files
+        file.write(uploaded.to_json)
+      end
+    end
+
     def always_upload_files
       expand_file_names(self.config.always_upload) + get_manifest_path
     end
@@ -243,8 +273,6 @@ module AssetSync
     end
 
     def upload_files
-      # get a fresh list of remote files
-      remote_files = ignore_existing_remote_files? ? [] : get_remote_files
       # fixes: https://github.com/rumblelabs/asset_sync/issues/19
       local_files_to_upload = local_files - ignored_files - remote_files + always_upload_files
       local_files_to_upload = (local_files_to_upload + get_non_fingerprinted(local_files_to_upload)).uniq
@@ -279,6 +307,8 @@ module AssetSync
         data = cdn.post_invalidation(self.config.cdn_distribution_id, files_to_invalidate)
         log "Invalidation id: #{data.body["Id"]}"
       end
+
+      update_remote_file_list_cache(local_files_to_upload)
     end
 
     def sync

data/lib/asset_sync/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module AssetSync
-  VERSION = "2.10.0"
+  VERSION = "2.11.0"
 end

data/lib/generators/asset_sync/templates/asset_sync.rb

@@ -66,6 +66,9 @@ if defined?(AssetSync)
     # Upload files concurrently
     # config.concurrent_uploads = false
     #
+    # Path to cache file to skip scanning remote
+    # config.remote_file_list_cache_file_path = './.asset_sync_remote_file_list_cache.json'
+    #
     # Fail silently.  Useful for environments such as Heroku
     # config.fail_silently = true
     #

data/spec/unit/storage_spec.rb

@@ -87,6 +87,54 @@ describe AssetSync::Storage do
       storage.upload_files
     end
 
+    it 'should allow remote_file_list_cache_file_path configuration' do
+      file_path = './foo.json'
+      @config.remote_file_list_cache_file_path = file_path
+      storage = AssetSync::Storage.new(@config)
+
+      allow(storage).to receive(:get_local_files).and_return(@local_files)
+      File.write(file_path, @remote_files.to_json)
+      expect(storage).not_to receive(:get_remote_files)
+      allow(File).to receive(:file?).and_return(true) # Pretend they all exist
+
+      (@local_files - @remote_files + storage.always_upload_files).each do |file|
+        expect(storage).to receive(:upload_file).with(file)
+      end
+
+      expect(storage).not_to receive(:warn)
+      storage.upload_files
+
+      # update remote_file_list_cache corretly
+      updated = JSON.parse(File.read(file_path))
+      expect(updated.sort.uniq).to eq (@remote_files + @local_files + storage.always_upload_files).sort.uniq
+
+      File.delete(file_path)
+    end
+
+    it 'should work with broken cache' do
+      file_path = './foo.json'
+      @config.remote_file_list_cache_file_path = file_path
+
+      storage = AssetSync::Storage.new(@config)
+
+      File.write(file_path, 'some non-json text file content')
+
+      allow(storage).to receive(:get_local_files).and_return(@local_files)
+      allow(storage).to receive(:get_remote_files).and_return(@remote_files)
+      allow(File).to receive(:file?).and_return(true) # Pretend they all exist
+
+      (@local_files - @remote_files + storage.always_upload_files).each do |file|
+        expect(storage).to receive(:upload_file).with(file)
+      end
+
+      # when broken, warning message should be prompted
+      expect(storage).to receive(:warn)
+
+      storage.upload_files
+
+      File.delete(file_path)
+    end
+
     it 'should upload updated non-fingerprinted files' do
       @local_files = [
         'public/image.png',

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: asset_sync
 version: !ruby/object:Gem::Version
-  version: 2.10.0
+  version: 2.11.0
 platform: ruby
 authors:
 - Simon Hamilton
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-02-26 00:00:00.000000000 Z
+date: 2020-03-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fog-core