asset_sync 2.9.0 → 2.12.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 76d23b058e01eb57bee9ae81d0a64eb9c36a9f46b00d33909ba41f0f19bb1665
-  data.tar.gz: d3693e883569e95091090c80a8d0d071e90c7d59f2763d690adf837ab14a5388
+  metadata.gz: 1fbb82cb5023ce3d6e1f088aa62fa32718f8e743977c31b773e3f2a324a0a397
+  data.tar.gz: 504e104e3dfe839bb19b0794a7af8f4bb932fd7f7500347374c14b8862e5ae43
 SHA512:
-  metadata.gz: 77a8932affc52ed2f4bdedd6e3328ed84ad5442e109959f952cb09aa0414354ab263b8f0a381f678b87c29545668a052767d78ce0b26bd2443f8d80e919e18b3
-  data.tar.gz: 52d878bc1cc42e22d171884cb794a6c4479625a7f4184de982b3956e4d614d7d365091ee0273b406f2b81b619a093d747c1464b7b81beab1f96791536e817836
+  metadata.gz: 2945ac417bfcc350068b1c9ce3c59781f0cd7bb50bd606919803e55602a81b06d6d6fed24de0c7e62ca695a83901481d296a9ae20031b5ab1e9edfc61a69823a
+  data.tar.gz: b96f8274e66d1d7ec2ab182fdcb3833edd7c24b9dc9d9fd9d038a3db8608dc1102a23b2998ec5c9749e89fe6d12212488dc9733ec927328fcc0f388bf84df357

data/CHANGELOG.md

@@ -18,11 +18,51 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 - Nothing
 
 
+## [2.12.1] - 2020-06-17
+
+### Fixed
+
+- Fix initializer template in generator
+  (https://github.com/AssetSync/asset_sync/pull/404)
+
+
+## [2.12.0] - 2020-06-11
+
+### Added
+
+- Add option `aws_session_token` to support AWS Temporary Security Credentials
+  (https://github.com/AssetSync/asset_sync/pull/403)
+
+
+## [2.11.0] - 2020-03-13
+
+### Added
+
+- Add option `remote_file_list_cache_file_path` to skip scanning remote
+  (https://github.com/AssetSync/asset_sync/pull/400)
+
+
+## [2.10.0] - 2020-02-26
+
+### Added
+
+- Add option `concurrent_uploads_max_threads` to limit number of threads for uploading files
+  (https://github.com/AssetSync/asset_sync/pull/398)
+
+
+## [2.9.1] - 2020-02-20
+
+### Fixed
+
+- Fix uploading of sprockets manifest file
+  (https://github.com/AssetSync/asset_sync/pull/397)
+
+
 ## [2.9.0] - 2020-01-15
 
 ### Added
 
-- Add option `concurrent_uploads` to improve speed of uploading 
+- Add option `concurrent_uploads` to improve speed of uploading
   (https://github.com/AssetSync/asset_sync/pull/393)
 
 
@@ -938,7 +978,12 @@ Changes:
     * Merge branch 'sinatra'
 
 
-[Unreleased]: https://github.com/AssetSync/asset_sync/compare/v2.9.0...HEAD
+[Unreleased]: https://github.com/AssetSync/asset_sync/compare/v2.12.1...HEAD
+[2.12.1]: https://github.com/AssetSync/asset_sync/compare/v2.12.0...v2.12.1
+[2.12.0]: https://github.com/AssetSync/asset_sync/compare/v2.11.0...v2.12.0
+[2.11.0]: https://github.com/AssetSync/asset_sync/compare/v2.10.0...v2.11.0
+[2.10.0]: https://github.com/AssetSync/asset_sync/compare/v2.9.1...v2.10.0
+[2.9.1]: https://github.com/AssetSync/asset_sync/compare/v2.9.0...v2.9.1
 [2.9.0]: https://github.com/AssetSync/asset_sync/compare/v2.8.2...v2.9.0
 [2.8.2]: https://github.com/AssetSync/asset_sync/compare/v2.8.1...v2.8.2
 [2.8.1]: https://github.com/AssetSync/asset_sync/compare/v2.8.0...v2.8.1

data/README.md

@@ -208,6 +208,7 @@ AssetSync.configure do |config|
   config.fog_directory = ENV['FOG_DIRECTORY']
   config.aws_access_key_id = ENV['AWS_ACCESS_KEY_ID']
   config.aws_secret_access_key = ENV['AWS_SECRET_ACCESS_KEY']
+  config.aws_session_token = ENV['AWS_SESSION_TOKEN'] if ENV.key?('AWS_SESSION_TOKEN')
 
   # Don't delete files from the store
   # config.existing_remote_files = 'keep'
@@ -244,6 +245,12 @@ AssetSync.configure do |config|
   #
   # Upload files concurrently
   # config.concurrent_uploads = false
+  # 
+  # Number of threads when concurrent_uploads is enabled
+  # config.concurrent_uploads_max_threads = 10
+  #
+  # Path to cache file to skip scanning remote
+  # config.remote_file_list_cache_file_path = './.asset_sync_remote_file_list_cache.json'
   #
   # Fail silently.  Useful for environments such as Heroku
   # config.fail_silently = true
@@ -342,6 +349,8 @@ AssetSync.config.gzip_compression == ENV['ASSET_SYNC_GZIP_COMPRESSION']
 * **manifest**: (`true, false`) when enabled, will use the `manifest.yml` generated by Rails to get the list of local files to upload. **experimental**. **default:** `'false'`
 * **include_manifest**: (`true, false`) when enabled, will upload the `manifest.yml` generated by Rails. **default:** `'false'`
 * **concurrent_uploads**: (`true, false`) when enabled, will upload the files in different Threads, this greatly improves the upload speed. **default:** `'false'`
+* **concurrent_uploads_max_threads**: when concurrent_uploads is enabled, this determines the number of threads that will be created. **default:** `10`
+* **remote_file_list_cache_file_path**: if present, use this path to cache remote file list to skip scanning remote **default:** `nil`
 * **enabled**: (`true, false`) when false, will disable asset sync. **default:** `'true'` (enabled)
 * **ignored\_files**: an array of files to ignore e.g. `['ignore_me.js', %r(ignore_some/\d{32}\.css)]` Useful if there are some files that are created dynamically on the server and you don't want to upload on deploy **default**: `[]`
 * **cache\_asset\_regexps**: an array of files to add cache headers e.g. `['cache_me.js', %r(cache_some\.\d{8}\.css)]` Useful if there are some files that are added to sprockets assets list and need to be set as 'Cacheable' on uploaded server.  Only rails compiled regexp is matched internally **default**: `[]`
@@ -540,6 +549,10 @@ AssetSync.configure do |config|
   config.aws_access_key_id = ENV['AWS_ACCESS_KEY_ID']
   config.aws_secret_access_key = ENV['AWS_SECRET_ACCESS_KEY']
   config.prefix = 'assets'
+  # Can be a `Pathname` or `String`
+  # Will be converted into an `Pathname`
+  # If relative, will be converted into an absolute path 
+  # via `::Rails.root` or `::Dir.pwd`
   config.public_path = Pathname('./public')
 end
 ```

data/gemfiles/rails_6_0.gemfile

@@ -5,6 +5,6 @@ source "https://rubygems.org"
 gem "rcov", platforms: :mri_18, group: [:development, :test]
 gem "simplecov", platforms: [:jruby, :mri_19, :ruby_19, :mri_20, :rbx], group: [:development, :test], require: false
 gem "jruby-openssl", platform: :jruby
-gem "rails", "~> 6.0.0.beta1"
+gem "rails", "~> 6.0.0"
 
 gemspec path: "../"

data/lib/asset_sync/config.rb

@@ -27,7 +27,8 @@ module AssetSync
     attr_accessor :cache_asset_regexps
     attr_accessor :include_manifest
     attr_accessor :concurrent_uploads
-    attr_writer :public_path
+    attr_accessor :concurrent_uploads_max_threads
+    attr_accessor :remote_file_list_cache_file_path
 
     # FOG configuration
     attr_accessor :fog_provider          # Currently Supported ['AWS', 'Rackspace']
@@ -36,7 +37,7 @@ module AssetSync
     attr_reader   :fog_public            # e.g. true, false, "default"
 
     # Amazon AWS
-    attr_accessor :aws_access_key_id, :aws_secret_access_key, :aws_reduced_redundancy, :aws_iam_roles, :aws_signature_version
+    attr_accessor :aws_access_key_id, :aws_secret_access_key, :aws_session_token, :aws_reduced_redundancy, :aws_iam_roles, :aws_signature_version
     attr_accessor :fog_host              # e.g. 's3.amazonaws.com'
     attr_accessor :fog_port              # e.g. '9000'
     attr_accessor :fog_path_style        # e.g. true
@@ -87,6 +88,8 @@ module AssetSync
       self.cache_asset_regexps = []
       self.include_manifest = false
       self.concurrent_uploads = false
+      self.concurrent_uploads_max_threads = 10
+      self.remote_file_list_cache_file_path = nil
       @additional_local_file_paths_procs = []
 
       load_yml! if defined?(::Rails) && yml_exists?
@@ -175,6 +178,19 @@ module AssetSync
       @public_path || ::Rails.public_path
     end
 
+    def public_path=(path)
+      # Generate absolute path even when relative path passed in
+      # Required for generating relative sprockets manifest path
+      pathname = Pathname(path)
+      @public_path = if pathname.absolute?
+        pathname
+      elsif defined?(::Rails.root)
+        ::Rails.root.join(pathname)
+      else
+        Pathname(::Dir.pwd).join(pathname)
+      end
+    end
+
     def load_yml!
       self.enabled                = yml["enabled"] if yml.has_key?('enabled')
       self.fog_provider           = yml["fog_provider"]
@@ -187,6 +203,7 @@ module AssetSync
       self.fog_scheme             = yml["fog_scheme"]
       self.aws_access_key_id      = yml["aws_access_key_id"]
       self.aws_secret_access_key  = yml["aws_secret_access_key"]
+      self.aws_session_token      = yml["aws_session_token"] if yml.has_key?("aws_session_token")
       self.aws_reduced_redundancy = yml["aws_reduced_redundancy"]
       self.aws_iam_roles          = yml["aws_iam_roles"]
       self.aws_signature_version  = yml["aws_signature_version"]
@@ -210,6 +227,8 @@ module AssetSync
       self.cache_asset_regexps    = yml['cache_asset_regexps'] if yml.has_key?("cache_asset_regexps")
       self.include_manifest       = yml['include_manifest'] if yml.has_key?("include_manifest")
       self.concurrent_uploads     = yml['concurrent_uploads'] if yml.has_key?('concurrent_uploads')
+      self.concurrent_uploads_max_threads = yml['concurrent_uploads_max_threads'] if yml.has_key?('concurrent_uploads_max_threads')
+      self.remote_file_list_cache_file_path = yml['remote_file_list_cache_file_path'] if yml.has_key?('remote_file_list_cache_file_path')
 
       self.azure_storage_account_name = yml['azure_storage_account_name'] if yml.has_key?("azure_storage_account_name")
       self.azure_storage_access_key   = yml['azure_storage_access_key'] if yml.has_key?("azure_storage_access_key")
@@ -242,6 +261,7 @@ module AssetSync
             :aws_access_key_id => aws_access_key_id,
             :aws_secret_access_key => aws_secret_access_key
           })
+          options.merge!({:aws_session_token => aws_session_token}) if aws_session_token
         end
         options.merge!({:host => fog_host}) if fog_host
         options.merge!({:port => fog_port}) if fog_port

data/lib/asset_sync/engine.rb

@@ -23,6 +23,7 @@ module AssetSync
 
           config.aws_access_key_id = ENV['AWS_ACCESS_KEY_ID'] if ENV.has_key?('AWS_ACCESS_KEY_ID')
           config.aws_secret_access_key = ENV['AWS_SECRET_ACCESS_KEY'] if ENV.has_key?('AWS_SECRET_ACCESS_KEY')
+          config.aws_session_token = ENV['AWS_SESSION_TOKEN'] if ENV.has_key?('AWS_SESSION_TOKEN')
           config.aws_signature_version = ENV['AWS_SIGNATURE_VERSION'] if ENV.has_key?('AWS_SIGNATURE_VERSION')
           config.aws_reduced_redundancy = ENV['AWS_REDUCED_REDUNDANCY'] == true  if ENV.has_key?('AWS_REDUCED_REDUNDANCY')
 
@@ -43,6 +44,7 @@ module AssetSync
           config.manifest = (ENV['ASSET_SYNC_MANIFEST'] == 'true') if ENV.has_key?('ASSET_SYNC_MANIFEST')
           config.include_manifest = (ENV['ASSET_SYNC_INCLUDE_MANIFEST'] == 'true') if ENV.has_key?('ASSET_SYNC_INCLUDE_MANIFEST')
           config.concurrent_uploads = (ENV['ASSET_SYNC_CONCURRENT_UPLOADS'] == 'true') if ENV.has_key?('ASSET_SYNC_CONCURRENT_UPLOADS')
+          config.remote_file_list_cache_file_path = ENV['ASSET_SYNC_REMOTE_FILE_LIST_CACHE_FILE_PATH'] if ENV.has_key?('ASSET_SYNC_REMOTE_FILE_LIST_CACHE_FILE_PATH')
         end
 
         config.prefix = ENV['ASSET_SYNC_PREFIX'] if ENV.has_key?('ASSET_SYNC_PREFIX')

data/lib/asset_sync/storage.rb

@@ -37,6 +37,10 @@ module AssetSync
       self.config.public_path
     end
 
+    def remote_file_list_cache_file_path
+      self.config.remote_file_list_cache_file_path
+    end
+
     def ignored_files
       expand_file_names(self.config.ignored_files)
     end
@@ -58,6 +62,32 @@ module AssetSync
         (get_local_files + config.additional_local_file_paths).uniq
     end
 
+    def remote_files
+      return [] if ignore_existing_remote_files?
+      return @remote_files if @remote_files
+
+      if remote_file_list_cache_file_path && File.file?(remote_file_list_cache_file_path)
+        begin
+          content = File.read(remote_file_list_cache_file_path)
+          return @remote_files = JSON.parse(content)
+        rescue JSON::ParserError
+          warn "Failed to parse #{remote_file_list_cache_file_path} as json"
+        end
+      end
+
+      @remote_files = get_remote_files
+    end
+
+    def update_remote_file_list_cache(local_files_to_upload)
+      return unless remote_file_list_cache_file_path
+      return if ignore_existing_remote_files?
+
+      File.open(self.remote_file_list_cache_file_path, 'w') do |file|
+        uploaded = local_files_to_upload + remote_files
+        file.write(uploaded.to_json)
+      end
+    end
+
     def always_upload_files
       expand_file_names(self.config.always_upload) + get_manifest_path
     end
@@ -243,24 +273,30 @@ module AssetSync
     end
 
     def upload_files
-      # get a fresh list of remote files
-      remote_files = ignore_existing_remote_files? ? [] : get_remote_files
       # fixes: https://github.com/rumblelabs/asset_sync/issues/19
       local_files_to_upload = local_files - ignored_files - remote_files + always_upload_files
       local_files_to_upload = (local_files_to_upload + get_non_fingerprinted(local_files_to_upload)).uniq
+      # Only files.
+      local_files_to_upload = local_files_to_upload.select { |f| File.file? "#{path}/#{f}" }
 
       if self.config.concurrent_uploads
-        threads = ThreadGroup.new
+        jobs = Queue.new
+        local_files_to_upload.each { |f| jobs.push(f) }
+        jobs.close
+
+        num_threads = [self.config.concurrent_uploads_max_threads, local_files_to_upload.length].min
         # Upload new files
-        local_files_to_upload.each do |f|
-          next unless File.file? "#{path}/#{f}" # Only files.
-          threads.add(Thread.new { upload_file f })
+        workers = Array.new(num_threads) do
+          Thread.new do
+            while f = jobs.pop
+              upload_file(f)
+            end
+          end
         end
-        sleep 1 while threads.list.any? # wait for threads to finish uploading
+        workers.map(&:join)
       else
         # Upload new files
         local_files_to_upload.each do |f|
-          next unless File.file? "#{path}/#{f}" # Only files.
           upload_file f
         end
       end
@@ -271,6 +307,8 @@ module AssetSync
         data = cdn.post_invalidation(self.config.cdn_distribution_id, files_to_invalidate)
         log "Invalidation id: #{data.body["Id"]}"
       end
+
+      update_remote_file_list_cache(local_files_to_upload)
     end
 
     def sync

data/lib/asset_sync/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module AssetSync
-  VERSION = "2.9.0"
+  VERSION = "2.12.1"
 end

data/lib/generators/asset_sync/install_generator.rb

@@ -35,6 +35,10 @@ module AssetSync
       "<%= ENV['AWS_SECRET_ACCESS_KEY'] %>"
     end
 
+    def aws_session_token
+      "<%= ENV['AWS_SESSION_TOKEN'] %>"
+    end
+
     def google_storage_access_key_id
       "<%= ENV['GOOGLE_STORAGE_ACCESS_KEY_ID'] %>"
     end

data/lib/generators/asset_sync/templates/asset_sync.rb

@@ -4,6 +4,7 @@ if defined?(AssetSync)
     config.fog_provider = 'AWS'
     config.aws_access_key_id = ENV['AWS_ACCESS_KEY_ID']
     config.aws_secret_access_key = ENV['AWS_SECRET_ACCESS_KEY']
+    config.aws_session_token = ENV['AWS_SESSION_TOKEN'] if ENV.key?('AWS_SESSION_TOKEN')
     # To use AWS reduced redundancy storage.
     # config.aws_reduced_redundancy = true
     #
@@ -66,6 +67,9 @@ if defined?(AssetSync)
     # Upload files concurrently
     # config.concurrent_uploads = false
     #
+    # Path to cache file to skip scanning remote
+    # config.remote_file_list_cache_file_path = './.asset_sync_remote_file_list_cache.json'
+    #
     # Fail silently.  Useful for environments such as Heroku
     # config.fail_silently = true
     #

data/spec/unit/railsless_spec.rb

@@ -14,7 +14,7 @@ describe AssetSync do
         config.fog_region = 'eu-west-1'
         config.existing_remote_files = "keep"
         config.prefix = "assets"
-        config.public_path = Pathname("./public")
+        config.public_path = "./public"
       end
     end
 
@@ -22,8 +22,9 @@ describe AssetSync do
       expect(AssetSync.config.prefix).to eq("assets")
     end
 
-    it "should have prefix of assets" do
-      expect(AssetSync.config.public_path.to_s).to eq("./public")
+    it "should have public_path" do
+      expect(AssetSync.config.public_path.to_s).to be_end_with("/public")
+      expect(AssetSync.config.public_path).to be_absolute
     end
 
     it "should default AssetSync to enabled" do

data/spec/unit/storage_spec.rb

@@ -70,6 +70,71 @@ describe AssetSync::Storage do
       storage.upload_files
     end
 
+    it 'should allow custom number of threads' do
+      @config.concurrent_uploads = true
+      @config.concurrent_uploads_max_threads = 2
+      storage = AssetSync::Storage.new(@config)
+
+      allow(storage).to receive(:get_local_files).and_return(@local_files)
+      allow(storage).to receive(:get_remote_files).and_return(@remote_files)
+      allow(File).to receive(:file?).and_return(true) # Pretend they all exist
+
+      expect(Thread).to receive(:new).exactly(2).times.and_call_original
+      (@local_files - @remote_files + storage.always_upload_files).each do |file|
+        expect(storage).to receive(:upload_file).with(file)
+      end
+
+      storage.upload_files
+    end
+
+    it 'should allow remote_file_list_cache_file_path configuration' do
+      file_path = './foo.json'
+      @config.remote_file_list_cache_file_path = file_path
+      storage = AssetSync::Storage.new(@config)
+
+      allow(storage).to receive(:get_local_files).and_return(@local_files)
+      File.write(file_path, @remote_files.to_json)
+      expect(storage).not_to receive(:get_remote_files)
+      allow(File).to receive(:file?).and_return(true) # Pretend they all exist
+
+      (@local_files - @remote_files + storage.always_upload_files).each do |file|
+        expect(storage).to receive(:upload_file).with(file)
+      end
+
+      expect(storage).not_to receive(:warn)
+      storage.upload_files
+
+      # update remote_file_list_cache corretly
+      updated = JSON.parse(File.read(file_path))
+      expect(updated.sort.uniq).to eq (@remote_files + @local_files + storage.always_upload_files).sort.uniq
+
+      File.delete(file_path)
+    end
+
+    it 'should work with broken cache' do
+      file_path = './foo.json'
+      @config.remote_file_list_cache_file_path = file_path
+
+      storage = AssetSync::Storage.new(@config)
+
+      File.write(file_path, 'some non-json text file content')
+
+      allow(storage).to receive(:get_local_files).and_return(@local_files)
+      allow(storage).to receive(:get_remote_files).and_return(@remote_files)
+      allow(File).to receive(:file?).and_return(true) # Pretend they all exist
+
+      (@local_files - @remote_files + storage.always_upload_files).each do |file|
+        expect(storage).to receive(:upload_file).with(file)
+      end
+
+      # when broken, warning message should be prompted
+      expect(storage).to receive(:warn)
+
+      storage.upload_files
+
+      File.delete(file_path)
+    end
+
     it 'should upload updated non-fingerprinted files' do
       @local_files = [
         'public/image.png',

metadata

@@ -1,17 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: asset_sync
 version: !ruby/object:Gem::Version
-  version: 2.9.0
+  version: 2.12.1
 platform: ruby
 authors:
 - Simon Hamilton
 - David Rice
 - Phil McClure
 - Toby Osbourn
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-01-15 00:00:00.000000000 Z
+date: 2020-06-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fog-core
@@ -241,7 +241,7 @@ homepage: https://github.com/rumblelabs/asset_sync
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -256,8 +256,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: Synchronises Assets in a Rails 3 application and Amazon S3/Cloudfront and
   Rackspace Cloudfiles