asset_sync 2.4.0 → 2.15.2

data/asset_sync.gemspec

@@ -15,8 +15,6 @@ Gem::Specification.new do |s|
 
   s.license = 'MIT'
 
-  s.rubyforge_project = "asset_sync"
-
   s.add_dependency("fog-core")
   s.add_dependency('unf')
   s.add_dependency('activemodel', ">= 4.1.0")
@@ -24,14 +22,19 @@ Gem::Specification.new do |s|
 
   s.add_development_dependency "rspec"
   s.add_development_dependency "bundler"
-  s.add_development_dependency "jeweler"
+  s.add_development_dependency "coveralls", ">= 0.7"
+
+  s.add_development_dependency('mime-types', ">= 3.0")
 
   s.add_development_dependency "fog-aws"
-  s.add_development_dependency "fog-azure-rm"
+  s.add_development_dependency "gitlab-fog-azure-rm"
+  s.add_development_dependency "fog-backblaze"
 
   s.add_development_dependency "uglifier"
   s.add_development_dependency "appraisal"
 
+  s.add_development_dependency "gem-release"
+
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }

data/gemfiles/{rails_4_1.gemfile → rails_5_2.gemfile}

@@ -5,6 +5,6 @@ source "https://rubygems.org"
 gem "rcov", platforms: :mri_18, group: [:development, :test]
 gem "simplecov", platforms: [:jruby, :mri_19, :ruby_19, :mri_20, :rbx], group: [:development, :test], require: false
 gem "jruby-openssl", platform: :jruby
-gem "rails", "~> 4.1.0"
+gem "rails", "~> 5.2.0"
 
 gemspec path: "../"

data/gemfiles/{rails_4_2.gemfile → rails_6_0.gemfile}

@@ -5,6 +5,6 @@ source "https://rubygems.org"
 gem "rcov", platforms: :mri_18, group: [:development, :test]
 gem "simplecov", platforms: [:jruby, :mri_19, :ruby_19, :mri_20, :rbx], group: [:development, :test], require: false
 gem "jruby-openssl", platform: :jruby
-gem "rails", "~> 4.2.0"
+gem "rails", "~> 6.0.0"
 
 gemspec path: "../"

data/gemfiles/{rails_5_0.gemfile → rails_6_1.gemfile}

@@ -5,6 +5,6 @@ source "https://rubygems.org"
 gem "rcov", platforms: :mri_18, group: [:development, :test]
 gem "simplecov", platforms: [:jruby, :mri_19, :ruby_19, :mri_20, :rbx], group: [:development, :test], require: false
 gem "jruby-openssl", platform: :jruby
-gem "rails", "~> 5.0.0"
+gem "rails", "~> 6.1.0"
 
 gemspec path: "../"

data/lib/asset_sync/asset_sync.rb

@@ -1,3 +1,5 @@
+require "yaml"
+
 module AssetSync
 
   class << self
@@ -60,6 +62,14 @@ module AssetSync
       stdout.puts msg unless config.log_silently?
     end
 
+    def load_yaml(yaml)
+      if YAML.respond_to?(:unsafe_load)
+        YAML.unsafe_load(yaml)
+      else
+        YAML.load(yaml)
+      end
+    end
+
     def enabled?
       config.enabled?
     end

data/lib/asset_sync/config.rb

@@ -1,6 +1,7 @@
+# frozen_string_literal: true
+
 require "active_model"
 require "erb"
-require "yaml"
 
 module AssetSync
   class Config
@@ -17,7 +18,6 @@ module AssetSync
     attr_accessor :always_upload
     attr_accessor :ignored_files
     attr_accessor :prefix
-    attr_accessor :public_path
     attr_accessor :enabled
     attr_accessor :custom_headers
     attr_accessor :run_on_precompile
@@ -25,15 +25,28 @@ module AssetSync
     attr_accessor :cdn_distribution_id
     attr_accessor :cache_asset_regexps
     attr_accessor :include_manifest
+    attr_accessor :concurrent_uploads
+    attr_accessor :concurrent_uploads_max_threads
+    attr_accessor :remote_file_list_cache_file_path
 
     # FOG configuration
     attr_accessor :fog_provider          # Currently Supported ['AWS', 'Rackspace']
     attr_accessor :fog_directory         # e.g. 'the-bucket-name'
     attr_accessor :fog_region            # e.g. 'eu-west-1'
+    attr_reader   :fog_public            # e.g. true, false, "default"
 
     # Amazon AWS
-    attr_accessor :aws_access_key_id, :aws_secret_access_key, :aws_reduced_redundancy, :aws_iam_roles, :aws_signature_version
+    attr_accessor :aws_access_key_id
+    attr_accessor :aws_secret_access_key
+    attr_accessor :aws_session_token
+    attr_accessor :aws_reduced_redundancy
+    attr_accessor :aws_iam_roles
+    attr_accessor :aws_signature_version
+    attr_accessor :aws_acl
+
+    # Fog
     attr_accessor :fog_host              # e.g. 's3.amazonaws.com'
+    attr_accessor :fog_port              # e.g. '9000'
     attr_accessor :fog_path_style        # e.g. true
     attr_accessor :fog_scheme            # e.g. 'http'
 
@@ -41,12 +54,20 @@ module AssetSync
     attr_accessor :rackspace_username, :rackspace_api_key, :rackspace_auth_url
 
     # Google Storage
-    attr_accessor :google_storage_secret_access_key, :google_storage_access_key_id
+    attr_accessor :google_storage_secret_access_key, :google_storage_access_key_id  # when using S3 interop
+    attr_accessor :google_json_key_location # when using service accounts
+    attr_accessor :google_json_key_string # when using service accounts
+    attr_accessor :google_project # when using service accounts
 
     # Azure Blob with Fog::AzureRM
     attr_accessor :azure_storage_account_name
     attr_accessor :azure_storage_access_key
 
+    # Backblaze B2 with Fog::Backblaze
+    attr_accessor :b2_key_id
+    attr_accessor :b2_key_token
+    attr_accessor :b2_bucket_id
+
     validates :existing_remote_files, :inclusion => { :in => %w(keep delete ignore) }
 
     validates :fog_provider,          :presence => true
@@ -56,11 +77,19 @@ module AssetSync
     validates :aws_secret_access_key, :presence => true, :if => proc {aws? && !aws_iam?}
     validates :rackspace_username,    :presence => true, :if => :rackspace?
     validates :rackspace_api_key,     :presence => true, :if => :rackspace?
-    validates :google_storage_secret_access_key,  :presence => true, :if => :google?
-    validates :google_storage_access_key_id,      :presence => true, :if => :google?
+    validates :google_storage_secret_access_key,  :presence => true, :if => :google_interop?
+    validates :google_storage_access_key_id,      :presence => true, :if => :google_interop?
+    validates :google_project,                    :presence => true, :if => :google_service_account?
+    validate(:if => :google_service_account?) do
+      unless google_json_key_location.present? || google_json_key_string.present?
+        errors.add(:base, 'must provide either google_json_key_location or google_json_key_string if using Google service account')
+      end
+    end
+    validates :concurrent_uploads,    :inclusion => { :in => [true, false] }
 
     def initialize
       self.fog_region = nil
+      self.fog_public = true
       self.existing_remote_files = 'keep'
       self.gzip_compression = false
       self.manifest = false
@@ -75,6 +104,9 @@ module AssetSync
       self.invalidate = []
       self.cache_asset_regexps = []
       self.include_manifest = false
+      self.concurrent_uploads = false
+      self.concurrent_uploads_max_threads = 10
+      self.remote_file_list_cache_file_path = nil
       @additional_local_file_paths_procs = []
 
       load_yml! if defined?(::Rails) && yml_exists?
@@ -126,10 +158,22 @@ module AssetSync
       fog_provider =~ /google/i
     end
 
+    def google_interop?
+      google? && google_json_key_location.nil? && google_json_key_string.nil?
+    end
+
+    def google_service_account?
+      google? && (google_json_key_location || google_json_key_string)
+    end
+
     def azure_rm?
       fog_provider =~ /azurerm/i
     end
 
+    def backblaze?
+      fog_provider =~ /backblaze/i
+    end
+
     def cache_asset_regexp=(cache_asset_regexp)
       self.cache_asset_regexps = [cache_asset_regexp]
     end
@@ -139,7 +183,7 @@ module AssetSync
     end
 
     def yml
-      @yml ||= ::YAML.load(::ERB.new(IO.read(yml_path)).result)[::Rails.env] || {}
+      @yml ||= ::AssetSync.load_yaml(::ERB.new(IO.read(yml_path)).result)[::Rails.env] || {}
     end
 
     def yml_path
@@ -155,40 +199,68 @@ module AssetSync
       @public_path || ::Rails.public_path
     end
 
+    def public_path=(path)
+      # Generate absolute path even when relative path passed in
+      # Required for generating relative sprockets manifest path
+      pathname = Pathname(path)
+      @public_path = if pathname.absolute?
+        pathname
+      elsif defined?(::Rails.root)
+        ::Rails.root.join(pathname)
+      else
+        Pathname(::Dir.pwd).join(pathname)
+      end
+    end
+
     def load_yml!
       self.enabled                = yml["enabled"] if yml.has_key?('enabled')
       self.fog_provider           = yml["fog_provider"]
       self.fog_host               = yml["fog_host"]
+      self.fog_port               = yml["fog_port"]
       self.fog_directory          = yml["fog_directory"]
       self.fog_region             = yml["fog_region"]
+      self.fog_public             = yml["fog_public"] if yml.has_key?("fog_public")
       self.fog_path_style         = yml["fog_path_style"]
       self.fog_scheme             = yml["fog_scheme"]
       self.aws_access_key_id      = yml["aws_access_key_id"]
       self.aws_secret_access_key  = yml["aws_secret_access_key"]
+      self.aws_session_token      = yml["aws_session_token"] if yml.has_key?("aws_session_token")
       self.aws_reduced_redundancy = yml["aws_reduced_redundancy"]
       self.aws_iam_roles          = yml["aws_iam_roles"]
       self.aws_signature_version  = yml["aws_signature_version"]
+      self.aws_acl                = yml["aws_acl"]
       self.rackspace_username     = yml["rackspace_username"]
       self.rackspace_auth_url     = yml["rackspace_auth_url"] if yml.has_key?("rackspace_auth_url")
       self.rackspace_api_key      = yml["rackspace_api_key"]
-      self.google_storage_secret_access_key = yml["google_storage_secret_access_key"]
-      self.google_storage_access_key_id     = yml["google_storage_access_key_id"]
+      self.google_json_key_location = yml["google_json_key_location"] if yml.has_key?("google_json_key_location")
+      self.google_project = yml["google_project"] if yml.has_key?("google_project")
+      self.google_storage_secret_access_key = yml["google_storage_secret_access_key"] if yml.has_key?("google_storage_secret_access_key")
+      self.google_storage_access_key_id     = yml["google_storage_access_key_id"] if yml.has_key?("google_storage_access_key_id")
+      self.google_json_key_string           = yml["google_json_key_string"] if yml.has_key?("google_json_key_string")
       self.existing_remote_files  = yml["existing_remote_files"] if yml.has_key?("existing_remote_files")
       self.gzip_compression       = yml["gzip_compression"] if yml.has_key?("gzip_compression")
       self.manifest               = yml["manifest"] if yml.has_key?("manifest")
       self.fail_silently          = yml["fail_silently"] if yml.has_key?("fail_silently")
+      self.log_silently           = yml["log_silently"] if yml.has_key?("log_silently")
       self.always_upload          = yml["always_upload"] if yml.has_key?("always_upload")
       self.ignored_files          = yml["ignored_files"] if yml.has_key?("ignored_files")
-      self.custom_headers          = yml["custom_headers"] if yml.has_key?("custom_headers")
+      self.custom_headers         = yml["custom_headers"] if yml.has_key?("custom_headers")
       self.run_on_precompile      = yml["run_on_precompile"] if yml.has_key?("run_on_precompile")
       self.invalidate             = yml["invalidate"] if yml.has_key?("invalidate")
       self.cdn_distribution_id    = yml['cdn_distribution_id'] if yml.has_key?("cdn_distribution_id")
       self.cache_asset_regexps    = yml['cache_asset_regexps'] if yml.has_key?("cache_asset_regexps")
       self.include_manifest       = yml['include_manifest'] if yml.has_key?("include_manifest")
+      self.concurrent_uploads     = yml['concurrent_uploads'] if yml.has_key?('concurrent_uploads')
+      self.concurrent_uploads_max_threads = yml['concurrent_uploads_max_threads'] if yml.has_key?('concurrent_uploads_max_threads')
+      self.remote_file_list_cache_file_path = yml['remote_file_list_cache_file_path'] if yml.has_key?('remote_file_list_cache_file_path')
 
       self.azure_storage_account_name = yml['azure_storage_account_name'] if yml.has_key?("azure_storage_account_name")
       self.azure_storage_access_key   = yml['azure_storage_access_key'] if yml.has_key?("azure_storage_access_key")
 
+      self.b2_key_id      = yml['b2_key_id']    if yml.has_key?("b2_key_id")
+      self.b2_key_token   = yml['b2_key_token'] if yml.has_key?("b2_key_token")
+      self.b2_bucket_id   = yml['b2_bucket_id'] if yml.has_key?("b2_bucket_id")
+
       # TODO deprecate the other old style config settings. FML.
       self.aws_access_key_id      = yml["aws_access_key"] if yml.has_key?("aws_access_key")
       self.aws_secret_access_key  = yml["aws_access_secret"] if yml.has_key?("aws_access_secret")
@@ -217,8 +289,10 @@ module AssetSync
             :aws_access_key_id => aws_access_key_id,
             :aws_secret_access_key => aws_secret_access_key
           })
+          options.merge!({:aws_session_token => aws_session_token}) if aws_session_token
         end
         options.merge!({:host => fog_host}) if fog_host
+        options.merge!({:port => fog_port}) if fog_port
         options.merge!({:scheme => fog_scheme}) if fog_scheme
         options.merge!({:aws_signature_version => aws_signature_version}) if aws_signature_version
         options.merge!({:path_style => fog_path_style}) if fog_path_style
@@ -231,10 +305,16 @@ module AssetSync
         options.merge!({ :rackspace_region => fog_region }) if fog_region
         options.merge!({ :rackspace_auth_url => rackspace_auth_url }) if rackspace_auth_url
       elsif google?
-        options.merge!({
-          :google_storage_secret_access_key => google_storage_secret_access_key,
-          :google_storage_access_key_id => google_storage_access_key_id
-        })
+        if google_json_key_location
+          options.merge!({:google_json_key_location => google_json_key_location, :google_project => google_project})
+        elsif google_json_key_string
+          options.merge!({:google_json_key_string => google_json_key_string, :google_project => google_project})
+        else
+          options.merge!({
+            :google_storage_secret_access_key => google_storage_secret_access_key,
+            :google_storage_access_key_id => google_storage_access_key_id
+          })
+        end
         options.merge!({:region => fog_region}) if fog_region
       elsif azure_rm?
         require 'fog/azurerm'
@@ -243,11 +323,18 @@ module AssetSync
           :azure_storage_access_key   => azure_storage_access_key,
         })
         options.merge!({:environment => fog_region}) if fog_region
+      elsif backblaze?
+        require 'fog/backblaze'
+        options.merge!({
+          :b2_key_id      => b2_key_id,
+          :b2_key_token   => b2_key_token,
+          :b2_bucket_id   => b2_bucket_id,
+        })
       else
         raise ArgumentError, "AssetSync Unknown provider: #{fog_provider} only AWS, Rackspace and Google are supported currently."
       end
 
-      return options
+      options
     end
 
     # @api
@@ -268,6 +355,15 @@ module AssetSync
       end
     end
 
+    #@api
+    def file_ext_to_mime_type_overrides
+      @file_ext_to_mime_type_overrides ||= FileExtToMimeTypeOverrides.new
+    end
+
+    def fog_public=(new_val)
+      @fog_public = FogPublicValue.new(new_val)
+    end
+
   private
 
     # This is a proc to get additional local files paths
@@ -277,5 +373,57 @@ module AssetSync
     def default_manifest_directory
       File.join(::Rails.public_path, assets_prefix)
     end
+
+
+    # @api private
+    class FileExtToMimeTypeOverrides
+      def initialize
+        # The default is to prevent new mime type `application/ecmascript` to be returned
+        # which disables compression on some CDNs
+        @overrides = {
+          "js" => "application/javascript",
+        }
+      end
+
+      # @api
+      def add(ext, mime_type)
+        # Symbol / Mime type object might be passed in
+        # But we want strings only
+        @overrides.store(
+          ext.to_s, mime_type.to_s,
+        )
+      end
+
+      # @api
+      def clear
+        @overrides = {}
+      end
+
+
+      # @api private
+      def key?(key)
+        @overrides.key?(key)
+      end
+
+      # @api private
+      def fetch(key)
+        @overrides.fetch(key)
+      end
+    end
+
+    # @api private
+    class FogPublicValue
+      def initialize(val)
+        @value = val
+      end
+
+      def use_explicit_value?
+        @value.to_s != "default"
+      end
+
+      def to_bool
+        !!@value
+      end
+    end
   end
 end

data/lib/asset_sync/engine.rb

@@ -17,12 +17,15 @@ module AssetSync
           config.fog_directory = ENV['FOG_DIRECTORY'] if ENV.has_key?('FOG_DIRECTORY')
           config.fog_region = ENV['FOG_REGION'] if ENV.has_key?('FOG_REGION')
           config.fog_host = ENV['FOG_HOST'] if ENV.has_key?('FOG_HOST')
+          config.fog_port = ENV['FOG_PORT'] if ENV.has_key?('FOG_PORT')
           config.fog_scheme = ENV['FOG_SCHEMA'] if ENV.has_key?('FOG_SCHEMA')
           config.fog_path_style = ENV['FOG_PATH_STYLE'] if ENV.has_key?('FOG_PATH_STYLE')
 
           config.aws_access_key_id = ENV['AWS_ACCESS_KEY_ID'] if ENV.has_key?('AWS_ACCESS_KEY_ID')
           config.aws_secret_access_key = ENV['AWS_SECRET_ACCESS_KEY'] if ENV.has_key?('AWS_SECRET_ACCESS_KEY')
+          config.aws_session_token = ENV['AWS_SESSION_TOKEN'] if ENV.has_key?('AWS_SESSION_TOKEN')
           config.aws_signature_version = ENV['AWS_SIGNATURE_VERSION'] if ENV.has_key?('AWS_SIGNATURE_VERSION')
+          config.aws_acl = ENV['AWS_ACL'] if ENV.has_key?('AWS_ACL')
           config.aws_reduced_redundancy = ENV['AWS_REDUCED_REDUNDANCY'] == true  if ENV.has_key?('AWS_REDUCED_REDUNDANCY')
 
           config.rackspace_username = ENV['RACKSPACE_USERNAME'] if ENV.has_key?('RACKSPACE_USERNAME')
@@ -34,6 +37,10 @@ module AssetSync
           config.azure_storage_account_name = ENV['AZURE_STORAGE_ACCOUNT_NAME'] if ENV.has_key?('AZURE_STORAGE_ACCOUNT_NAME')
           config.azure_storage_access_key = ENV['AZURE_STORAGE_ACCESS_KEY'] if ENV.has_key?('AZURE_STORAGE_ACCESS_KEY')
 
+          config.b2_key_id    = ENV['B2_KEY_ID']    if ENV.has_key?('B2_KEY_ID')
+          config.b2_key_token = ENV['B2_KEY_TOKEN'] if ENV.has_key?('B2_KEY_TOKEN')
+          config.b2_bucket_id = ENV['B2_BUCKET_ID'] if ENV.has_key?('B2_BUCKET_ID')
+
           config.enabled = (ENV['ASSET_SYNC_ENABLED'] == 'true') if ENV.has_key?('ASSET_SYNC_ENABLED')
 
           config.existing_remote_files = ENV['ASSET_SYNC_EXISTING_REMOTE_FILES'] || "keep"
@@ -41,6 +48,8 @@ module AssetSync
           config.gzip_compression = (ENV['ASSET_SYNC_GZIP_COMPRESSION'] == 'true') if ENV.has_key?('ASSET_SYNC_GZIP_COMPRESSION')
           config.manifest = (ENV['ASSET_SYNC_MANIFEST'] == 'true') if ENV.has_key?('ASSET_SYNC_MANIFEST')
           config.include_manifest = (ENV['ASSET_SYNC_INCLUDE_MANIFEST'] == 'true') if ENV.has_key?('ASSET_SYNC_INCLUDE_MANIFEST')
+          config.concurrent_uploads = (ENV['ASSET_SYNC_CONCURRENT_UPLOADS'] == 'true') if ENV.has_key?('ASSET_SYNC_CONCURRENT_UPLOADS')
+          config.remote_file_list_cache_file_path = ENV['ASSET_SYNC_REMOTE_FILE_LIST_CACHE_FILE_PATH'] if ENV.has_key?('ASSET_SYNC_REMOTE_FILE_LIST_CACHE_FILE_PATH')
         end
 
         config.prefix = ENV['ASSET_SYNC_PREFIX'] if ENV.has_key?('ASSET_SYNC_PREFIX')

data/lib/asset_sync/multi_mime.rb

@@ -4,11 +4,16 @@ module AssetSync
   class MultiMime
 
     def self.lookup(ext)
+      overrides =
+        ::AssetSync.config.file_ext_to_mime_type_overrides
+      if overrides.key?(ext)
+        return overrides.fetch(ext)
+      end
 
       if defined?(::MIME::Types)
-        ::MIME::Types.type_for(ext).first
+        ::MIME::Types.type_for(ext).first.to_s
       elsif defined?(::Mime::Type)
-        ::Mime::Type.lookup_by_extension(ext)
+        ::Mime::Type.lookup_by_extension(ext).to_s
       elsif defined?(::Rack::Mime)
         ext_with_dot = ".#{ext}"
         ::Rack::Mime.mime_type(ext_with_dot)

data/lib/asset_sync/storage.rb

@@ -4,7 +4,7 @@ require "asset_sync/multi_mime"
 
 module AssetSync
   class Storage
-    REGEXP_FINGERPRINTED_FILES = /^(.*)\/([^-]+)-[^\.]+\.([^\.]+)$/
+    REGEXP_FINGERPRINTED_FILES = /\A(.*)\/(.+)-[^\.]+\.([^\.]+)\z/m
     REGEXP_ASSETS_TO_CACHE_CONTROL = /-[0-9a-fA-F]{32,}$/
 
     class BucketNotFound < StandardError;
@@ -22,7 +22,13 @@ module AssetSync
 
     def bucket
       # fixes: https://github.com/rumblelabs/asset_sync/issues/18
-      @bucket ||= connection.directories.get(self.config.fog_directory, :prefix => self.config.assets_prefix)
+
+      @bucket ||= if self.config.backblaze?
+                    connection.directories.get(self.config.fog_directory)
+                  else
+                    connection.directories.get(self.config.fog_directory, :prefix => self.config.assets_prefix)
+                  end
+
     end
 
     def log(msg)
@@ -37,13 +43,17 @@ module AssetSync
       self.config.public_path
     end
 
+    def remote_file_list_cache_file_path
+      self.config.remote_file_list_cache_file_path
+    end
+
     def ignored_files
       expand_file_names(self.config.ignored_files)
     end
 
     def get_manifest_path
       return [] unless self.config.include_manifest
-        
+
       if ActionView::Base.respond_to?(:assets_manifest)
         manifest = Sprockets::Manifest.new(ActionView::Base.assets_manifest.environment, ActionView::Base.assets_manifest.dir)
         manifest_path = manifest.filename
@@ -58,6 +68,32 @@ module AssetSync
         (get_local_files + config.additional_local_file_paths).uniq
     end
 
+    def remote_files
+      return [] if ignore_existing_remote_files?
+      return @remote_files if @remote_files
+
+      if remote_file_list_cache_file_path && File.file?(remote_file_list_cache_file_path)
+        begin
+          content = File.read(remote_file_list_cache_file_path)
+          return @remote_files = JSON.parse(content)
+        rescue JSON::ParserError
+          warn "Failed to parse #{remote_file_list_cache_file_path} as json"
+        end
+      end
+
+      @remote_files = get_remote_files
+    end
+
+    def update_remote_file_list_cache(local_files_to_upload)
+      return unless remote_file_list_cache_file_path
+      return if ignore_existing_remote_files?
+
+      File.open(self.remote_file_list_cache_file_path, 'w') do |file|
+        uploaded = local_files_to_upload + remote_files
+        file.write(uploaded.to_json)
+      end
+    end
+
     def always_upload_files
       expand_file_names(self.config.always_upload) + get_manifest_path
     end
@@ -81,7 +117,7 @@ module AssetSync
           return manifest.assets.values.map { |f| File.join(self.config.assets_prefix, f) }
         elsif File.exist?(self.config.manifest_path)
           log "Using: Manifest #{self.config.manifest_path}"
-          yml = YAML.load(IO.read(self.config.manifest_path))
+          yml = AssetSync.load_yaml(IO.read(self.config.manifest_path))
 
           return yml.map do |original, compiled|
             # Upload font originals and compiled
@@ -132,9 +168,15 @@ module AssetSync
       from_remote_files_to_delete = remote_files - local_files - ignored_files - always_upload_files
 
       log "Flagging #{from_remote_files_to_delete.size} file(s) for deletion"
-      # Delete unneeded remote files
-      bucket.files.each do |f|
-        delete_file(f, from_remote_files_to_delete)
+      # Delete unneeded remote files, if we are on aws delete in bulk else use sequential delete
+      if self.config.aws? && connection.respond_to?(:delete_multiple_objects)
+        from_remote_files_to_delete.each_slice(500) do |slice|
+          connection.delete_multiple_objects(config.fog_directory, slice)
+        end
+      else
+        bucket.files.each do |f|
+          delete_file(f, from_remote_files_to_delete)
+        end
       end
     end
 
@@ -148,10 +190,19 @@ module AssetSync
       file = {
         :key => f,
         :body => file_handle,
-        :public => true,
         :content_type => mime
       }
 
+      # region fog_public
+
+      if config.aws? && config.aws_acl
+        file[:acl] = config.aws_acl
+      elsif config.fog_public.use_explicit_value?
+        file[:public] = config.fog_public.to_bool
+      end
+
+      # endregion fog_public
+
       uncompressed_filename = f.sub(/\.gz\z/, '')
       basename = File.basename(uncompressed_filename, File.extname(uncompressed_filename))
 
@@ -224,28 +275,38 @@ module AssetSync
         })
       end
 
-      if config.azure_rm?
-        # converts content_type from MIME::Type to String.
-        # because Azure::Storage (called from Fog::AzureRM) expects content_type as a String like "application/json; charset=utf-8"
-        file[:content_type] = file[:content_type].content_type if file[:content_type].is_a?(::MIME::Type)
-      end
-
       bucket.files.create( file ) unless ignore
       file_handle.close
       gzip_file_handle.close if gzip_file_handle
     end
 
     def upload_files
-      # get a fresh list of remote files
-      remote_files = ignore_existing_remote_files? ? [] : get_remote_files
       # fixes: https://github.com/rumblelabs/asset_sync/issues/19
       local_files_to_upload = local_files - ignored_files - remote_files + always_upload_files
       local_files_to_upload = (local_files_to_upload + get_non_fingerprinted(local_files_to_upload)).uniq
-
-      # Upload new files
-      local_files_to_upload.each do |f|
-        next unless File.file? "#{path}/#{f}" # Only files.
-        upload_file f
+      # Only files.
+      local_files_to_upload = local_files_to_upload.select { |f| File.file? "#{path}/#{f}" }
+
+      if self.config.concurrent_uploads
+        jobs = Queue.new
+        local_files_to_upload.each { |f| jobs.push(f) }
+        jobs.close
+
+        num_threads = [self.config.concurrent_uploads_max_threads, local_files_to_upload.length].min
+        # Upload new files
+        workers = Array.new(num_threads) do
+          Thread.new do
+            while f = jobs.pop
+              upload_file(f)
+            end
+          end
+        end
+        workers.map(&:join)
+      else
+        # Upload new files
+        local_files_to_upload.each do |f|
+          upload_file f
+        end
       end
 
       if self.config.cdn_distribution_id && files_to_invalidate.any?
@@ -254,6 +315,8 @@ module AssetSync
         data = cdn.post_invalidation(self.config.cdn_distribution_id, files_to_invalidate)
         log "Invalidation id: #{data.body["Id"]}"
       end
+
+      update_remote_file_list_cache(local_files_to_upload)
     end
 
     def sync

data/lib/asset_sync/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module AssetSync
-  VERSION = "2.4.0".freeze
+  VERSION = "2.15.2"
 end