asset_sync 2.8.2 → 2.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +0 -11
- data/Appraisals +1 -5
- data/CHANGELOG.md +46 -1
- data/README.md +18 -1
- data/gemfiles/rails_6_0.gemfile +1 -1
- data/lib/asset_sync/config.rb +26 -2
- data/lib/asset_sync/engine.rb +3 -0
- data/lib/asset_sync/storage.rb +55 -7
- data/lib/asset_sync/version.rb +1 -1
- data/lib/generators/asset_sync/install_generator.rb +4 -0
- data/lib/generators/asset_sync/templates/asset_sync.rb +7 -0
- data/spec/unit/railsless_spec.rb +4 -3
- data/spec/unit/storage_spec.rb +82 -1
- metadata +6 -7
- data/gemfiles/rails_4_2.gemfile +0 -10
    
        checksums.yaml
    CHANGED
    
    | @@ -1,7 +1,7 @@ | |
| 1 1 | 
             
            ---
         | 
| 2 2 | 
             
            SHA256:
         | 
| 3 | 
            -
              metadata.gz:  | 
| 4 | 
            -
              data.tar.gz:  | 
| 3 | 
            +
              metadata.gz: c788619973e3b9413653db07eb712d4ce2ed2cbea2560a5305770f77fc5a32ae
         | 
| 4 | 
            +
              data.tar.gz: ecc0ea7cb6a0b89375a937d7528069410fc1e1ed01b0be3463cd0a81e741e99c
         | 
| 5 5 | 
             
            SHA512:
         | 
| 6 | 
            -
              metadata.gz:  | 
| 7 | 
            -
              data.tar.gz:  | 
| 6 | 
            +
              metadata.gz: 7df91a8c7aec03b61936a4fbeea5734a419ae900f74fa59b800a061221cdaab1a3299992f2fec222f2778d510ae2988925552b0a699aef92a9b7acd2a9e2dbad
         | 
| 7 | 
            +
              data.tar.gz: 922efce815fa4b93366baaa889d3816d3f658693c0993255b29ee1d9744b64d0a43d36f45a6e992da40edf9c651398fd5c30cf5bd93f0218b5355f7492119309
         | 
    
        data/.travis.yml
    CHANGED
    
    | @@ -14,7 +14,6 @@ rvm: | |
| 14 14 | 
             
              - jruby
         | 
| 15 15 | 
             
              - jruby-head
         | 
| 16 16 | 
             
            gemfile:
         | 
| 17 | 
            -
              - gemfiles/rails_4_2.gemfile
         | 
| 18 17 | 
             
              - gemfiles/rails_5_0.gemfile
         | 
| 19 18 | 
             
              - gemfiles/rails_5_1.gemfile
         | 
| 20 19 | 
             
              - gemfiles/rails_5_2.gemfile
         | 
| @@ -33,16 +32,6 @@ env: | |
| 33 32 | 
             
            matrix:
         | 
| 34 33 | 
             
              fast_finish: true
         | 
| 35 34 | 
             
              allow_failures:
         | 
| 36 | 
            -
                # bundler version conflict
         | 
| 37 | 
            -
                - rvm: 2.3
         | 
| 38 | 
            -
                  gemfile: gemfiles/rails_4_2.gemfile
         | 
| 39 | 
            -
                - rvm: 2.4
         | 
| 40 | 
            -
                  gemfile: gemfiles/rails_4_2.gemfile
         | 
| 41 | 
            -
                - rvm: 2.5
         | 
| 42 | 
            -
                  gemfile: gemfiles/rails_4_2.gemfile
         | 
| 43 | 
            -
                - rvm: jruby
         | 
| 44 | 
            -
                  gemfile: gemfiles/rails_4_2.gemfile
         | 
| 45 | 
            -
                - gemfile: gemfiles/rails_6_0.gemfile
         | 
| 46 35 | 
             
                - rvm: ruby-head
         | 
| 47 36 | 
             
                - rvm: jruby-head
         | 
| 48 37 | 
             
              exclude:
         | 
    
        data/Appraisals
    CHANGED
    
    | @@ -1,8 +1,4 @@ | |
| 1 1 |  | 
| 2 | 
            -
            appraise "rails_4_2" do
         | 
| 3 | 
            -
              gem "rails", "~> 4.2.0"
         | 
| 4 | 
            -
            end
         | 
| 5 | 
            -
             | 
| 6 2 | 
             
            appraise "rails_5_0" do
         | 
| 7 3 | 
             
              gem "rails", "~> 5.0.0"
         | 
| 8 4 | 
             
            end
         | 
| @@ -16,5 +12,5 @@ appraise "rails_5_2" do | |
| 16 12 | 
             
            end
         | 
| 17 13 |  | 
| 18 14 | 
             
            appraise "rails_6_0" do
         | 
| 19 | 
            -
              gem "rails", "~> 6.0.0 | 
| 15 | 
            +
              gem "rails", "~> 6.0.0"
         | 
| 20 16 | 
             
            end
         | 
    
        data/CHANGELOG.md
    CHANGED
    
    | @@ -18,6 +18,46 @@ This project adheres to [Semantic Versioning](http://semver.org/). | |
| 18 18 | 
             
            - Nothing
         | 
| 19 19 |  | 
| 20 20 |  | 
| 21 | 
            +
            ## [2.12.0] - 2020-06-11
         | 
| 22 | 
            +
             | 
| 23 | 
            +
            ### Added
         | 
| 24 | 
            +
             | 
| 25 | 
            +
            - Add option `aws_session_token` to support AWS Temporary Security Credentials
         | 
| 26 | 
            +
              (https://github.com/AssetSync/asset_sync/pull/403)
         | 
| 27 | 
            +
             | 
| 28 | 
            +
             | 
| 29 | 
            +
            ## [2.11.0] - 2020-03-13
         | 
| 30 | 
            +
             | 
| 31 | 
            +
            ### Added
         | 
| 32 | 
            +
             | 
| 33 | 
            +
            - Add option `remote_file_list_cache_file_path` to skip scanning remote
         | 
| 34 | 
            +
              (https://github.com/AssetSync/asset_sync/pull/400)
         | 
| 35 | 
            +
             | 
| 36 | 
            +
             | 
| 37 | 
            +
            ## [2.10.0] - 2020-02-26
         | 
| 38 | 
            +
             | 
| 39 | 
            +
            ### Added
         | 
| 40 | 
            +
             | 
| 41 | 
            +
            - Add option `concurrent_uploads_max_threads` to limit number of threads for uploading files
         | 
| 42 | 
            +
              (https://github.com/AssetSync/asset_sync/pull/398)
         | 
| 43 | 
            +
             | 
| 44 | 
            +
             | 
| 45 | 
            +
            ## [2.9.1] - 2020-02-20
         | 
| 46 | 
            +
             | 
| 47 | 
            +
            ### Fixed
         | 
| 48 | 
            +
             | 
| 49 | 
            +
            - Fix uploading of sprockets manifest file
         | 
| 50 | 
            +
              (https://github.com/AssetSync/asset_sync/pull/397)
         | 
| 51 | 
            +
             | 
| 52 | 
            +
             | 
| 53 | 
            +
            ## [2.9.0] - 2020-01-15
         | 
| 54 | 
            +
             | 
| 55 | 
            +
            ### Added
         | 
| 56 | 
            +
             | 
| 57 | 
            +
            - Add option `concurrent_uploads` to improve speed of uploading
         | 
| 58 | 
            +
              (https://github.com/AssetSync/asset_sync/pull/393)
         | 
| 59 | 
            +
             | 
| 60 | 
            +
             | 
| 21 61 | 
             
            ## [2.8.2] - 2019-12-27
         | 
| 22 62 |  | 
| 23 63 | 
             
            ### Changed
         | 
| @@ -930,7 +970,12 @@ Changes: | |
| 930 970 | 
             
                * Merge branch 'sinatra'
         | 
| 931 971 |  | 
| 932 972 |  | 
| 933 | 
            -
            [Unreleased]: https://github.com/AssetSync/asset_sync/compare/v2. | 
| 973 | 
            +
            [Unreleased]: https://github.com/AssetSync/asset_sync/compare/v2.12.0...HEAD
         | 
| 974 | 
            +
            [2.12.0]: https://github.com/AssetSync/asset_sync/compare/v2.11.0...v2.12.0
         | 
| 975 | 
            +
            [2.11.0]: https://github.com/AssetSync/asset_sync/compare/v2.10.0...v2.11.0
         | 
| 976 | 
            +
            [2.10.0]: https://github.com/AssetSync/asset_sync/compare/v2.9.1...v2.10.0
         | 
| 977 | 
            +
            [2.9.1]: https://github.com/AssetSync/asset_sync/compare/v2.9.0...v2.9.1
         | 
| 978 | 
            +
            [2.9.0]: https://github.com/AssetSync/asset_sync/compare/v2.8.2...v2.9.0
         | 
| 934 979 | 
             
            [2.8.2]: https://github.com/AssetSync/asset_sync/compare/v2.8.1...v2.8.2
         | 
| 935 980 | 
             
            [2.8.1]: https://github.com/AssetSync/asset_sync/compare/v2.8.0...v2.8.1
         | 
| 936 981 | 
             
            [2.8.0]: https://github.com/AssetSync/asset_sync/compare/v2.7.0...v2.8.0
         | 
    
        data/README.md
    CHANGED
    
    | @@ -141,7 +141,7 @@ The Built-in Initializer will configure **AssetSync** based on the contents of y | |
| 141 141 |  | 
| 142 142 | 
             
            Add your configuration details to **heroku**
         | 
| 143 143 |  | 
| 144 | 
            -
             | 
| 144 | 
            +
            ``` bash
         | 
| 145 145 | 
             
            heroku config:add AWS_ACCESS_KEY_ID=xxxx
         | 
| 146 146 | 
             
            heroku config:add AWS_SECRET_ACCESS_KEY=xxxx
         | 
| 147 147 | 
             
            heroku config:add FOG_DIRECTORY=xxxx
         | 
| @@ -208,6 +208,7 @@ AssetSync.configure do |config| | |
| 208 208 | 
             
              config.fog_directory = ENV['FOG_DIRECTORY']
         | 
| 209 209 | 
             
              config.aws_access_key_id = ENV['AWS_ACCESS_KEY_ID']
         | 
| 210 210 | 
             
              config.aws_secret_access_key = ENV['AWS_SECRET_ACCESS_KEY']
         | 
| 211 | 
            +
              config.aws_session_token = ENV['AWS_SESSION_TOKEN'] if ENV.key?['AWS_SESSION_TOKEN']
         | 
| 211 212 |  | 
| 212 213 | 
             
              # Don't delete files from the store
         | 
| 213 214 | 
             
              # config.existing_remote_files = 'keep'
         | 
| @@ -242,6 +243,15 @@ AssetSync.configure do |config| | |
| 242 243 | 
             
              # Upload the manifest file also.
         | 
| 243 244 | 
             
              # config.include_manifest = false
         | 
| 244 245 | 
             
              #
         | 
| 246 | 
            +
              # Upload files concurrently
         | 
| 247 | 
            +
              # config.concurrent_uploads = false
         | 
| 248 | 
            +
              # 
         | 
| 249 | 
            +
              # Number of threads when concurrent_uploads is enabled
         | 
| 250 | 
            +
              # config.concurrent_uploads_max_threads = 10
         | 
| 251 | 
            +
              #
         | 
| 252 | 
            +
              # Path to cache file to skip scanning remote
         | 
| 253 | 
            +
              # config.remote_file_list_cache_file_path = './.asset_sync_remote_file_list_cache.json'
         | 
| 254 | 
            +
              #
         | 
| 245 255 | 
             
              # Fail silently.  Useful for environments such as Heroku
         | 
| 246 256 | 
             
              # config.fail_silently = true
         | 
| 247 257 | 
             
              #
         | 
| @@ -338,6 +348,9 @@ AssetSync.config.gzip_compression == ENV['ASSET_SYNC_GZIP_COMPRESSION'] | |
| 338 348 | 
             
            * **gzip\_compression**: (`true, false`) when enabled, will automatically replace files that have a gzip compressed equivalent with the compressed version. **default:** `'false'`
         | 
| 339 349 | 
             
            * **manifest**: (`true, false`) when enabled, will use the `manifest.yml` generated by Rails to get the list of local files to upload. **experimental**. **default:** `'false'`
         | 
| 340 350 | 
             
            * **include_manifest**: (`true, false`) when enabled, will upload the `manifest.yml` generated by Rails. **default:** `'false'`
         | 
| 351 | 
            +
            * **concurrent_uploads**: (`true, false`) when enabled, will upload the files in different Threads, this greatly improves the upload speed. **default:** `'false'`
         | 
| 352 | 
            +
            * **concurrent_uploads_max_threads**: when concurrent_uploads is enabled, this determines the number of threads that will be created. **default:** `10`
         | 
| 353 | 
            +
            * **remote_file_list_cache_file_path**: if present, use this path to cache remote file list to skip scanning remote **default:** `nil`
         | 
| 341 354 | 
             
            * **enabled**: (`true, false`) when false, will disable asset sync. **default:** `'true'` (enabled)
         | 
| 342 355 | 
             
            * **ignored\_files**: an array of files to ignore e.g. `['ignore_me.js', %r(ignore_some/\d{32}\.css)]` Useful if there are some files that are created dynamically on the server and you don't want to upload on deploy **default**: `[]`
         | 
| 343 356 | 
             
            * **cache\_asset\_regexps**: an array of files to add cache headers e.g. `['cache_me.js', %r(cache_some\.\d{8}\.css)]` Useful if there are some files that are added to sprockets assets list and need to be set as 'Cacheable' on uploaded server.  Only rails compiled regexp is matched internally **default**: `[]`
         | 
| @@ -536,6 +549,10 @@ AssetSync.configure do |config| | |
| 536 549 | 
             
              config.aws_access_key_id = ENV['AWS_ACCESS_KEY_ID']
         | 
| 537 550 | 
             
              config.aws_secret_access_key = ENV['AWS_SECRET_ACCESS_KEY']
         | 
| 538 551 | 
             
              config.prefix = 'assets'
         | 
| 552 | 
            +
              # Can be a `Pathname` or `String`
         | 
| 553 | 
            +
              # Will be converted into an `Pathname`
         | 
| 554 | 
            +
              # If relative, will be converted into an absolute path 
         | 
| 555 | 
            +
              # via `::Rails.root` or `::Dir.pwd`
         | 
| 539 556 | 
             
              config.public_path = Pathname('./public')
         | 
| 540 557 | 
             
            end
         | 
| 541 558 | 
             
            ```
         | 
    
        data/gemfiles/rails_6_0.gemfile
    CHANGED
    
    | @@ -5,6 +5,6 @@ source "https://rubygems.org" | |
| 5 5 | 
             
            gem "rcov", platforms: :mri_18, group: [:development, :test]
         | 
| 6 6 | 
             
            gem "simplecov", platforms: [:jruby, :mri_19, :ruby_19, :mri_20, :rbx], group: [:development, :test], require: false
         | 
| 7 7 | 
             
            gem "jruby-openssl", platform: :jruby
         | 
| 8 | 
            -
            gem "rails", "~> 6.0.0 | 
| 8 | 
            +
            gem "rails", "~> 6.0.0"
         | 
| 9 9 |  | 
| 10 10 | 
             
            gemspec path: "../"
         | 
    
        data/lib/asset_sync/config.rb
    CHANGED
    
    | @@ -26,7 +26,9 @@ module AssetSync | |
| 26 26 | 
             
                attr_accessor :cdn_distribution_id
         | 
| 27 27 | 
             
                attr_accessor :cache_asset_regexps
         | 
| 28 28 | 
             
                attr_accessor :include_manifest
         | 
| 29 | 
            -
                 | 
| 29 | 
            +
                attr_accessor :concurrent_uploads
         | 
| 30 | 
            +
                attr_accessor :concurrent_uploads_max_threads
         | 
| 31 | 
            +
                attr_accessor :remote_file_list_cache_file_path
         | 
| 30 32 |  | 
| 31 33 | 
             
                # FOG configuration
         | 
| 32 34 | 
             
                attr_accessor :fog_provider          # Currently Supported ['AWS', 'Rackspace']
         | 
| @@ -35,7 +37,7 @@ module AssetSync | |
| 35 37 | 
             
                attr_reader   :fog_public            # e.g. true, false, "default"
         | 
| 36 38 |  | 
| 37 39 | 
             
                # Amazon AWS
         | 
| 38 | 
            -
                attr_accessor :aws_access_key_id, :aws_secret_access_key, :aws_reduced_redundancy, :aws_iam_roles, :aws_signature_version
         | 
| 40 | 
            +
                attr_accessor :aws_access_key_id, :aws_secret_access_key, :aws_session_token, :aws_reduced_redundancy, :aws_iam_roles, :aws_signature_version
         | 
| 39 41 | 
             
                attr_accessor :fog_host              # e.g. 's3.amazonaws.com'
         | 
| 40 42 | 
             
                attr_accessor :fog_port              # e.g. '9000'
         | 
| 41 43 | 
             
                attr_accessor :fog_path_style        # e.g. true
         | 
| @@ -66,6 +68,7 @@ module AssetSync | |
| 66 68 | 
             
                validates :google_storage_access_key_id,      :presence => true, :if => :google_interop?
         | 
| 67 69 | 
             
                validates :google_json_key_location,          :presence => true, :if => :google_service_account?
         | 
| 68 70 | 
             
                validates :google_project,                    :presence => true, :if => :google_service_account?
         | 
| 71 | 
            +
                validates :concurrent_uploads,    :inclusion => { :in => [true, false] }
         | 
| 69 72 |  | 
| 70 73 | 
             
                def initialize
         | 
| 71 74 | 
             
                  self.fog_region = nil
         | 
| @@ -84,6 +87,9 @@ module AssetSync | |
| 84 87 | 
             
                  self.invalidate = []
         | 
| 85 88 | 
             
                  self.cache_asset_regexps = []
         | 
| 86 89 | 
             
                  self.include_manifest = false
         | 
| 90 | 
            +
                  self.concurrent_uploads = false
         | 
| 91 | 
            +
                  self.concurrent_uploads_max_threads = 10
         | 
| 92 | 
            +
                  self.remote_file_list_cache_file_path = nil
         | 
| 87 93 | 
             
                  @additional_local_file_paths_procs = []
         | 
| 88 94 |  | 
| 89 95 | 
             
                  load_yml! if defined?(::Rails) && yml_exists?
         | 
| @@ -172,6 +178,19 @@ module AssetSync | |
| 172 178 | 
             
                  @public_path || ::Rails.public_path
         | 
| 173 179 | 
             
                end
         | 
| 174 180 |  | 
| 181 | 
            +
                def public_path=(path)
         | 
| 182 | 
            +
                  # Generate absolute path even when relative path passed in
         | 
| 183 | 
            +
                  # Required for generating relative sprockets manifest path
         | 
| 184 | 
            +
                  pathname = Pathname(path)
         | 
| 185 | 
            +
                  @public_path = if pathname.absolute?
         | 
| 186 | 
            +
                    pathname
         | 
| 187 | 
            +
                  elsif defined?(::Rails.root)
         | 
| 188 | 
            +
                    ::Rails.root.join(pathname)
         | 
| 189 | 
            +
                  else
         | 
| 190 | 
            +
                    Pathname(::Dir.pwd).join(pathname)
         | 
| 191 | 
            +
                  end
         | 
| 192 | 
            +
                end
         | 
| 193 | 
            +
             | 
| 175 194 | 
             
                def load_yml!
         | 
| 176 195 | 
             
                  self.enabled                = yml["enabled"] if yml.has_key?('enabled')
         | 
| 177 196 | 
             
                  self.fog_provider           = yml["fog_provider"]
         | 
| @@ -184,6 +203,7 @@ module AssetSync | |
| 184 203 | 
             
                  self.fog_scheme             = yml["fog_scheme"]
         | 
| 185 204 | 
             
                  self.aws_access_key_id      = yml["aws_access_key_id"]
         | 
| 186 205 | 
             
                  self.aws_secret_access_key  = yml["aws_secret_access_key"]
         | 
| 206 | 
            +
                  self.aws_session_token      = yml["aws_session_token"] if yml.has_key?("aws_session_token")
         | 
| 187 207 | 
             
                  self.aws_reduced_redundancy = yml["aws_reduced_redundancy"]
         | 
| 188 208 | 
             
                  self.aws_iam_roles          = yml["aws_iam_roles"]
         | 
| 189 209 | 
             
                  self.aws_signature_version  = yml["aws_signature_version"]
         | 
| @@ -206,6 +226,9 @@ module AssetSync | |
| 206 226 | 
             
                  self.cdn_distribution_id    = yml['cdn_distribution_id'] if yml.has_key?("cdn_distribution_id")
         | 
| 207 227 | 
             
                  self.cache_asset_regexps    = yml['cache_asset_regexps'] if yml.has_key?("cache_asset_regexps")
         | 
| 208 228 | 
             
                  self.include_manifest       = yml['include_manifest'] if yml.has_key?("include_manifest")
         | 
| 229 | 
            +
                  self.concurrent_uploads     = yml['concurrent_uploads'] if yml.has_key?('concurrent_uploads')
         | 
| 230 | 
            +
                  self.concurrent_uploads_max_threads = yml['concurrent_uploads_max_threads'] if yml.has_key?('concurrent_uploads_max_threads')
         | 
| 231 | 
            +
                  self.remote_file_list_cache_file_path = yml['remote_file_list_cache_file_path'] if yml.has_key?('remote_file_list_cache_file_path')
         | 
| 209 232 |  | 
| 210 233 | 
             
                  self.azure_storage_account_name = yml['azure_storage_account_name'] if yml.has_key?("azure_storage_account_name")
         | 
| 211 234 | 
             
                  self.azure_storage_access_key   = yml['azure_storage_access_key'] if yml.has_key?("azure_storage_access_key")
         | 
| @@ -238,6 +261,7 @@ module AssetSync | |
| 238 261 | 
             
                        :aws_access_key_id => aws_access_key_id,
         | 
| 239 262 | 
             
                        :aws_secret_access_key => aws_secret_access_key
         | 
| 240 263 | 
             
                      })
         | 
| 264 | 
            +
                      options.merge!({:aws_session_token => aws_session_token}) if aws_session_token
         | 
| 241 265 | 
             
                    end
         | 
| 242 266 | 
             
                    options.merge!({:host => fog_host}) if fog_host
         | 
| 243 267 | 
             
                    options.merge!({:port => fog_port}) if fog_port
         | 
    
        data/lib/asset_sync/engine.rb
    CHANGED
    
    | @@ -23,6 +23,7 @@ module AssetSync | |
| 23 23 |  | 
| 24 24 | 
             
                      config.aws_access_key_id = ENV['AWS_ACCESS_KEY_ID'] if ENV.has_key?('AWS_ACCESS_KEY_ID')
         | 
| 25 25 | 
             
                      config.aws_secret_access_key = ENV['AWS_SECRET_ACCESS_KEY'] if ENV.has_key?('AWS_SECRET_ACCESS_KEY')
         | 
| 26 | 
            +
                      config.aws_session_token = ENV['AWS_SESSION_TOKEN'] if ENV.has_key?('AWS_SESSION_TOKEN')
         | 
| 26 27 | 
             
                      config.aws_signature_version = ENV['AWS_SIGNATURE_VERSION'] if ENV.has_key?('AWS_SIGNATURE_VERSION')
         | 
| 27 28 | 
             
                      config.aws_reduced_redundancy = ENV['AWS_REDUCED_REDUNDANCY'] == true  if ENV.has_key?('AWS_REDUCED_REDUNDANCY')
         | 
| 28 29 |  | 
| @@ -42,6 +43,8 @@ module AssetSync | |
| 42 43 | 
             
                      config.gzip_compression = (ENV['ASSET_SYNC_GZIP_COMPRESSION'] == 'true') if ENV.has_key?('ASSET_SYNC_GZIP_COMPRESSION')
         | 
| 43 44 | 
             
                      config.manifest = (ENV['ASSET_SYNC_MANIFEST'] == 'true') if ENV.has_key?('ASSET_SYNC_MANIFEST')
         | 
| 44 45 | 
             
                      config.include_manifest = (ENV['ASSET_SYNC_INCLUDE_MANIFEST'] == 'true') if ENV.has_key?('ASSET_SYNC_INCLUDE_MANIFEST')
         | 
| 46 | 
            +
                      config.concurrent_uploads = (ENV['ASSET_SYNC_CONCURRENT_UPLOADS'] == 'true') if ENV.has_key?('ASSET_SYNC_CONCURRENT_UPLOADS')
         | 
| 47 | 
            +
                      config.remote_file_list_cache_file_path = ENV['ASSET_SYNC_REMOTE_FILE_LIST_CACHE_FILE_PATH'] if ENV.has_key?('ASSET_SYNC_REMOTE_FILE_LIST_CACHE_FILE_PATH')
         | 
| 45 48 | 
             
                    end
         | 
| 46 49 |  | 
| 47 50 | 
             
                    config.prefix = ENV['ASSET_SYNC_PREFIX'] if ENV.has_key?('ASSET_SYNC_PREFIX')
         | 
    
        data/lib/asset_sync/storage.rb
    CHANGED
    
    | @@ -37,6 +37,10 @@ module AssetSync | |
| 37 37 | 
             
                  self.config.public_path
         | 
| 38 38 | 
             
                end
         | 
| 39 39 |  | 
| 40 | 
            +
                def remote_file_list_cache_file_path
         | 
| 41 | 
            +
                  self.config.remote_file_list_cache_file_path
         | 
| 42 | 
            +
                end
         | 
| 43 | 
            +
             | 
| 40 44 | 
             
                def ignored_files
         | 
| 41 45 | 
             
                  expand_file_names(self.config.ignored_files)
         | 
| 42 46 | 
             
                end
         | 
| @@ -58,6 +62,32 @@ module AssetSync | |
| 58 62 | 
             
                    (get_local_files + config.additional_local_file_paths).uniq
         | 
| 59 63 | 
             
                end
         | 
| 60 64 |  | 
| 65 | 
            +
                def remote_files
         | 
| 66 | 
            +
                  return [] if ignore_existing_remote_files?
         | 
| 67 | 
            +
                  return @remote_files if @remote_files
         | 
| 68 | 
            +
             | 
| 69 | 
            +
                  if remote_file_list_cache_file_path && File.file?(remote_file_list_cache_file_path)
         | 
| 70 | 
            +
                    begin
         | 
| 71 | 
            +
                      content = File.read(remote_file_list_cache_file_path)
         | 
| 72 | 
            +
                      return @remote_files = JSON.parse(content)
         | 
| 73 | 
            +
                    rescue JSON::ParserError
         | 
| 74 | 
            +
                      warn "Failed to parse #{remote_file_list_cache_file_path} as json"
         | 
| 75 | 
            +
                    end
         | 
| 76 | 
            +
                  end
         | 
| 77 | 
            +
             | 
| 78 | 
            +
                  @remote_files = get_remote_files
         | 
| 79 | 
            +
                end
         | 
| 80 | 
            +
             | 
| 81 | 
            +
                def update_remote_file_list_cache(local_files_to_upload)
         | 
| 82 | 
            +
                  return unless remote_file_list_cache_file_path
         | 
| 83 | 
            +
                  return if ignore_existing_remote_files?
         | 
| 84 | 
            +
             | 
| 85 | 
            +
                  File.open(self.remote_file_list_cache_file_path, 'w') do |file|
         | 
| 86 | 
            +
                    uploaded = local_files_to_upload + remote_files
         | 
| 87 | 
            +
                    file.write(uploaded.to_json)
         | 
| 88 | 
            +
                  end
         | 
| 89 | 
            +
                end
         | 
| 90 | 
            +
             | 
| 61 91 | 
             
                def always_upload_files
         | 
| 62 92 | 
             
                  expand_file_names(self.config.always_upload) + get_manifest_path
         | 
| 63 93 | 
             
                end
         | 
| @@ -243,16 +273,32 @@ module AssetSync | |
| 243 273 | 
             
                end
         | 
| 244 274 |  | 
| 245 275 | 
             
                def upload_files
         | 
| 246 | 
            -
                  # get a fresh list of remote files
         | 
| 247 | 
            -
                  remote_files = ignore_existing_remote_files? ? [] : get_remote_files
         | 
| 248 276 | 
             
                  # fixes: https://github.com/rumblelabs/asset_sync/issues/19
         | 
| 249 277 | 
             
                  local_files_to_upload = local_files - ignored_files - remote_files + always_upload_files
         | 
| 250 278 | 
             
                  local_files_to_upload = (local_files_to_upload + get_non_fingerprinted(local_files_to_upload)).uniq
         | 
| 251 | 
            -
             | 
| 252 | 
            -
                   | 
| 253 | 
            -
             | 
| 254 | 
            -
             | 
| 255 | 
            -
                     | 
| 279 | 
            +
                  # Only files.
         | 
| 280 | 
            +
                  local_files_to_upload = local_files_to_upload.select { |f| File.file? "#{path}/#{f}" }
         | 
| 281 | 
            +
             | 
| 282 | 
            +
                  if self.config.concurrent_uploads
         | 
| 283 | 
            +
                    jobs = Queue.new
         | 
| 284 | 
            +
                    local_files_to_upload.each { |f| jobs.push(f) }
         | 
| 285 | 
            +
                    jobs.close
         | 
| 286 | 
            +
             | 
| 287 | 
            +
                    num_threads = [self.config.concurrent_uploads_max_threads, local_files_to_upload.length].min
         | 
| 288 | 
            +
                    # Upload new files
         | 
| 289 | 
            +
                    workers = Array.new(num_threads) do
         | 
| 290 | 
            +
                      Thread.new do
         | 
| 291 | 
            +
                        while f = jobs.pop
         | 
| 292 | 
            +
                          upload_file(f)
         | 
| 293 | 
            +
                        end
         | 
| 294 | 
            +
                      end
         | 
| 295 | 
            +
                    end
         | 
| 296 | 
            +
                    workers.map(&:join)
         | 
| 297 | 
            +
                  else
         | 
| 298 | 
            +
                    # Upload new files
         | 
| 299 | 
            +
                    local_files_to_upload.each do |f|
         | 
| 300 | 
            +
                      upload_file f
         | 
| 301 | 
            +
                    end
         | 
| 256 302 | 
             
                  end
         | 
| 257 303 |  | 
| 258 304 | 
             
                  if self.config.cdn_distribution_id && files_to_invalidate.any?
         | 
| @@ -261,6 +307,8 @@ module AssetSync | |
| 261 307 | 
             
                    data = cdn.post_invalidation(self.config.cdn_distribution_id, files_to_invalidate)
         | 
| 262 308 | 
             
                    log "Invalidation id: #{data.body["Id"]}"
         | 
| 263 309 | 
             
                  end
         | 
| 310 | 
            +
             | 
| 311 | 
            +
                  update_remote_file_list_cache(local_files_to_upload)
         | 
| 264 312 | 
             
                end
         | 
| 265 313 |  | 
| 266 314 | 
             
                def sync
         | 
    
        data/lib/asset_sync/version.rb
    CHANGED
    
    
| @@ -4,6 +4,7 @@ if defined?(AssetSync) | |
| 4 4 | 
             
                config.fog_provider = 'AWS'
         | 
| 5 5 | 
             
                config.aws_access_key_id = ENV['AWS_ACCESS_KEY_ID']
         | 
| 6 6 | 
             
                config.aws_secret_access_key = ENV['AWS_SECRET_ACCESS_KEY']
         | 
| 7 | 
            +
                config.aws_session_token = ENV['AWS_SESSION_TOKEN'] if ENV.key?['AWS_SESSION_TOKEN']
         | 
| 7 8 | 
             
                # To use AWS reduced redundancy storage.
         | 
| 8 9 | 
             
                # config.aws_reduced_redundancy = true
         | 
| 9 10 | 
             
                #
         | 
| @@ -63,6 +64,12 @@ if defined?(AssetSync) | |
| 63 64 | 
             
                # Upload the manifest file also.
         | 
| 64 65 | 
             
                # config.include_manifest = false
         | 
| 65 66 | 
             
                #
         | 
| 67 | 
            +
                # Upload files concurrently
         | 
| 68 | 
            +
                # config.concurrent_uploads = false
         | 
| 69 | 
            +
                #
         | 
| 70 | 
            +
                # Path to cache file to skip scanning remote
         | 
| 71 | 
            +
                # config.remote_file_list_cache_file_path = './.asset_sync_remote_file_list_cache.json'
         | 
| 72 | 
            +
                #
         | 
| 66 73 | 
             
                # Fail silently.  Useful for environments such as Heroku
         | 
| 67 74 | 
             
                # config.fail_silently = true
         | 
| 68 75 | 
             
                #
         | 
    
        data/spec/unit/railsless_spec.rb
    CHANGED
    
    | @@ -14,7 +14,7 @@ describe AssetSync do | |
| 14 14 | 
             
                    config.fog_region = 'eu-west-1'
         | 
| 15 15 | 
             
                    config.existing_remote_files = "keep"
         | 
| 16 16 | 
             
                    config.prefix = "assets"
         | 
| 17 | 
            -
                    config.public_path =  | 
| 17 | 
            +
                    config.public_path = "./public"
         | 
| 18 18 | 
             
                  end
         | 
| 19 19 | 
             
                end
         | 
| 20 20 |  | 
| @@ -22,8 +22,9 @@ describe AssetSync do | |
| 22 22 | 
             
                  expect(AssetSync.config.prefix).to eq("assets")
         | 
| 23 23 | 
             
                end
         | 
| 24 24 |  | 
| 25 | 
            -
                it "should have  | 
| 26 | 
            -
                  expect(AssetSync.config.public_path.to_s).to  | 
| 25 | 
            +
                it "should have public_path" do
         | 
| 26 | 
            +
                  expect(AssetSync.config.public_path.to_s).to be_end_with("/public")
         | 
| 27 | 
            +
                  expect(AssetSync.config.public_path).to be_absolute
         | 
| 27 28 | 
             
                end
         | 
| 28 29 |  | 
| 29 30 | 
             
                it "should default AssetSync to enabled" do
         | 
    
        data/spec/unit/storage_spec.rb
    CHANGED
    
    | @@ -54,6 +54,87 @@ describe AssetSync::Storage do | |
| 54 54 | 
             
                  storage.upload_files
         | 
| 55 55 | 
             
                end
         | 
| 56 56 |  | 
| 57 | 
            +
                it 'should upload files concurrently if enabled' do
         | 
| 58 | 
            +
                  @config.concurrent_uploads = true
         | 
| 59 | 
            +
                  storage = AssetSync::Storage.new(@config)
         | 
| 60 | 
            +
             | 
| 61 | 
            +
                  allow(storage).to receive(:get_local_files).and_return(@local_files)
         | 
| 62 | 
            +
                  allow(storage).to receive(:get_remote_files).and_return(@remote_files)
         | 
| 63 | 
            +
                  allow(File).to receive(:file?).and_return(true) # Pretend they all exist
         | 
| 64 | 
            +
             | 
| 65 | 
            +
                  expect(Thread).to receive(:new).exactly(3).times.and_call_original
         | 
| 66 | 
            +
                  (@local_files - @remote_files + storage.always_upload_files).each do |file|
         | 
| 67 | 
            +
                    expect(storage).to receive(:upload_file).with(file)
         | 
| 68 | 
            +
                  end
         | 
| 69 | 
            +
             | 
| 70 | 
            +
                  storage.upload_files
         | 
| 71 | 
            +
                end
         | 
| 72 | 
            +
             | 
| 73 | 
            +
                it 'should allow custom number of threads' do
         | 
| 74 | 
            +
                  @config.concurrent_uploads = true
         | 
| 75 | 
            +
                  @config.concurrent_uploads_max_threads = 2
         | 
| 76 | 
            +
                  storage = AssetSync::Storage.new(@config)
         | 
| 77 | 
            +
             | 
| 78 | 
            +
                  allow(storage).to receive(:get_local_files).and_return(@local_files)
         | 
| 79 | 
            +
                  allow(storage).to receive(:get_remote_files).and_return(@remote_files)
         | 
| 80 | 
            +
                  allow(File).to receive(:file?).and_return(true) # Pretend they all exist
         | 
| 81 | 
            +
             | 
| 82 | 
            +
                  expect(Thread).to receive(:new).exactly(2).times.and_call_original
         | 
| 83 | 
            +
                  (@local_files - @remote_files + storage.always_upload_files).each do |file|
         | 
| 84 | 
            +
                    expect(storage).to receive(:upload_file).with(file)
         | 
| 85 | 
            +
                  end
         | 
| 86 | 
            +
             | 
| 87 | 
            +
                  storage.upload_files
         | 
| 88 | 
            +
                end
         | 
| 89 | 
            +
             | 
| 90 | 
            +
                it 'should allow remote_file_list_cache_file_path configuration' do
         | 
| 91 | 
            +
                  file_path = './foo.json'
         | 
| 92 | 
            +
                  @config.remote_file_list_cache_file_path = file_path
         | 
| 93 | 
            +
                  storage = AssetSync::Storage.new(@config)
         | 
| 94 | 
            +
             | 
| 95 | 
            +
                  allow(storage).to receive(:get_local_files).and_return(@local_files)
         | 
| 96 | 
            +
                  File.write(file_path, @remote_files.to_json)
         | 
| 97 | 
            +
                  expect(storage).not_to receive(:get_remote_files)
         | 
| 98 | 
            +
                  allow(File).to receive(:file?).and_return(true) # Pretend they all exist
         | 
| 99 | 
            +
             | 
| 100 | 
            +
                  (@local_files - @remote_files + storage.always_upload_files).each do |file|
         | 
| 101 | 
            +
                    expect(storage).to receive(:upload_file).with(file)
         | 
| 102 | 
            +
                  end
         | 
| 103 | 
            +
             | 
| 104 | 
            +
                  expect(storage).not_to receive(:warn)
         | 
| 105 | 
            +
                  storage.upload_files
         | 
| 106 | 
            +
             | 
| 107 | 
            +
                  # update remote_file_list_cache corretly
         | 
| 108 | 
            +
                  updated = JSON.parse(File.read(file_path))
         | 
| 109 | 
            +
                  expect(updated.sort.uniq).to eq (@remote_files + @local_files + storage.always_upload_files).sort.uniq
         | 
| 110 | 
            +
             | 
| 111 | 
            +
                  File.delete(file_path)
         | 
| 112 | 
            +
                end
         | 
| 113 | 
            +
             | 
| 114 | 
            +
                it 'should work with broken cache' do
         | 
| 115 | 
            +
                  file_path = './foo.json'
         | 
| 116 | 
            +
                  @config.remote_file_list_cache_file_path = file_path
         | 
| 117 | 
            +
             | 
| 118 | 
            +
                  storage = AssetSync::Storage.new(@config)
         | 
| 119 | 
            +
             | 
| 120 | 
            +
                  File.write(file_path, 'some non-json text file content')
         | 
| 121 | 
            +
             | 
| 122 | 
            +
                  allow(storage).to receive(:get_local_files).and_return(@local_files)
         | 
| 123 | 
            +
                  allow(storage).to receive(:get_remote_files).and_return(@remote_files)
         | 
| 124 | 
            +
                  allow(File).to receive(:file?).and_return(true) # Pretend they all exist
         | 
| 125 | 
            +
             | 
| 126 | 
            +
                  (@local_files - @remote_files + storage.always_upload_files).each do |file|
         | 
| 127 | 
            +
                    expect(storage).to receive(:upload_file).with(file)
         | 
| 128 | 
            +
                  end
         | 
| 129 | 
            +
             | 
| 130 | 
            +
                  # when broken, warning message should be prompted
         | 
| 131 | 
            +
                  expect(storage).to receive(:warn)
         | 
| 132 | 
            +
             | 
| 133 | 
            +
                  storage.upload_files
         | 
| 134 | 
            +
             | 
| 135 | 
            +
                  File.delete(file_path)
         | 
| 136 | 
            +
                end
         | 
| 137 | 
            +
             | 
| 57 138 | 
             
                it 'should upload updated non-fingerprinted files' do
         | 
| 58 139 | 
             
                  @local_files = [
         | 
| 59 140 | 
             
                    'public/image.png',
         | 
| @@ -125,7 +206,7 @@ describe AssetSync::Storage do | |
| 125 206 | 
             
                  end
         | 
| 126 207 | 
             
                end
         | 
| 127 208 |  | 
| 128 | 
            -
                it 'should upload additonal | 
| 209 | 
            +
                it 'should upload additonal files' do
         | 
| 129 210 | 
             
                  @local_files = [
         | 
| 130 211 | 
             
                    'public/image.png',
         | 
| 131 212 | 
             
                    'public/image-82389298328.png',
         | 
    
        metadata
    CHANGED
    
    | @@ -1,17 +1,17 @@ | |
| 1 1 | 
             
            --- !ruby/object:Gem::Specification
         | 
| 2 2 | 
             
            name: asset_sync
         | 
| 3 3 | 
             
            version: !ruby/object:Gem::Version
         | 
| 4 | 
            -
              version: 2. | 
| 4 | 
            +
              version: 2.12.0
         | 
| 5 5 | 
             
            platform: ruby
         | 
| 6 6 | 
             
            authors:
         | 
| 7 7 | 
             
            - Simon Hamilton
         | 
| 8 8 | 
             
            - David Rice
         | 
| 9 9 | 
             
            - Phil McClure
         | 
| 10 10 | 
             
            - Toby Osbourn
         | 
| 11 | 
            -
            autorequire: | 
| 11 | 
            +
            autorequire:
         | 
| 12 12 | 
             
            bindir: bin
         | 
| 13 13 | 
             
            cert_chain: []
         | 
| 14 | 
            -
            date:  | 
| 14 | 
            +
            date: 2020-06-11 00:00:00.000000000 Z
         | 
| 15 15 | 
             
            dependencies:
         | 
| 16 16 | 
             
            - !ruby/object:Gem::Dependency
         | 
| 17 17 | 
             
              name: fog-core
         | 
| @@ -203,7 +203,6 @@ files: | |
| 203 203 | 
             
            - UPGRADING.md
         | 
| 204 204 | 
             
            - asset_sync.gemspec
         | 
| 205 205 | 
             
            - docs/heroku.md
         | 
| 206 | 
            -
            - gemfiles/rails_4_2.gemfile
         | 
| 207 206 | 
             
            - gemfiles/rails_5_0.gemfile
         | 
| 208 207 | 
             
            - gemfiles/rails_5_1.gemfile
         | 
| 209 208 | 
             
            - gemfiles/rails_5_2.gemfile
         | 
| @@ -242,7 +241,7 @@ homepage: https://github.com/rumblelabs/asset_sync | |
| 242 241 | 
             
            licenses:
         | 
| 243 242 | 
             
            - MIT
         | 
| 244 243 | 
             
            metadata: {}
         | 
| 245 | 
            -
            post_install_message: | 
| 244 | 
            +
            post_install_message:
         | 
| 246 245 | 
             
            rdoc_options: []
         | 
| 247 246 | 
             
            require_paths:
         | 
| 248 247 | 
             
            - lib
         | 
| @@ -257,8 +256,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement | |
| 257 256 | 
             
                - !ruby/object:Gem::Version
         | 
| 258 257 | 
             
                  version: '0'
         | 
| 259 258 | 
             
            requirements: []
         | 
| 260 | 
            -
            rubygems_version: 3.1. | 
| 261 | 
            -
            signing_key: | 
| 259 | 
            +
            rubygems_version: 3.1.4
         | 
| 260 | 
            +
            signing_key:
         | 
| 262 261 | 
             
            specification_version: 4
         | 
| 263 262 | 
             
            summary: Synchronises Assets in a Rails 3 application and Amazon S3/Cloudfront and
         | 
| 264 263 | 
             
              Rackspace Cloudfiles
         | 
    
        data/gemfiles/rails_4_2.gemfile
    DELETED
    
    | @@ -1,10 +0,0 @@ | |
| 1 | 
            -
            # This file was generated by Appraisal
         | 
| 2 | 
            -
             | 
| 3 | 
            -
            source "https://rubygems.org"
         | 
| 4 | 
            -
             | 
| 5 | 
            -
            gem "rcov", platforms: :mri_18, group: [:development, :test]
         | 
| 6 | 
            -
            gem "simplecov", platforms: [:jruby, :mri_19, :ruby_19, :mri_20, :rbx], group: [:development, :test], require: false
         | 
| 7 | 
            -
            gem "jruby-openssl", platform: :jruby
         | 
| 8 | 
            -
            gem "rails", "~> 4.2.0"
         | 
| 9 | 
            -
             | 
| 10 | 
            -
            gemspec path: "../"
         |