asset_sync 2.9.1 → 2.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 454e84d789424886800c6c11f0576701eba664d42cfbf929b3b223d6d60383d0
-  data.tar.gz: 349569046fa6e19ff1ac6a27527f6e7de300b5953b996ec04c8f2e9ecf272e13
+  metadata.gz: e9d768c5e2075b08498c16af32ad4fb3f1006d4c68bd88846a60c843309a8ef3
+  data.tar.gz: 87a0b335fd8fc8903f30ce171d78d7beff4a87cb7fe0962bdf0c68dcf537c277
 SHA512:
-  metadata.gz: e8392af4e3c757f407b831b12ed76dc200da5a7aa5ddc9601c573be0fd965336fc9cfd4affb49d8420eb7edc95dcf1dba697e5ae59dfa5de7d445dd51efbf236
-  data.tar.gz: 4a4a1de27d7e42edc656e6ca792eaa62afe55484bfa76c1d565148af685698ad32265eb7c5752fa3ac1fe2f0216bfefb1c25e9ea3fd97fcb4796deef4cf3c689
+  metadata.gz: 24dba26c0c958ce65353daccf498fba8b5689a778d9e091325eeb323677f8057ade739e37c60a7c1e8d66d3bae36d87601da6d2bd569bf06559df89cdfca24f9
+  data.tar.gz: eca6579e50a37d8ce887cb5a44c068db42a387d04b58badb9b371d77944fd7f1a95c57132a0af4863f63efe62a7403f68b74d512145da78ca4aeee060344e17c

data/.travis.yml

@@ -2,6 +2,9 @@
 # http://docs.travis-ci.com/user/workers/container-based-infrastructure/
 sudo: false
 language: ruby
+arch: 
+  - amd64
+  - ppc64le
 cache:
   bundler: true
 rvm:

data/CHANGELOG.md

@@ -18,6 +18,46 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 - Nothing
 
 
+## [2.13.0] - 2020-12-14
+
+### Added
+
+- Add Backblaze B2 Cloud Storage support
+  (https://github.com/AssetSync/asset_sync/pull/410)
+
+
+## [2.12.1] - 2020-06-17
+
+### Fixed
+
+- Fix initializer template in generator
+  (https://github.com/AssetSync/asset_sync/pull/404)
+
+
+## [2.12.0] - 2020-06-11
+
+### Added
+
+- Add option `aws_session_token` to support AWS Temporary Security Credentials
+  (https://github.com/AssetSync/asset_sync/pull/403)
+
+
+## [2.11.0] - 2020-03-13
+
+### Added
+
+- Add option `remote_file_list_cache_file_path` to skip scanning remote
+  (https://github.com/AssetSync/asset_sync/pull/400)
+
+
+## [2.10.0] - 2020-02-26
+
+### Added
+
+- Add option `concurrent_uploads_max_threads` to limit number of threads for uploading files
+  (https://github.com/AssetSync/asset_sync/pull/398)
+
+
 ## [2.9.1] - 2020-02-20
 
 ### Fixed
@@ -946,7 +986,13 @@ Changes:
     * Merge branch 'sinatra'
 
 
-[Unreleased]: https://github.com/AssetSync/asset_sync/compare/v2.9.0...HEAD
+[Unreleased]: https://github.com/AssetSync/asset_sync/compare/v2.13.0...HEAD
+[2.13.0]: https://github.com/AssetSync/asset_sync/compare/v2.12.1...v2.13.0
+[2.12.1]: https://github.com/AssetSync/asset_sync/compare/v2.12.0...v2.12.1
+[2.12.0]: https://github.com/AssetSync/asset_sync/compare/v2.11.0...v2.12.0
+[2.11.0]: https://github.com/AssetSync/asset_sync/compare/v2.10.0...v2.11.0
+[2.10.0]: https://github.com/AssetSync/asset_sync/compare/v2.9.1...v2.10.0
+[2.9.1]: https://github.com/AssetSync/asset_sync/compare/v2.9.0...v2.9.1
 [2.9.0]: https://github.com/AssetSync/asset_sync/compare/v2.8.2...v2.9.0
 [2.8.2]: https://github.com/AssetSync/asset_sync/compare/v2.8.1...v2.8.2
 [2.8.1]: https://github.com/AssetSync/asset_sync/compare/v2.8.0...v2.8.1

data/README.md

@@ -38,6 +38,13 @@ gem "asset_sync"
 gem "fog-azure-rm"
 ```
 
+To use Backblaze B2, insert these.
+
+``` ruby
+gem "asset_sync"
+gem "fog-backblaze"
+```
+
 
 ### Extended Installation (Faster sync with turbosprockets)
 
@@ -77,6 +84,13 @@ Or, to use Azure Blob storage, configure as this.
   config.action_controller.asset_host = "//#{ENV['AZURE_STORAGE_ACCOUNT_NAME']}.blob.core.windows.net/#{ENV['FOG_DIRECTORY']}"
 ```
 
+Or, to use Backblaze B2, configure as this.
+
+``` ruby
+  #config/environments/production.rb
+  config.action_controller.asset_host = "//f000.backblazeb2.com/file/#{ENV['FOG_DIRECTORY']}"
+```
+
 
 On **HTTPS**: the exclusion of any protocol in the asset host declaration above will allow browsers to choose the transport mechanism on the fly. So if your application is available under both HTTP and HTTPS the assets will be served to match.
 
@@ -170,7 +184,7 @@ heroku config:add FOG_DIRECTORY=xxxx
 heroku config:add FOG_PROVIDER=Rackspace
 ```
 
-Google Storage Cloud configuration is supported as well. The preferred option is using the [GCS JSON API](https://github.com/fog/fog-google#storage) which requires that you create an appropriate service account, generate the signatures and make them accessible to asset sync at the prescribed location 
+Google Storage Cloud configuration is supported as well. The preferred option is using the [GCS JSON API](https://github.com/fog/fog-google#storage) which requires that you create an appropriate service account, generate the signatures and make them accessible to asset sync at the prescribed location
 
 ```bash
 heroku config:add FOG_PROVIDER=Google
@@ -199,6 +213,7 @@ Run the included Rake task to generate a starting point.
     rails g asset_sync:install --provider=Rackspace
     rails g asset_sync:install --provider=AWS
     rails g asset_sync:install --provider=AzureRM
+    rails g asset_sync:install --provider=Backblaze
 
 The generator will create a Rails initializer at `config/initializers/asset_sync.rb`.
 
@@ -208,6 +223,7 @@ AssetSync.configure do |config|
   config.fog_directory = ENV['FOG_DIRECTORY']
   config.aws_access_key_id = ENV['AWS_ACCESS_KEY_ID']
   config.aws_secret_access_key = ENV['AWS_SECRET_ACCESS_KEY']
+  config.aws_session_token = ENV['AWS_SESSION_TOKEN'] if ENV.key?('AWS_SESSION_TOKEN')
 
   # Don't delete files from the store
   # config.existing_remote_files = 'keep'
@@ -245,6 +261,12 @@ AssetSync.configure do |config|
   # Upload files concurrently
   # config.concurrent_uploads = false
   #
+  # Number of threads when concurrent_uploads is enabled
+  # config.concurrent_uploads_max_threads = 10
+  #
+  # Path to cache file to skip scanning remote
+  # config.remote_file_list_cache_file_path = './.asset_sync_remote_file_list_cache.json'
+  #
   # Fail silently.  Useful for environments such as Heroku
   # config.fail_silently = true
   #
@@ -267,6 +289,7 @@ Run the included Rake task to generate a starting point.
     rails g asset_sync:install --use-yml --provider=Rackspace
     rails g asset_sync:install --use-yml --provider=AWS
     rails g asset_sync:install --use-yml --provider=AzureRM
+    rails g asset_sync:install --use-yml --provider=Backblaze
 
 The generator will create a YAML file at `config/asset_sync.yml`.
 
@@ -342,6 +365,8 @@ AssetSync.config.gzip_compression == ENV['ASSET_SYNC_GZIP_COMPRESSION']
 * **manifest**: (`true, false`) when enabled, will use the `manifest.yml` generated by Rails to get the list of local files to upload. **experimental**. **default:** `'false'`
 * **include_manifest**: (`true, false`) when enabled, will upload the `manifest.yml` generated by Rails. **default:** `'false'`
 * **concurrent_uploads**: (`true, false`) when enabled, will upload the files in different Threads, this greatly improves the upload speed. **default:** `'false'`
+* **concurrent_uploads_max_threads**: when concurrent_uploads is enabled, this determines the number of threads that will be created. **default:** `10`
+* **remote_file_list_cache_file_path**: if present, use this path to cache remote file list to skip scanning remote **default:** `nil`
 * **enabled**: (`true, false`) when false, will disable asset sync. **default:** `'true'` (enabled)
 * **ignored\_files**: an array of files to ignore e.g. `['ignore_me.js', %r(ignore_some/\d{32}\.css)]` Useful if there are some files that are created dynamically on the server and you don't want to upload on deploy **default**: `[]`
 * **cache\_asset\_regexps**: an array of files to add cache headers e.g. `['cache_me.js', %r(cache_some\.\d{8}\.css)]` Useful if there are some files that are added to sprockets assets list and need to be set as 'Cacheable' on uploaded server.  Only rails compiled regexp is matched internally **default**: `[]`
@@ -374,7 +399,7 @@ To customize the overrides:
 AssetSync.configure do |config|
   # Clear the default overrides
   config.file_ext_to_mime_type_overrides.clear
-  
+
   # Add/Edit overrides
   # Will call `#to_s` for inputs
   config.file_ext_to_mime_type_overrides.add(:js, :"application/x-javascript")
@@ -383,7 +408,7 @@ end
 The blocks are run when local files are being scanned and uploaded
 
 #### Fog (Required)
-* **fog\_provider**: your storage provider *AWS* (S3) or *Rackspace* (Cloud Files) or *Google* (Google Storage) or *AzureRM* (Azure Blob)
+* **fog\_provider**: your storage provider *AWS* (S3) or *Rackspace* (Cloud Files) or *Google* (Google Storage) or *AzureRM* (Azure Blob) or *Backblaze* (Backblaze B2)
 * **fog\_directory**: your bucket name
 
 #### Fog (Optional)
@@ -417,6 +442,11 @@ When using the S3 API
 * **azure\_storage\_account\_name**: your Azure Blob access key
 * **azure\_storage\_access\_key**: your Azure Blob access secret
 
+#### Backblaze B2
+* **b2\_key\_id**: Your Backblaze B2 key ID
+* **b2\_key\_token**: Your Backblaze B2 key token
+* **b2\_bucket\_id**: Your Backblaze B2 bucket ID
+
 #### Rackspace (Optional)
 
 * **rackspace\_auth\_url**: Rackspace auth URL, for Rackspace London use: `https://lon.identity.api.rackspacecloud.com/v2.0`
@@ -542,7 +572,7 @@ AssetSync.configure do |config|
   config.prefix = 'assets'
   # Can be a `Pathname` or `String`
   # Will be converted into an `Pathname`
-  # If relative, will be converted into an absolute path 
+  # If relative, will be converted into an absolute path
   # via `::Rails.root` or `::Dir.pwd`
   config.public_path = Pathname('./public')
 end
@@ -613,7 +643,7 @@ Make sure you have a .env file with these details:-
     AWS_SECRET_ACCESS_KEY=<yoursecretkey>
     FOG_DIRECTORY=<yourbucket>
     FOG_REGION=<youbucketregion>
-    
+
     # for AzureRM provider
     AZURE_STORAGE_ACCOUNT_NAME=<youraccountname>
     AZURE_STORAGE_ACCESS_KEY=<youraccesskey>

data/asset_sync.gemspec

@@ -28,6 +28,7 @@ Gem::Specification.new do |s|
 
   s.add_development_dependency "fog-aws"
   s.add_development_dependency "fog-azure-rm"
+  s.add_development_dependency "fog-backblaze"
 
   s.add_development_dependency "uglifier"
   s.add_development_dependency "appraisal"

data/lib/asset_sync/config.rb

@@ -27,6 +27,8 @@ module AssetSync
     attr_accessor :cache_asset_regexps
     attr_accessor :include_manifest
     attr_accessor :concurrent_uploads
+    attr_accessor :concurrent_uploads_max_threads
+    attr_accessor :remote_file_list_cache_file_path
 
     # FOG configuration
     attr_accessor :fog_provider          # Currently Supported ['AWS', 'Rackspace']
@@ -35,7 +37,7 @@ module AssetSync
     attr_reader   :fog_public            # e.g. true, false, "default"
 
     # Amazon AWS
-    attr_accessor :aws_access_key_id, :aws_secret_access_key, :aws_reduced_redundancy, :aws_iam_roles, :aws_signature_version
+    attr_accessor :aws_access_key_id, :aws_secret_access_key, :aws_session_token, :aws_reduced_redundancy, :aws_iam_roles, :aws_signature_version
     attr_accessor :fog_host              # e.g. 's3.amazonaws.com'
     attr_accessor :fog_port              # e.g. '9000'
     attr_accessor :fog_path_style        # e.g. true
@@ -53,6 +55,11 @@ module AssetSync
     attr_accessor :azure_storage_account_name
     attr_accessor :azure_storage_access_key
 
+    # Backblaze B2 with Fog::Backblaze
+    attr_accessor :b2_key_id
+    attr_accessor :b2_key_token
+    attr_accessor :b2_bucket_id
+
     validates :existing_remote_files, :inclusion => { :in => %w(keep delete ignore) }
 
     validates :fog_provider,          :presence => true
@@ -86,6 +93,8 @@ module AssetSync
       self.cache_asset_regexps = []
       self.include_manifest = false
       self.concurrent_uploads = false
+      self.concurrent_uploads_max_threads = 10
+      self.remote_file_list_cache_file_path = nil
       @additional_local_file_paths_procs = []
 
       load_yml! if defined?(::Rails) && yml_exists?
@@ -149,6 +158,10 @@ module AssetSync
       fog_provider =~ /azurerm/i
     end
 
+    def backblaze?
+      fog_provider =~ /backblaze/i
+    end
+
     def cache_asset_regexp=(cache_asset_regexp)
       self.cache_asset_regexps = [cache_asset_regexp]
     end
@@ -199,6 +212,7 @@ module AssetSync
       self.fog_scheme             = yml["fog_scheme"]
       self.aws_access_key_id      = yml["aws_access_key_id"]
       self.aws_secret_access_key  = yml["aws_secret_access_key"]
+      self.aws_session_token      = yml["aws_session_token"] if yml.has_key?("aws_session_token")
       self.aws_reduced_redundancy = yml["aws_reduced_redundancy"]
       self.aws_iam_roles          = yml["aws_iam_roles"]
       self.aws_signature_version  = yml["aws_signature_version"]
@@ -222,10 +236,16 @@ module AssetSync
       self.cache_asset_regexps    = yml['cache_asset_regexps'] if yml.has_key?("cache_asset_regexps")
       self.include_manifest       = yml['include_manifest'] if yml.has_key?("include_manifest")
       self.concurrent_uploads     = yml['concurrent_uploads'] if yml.has_key?('concurrent_uploads')
+      self.concurrent_uploads_max_threads = yml['concurrent_uploads_max_threads'] if yml.has_key?('concurrent_uploads_max_threads')
+      self.remote_file_list_cache_file_path = yml['remote_file_list_cache_file_path'] if yml.has_key?('remote_file_list_cache_file_path')
 
       self.azure_storage_account_name = yml['azure_storage_account_name'] if yml.has_key?("azure_storage_account_name")
       self.azure_storage_access_key   = yml['azure_storage_access_key'] if yml.has_key?("azure_storage_access_key")
 
+      self.b2_key_id      = yml['b2_key_id']    if yml.has_key?("b2_key_id")
+      self.b2_key_token   = yml['b2_key_token'] if yml.has_key?("b2_key_token")
+      self.b2_bucket_id   = yml['b2_bucket_id'] if yml.has_key?("b2_bucket_id")
+
       # TODO deprecate the other old style config settings. FML.
       self.aws_access_key_id      = yml["aws_access_key"] if yml.has_key?("aws_access_key")
       self.aws_secret_access_key  = yml["aws_access_secret"] if yml.has_key?("aws_access_secret")
@@ -254,6 +274,7 @@ module AssetSync
             :aws_access_key_id => aws_access_key_id,
             :aws_secret_access_key => aws_secret_access_key
           })
+          options.merge!({:aws_session_token => aws_session_token}) if aws_session_token
         end
         options.merge!({:host => fog_host}) if fog_host
         options.merge!({:port => fog_port}) if fog_port
@@ -285,6 +306,13 @@ module AssetSync
           :azure_storage_access_key   => azure_storage_access_key,
         })
         options.merge!({:environment => fog_region}) if fog_region
+      elsif backblaze?
+        require 'fog/backblaze'
+        options.merge!({
+          :b2_key_id      => b2_key_id,
+          :b2_key_token   => b2_key_token,
+          :b2_bucket_id   => b2_bucket_id,
+        })
       else
         raise ArgumentError, "AssetSync Unknown provider: #{fog_provider} only AWS, Rackspace and Google are supported currently."
       end

data/lib/asset_sync/engine.rb

@@ -23,6 +23,7 @@ module AssetSync
 
           config.aws_access_key_id = ENV['AWS_ACCESS_KEY_ID'] if ENV.has_key?('AWS_ACCESS_KEY_ID')
           config.aws_secret_access_key = ENV['AWS_SECRET_ACCESS_KEY'] if ENV.has_key?('AWS_SECRET_ACCESS_KEY')
+          config.aws_session_token = ENV['AWS_SESSION_TOKEN'] if ENV.has_key?('AWS_SESSION_TOKEN')
           config.aws_signature_version = ENV['AWS_SIGNATURE_VERSION'] if ENV.has_key?('AWS_SIGNATURE_VERSION')
           config.aws_reduced_redundancy = ENV['AWS_REDUCED_REDUNDANCY'] == true  if ENV.has_key?('AWS_REDUCED_REDUNDANCY')
 
@@ -35,6 +36,10 @@ module AssetSync
           config.azure_storage_account_name = ENV['AZURE_STORAGE_ACCOUNT_NAME'] if ENV.has_key?('AZURE_STORAGE_ACCOUNT_NAME')
           config.azure_storage_access_key = ENV['AZURE_STORAGE_ACCESS_KEY'] if ENV.has_key?('AZURE_STORAGE_ACCESS_KEY')
 
+          config.b2_key_id    = ENV['B2_KEY_ID']    if ENV.has_key?('B2_KEY_ID')
+          config.b2_key_token = ENV['B2_KEY_TOKEN'] if ENV.has_key?('B2_KEY_TOKEN')
+          config.b2_bucket_id = ENV['B2_BUCKET_ID'] if ENV.has_key?('B2_BUCKET_ID')
+
           config.enabled = (ENV['ASSET_SYNC_ENABLED'] == 'true') if ENV.has_key?('ASSET_SYNC_ENABLED')
 
           config.existing_remote_files = ENV['ASSET_SYNC_EXISTING_REMOTE_FILES'] || "keep"
@@ -43,6 +48,7 @@ module AssetSync
           config.manifest = (ENV['ASSET_SYNC_MANIFEST'] == 'true') if ENV.has_key?('ASSET_SYNC_MANIFEST')
           config.include_manifest = (ENV['ASSET_SYNC_INCLUDE_MANIFEST'] == 'true') if ENV.has_key?('ASSET_SYNC_INCLUDE_MANIFEST')
           config.concurrent_uploads = (ENV['ASSET_SYNC_CONCURRENT_UPLOADS'] == 'true') if ENV.has_key?('ASSET_SYNC_CONCURRENT_UPLOADS')
+          config.remote_file_list_cache_file_path = ENV['ASSET_SYNC_REMOTE_FILE_LIST_CACHE_FILE_PATH'] if ENV.has_key?('ASSET_SYNC_REMOTE_FILE_LIST_CACHE_FILE_PATH')
         end
 
         config.prefix = ENV['ASSET_SYNC_PREFIX'] if ENV.has_key?('ASSET_SYNC_PREFIX')

data/lib/asset_sync/storage.rb

@@ -22,7 +22,13 @@ module AssetSync
 
     def bucket
       # fixes: https://github.com/rumblelabs/asset_sync/issues/18
-      @bucket ||= connection.directories.get(self.config.fog_directory, :prefix => self.config.assets_prefix)
+
+      @bucket ||= if self.config.backblaze?
+                    connection.directories.get(self.config.fog_directory)
+                  else
+                    connection.directories.get(self.config.fog_directory, :prefix => self.config.assets_prefix)
+                  end
+
     end
 
     def log(msg)
@@ -37,6 +43,10 @@ module AssetSync
       self.config.public_path
     end
 
+    def remote_file_list_cache_file_path
+      self.config.remote_file_list_cache_file_path
+    end
+
     def ignored_files
       expand_file_names(self.config.ignored_files)
     end
@@ -58,6 +68,32 @@ module AssetSync
         (get_local_files + config.additional_local_file_paths).uniq
     end
 
+    def remote_files
+      return [] if ignore_existing_remote_files?
+      return @remote_files if @remote_files
+
+      if remote_file_list_cache_file_path && File.file?(remote_file_list_cache_file_path)
+        begin
+          content = File.read(remote_file_list_cache_file_path)
+          return @remote_files = JSON.parse(content)
+        rescue JSON::ParserError
+          warn "Failed to parse #{remote_file_list_cache_file_path} as json"
+        end
+      end
+
+      @remote_files = get_remote_files
+    end
+
+    def update_remote_file_list_cache(local_files_to_upload)
+      return unless remote_file_list_cache_file_path
+      return if ignore_existing_remote_files?
+
+      File.open(self.remote_file_list_cache_file_path, 'w') do |file|
+        uploaded = local_files_to_upload + remote_files
+        file.write(uploaded.to_json)
+      end
+    end
+
     def always_upload_files
       expand_file_names(self.config.always_upload) + get_manifest_path
     end
@@ -243,24 +279,30 @@ module AssetSync
     end
 
     def upload_files
-      # get a fresh list of remote files
-      remote_files = ignore_existing_remote_files? ? [] : get_remote_files
       # fixes: https://github.com/rumblelabs/asset_sync/issues/19
       local_files_to_upload = local_files - ignored_files - remote_files + always_upload_files
       local_files_to_upload = (local_files_to_upload + get_non_fingerprinted(local_files_to_upload)).uniq
+      # Only files.
+      local_files_to_upload = local_files_to_upload.select { |f| File.file? "#{path}/#{f}" }
 
       if self.config.concurrent_uploads
-        threads = ThreadGroup.new
+        jobs = Queue.new
+        local_files_to_upload.each { |f| jobs.push(f) }
+        jobs.close
+
+        num_threads = [self.config.concurrent_uploads_max_threads, local_files_to_upload.length].min
         # Upload new files
-        local_files_to_upload.each do |f|
-          next unless File.file? "#{path}/#{f}" # Only files.
-          threads.add(Thread.new { upload_file f })
+        workers = Array.new(num_threads) do
+          Thread.new do
+            while f = jobs.pop
+              upload_file(f)
+            end
+          end
         end
-        sleep 1 while threads.list.any? # wait for threads to finish uploading
+        workers.map(&:join)
       else
         # Upload new files
         local_files_to_upload.each do |f|
-          next unless File.file? "#{path}/#{f}" # Only files.
           upload_file f
         end
       end
@@ -271,6 +313,8 @@ module AssetSync
         data = cdn.post_invalidation(self.config.cdn_distribution_id, files_to_invalidate)
         log "Invalidation id: #{data.body["Id"]}"
       end
+
+      update_remote_file_list_cache(local_files_to_upload)
     end
 
     def sync

data/lib/asset_sync/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module AssetSync
-  VERSION = "2.9.1"
+  VERSION = "2.13.0"
 end

data/lib/generators/asset_sync/install_generator.rb

@@ -5,7 +5,7 @@ module AssetSync
 
     # Commandline options can be defined here using Thor-like options:
     class_option :use_yml,   :type => :boolean, :default => false, :desc => "Use YML file instead of Rails Initializer"
-    class_option :provider,  :type => :string,  :default => "AWS",  :desc => "Generate with support for 'AWS', 'Rackspace', 'Google', or 'AzureRM"
+    class_option :provider,  :type => :string,  :default => "AWS",  :desc => "Generate with support for 'AWS', 'Rackspace', 'Google', 'AzureRM', or 'Backblaze'"
 
     def self.source_root
       @source_root ||= File.join(File.dirname(__FILE__), 'templates')
@@ -27,6 +27,10 @@ module AssetSync
       options[:provider] == 'AzureRM'
     end
 
+    def backblaze?
+      options[:provider] == 'Backblaze'
+    end
+
     def aws_access_key_id
       "<%= ENV['AWS_ACCESS_KEY_ID'] %>"
     end
@@ -35,6 +39,10 @@ module AssetSync
       "<%= ENV['AWS_SECRET_ACCESS_KEY'] %>"
     end
 
+    def aws_session_token
+      "<%= ENV['AWS_SESSION_TOKEN'] %>"
+    end
+
     def google_storage_access_key_id
       "<%= ENV['GOOGLE_STORAGE_ACCESS_KEY_ID'] %>"
     end
@@ -59,6 +67,18 @@ module AssetSync
       "<%= ENV['AZURE_STORAGE_ACCESS_KEY'] %>"
     end
 
+    def b2_key_id
+      "<%= ENV['B2_KEY_ID'] %>"
+    end
+
+    def b2_key_token
+      "<%= ENV['B2_KEY_TOKEN'] %>"
+    end
+
+    def b2_bucket_id
+      "<%= ENV['B2_BUCKET_ID'] %>"
+    end
+
     def app_name
       @app_name ||= Rails.application.is_a?(Rails::Application) && Rails.application.class.name.sub(/::Application$/, "").downcase
     end

data/lib/generators/asset_sync/templates/asset_sync.rb

@@ -4,6 +4,7 @@ if defined?(AssetSync)
     config.fog_provider = 'AWS'
     config.aws_access_key_id = ENV['AWS_ACCESS_KEY_ID']
     config.aws_secret_access_key = ENV['AWS_SECRET_ACCESS_KEY']
+    config.aws_session_token = ENV['AWS_SESSION_TOKEN'] if ENV.key?('AWS_SESSION_TOKEN')
     # To use AWS reduced redundancy storage.
     # config.aws_reduced_redundancy = true
     #
@@ -34,6 +35,12 @@ if defined?(AssetSync)
     config.azure_storage_account_name = ENV['AZURE_STORAGE_ACCOUNT_NAME']
     config.azure_storage_access_key = ENV['AZURE_STORAGE_ACCESS_KEY']
 
+    <%- elsif backblaze? -%>
+    config.fog_provider = 'Backblaze'
+    config.b2_key_id      = ENV['B2_KEY_ID']
+    config.b2_key_token   = ENV['B2_KEY_TOKEN']
+    config.b2_bucket_id   = ENV['B2_BUCKET_ID']
+
     # config.fog_directory specifies container name of Azure Blob storage
     <%- end -%>
     config.fog_directory = ENV['FOG_DIRECTORY']
@@ -66,6 +73,9 @@ if defined?(AssetSync)
     # Upload files concurrently
     # config.concurrent_uploads = false
     #
+    # Path to cache file to skip scanning remote
+    # config.remote_file_list_cache_file_path = './.asset_sync_remote_file_list_cache.json'
+    #
     # Fail silently.  Useful for environments such as Heroku
     # config.fail_silently = true
     #

data/lib/generators/asset_sync/templates/asset_sync.yml

@@ -32,8 +32,13 @@ defaults: &defaults
   fog_provider: 'AzureRM'
   azure_storage_account_name: "<%= azure_storage_account_name %>"
   azure_storage_access_key: "<%= azure_storage_access_key %>"
-
   # fog_directory specifies container name of Azure Blob storage
+  <%- elsif backblaze? -%>
+  fog_provider:   Backblaze
+  b2_key_id:      "<%= b2_key_id %>"
+  b2_key_token:   "<%= b2_key_token %>"
+  b2_bucket_id:   "<%= b2_bucket_id %>"
+  # fog_directory specifies container name of Backblaze B2 Bucket
   <%- end -%>
   fog_directory: "<%= app_name %>-assets"
 

data/spec/fixtures/backblaze_with_yml/config/asset_sync.yml

@@ -0,0 +1,20 @@
+defaults: &defaults
+  fog_provider: "Backblaze"
+  b2_key_id: 'xxxx'
+  b2_key_token: 'zzzz'
+  b2_bucket_id: '1234'
+
+development:
+  <<: *defaults
+  fog_directory: "rails_app_development"
+  existing_remote_files: keep
+
+test:
+  <<: *defaults
+  fog_directory: "rails_app_test"
+  existing_remote_files: keep
+
+production:
+  <<: *defaults
+  fog_directory: "rails_app_production"
+  existing_remote_files: delete

data/spec/integration/backblaze_intergration_spec.rb

@@ -0,0 +1,74 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+require "fog/backblaze"
+
+def bucket(name)
+  options = {
+    :provider       => 'Backblaze',
+    :b2_key_id      => ENV['B2_KEY_ID'],
+    :b2_key_token   => ENV['B2_KEY_TOKEN'],
+    :b2_bucket_id   => ENV['B2_BUCKET_ID']
+  }
+  options.merge!({ :environment => ENV['FOG_REGION'] }) if ENV.has_key?('FOG_REGION')
+
+  connection = Fog::Storage.new(options)
+  connection.directories.get(ENV['FOG_DIRECTORY'])
+end
+
+def execute(command)
+  app_path = File.expand_path("../../dummy_app", __FILE__)
+  Dir.chdir app_path
+  `#{command}`
+end
+
+describe "AssetSync" do
+
+  before(:each) do
+    @prefix = SecureRandom.hex(6)
+  end
+
+  let(:app_js_regex){
+    /#{@prefix}\/application-[a-zA-Z0-9]*.js$/
+  }
+
+  let(:app_js_gz_regex){
+    /#{@prefix}\/application-[a-zA-Z0-9]*.js.gz$/
+  }
+
+  let(:files){ bucket(@prefix).files }
+
+
+  after(:each) do
+    @directory = bucket(@prefix)
+    @directory.files.each do |f|
+      f.destroy
+    end
+  end
+
+  it "sync" do
+    execute "rake ASSET_SYNC_PREFIX=#{@prefix} assets:precompile"
+
+    files = bucket(@prefix).files
+
+    app_js = files.select{ |f| f.key =~ app_js_regex }.first
+    expect(app_js.content_type).to eq("application/javascript")
+
+    app_js_gz = files.select{ |f| f.key =~ app_js_gz_regex }.first
+    expect(app_js_gz.content_type).to eq("application/javascript")
+    expect(app_js_gz.content_encoding).to eq("gzip")
+  end
+
+  it "sync with enabled=false" do
+    execute "rake ASSET_SYNC_PREFIX=#{@prefix} ASSET_SYNC_ENABLED=false assets:precompile"
+    expect(bucket(@prefix).files.size).to eq(0)
+  end
+
+  it "sync with gzip_compression=true" do
+    execute "rake ASSET_SYNC_PREFIX=#{@prefix} ASSET_SYNC_GZIP_COMPRESSION=true assets:precompile"
+    # bucket(@prefix).files.size.should == 3
+
+    app_js_path = files.select{ |f| f.key =~ app_js_regex }.first
+    app_js = files.get( app_js_path.key )
+    expect(app_js.content_type).to eq("application/javascript")
+  end
+
+end

data/spec/unit/backblaze_spec.rb

@@ -0,0 +1,150 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+describe AssetSync do
+  include_context "mock Rails without_yml"
+
+  describe 'with initializer' do
+    before(:each) do
+      AssetSync.config = AssetSync::Config.new
+      AssetSync.configure do |config|
+        config.fog_provider = 'Backblaze'
+        config.b2_key_id = 'aaaa'
+        config.b2_key_token = 'bbbb'
+        config.b2_bucket_id = '4567'
+        config.fog_directory = 'mybucket'
+        config.existing_remote_files = "keep"
+      end
+    end
+
+    it "should configure provider as Backblaze" do
+      expect(AssetSync.config.fog_provider).to eq('Backblaze')
+    end
+
+    it "should should keep existing remote files" do
+      expect(AssetSync.config.existing_remote_files?).to eq(true)
+    end
+
+    it "should configure b2_key_id" do
+      expect(AssetSync.config.b2_key_id).to eq("aaaa")
+    end
+
+    it "should configure b2_key_token" do
+      expect(AssetSync.config.b2_key_token).to eq("bbbb")
+    end
+
+    it "should configure b2_bucket_id" do
+      expect(AssetSync.config.b2_bucket_id).to eq("4567")
+    end
+
+    it "should configure fog_directory" do
+      expect(AssetSync.config.fog_directory).to eq("mybucket")
+    end
+
+    it "should configure existing_remote_files" do
+      expect(AssetSync.config.existing_remote_files).to eq("keep")
+    end
+
+    it "should default gzip_compression to false" do
+      expect(AssetSync.config.gzip_compression).to be_falsey
+    end
+
+    it "should default manifest to false" do
+      expect(AssetSync.config.manifest).to be_falsey
+    end
+  end
+
+  describe 'from yml' do
+    before(:each) do
+      set_rails_root('backblaze_with_yml')
+      AssetSync.config = AssetSync::Config.new
+    end
+
+    it "should configure b2_key_id" do
+      expect(AssetSync.config.b2_key_id).to eq("xxxx")
+    end
+
+    it "should configure b2_key_token" do
+      expect(AssetSync.config.b2_key_token).to eq("zzzz")
+    end
+
+    it "should configure b2_bucket_id" do
+      expect(AssetSync.config.b2_bucket_id).to eq("1234")
+    end
+
+    it "should configure fog_directory" do
+      expect(AssetSync.config.fog_directory).to eq("rails_app_test")
+    end
+
+    it "should configure existing_remote_files" do
+      expect(AssetSync.config.existing_remote_files).to eq("keep")
+    end
+
+    it "should default gzip_compression to false" do
+      expect(AssetSync.config.gzip_compression).to be_falsey
+    end
+
+    it "should default manifest to false" do
+      expect(AssetSync.config.manifest).to be_falsey
+    end
+  end
+
+  describe 'with no configuration' do
+    before(:each) do
+      AssetSync.config = AssetSync::Config.new
+    end
+
+    it "should be invalid" do
+      expect{ AssetSync.sync }.to raise_error(::AssetSync::Config::Invalid)
+    end
+  end
+
+  describe 'with fail_silent configuration' do
+    before(:each) do
+      allow(AssetSync).to receive(:stderr).and_return(StringIO.new)
+      AssetSync.config = AssetSync::Config.new
+      AssetSync.configure do |config|
+        config.fail_silently = true
+      end
+    end
+
+    it "should not raise an invalid exception" do
+      expect{ AssetSync.sync }.not_to raise_error
+    end
+  end
+
+  describe 'with gzip_compression enabled' do
+    before(:each) do
+      AssetSync.config = AssetSync::Config.new
+      AssetSync.config.gzip_compression = true
+    end
+
+    it "config.gzip? should be true" do
+      expect(AssetSync.config.gzip?).to be_truthy
+    end
+  end
+
+  describe 'with manifest enabled' do
+    before(:each) do
+      AssetSync.config = AssetSync::Config.new
+      AssetSync.config.manifest = true
+    end
+
+    it "config.manifest should be true" do
+      expect(AssetSync.config.manifest).to be_truthy
+    end
+
+    it "config.manifest_path should default to public/assets.." do
+      expect(AssetSync.config.manifest_path).to match(/public\/assets\/manifest.yml/)
+    end
+
+    it "config.manifest_path should default to public/assets.." do
+      Rails.application.config.assets.manifest = "/var/assets"
+      expect(AssetSync.config.manifest_path).to eq("/var/assets/manifest.yml")
+    end
+
+    it "config.manifest_path should default to public/custom_assets.." do
+      Rails.application.config.assets.prefix = 'custom_assets'
+      expect(AssetSync.config.manifest_path).to match(/public\/custom_assets\/manifest.yml/)
+    end
+  end
+end

data/spec/unit/storage_spec.rb

@@ -70,6 +70,71 @@ describe AssetSync::Storage do
       storage.upload_files
     end
 
+    it 'should allow custom number of threads' do
+      @config.concurrent_uploads = true
+      @config.concurrent_uploads_max_threads = 2
+      storage = AssetSync::Storage.new(@config)
+
+      allow(storage).to receive(:get_local_files).and_return(@local_files)
+      allow(storage).to receive(:get_remote_files).and_return(@remote_files)
+      allow(File).to receive(:file?).and_return(true) # Pretend they all exist
+
+      expect(Thread).to receive(:new).exactly(2).times.and_call_original
+      (@local_files - @remote_files + storage.always_upload_files).each do |file|
+        expect(storage).to receive(:upload_file).with(file)
+      end
+
+      storage.upload_files
+    end
+
+    it 'should allow remote_file_list_cache_file_path configuration' do
+      file_path = './foo.json'
+      @config.remote_file_list_cache_file_path = file_path
+      storage = AssetSync::Storage.new(@config)
+
+      allow(storage).to receive(:get_local_files).and_return(@local_files)
+      File.write(file_path, @remote_files.to_json)
+      expect(storage).not_to receive(:get_remote_files)
+      allow(File).to receive(:file?).and_return(true) # Pretend they all exist
+
+      (@local_files - @remote_files + storage.always_upload_files).each do |file|
+        expect(storage).to receive(:upload_file).with(file)
+      end
+
+      expect(storage).not_to receive(:warn)
+      storage.upload_files
+
+      # update remote_file_list_cache corretly
+      updated = JSON.parse(File.read(file_path))
+      expect(updated.sort.uniq).to eq (@remote_files + @local_files + storage.always_upload_files).sort.uniq
+
+      File.delete(file_path)
+    end
+
+    it 'should work with broken cache' do
+      file_path = './foo.json'
+      @config.remote_file_list_cache_file_path = file_path
+
+      storage = AssetSync::Storage.new(@config)
+
+      File.write(file_path, 'some non-json text file content')
+
+      allow(storage).to receive(:get_local_files).and_return(@local_files)
+      allow(storage).to receive(:get_remote_files).and_return(@remote_files)
+      allow(File).to receive(:file?).and_return(true) # Pretend they all exist
+
+      (@local_files - @remote_files + storage.always_upload_files).each do |file|
+        expect(storage).to receive(:upload_file).with(file)
+      end
+
+      # when broken, warning message should be prompted
+      expect(storage).to receive(:warn)
+
+      storage.upload_files
+
+      File.delete(file_path)
+    end
+
     it 'should upload updated non-fingerprinted files' do
       @local_files = [
         'public/image.png',
@@ -371,6 +436,7 @@ describe AssetSync::Storage do
         allow(directory).to receive(:files).and_return([file])
         allow(file).to receive(:key).and_return('public/image.png')
         allow(connection).to receive(:directories).and_return(directories)
+        allow(config).to receive(:backblaze?).and_return(false)
         expect(connection).not_to receive(:delete_multiple_objects)
         expect(file).to receive(:destroy)
 

metadata

@@ -1,17 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: asset_sync
 version: !ruby/object:Gem::Version
-  version: 2.9.1
+  version: 2.13.0
 platform: ruby
 authors:
 - Simon Hamilton
 - David Rice
 - Phil McClure
 - Toby Osbourn
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-02-20 00:00:00.000000000 Z
+date: 2020-12-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fog-core
@@ -153,6 +153,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: fog-backblaze
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: uglifier
   requirement: !ruby/object:Gem::Requirement
@@ -223,15 +237,18 @@ files:
 - spec/dummy_app/app/assets/javascripts/application.js
 - spec/fixtures/aws_with_yml/config/asset_sync.yml
 - spec/fixtures/azure_rm_with_yml/config/asset_sync.yml
+- spec/fixtures/backblaze_with_yml/config/asset_sync.yml
 - spec/fixtures/google_with_service_account_yml/config/asset_sync.yml
 - spec/fixtures/google_with_yml/config/asset_sync.yml
 - spec/fixtures/rackspace_with_yml/config/asset_sync.yml
 - spec/fixtures/with_invalid_yml/config/asset_sync.yml
 - spec/integration/aws_integration_spec.rb
 - spec/integration/azure_rm_integration_spec.rb
+- spec/integration/backblaze_intergration_spec.rb
 - spec/spec_helper.rb
 - spec/unit/asset_sync_spec.rb
 - spec/unit/azure_rm_spec.rb
+- spec/unit/backblaze_spec.rb
 - spec/unit/google_spec.rb
 - spec/unit/multi_mime_spec.rb
 - spec/unit/rackspace_spec.rb
@@ -241,7 +258,7 @@ homepage: https://github.com/rumblelabs/asset_sync
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -256,8 +273,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.2.0
+signing_key:
 specification_version: 4
 summary: Synchronises Assets in a Rails 3 application and Amazon S3/Cloudfront and
   Rackspace Cloudfiles
@@ -266,15 +283,18 @@ test_files:
 - spec/dummy_app/app/assets/javascripts/application.js
 - spec/fixtures/aws_with_yml/config/asset_sync.yml
 - spec/fixtures/azure_rm_with_yml/config/asset_sync.yml
+- spec/fixtures/backblaze_with_yml/config/asset_sync.yml
 - spec/fixtures/google_with_service_account_yml/config/asset_sync.yml
 - spec/fixtures/google_with_yml/config/asset_sync.yml
 - spec/fixtures/rackspace_with_yml/config/asset_sync.yml
 - spec/fixtures/with_invalid_yml/config/asset_sync.yml
 - spec/integration/aws_integration_spec.rb
 - spec/integration/azure_rm_integration_spec.rb
+- spec/integration/backblaze_intergration_spec.rb
 - spec/spec_helper.rb
 - spec/unit/asset_sync_spec.rb
 - spec/unit/azure_rm_spec.rb
+- spec/unit/backblaze_spec.rb
 - spec/unit/google_spec.rb
 - spec/unit/multi_mime_spec.rb
 - spec/unit/rackspace_spec.rb