asset_sync 0.5.4 → 1.0.0

data/lib/asset_sync/engine.rb

@@ -13,33 +13,34 @@ module AssetSync
       elsif !File.exists?( app_initializer ) && !File.exists?( app_yaml )
         AssetSync.log "AssetSync: using default configuration from built-in initializer"
         AssetSync.configure do |config|
-          config.fog_provider = ENV['FOG_PROVIDER']
-          config.fog_directory = ENV['FOG_DIRECTORY']
-          config.fog_region = ENV['FOG_REGION']
+          config.fog_provider = ENV['FOG_PROVIDER'] if ENV.has_key?('FOG_PROVIDER')
+          config.fog_directory = ENV['FOG_DIRECTORY'] if ENV.has_key?('FOG_DIRECTORY')
+          config.fog_region = ENV['FOG_REGION'] if ENV.has_key?('FOG_REGION')
 
-          config.aws_access_key_id = ENV['AWS_ACCESS_KEY_ID']
-          config.aws_secret_access_key = ENV['AWS_SECRET_ACCESS_KEY']
+          config.aws_access_key_id = ENV['AWS_ACCESS_KEY_ID'] if ENV.has_key?('AWS_ACCESS_KEY_ID')
+          config.aws_secret_access_key = ENV['AWS_SECRET_ACCESS_KEY'] if ENV.has_key?('AWS_SECRET_ACCESS_KEY')
+          config.aws_reduced_redundancy = ENV['AWS_REDUCED_REDUNDANCY'] == true  if ENV.has_key?('AWS_REDUCED_REDUNDANCY')
 
-          config.rackspace_username = ENV['RACKSPACE_USERNAME']
-          config.rackspace_api_key = ENV['RACKSPACE_API_KEY']
+          config.rackspace_username = ENV['RACKSPACE_USERNAME'] if ENV.has_key?('RACKSPACE_USERNAME')
+          config.rackspace_api_key = ENV['RACKSPACE_API_KEY'] if ENV.has_key?('RACKSPACE_API_KEY')
 
-          config.google_storage_access_key_id = ENV['GOOGLE_STORAGE_ACCESS_KEY_ID']
-          config.google_storage_secret_access_key = ENV['GOOGLE_STORAGE_SECRET_ACCESS_KEY']
+          config.google_storage_access_key_id = ENV['GOOGLE_STORAGE_ACCESS_KEY_ID'] if ENV.has_key?('GOOGLE_STORAGE_ACCESS_KEY_ID')
+          config.google_storage_secret_access_key = ENV['GOOGLE_STORAGE_SECRET_ACCESS_KEY'] if ENV.has_key?('GOOGLE_STORAGE_SECRET_ACCESS_KEY')
+
+          config.enabled = (ENV['ASSET_SYNC_ENABLED'] == 'true') if ENV.has_key?('ASSET_SYNC_ENABLED')
 
-          if ENV.has_key? 'ASSET_SYNC_ENABLED'
-            config.enabled = ENV['ASSET_SYNC_ENABLED'] == 'true'
-          end
           config.existing_remote_files = ENV['ASSET_SYNC_EXISTING_REMOTE_FILES'] || "keep"
-          config.gzip_compression = ENV['ASSET_SYNC_GZIP_COMPRESSION'] == 'true'
-          config.manifest = ENV['ASSET_SYNC_MANIFEST'] == 'true'
-        end
 
-        if ENV.has_key? 'ASSET_SYNC_PREFIX'
-          config.prefix = ENV['ASSET_SYNC_PREFIX']
+          config.gzip_compression = (ENV['ASSET_SYNC_GZIP_COMPRESSION'] == 'true') if ENV.has_key?('ASSET_SYNC_GZIP_COMPRESSION')
+          config.manifest = (ENV['ASSET_SYNC_MANIFEST'] == 'true') if ENV.has_key?('ASSET_SYNC_MANIFEST')
         end
+
+        config.prefix = ENV['ASSET_SYNC_PREFIX'] if ENV.has_key?('ASSET_SYNC_PREFIX')
+
         config.existing_remote_files = ENV['ASSET_SYNC_EXISTING_REMOTE_FILES'] || "keep"
-        config.gzip_compression = ENV['ASSET_SYNC_GZIP_COMPRESSION'] == 'true'
-        config.manifest = ENV['ASSET_SYNC_MANIFEST'] == 'true'
+
+        config.gzip_compression = (ENV['ASSET_SYNC_GZIP_COMPRESSION'] == 'true') if ENV.has_key?('ASSET_SYNC_GZIP_COMPRESSION')
+        config.manifest = (ENV['ASSET_SYNC_MANIFEST'] == 'true') if ENV.has_key?('ASSET_SYNC_MANIFEST')
 
       end
 

data/lib/asset_sync/storage.rb

@@ -1,7 +1,9 @@
 module AssetSync
   class Storage
+    REGEXP_FINGERPRINTED_FILES = /^(.*)\/([^-]+)-[^\.]+\.([^\.]+)$/
 
-    class BucketNotFound < StandardError; end
+    class BucketNotFound < StandardError;
+    end
 
     attr_accessor :config
 
@@ -34,37 +36,57 @@ module AssetSync
       files = []
       Array(self.config.ignored_files).each do |ignore|
         case ignore
-        when Regexp
-          files += self.local_files.select do |file|
-            file =~ ignore
-          end
-        when String
-          files += self.local_files.select do |file|
-            file.split('/').last == ignore
-          end
-        else
-          log "Error: please define ignored_files as string or regular expression. #{ignore} (#{ignore.class}) ignored."
+          when Regexp
+            files += self.local_files.select do |file|
+              file =~ ignore
+            end
+          when String
+            files += self.local_files.select do |file|
+              file.split('/').last == ignore
+            end
+          else
+            log "Error: please define ignored_files as string or regular expression. #{ignore} (#{ignore.class}) ignored."
         end
       end
       files.uniq
     end
 
     def local_files
-      @local_files ||= get_local_files
+      @local_files ||= get_local_files.uniq
     end
 
     def always_upload_files
       self.config.always_upload.map { |f| File.join(self.config.assets_prefix, f) }
     end
 
+    def files_with_custom_headers
+      self.config.custom_headers.inject({}) { |h,(k, v)| h[File.join(self.config.assets_prefix, k)] = v; h; }
+    end
+
+    def files_to_invalidate
+      self.config.invalidate.map { |filename| File.join("/", self.config.assets_prefix, filename) }
+    end
+
     def get_local_files
       if self.config.manifest
-        if File.exists?(self.config.manifest_path)
-          yml = YAML.load(IO.read(self.config.manifest_path))
+        if ActionView::Base.respond_to?(:assets_manifest)
+          log "Using: Rails 4.0 manifest access"
+          manifest = Sprockets::Manifest.new(ActionView::Base.assets_manifest.environment, ActionView::Base.assets_manifest.dir)
+          return manifest.assets.values.map { |f| File.join(self.config.assets_prefix, f) }
+        elsif File.exists?(self.config.manifest_path)
           log "Using: Manifest #{self.config.manifest_path}"
-          return yml.values.map { |f| File.join(self.config.assets_prefix, f) }
+          yml = YAML.load(IO.read(self.config.manifest_path))
+   
+          return yml.map do |original, compiled|
+            # Upload font originals and compiled
+            if original =~ /^.+(eot|svg|ttf|woff)$/
+              [original, compiled]
+            else
+              compiled
+            end
+          end.flatten.map { |f| File.join(self.config.assets_prefix, f) }.uniq!
         else
-          log "Warning: manifest.yml not found at #{self.config.manifest_path}"
+          log "Warning: Manifest could not be found"
         end
       end
       log "Using: Directory Search of #{path}/#{self.config.assets_prefix}"
@@ -93,7 +115,7 @@ module AssetSync
       log "Fetching files to flag for delete"
       remote_files = get_remote_files
       # fixes: https://github.com/rumblelabs/asset_sync/issues/19
-      from_remote_files_to_delete = remote_files - local_files - ignored_files
+      from_remote_files_to_delete = remote_files - local_files - ignored_files - always_upload_files
 
       log "Flagging #{from_remote_files_to_delete.size} file(s) for deletion"
       # Delete unneeded remote files
@@ -111,11 +133,31 @@ module AssetSync
         :key => f,
         :body => File.open("#{path}/#{f}"),
         :public => true,
-        :cache_control => "public, max-age=#{one_year}",
-        :expires => CGI.rfc1123_date(Time.now + one_year),
         :content_type => mime
       }
 
+      if /-[0-9a-fA-F]{32}$/.match(File.basename(f,File.extname(f)))
+        file.merge!({
+          :cache_control => "public, max-age=#{one_year}",
+          :expires => CGI.rfc1123_date(Time.now + one_year)
+        })
+      end
+
+      # overwrite headers if applicable, you probably shouldn't specific key/body, but cache-control headers etc.
+
+      if files_with_custom_headers.has_key? f
+        file.merge! files_with_custom_headers[f]
+        log "Overwriting #{f} with custom headers #{files_with_custom_headers[f].to_s}"
+      elsif key = self.config.custom_headers.keys.detect {|k| f.match(Regexp.new(k))}
+        headers = {}
+        self.config.custom_headers[key].each do |key, value|
+          headers[key.to_sym] = value
+        end
+        file.merge! headers
+        log "Overwriting matching file #{f} with custom headers #{headers.to_s}"
+      end
+
+
       gzipped = "#{path}/#{f}.gz"
       ignore = false
 
@@ -126,15 +168,15 @@ module AssetSync
         ignore = true
       elsif config.gzip? && File.exists?(gzipped)
         original_size = File.size("#{path}/#{f}")
-        gzipped_size  = File.size(gzipped)
+        gzipped_size = File.size(gzipped)
 
         if gzipped_size < original_size
           percentage = ((gzipped_size.to_f/original_size.to_f)*100).round(2)
           file.merge!({
-            :key => f,
-            :body => File.open(gzipped),
-            :content_encoding => 'gzip'
-          })
+                        :key => f,
+                        :body => File.open(gzipped),
+                        :content_encoding => 'gzip'
+                      })
           log "Uploading: #{gzipped} in place of #{f} saving #{percentage}%"
         else
           percentage = ((original_size.to_f/gzipped_size.to_f)*100).round(2)
@@ -155,6 +197,12 @@ module AssetSync
         log "Uploading: #{f}"
       end
 
+      if config.aws? && config.aws_rrs?
+        file.merge!({
+          :storage_class => 'REDUCED_REDUNDANCY'
+        })
+      end
+
       file = bucket.files.create( file ) unless ignore
     end
 
@@ -163,12 +211,20 @@ module AssetSync
       remote_files = ignore_existing_remote_files? ? [] : get_remote_files
       # fixes: https://github.com/rumblelabs/asset_sync/issues/19
       local_files_to_upload = local_files - ignored_files - remote_files + always_upload_files
+      local_files_to_upload = (local_files_to_upload + get_non_fingerprinted(local_files_to_upload)).uniq
 
       # Upload new files
       local_files_to_upload.each do |f|
         next unless File.file? "#{path}/#{f}" # Only files.
         upload_file f
       end
+
+      if self.config.cdn_distribution_id && files_to_invalidate.any?
+        log "Invalidating Files"
+        cdn ||= Fog::CDN.new(self.config.fog_options.except(:region))
+        data = cdn.post_invalidation(self.config.cdn_distribution_id, files_to_invalidate)
+        log "Invalidation id: #{data.body["Id"]}"
+      end
     end
 
     def sync
@@ -184,5 +240,13 @@ module AssetSync
     def ignore_existing_remote_files?
       self.config.existing_remote_files == 'ignore'
     end
+
+    def get_non_fingerprinted(files)
+      files.map do |file|
+        match_data = file.match(REGEXP_FINGERPRINTED_FILES)
+        match_data && "#{match_data[1]}/#{match_data[2]}.#{match_data[3]}"
+      end.compact
+    end
+
   end
 end

data/lib/asset_sync/version.rb

@@ -1,3 +1,3 @@
 module AssetSync
-  VERSION = "0.5.4"
+  VERSION = "1.0.0"
 end

data/lib/generators/asset_sync/templates/asset_sync.rb

@@ -3,6 +3,8 @@ AssetSync.configure do |config|
   config.fog_provider = 'AWS'
   config.aws_access_key_id = ENV['AWS_ACCESS_KEY_ID']
   config.aws_secret_access_key = ENV['AWS_SECRET_ACCESS_KEY']
+  # To use AWS reduced redundancy storage.
+  # config.aws_reduced_redundancy = true
   <%- elsif google? -%>
   config.fog_provider = 'Google'
   config.google_storage_access_key_id = ENV['GOOGLE_STORAGE_ACCESS_KEY_ID']
@@ -16,7 +18,11 @@ AssetSync.configure do |config|
   # config.rackspace_auth_url = "lon.auth.api.rackspacecloud.com"
   <%- end -%>
   config.fog_directory = ENV['FOG_DIRECTORY']
-  
+
+  # Invalidate a file on a cdn after uploading files
+  # config.cdn_distribution_id = "12345"
+  # config.invalidate = ['file1.js']
+
   # Increase upload performance by configuring your region
   # config.fog_region = 'eu-west-1'
   #
@@ -26,7 +32,7 @@ AssetSync.configure do |config|
   # Automatically replace files with their equivalent gzip compressed version
   # config.gzip_compression = true
   #
-  # Use the Rails generated 'manifest.yml' file to produce the list of files to 
+  # Use the Rails generated 'manifest.yml' file to produce the list of files to
   # upload instead of searching the assets directory.
   # config.manifest = true
   #

data/lib/generators/asset_sync/templates/asset_sync.yml

@@ -3,6 +3,8 @@ defaults: &defaults
   fog_provider: 'AWS'
   aws_access_key_id: "<%= aws_access_key_id %>"
   aws_secret_access_key: "<%= aws_secret_access_key %>"
+  # To use AWS reduced redundancy storage.
+  # aws_reduced_redundancy: true
   <%- elsif google? -%>
   fog_provider: 'Google'
   google_storage_access_key_id: "<%= google_storage_access_key_id %>"
@@ -12,14 +14,14 @@ defaults: &defaults
   rackspace_username: "<%= rackspace_username %>"
   rackspace_api_key: "<%= rackspace_api_key %>"
   # if you need to change rackspace_auth_url (e.g. if you need to use Rackspace London)
-  # rackspace_auth_url: "lon.auth.api.rackspacecloud.com"
+  # rackspace_auth_url: "https://lon.identity.api.rackspacecloud.com/v2.0"
   <%- end -%>
   fog_directory: "<%= app_name %>-assets"
   # You may need to specify what region your storage bucket is in
   # fog_region: "eu-west-1"
   existing_remote_files: keep
   # To delete existing remote files.
-  # existing_remote_files: delete 
+  # existing_remote_files: delete
   # Automatically replace files with their equivalent gzip compressed version
   # gzip_compression: true
   # Fail silently.  Useful for environments such as Heroku

data/lib/tasks/asset_sync.rake

@@ -1,6 +1,23 @@
+namespace :assets do
+
+  desc 'Synchronize assets to remote (assumes assets are already compiled)'
+  task :sync => :environment do
+    AssetSync.sync
+  end
+  namespace :sync do
+    desc 'Delete out-of-sync files on remote'
+    task :clean => :environment do
+      AssetSync.clean
+    end
+  end
+
+end
+
 if Rake::Task.task_defined?("assets:precompile:nondigest")
   Rake::Task["assets:precompile:nondigest"].enhance do
-    AssetSync.sync
+    # Conditional execution needs to be inside the enhance block because the enhance block
+    # will get executed before yaml or Rails initializers.
+    Rake::Task["assets:sync"].invoke if defined?(AssetSync) && AssetSync.config.run_on_precompile
   end
 else
   Rake::Task["assets:precompile"].enhance do
@@ -8,6 +25,6 @@ else
     # RAILS_GROUP and RAILS_ENV are not defined. We need to reload the
     # assets environment in this case.
     # Rake::Task["assets:environment"].invoke if Rake::Task.task_defined?("assets:environment")
-    AssetSync.sync
+    Rake::Task["assets:sync"].invoke if defined?(AssetSync) && AssetSync.config.run_on_precompile
   end
 end

data/spec/dummy_app/Rakefile

@@ -14,6 +14,7 @@ module AssetSyncTest
   class Application < Rails::Application
     config.encoding = "utf-8"
     config.filter_parameters += [:password]
+    config.eager_load = false
     config.assets.enabled = true
     config.assets.version = '1.0'
     config.secret_token = 'bf196b4383deefa4e0120a6ef1d9af1cc45f5c4ebd04405'
@@ -26,4 +27,4 @@ module AssetSyncTest
 end
 AssetSyncTest::Application.initialize!
 AssetSyncTest::Application.load_tasks
-# Rake::Task['assets:precompile:all'].invoke
+# Rake::Task['assets:precompile:all'].invoke

data/spec/fixtures/aws_with_yml/config/asset_sync.yml

@@ -3,6 +3,7 @@ defaults: &defaults
   aws_access_key_id: "xxxx"
   aws_secret_access_key: "zzzz"
   region: "eu-west-1"
+  run_on_precompile: false
 
 development:
   <<: *defaults

data/spec/fixtures/with_invalid_yml/config/asset_sync.yml

@@ -0,0 +1,24 @@
+defaults: &defaults
+  fog_provider= "AWS"
+  aws_access_key_id: "xxxx"
+  aws_secret_access_key: "zzzz"
+  region: "eu-west-1"
+
+development:
+  <<: *defaults
+  fog_directory: "rails_app_development"
+  existing_remote_files: keep
+
+test:
+  <<: *defaults
+  fog_directory: "rails_app_test"
+  existing_remote_files: keep
+
+production:
+  <<: *defaults
+  fog_directory: "rails_app_production"
+  existing_remote_files: delete
+
+hybrid:
+  <<: *defaults
+  enabled: false

data/spec/integration/aws_integration_spec.rb

@@ -23,6 +23,17 @@ describe "AssetSync" do
     @prefix = SecureRandom.hex(6)
   end
 
+  let(:app_js_regex){ 
+    /#{@prefix}\/application-[a-zA-Z0-9]*.js$/ 
+  }
+
+  let(:app_js_gz_regex){ 
+    /#{@prefix}\/application-[a-zA-Z0-9]*.js.gz$/ 
+  }
+
+  let(:files){ bucket(@prefix).files }
+
+
   after(:each) do
     @directory = bucket(@prefix)
     @directory.files.each do |f|
@@ -32,12 +43,17 @@ describe "AssetSync" do
 
   it "sync" do
     execute "rake ASSET_SYNC_PREFIX=#{@prefix} assets:precompile"
-    bucket(@prefix).files.size.should == 5
+    # bucket(@prefix).files.size.should == 5
+
+    files = bucket(@prefix).files
+
+    app_js_path = files.select{ |f| f.key =~ app_js_regex }.first
+    app_js_gz_path = files.select{ |f| f.key =~ app_js_gz_regex }.first
 
-    app_js = bucket(@prefix).files.get("#{@prefix}/application.js")
+    app_js = files.get( app_js_path.key )
     app_js.content_type.should == "text/javascript"
 
-    app_js_gz = bucket(@prefix).files.get("#{@prefix}/application.js.gz")
+    app_js_gz = files.get( app_js_gz_path.key )
     app_js_gz.content_type.should == "text/javascript"
     app_js_gz.content_encoding.should == "gzip"
   end
@@ -49,9 +65,10 @@ describe "AssetSync" do
 
   it "sync with gzip_compression=true" do
     execute "rake ASSET_SYNC_PREFIX=#{@prefix} ASSET_SYNC_GZIP_COMPRESSION=true assets:precompile"
-    bucket(@prefix).files.size.should == 3
+    # bucket(@prefix).files.size.should == 3
 
-    app_js = bucket(@prefix).files.get("#{@prefix}/application.js")
+    app_js_path = files.select{ |f| f.key =~ app_js_regex }.first
+    app_js = files.get( app_js_path.key )
     app_js.content_type.should == "text/javascript"
   end
 

data/spec/spec_helper.rb

@@ -1,11 +1,13 @@
 require 'rubygems'
 require 'bundler'
 
-if RUBY_VERSION != '1.8.7'
+begin
   require 'simplecov'
   SimpleCov.start do
     add_filter 'spec'
   end
+rescue LoadError
+  # SimpleCov ain't available - continue
 end
 
 begin
@@ -30,7 +32,7 @@ shared_context "mock without Rails" do
     if defined? Rails
       Object.send(:remove_const, :Rails)
     end
-    AssetSync.stub!(:log)
+    AssetSync.stub(:log)
   end
 end
 
@@ -38,14 +40,14 @@ end
 shared_context "mock Rails" do
   before(:each) do
     unless defined? Rails
-      Rails = mock 'Rails'
+      Rails = double 'Rails'
     end
     Rails.stub(:env).and_return('test')
-    Rails.stub :application => mock('application')
-    Rails.application.stub :config => mock('config')
+    Rails.stub :application => double('application')
+    Rails.application.stub :config => double('config')
     Rails.application.config.stub :assets => ActiveSupport::OrderedOptions.new
     Rails.application.config.assets.prefix = '/assets'
-    AssetSync.stub!(:log)
+    AssetSync.stub(:log)
   end
 end