sq-asset_sync 2.0.0

data/lib/asset_sync/engine.rb

@@ -0,0 +1,53 @@
+module AssetSync
+  class Engine < Rails::Engine
+
+    engine_name "asset_sync"
+
+    initializer "asset_sync config", :group => :all do |app|
+      app_initializer = Rails.root.join('config', 'initializers', 'asset_sync.rb').to_s
+      app_yaml = Rails.root.join('config', 'asset_sync.yml').to_s
+
+      if File.exist?( app_initializer )
+        AssetSync.log "AssetSync: using #{app_initializer}"
+        load app_initializer
+      elsif !File.exist?( app_initializer ) && !File.exist?( app_yaml )
+        AssetSync.log "AssetSync: using default configuration from built-in initializer"
+        AssetSync.configure do |config|
+          config.fog_provider = ENV['FOG_PROVIDER'] if ENV.has_key?('FOG_PROVIDER')
+          config.fog_directory = ENV['FOG_DIRECTORY'] if ENV.has_key?('FOG_DIRECTORY')
+          config.fog_region = ENV['FOG_REGION'] if ENV.has_key?('FOG_REGION')
+
+          config.aws_access_key_id = ENV['AWS_ACCESS_KEY_ID'] if ENV.has_key?('AWS_ACCESS_KEY_ID')
+          config.aws_secret_access_key = ENV['AWS_SECRET_ACCESS_KEY'] if ENV.has_key?('AWS_SECRET_ACCESS_KEY')
+          config.aws_reduced_redundancy = ENV['AWS_REDUCED_REDUNDANCY'] == true  if ENV.has_key?('AWS_REDUCED_REDUNDANCY')
+
+          config.rackspace_username = ENV['RACKSPACE_USERNAME'] if ENV.has_key?('RACKSPACE_USERNAME')
+          config.rackspace_api_key = ENV['RACKSPACE_API_KEY'] if ENV.has_key?('RACKSPACE_API_KEY')
+
+          config.google_storage_access_key_id = ENV['GOOGLE_STORAGE_ACCESS_KEY_ID'] if ENV.has_key?('GOOGLE_STORAGE_ACCESS_KEY_ID')
+          config.google_storage_secret_access_key = ENV['GOOGLE_STORAGE_SECRET_ACCESS_KEY'] if ENV.has_key?('GOOGLE_STORAGE_SECRET_ACCESS_KEY')
+
+          config.enabled = (ENV['ASSET_SYNC_ENABLED'] == 'true') if ENV.has_key?('ASSET_SYNC_ENABLED')
+
+          config.existing_remote_files = ENV['ASSET_SYNC_EXISTING_REMOTE_FILES'] || "keep"
+
+          config.gzip_compression = (ENV['ASSET_SYNC_GZIP_COMPRESSION'] == 'true') if ENV.has_key?('ASSET_SYNC_GZIP_COMPRESSION')
+          config.manifest = (ENV['ASSET_SYNC_MANIFEST'] == 'true') if ENV.has_key?('ASSET_SYNC_MANIFEST')
+        end
+
+        config.prefix = ENV['ASSET_SYNC_PREFIX'] if ENV.has_key?('ASSET_SYNC_PREFIX')
+
+        config.existing_remote_files = ENV['ASSET_SYNC_EXISTING_REMOTE_FILES'] || "keep"
+
+        config.gzip_compression = (ENV['ASSET_SYNC_GZIP_COMPRESSION'] == 'true') if ENV.has_key?('ASSET_SYNC_GZIP_COMPRESSION')
+        config.manifest = (ENV['ASSET_SYNC_MANIFEST'] == 'true') if ENV.has_key?('ASSET_SYNC_MANIFEST')
+
+      end
+
+      if File.exist?( app_yaml )
+        AssetSync.log "AssetSync: YAML file found #{app_yaml} settings will be merged into the configuration"
+      end
+    end
+
+  end
+end

data/lib/asset_sync/multi_mime.rb

@@ -0,0 +1,16 @@
+require 'mime/types'
+
+module AssetSync
+  class MultiMime
+
+    def self.lookup(ext)
+      if defined?(Mime::Type)
+        Mime::Type.lookup_by_extension(ext)
+      elsif defined?(Rack::Mime)
+        ext_with_dot = ".#{ext}"
+        Rack::Mime.mime_type(ext_with_dot)
+      end || ::MIME::Types.type_for(ext).first
+    end
+
+  end
+end

data/lib/asset_sync/railtie.rb

@@ -0,0 +1,5 @@
+class Rails::Railtie::Configuration
+  def asset_sync
+    AssetSync.config
+  end
+end

data/lib/asset_sync/storage.rb

@@ -0,0 +1,291 @@
+module AssetSync
+  class Storage
+    REGEXP_FINGERPRINTED_FILES = /^(.*)\/([^-]+)-[^\.]+\.([^\.]+)$/
+
+    class BucketNotFound < StandardError;
+    end
+
+    attr_accessor :config
+
+    def initialize(cfg)
+      @config = cfg
+    end
+
+    def connection
+      @connection ||= Fog::Storage.new(self.config.fog_options)
+    end
+
+    def bucket
+      # fixes: https://github.com/rumblelabs/asset_sync/issues/18
+      @bucket ||= connection.directories.get(self.config.fog_directory, :prefix => self.config.assets_prefix)
+    end
+
+    def log(msg)
+      AssetSync.log(msg)
+    end
+
+    def keep_existing_remote_files?
+      self.config.existing_remote_files?
+    end
+
+    def path
+      self.config.public_path
+    end
+
+    def ignored_files
+      files = []
+      Array(self.config.ignored_files).each do |ignore|
+        case ignore
+          when Regexp
+            files += self.local_files.select do |file|
+              file =~ ignore
+            end
+          when String
+            files += self.local_files.select do |file|
+              file.split('/').last == ignore
+            end
+          else
+            log "Error: please define ignored_files as string or regular expression. #{ignore} (#{ignore.class}) ignored."
+        end
+      end
+      files.uniq
+    end
+
+    def local_files
+      @local_files ||= get_local_files.uniq
+    end
+
+    def always_upload_files
+      self.config.always_upload.map { |f| File.join(self.config.assets_prefix, f) }
+    end
+
+    def files_with_custom_headers
+      self.config.custom_headers.inject({}) { |h,(k, v)| h[File.join(self.config.assets_prefix, k)] = v; h; }
+    end
+
+    def files_to_invalidate
+      self.config.invalidate.map { |filename| File.join("/", self.config.assets_prefix, filename) }
+    end
+
+    def get_local_files
+      if self.config.manifest
+        if ActionView::Base.respond_to?(:assets_manifest)
+          log "Using: Rails 4.0 manifest access"
+          manifest = Sprockets::Manifest.new(ActionView::Base.assets_manifest.environment, ActionView::Base.assets_manifest.dir)
+          return manifest.assets.values.map { |f| File.join(self.config.assets_prefix, f) }
+        elsif File.exist?(self.config.manifest_path)
+          log "Using: Manifest #{self.config.manifest_path}"
+          yml = YAML.load(IO.read(self.config.manifest_path))
+
+          return yml.map do |original, compiled|
+            # Upload font originals and compiled
+            if original =~ /^.+(eot|svg|ttf|woff)$/
+              [original, compiled]
+            else
+              compiled
+            end
+          end.flatten.map { |f| File.join(self.config.assets_prefix, f) }.uniq!
+        else
+          log "Warning: Manifest could not be found"
+        end
+      end
+      log "Using: Directory Search of #{path}/#{self.config.assets_prefix}"
+      Dir.chdir(path) do
+        to_load = self.config.assets_prefix.present? ? "#{self.config.assets_prefix}/**/**" : '**/**'
+        Dir[to_load]
+      end
+    end
+
+    def get_remote_files
+      raise BucketNotFound.new("#{self.config.fog_provider} Bucket: #{self.config.fog_directory} not found.") unless bucket
+      # fixes: https://github.com/rumblelabs/asset_sync/issues/16
+      #        (work-around for https://github.com/fog/fog/issues/596)
+      files = []
+      bucket.files.each { |f| files << f.key }
+      return files
+    end
+
+    def delete_file(f, remote_files_to_delete)
+      if remote_files_to_delete.include?(f.key)
+        log "Deleting: #{f.key}"
+        f.destroy
+      end
+    end
+
+    def delete_extra_remote_files
+      log "Fetching files to flag for delete"
+      remote_files = get_remote_files
+      # fixes: https://github.com/rumblelabs/asset_sync/issues/19
+      from_remote_files_to_delete = remote_files - local_files - ignored_files - always_upload_files
+
+      log "Flagging #{from_remote_files_to_delete.size} file(s) for deletion"
+      # Delete unneeded remote files
+      bucket.files.each do |f|
+        delete_file(f, from_remote_files_to_delete)
+      end
+    end
+
+    def upload_file(f)
+      # TODO output files in debug logs as asset filename only.
+      one_year = 31557600
+      ext = File.extname(f)[1..-1]
+      mime = MultiMime.lookup(ext)
+      file = {
+        :key => f,
+        :body => File.open("#{path}/#{f}"),
+        :public => true,
+        :content_type => mime
+      }
+
+      uncompressed_filename = f.sub(/\.gz\z/, '')
+      basename = File.basename(uncompressed_filename, File.extname(uncompressed_filename))
+      if /-[0-9a-fA-F]{32,}$/.match(basename)
+        file.merge!({
+          :cache_control => "public, max-age=#{one_year}",
+          :expires => CGI.rfc1123_date(Time.now + one_year)
+        })
+      end
+
+      # overwrite headers if applicable, you probably shouldn't specific key/body, but cache-control headers etc.
+
+      if files_with_custom_headers.has_key? f
+        file.merge! files_with_custom_headers[f]
+        log "Overwriting #{f} with custom headers #{files_with_custom_headers[f].to_s}"
+      elsif key = self.config.custom_headers.keys.detect {|k| f.match(Regexp.new(k))}
+        headers = {}
+        self.config.custom_headers[key].each do |k, value|
+          headers[k.to_sym] = value
+        end
+        file.merge! headers
+        log "Overwriting matching file #{f} with custom headers #{headers.to_s}"
+      end
+
+
+      gzipped = "#{path}/#{f}.gz"
+      ignore = false
+
+      if config.gzip? && File.extname(f) == ".gz"
+        # Don't bother uploading gzipped assets if we are in gzip_compression mode
+        # as we will overwrite file.css with file.css.gz if it exists.
+        log "Ignoring: #{f}"
+        ignore = true
+      elsif config.gzip? && File.exist?(gzipped)
+        original_size = File.size("#{path}/#{f}")
+        gzipped_size = File.size(gzipped)
+
+        if gzipped_size < original_size
+          percentage = ((gzipped_size.to_f/original_size.to_f)*100).round(2)
+          file.merge!({
+                        :key => f,
+                        :body => File.open(gzipped),
+                        :content_encoding => 'gzip'
+                      })
+          log "Uploading: #{gzipped} in place of #{f} saving #{percentage}%"
+        else
+          percentage = ((original_size.to_f/gzipped_size.to_f)*100).round(2)
+          log "Uploading: #{f} instead of #{gzipped} (compression increases this file by #{percentage}%)"
+        end
+      else
+        if !config.gzip? && File.extname(f) == ".gz"
+          # set content encoding for gzipped files this allows cloudfront to properly handle requests with Accept-Encoding
+          # http://docs.amazonwebservices.com/AmazonCloudFront/latest/DeveloperGuide/ServingCompressedFiles.html
+          uncompressed_filename = f[0..-4]
+          ext = File.extname(uncompressed_filename)[1..-1]
+          mime = MultiMime.lookup(ext)
+          file.merge!({
+            :content_type     => mime,
+            :content_encoding => 'gzip'
+          })
+        end
+        log "Uploading: #{f}"
+      end
+
+      if config.aws? && config.aws_rrs?
+        file.merge!({
+          :storage_class => 'REDUCED_REDUNDANCY'
+        })
+      end
+
+      file = bucket.files.create( file ) unless ignore
+    end
+
+    def skip_uploading?(manifest_file)
+      unless manifest_file
+        log "Failed to find manifest file"
+        return false
+      end
+
+      log "Found manifest file #{manifest_file}"
+
+      begin
+        s3_manifest = connection.head_object(self.config.fog_directory, manifest_file)
+
+        # If the manifest is within 24 hours old and exists on the server, we're going to assume
+        # we can skip uploading the assets.
+        created_at = Time.parse(s3_manifest.headers['Last-Modified']).utc
+        log "S3 Manifest was created on #{created_at} (#{Time.now.utc})"
+        return true if (Time.now.utc - created_at) <= 86400
+
+      rescue => ex
+        log "Got #{ex.class}, #{ex.message} while trying to HEAD the manifest from S3"
+        return false
+      end
+
+      false
+    end
+
+    def upload_files
+      manifest_file = self.config.manifest_digest_path
+      if skip_uploading?(manifest_file)
+        log "Skipping file upload"
+        return
+      end
+
+      # get a fresh list of remote files
+      remote_files = ignore_existing_remote_files? ? [] : get_remote_files
+      # fixes: https://github.com/rumblelabs/asset_sync/issues/19
+      local_files_to_upload = local_files - ignored_files - remote_files + always_upload_files
+      local_files_to_upload = (local_files_to_upload + get_non_fingerprinted(local_files_to_upload)).uniq
+
+      local_files_to_upload.delete manifest_file if manifest_file
+
+      # Upload new files
+      local_files_to_upload.each do |f|
+        next unless File.file? "#{path}/#{f}" # Only files.
+        upload_file f
+      end
+
+      # At the very end, upload the manifest indicating we're done
+      upload_file manifest_file if manifest_file
+
+      if self.config.cdn_distribution_id && files_to_invalidate.any?
+        log "Invalidating Files"
+        cdn ||= Fog::CDN.new(self.config.fog_options.except(:region))
+        data = cdn.post_invalidation(self.config.cdn_distribution_id, files_to_invalidate)
+        log "Invalidation id: #{data.body["Id"]}"
+      end
+    end
+
+    def sync
+      # fixes: https://github.com/rumblelabs/asset_sync/issues/19
+      log "AssetSync: Syncing."
+      upload_files
+      delete_extra_remote_files unless keep_existing_remote_files?
+      log "AssetSync: Done."
+    end
+
+    private
+
+    def ignore_existing_remote_files?
+      self.config.existing_remote_files == 'ignore'
+    end
+
+    def get_non_fingerprinted(files)
+      files.map do |file|
+        match_data = file.match(REGEXP_FINGERPRINTED_FILES)
+        match_data && "#{match_data[1]}/#{match_data[2]}.#{match_data[3]}"
+      end.compact
+    end
+
+  end
+end

data/lib/asset_sync/version.rb

@@ -0,0 +1,3 @@
+module AssetSync
+  VERSION = "2.0.0"
+end

data/lib/generators/asset_sync/install_generator.rb

@@ -0,0 +1,67 @@
+require 'rails/generators'
+module AssetSync
+  class InstallGenerator < Rails::Generators::Base
+    desc "Install a config/asset_sync.yml and the asset:precompile rake task enhancer"
+
+    # Commandline options can be defined here using Thor-like options:
+    class_option :use_yml,   :type => :boolean, :default => false, :desc => "Use YML file instead of Rails Initializer"
+    class_option :provider,  :type => :string,  :default => "AWS",  :desc => "Generate with support for 'AWS', 'Rackspace', or 'Google'"
+
+    def self.source_root
+      @source_root ||= File.join(File.dirname(__FILE__), 'templates')
+    end
+
+    def aws?
+      options[:provider] == 'AWS'
+    end
+
+    def google?
+      options[:provider] == 'Google'
+    end
+
+    def rackspace?
+      options[:provider] == 'Rackspace'
+    end
+
+    def aws_access_key_id
+      "<%= ENV['AWS_ACCESS_KEY_ID'] %>"
+    end
+
+    def aws_secret_access_key
+      "<%= ENV['AWS_SECRET_ACCESS_KEY'] %>"
+    end
+
+    def google_storage_access_key_id
+      "<%= ENV['GOOGLE_STORAGE_ACCESS_KEY_ID'] %>"
+    end
+
+    def google_storage_secret_access_key
+      "<%= ENV['GOOGLE_STORAGE_SECRET_ACCESS_KEY'] %>"
+    end
+
+    def rackspace_username
+      "<%= ENV['RACKSPACE_USERNAME'] %>"
+    end
+
+    def rackspace_api_key
+      "<%= ENV['RACKSPACE_API_KEY'] %>"
+    end
+
+    def app_name
+      @app_name ||= Rails.application.is_a?(Rails::Application) && Rails.application.class.name.sub(/::Application$/, "").downcase
+    end
+
+    def generate_config
+      if options[:use_yml]
+        template "asset_sync.yml", "config/asset_sync.yml"
+      end
+    end
+
+    def generate_initializer
+      unless options[:use_yml]
+        template "asset_sync.rb", "config/initializers/asset_sync.rb"
+      end
+    end
+
+  end
+end

data/lib/generators/asset_sync/templates/asset_sync.rb

@@ -0,0 +1,41 @@
+AssetSync.configure do |config|
+  <%- if aws? -%>
+  config.fog_provider = 'AWS'
+  config.aws_access_key_id = ENV['AWS_ACCESS_KEY_ID']
+  config.aws_secret_access_key = ENV['AWS_SECRET_ACCESS_KEY']
+  # To use AWS reduced redundancy storage.
+  # config.aws_reduced_redundancy = true
+  <%- elsif google? -%>
+  config.fog_provider = 'Google'
+  config.google_storage_access_key_id = ENV['GOOGLE_STORAGE_ACCESS_KEY_ID']
+  config.google_storage_secret_access_key = ENV['GOOGLE_STORAGE_SECRET_ACCESS_KEY']
+  <%- elsif rackspace? -%>
+  config.fog_provider = 'Rackspace'
+  config.rackspace_username = ENV['RACKSPACE_USERNAME']
+  config.rackspace_api_key = ENV['RACKSPACE_API_KEY']
+
+  # if you need to change rackspace_auth_url (e.g. if you need to use Rackspace London)
+  # config.rackspace_auth_url = "lon.auth.api.rackspacecloud.com"
+  <%- end -%>
+  config.fog_directory = ENV['FOG_DIRECTORY']
+
+  # Invalidate a file on a cdn after uploading files
+  # config.cdn_distribution_id = "12345"
+  # config.invalidate = ['file1.js']
+
+  # Increase upload performance by configuring your region
+  # config.fog_region = 'eu-west-1'
+  #
+  # Don't delete files from the store
+  # config.existing_remote_files = "keep"
+  #
+  # Automatically replace files with their equivalent gzip compressed version
+  # config.gzip_compression = true
+  #
+  # Use the Rails generated 'manifest.yml' file to produce the list of files to
+  # upload instead of searching the assets directory.
+  # config.manifest = true
+  #
+  # Fail silently.  Useful for environments such as Heroku
+  # config.fail_silently = true
+end