branchable_cdn_assets 0.5.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 6ea31a15245850a8679508ce6256859f872e8e22
+  data.tar.gz: 34645624f7e366189c37c46c2ccd1841e032652c
+SHA512:
+  metadata.gz: d724f92695a224e3fc88dca3ca8075b18cdb420a6a0817205ceeceb9a3b3cc789907b4fcf9c0247221e9a913a2977fc6ea987c3547171865874f72435e68174c
+  data.tar.gz: fd962e24f6d4b906f7507d6636d6cd11f82a782112e418dc0421a67e66801a4ac9c11ed66a7d8ff9e697746f6129ae26f0fa1560470f6cfab6f44372155954bc

data/lib/branchable_cdn_assets.rb

@@ -0,0 +1,19 @@
+require 'tempfile'
+require 'yaml'
+
+require 'asgit'
+require 'here_or_there'
+require 'colorize'
+require 'fog'
+
+require_relative 'branchable_cdn_assets/version'
+require_relative 'branchable_cdn_assets/check_before'
+require_relative 'branchable_cdn_assets/shell'
+
+require_relative 'branchable_cdn_assets/file_manager'
+require_relative 'branchable_cdn_assets/manifest'
+require_relative 'branchable_cdn_assets/config'
+require_relative 'branchable_cdn_assets/cloudfront'
+
+module BranchableCDNAssets
+end

data/lib/branchable_cdn_assets/check_before.rb

@@ -0,0 +1,49 @@
+module BranchableCDNAssets
+
+  # this abstracts creating filter methods
+  # not offering inheritance currently
+  #
+  #     check_before :check, params={}
+  #
+  # to only check for specific methods,
+  # pass an array of methods to the :methods key
+  #
+  #     check_before :check, methods: ["foo", "bar"]
+  #
+  module CheckBefore
+
+    def before_checks
+      @_before_checks ||= []
+    end
+
+    # define a before check for a specific method
+    #
+    # @param method [Symbol] the method to check on
+    # @param check [Symbol] the check to run
+    # @param params [Hash] additional parameters for the check
+    # @return [Void]
+    def check_before check, params={}
+      before_checks << params.merge( check: check )
+    end
+
+
+    def self.extended base
+      base.class_eval do
+
+        # @param action [Symbol] the method to execute
+        # @param *args [Array] arguments to pass to the check method
+        def with_check action, *args
+          self.class.before_checks.select do |c|
+            c[:methods].nil? || c[:methods].include?(action)
+          end.each do |c|
+            send c[:check]
+          end
+
+          public_send(action, *args)
+        end
+
+      end
+    end
+
+  end
+end

data/lib/branchable_cdn_assets/cloudfront.rb

@@ -0,0 +1,28 @@
+module BranchableCDNAssets
+  class Cloudfront
+
+    attr_reader :access_key, :secret_key, :distribution_id,
+                :cdn
+
+    # handles communication with cloudfront
+    def initialize keys
+      @distribution_id = keys.fetch :distribution_id
+      @access_key      = keys.fetch :access_key
+      @secret_key      = keys.fetch :secret_key
+
+      @cdn = ::Fog::CDN.new provider: 'AWS',
+                            aws_access_key_id: access_key,
+                            aws_secret_access_key: secret_key
+    end
+
+    # invalidate a batch of files on fog
+    # @param files [Array]
+    def invalidate_files files
+      resp = cdn.post_invalidation( distribution_id, Array(files) )
+      resp.body["InvalidationBatch"]["Path"].each do |file|
+        puts "Posted an invalidation for #{file}".colorize( :green )
+      end
+    end
+
+  end
+end

data/lib/branchable_cdn_assets/config.rb

@@ -0,0 +1,80 @@
+require_relative 'config/environment_attribute_reader'
+
+module BranchableCDNAssets
+  class Config
+    extend EnvironmentAttributeReader
+
+    attr_reader :raw_data, :branch,
+                :production_branch, :cloudfront, :cdn_dir,
+                :env, :environments
+    env_attr_reader :host, :root, :url
+
+    def initialize data, branch=Asgit.current_branch
+      @raw_data = normalize_data data
+      @branch   = branch
+
+      @production_branch = raw_data.fetch :production_branch, 'master'
+      @cloudfront        = raw_data.fetch :cloudfront,        {}
+      @cdn_dir           = raw_data.fetch :dir,               'cdn'
+
+      @env          = env_for_branch
+      @environments = add_env_path_modifications_to_env_data raw_data.fetch(:environments, {})
+    end
+
+    private
+
+      def add_env_path_modifications_to_env_data environments
+        Hash[*environments.map do |env,data|
+          data = data.dup
+          unless env == :production
+            data[:root] = File.join( data[:root], branch, '/' )
+            data[:url]  = File.join( data[:url],  branch )
+          end
+
+          [env,data]
+        end.flatten]
+      end
+
+      def env_for_branch
+        if branch == production_branch
+          return :production
+        elsif raw_data.fetch(:environments).has_key?( branch.to_sym )
+          return branch.to_sym
+        else
+          return raw_data.fetch(:default_env, 'staging').to_sym
+        end
+      end
+
+      def normalize_data data
+        if data.respond_to?(:to_hash)
+          symbolize_keys data.to_hash
+        else
+          symbolize_keys read_config_file(data)
+        end
+      end
+
+      def read_config_file path
+        if File.exists?(path)
+          YAML.load IO.read(path)
+        else
+          raise "config file not found at #{path}"
+        end
+      end
+
+      def symbolize_keys(hash)
+        hash.inject({}) do |out, (key, value)|
+          k = case key
+              when String then key.to_sym
+              else key
+              end
+          v = case value
+              when Hash then symbolize_keys(value)
+              else value
+              end
+          out[k] = v
+          out
+        end
+      end
+
+  end
+end

data/lib/branchable_cdn_assets/config/environment_attribute_reader.rb

@@ -0,0 +1,17 @@
+module BranchableCDNAssets
+  class Config
+
+    module EnvironmentAttributeReader
+      def env_attr_reader *keys
+        keys.each do |key|
+          define_method key do
+            environments[env.to_sym].fetch(key) {
+              raise ArgumentError, "No key '#{key.to_s}' exists for '#{env}'"
+            }
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/branchable_cdn_assets/file_manager.rb

@@ -0,0 +1,222 @@
+require_relative 'file_manager/checks'
+
+module BranchableCDNAssets
+  class FileManager
+    extend  CheckBefore
+    include Checks
+
+    check_before :manifest_list_clean_for_master, methods: [ :push!, :pull!, :prune! ]
+    check_before :local_file_conflict,            methods: [ :push!, :pull! ]
+    check_before :ready_for_production,           methods: [ :move_to_production ]
+
+    attr_reader :config
+    attr_reader :root
+    attr_reader :branch
+    attr_reader :manifest
+
+    # handles moving files to/from our various cdn locations
+    def initialize config, branch=nil
+      @config   = config
+      @root     = config.cdn_dir
+      @branch   = branch || config.branch
+      @manifest = Manifest.new( File.join( @root, "#{@branch}.manifest" ) )
+    end
+
+    # @param where [Symbol] :local, :remote, :all (default)
+    # @return [Array]
+    def list where=:all
+      case where
+      when :local then
+        return list_local_files
+      when :remote then
+        return list_remote_files
+      when :both then
+        return list_local_files & list_remote_files
+      else
+        return list_local_files + list_remote_files
+      end
+    end
+
+    # destructive pull, removes references to remote
+    # files once they're pulled
+    # @return [Array] the pulled files
+    def pull!
+      pull_list = pull
+      @manifest.remove_files( pull_list )
+      @manifest.update_source_file!
+      return pull_list
+    end
+
+    # destructive push, removes the local versions of
+    # the pushed files and adds them to the manifest
+    # @return [Array] the pushed files
+    def push!
+      setup_remote
+
+      push_list = push
+      invalidate( push_list ) if config.env.to_sym == :production
+
+      @manifest.merge_files( push_list )
+      @manifest.update_source_file!
+
+      return push_list
+    end
+
+    # remove local files that exist on the remote
+    # @return [Array]
+    def prune!
+      puts "files to be removed:\n#{list(:both).join("\n")}"
+      abort unless Shell.get_input("do you want to continue? (y|n) ") == "y"
+
+      list(:both).each do |f|
+        Shell.run_local "rm #{File.join( root, f )}"
+      end
+      remove_empty_directories
+    end
+
+    # move the current cdn files to the production cdn
+    # guarded to only run from prod
+    def move_to_production branch
+      branch_cdn = FileManager.new( Config.new( config.raw_data, branch ) )
+      if branch_cdn.manifest.files.empty?
+        puts "no files to move from #{branch}"
+        abort
+      end
+      branch_cdn.pull!
+      push!
+    end
+
+    def remove_empty_directories
+      empty_directories.each do |dir|
+        Dir.rmdir( dir )
+      end
+    end
+
+    def find file
+      return :local if list(:local).include?(file)
+      return File.join( config.url, file ) if list(:remote).include?(file)
+
+      unless config.env == :production
+        production_config   = Config.new( config.raw_data, config.production_branch )
+        production_manifest = Manifest.new( File.join( root, "#{config.production_branch}.manifest" ) )
+        return File.join( production_config.url, file ) if production_manifest.files.include?(file)
+      end
+    end
+
+    private
+
+      def invalidate files
+        to_invalidate = ( files & list(:local) ).map do |f|
+          File.join( config.cloudfront[:path_prefix], f )
+        end
+
+        Cloudfront.new( config.cloudfront )
+                  .invalidate_files( to_invalidate )
+      end
+
+      # create the root dir on remote
+      def setup_remote
+        resp = Shell.run_remote "mkdir -p #{config.root}", hostname: config.host
+        raise resp.stderr unless resp.success?
+      end
+
+      def empty_directories
+        Dir[ File.join( Dir.pwd, root, '**/*' ) ].select do |d|
+          File.directory?(d) && (Dir.entries(d) - ['.','..']).empty?
+        end
+      end
+
+      # @return [Array] the pushed files (only those updated on remote)
+      def push
+        list_file = create_list_file( list(:local).join("\n") )
+        ensure_local_file_permissions
+        push_list = map_rsync_output_to_list rsync_files_to_remote(list_file)
+        list_file.unlink
+
+        push_list = resolve_conflicts(push_list)
+        puts "#{push_list.length} files pushed to #{branch}"
+        return push_list
+      end
+
+      def rsync_files_to_remote list_file
+        resp = Shell.run_local "rsync -aviz --files-from=#{list_file.path} " +
+                               "-e ssh #{root}/ #{config.host}:#{config.root}"
+
+        raise "rsync failed\n#{resp.stderr}" unless resp.success?
+        return resp.stdout
+      end
+
+      def ensure_local_file_permissions
+        list_local_files.each do |file|
+          Shell.run_local "chmod 644 #{File.join( root, file )}"
+        end
+      end
+
+      # Take files from the cdn and move them locally
+      # @return [Array] list of files pulled
+      def pull
+        # create list for rsync
+        list_file = create_list_file( list(:remote).join("\n") )
+        pull_list = map_rsync_output_to_list rsync_files_from_remote(list_file)
+        list_file.unlink
+
+        puts "#{pull_list.length} files pulled from #{branch}"
+        return pull_list
+      end
+
+      def rsync_files_from_remote list_file
+        resp = Shell.run_local "rsync -aviz --files-from=#{list_file.path}" +
+                               " -e ssh #{config.host}:#{config.root} #{root}/"
+
+        raise "rsync failed\n#{resp.stderr}" unless resp.success?
+        return resp.stdout
+      end
+
+      # @return [Array] resolved list
+      def resolve_conflicts push_list
+        conflicts = list(:local) - push_list
+        return push_list if conflicts.empty?
+
+        puts "these local files were not pushed by rsync:"
+        puts conflicts.join("\n")
+        if Shell.get_input("add to the #{branch} manifest? (y|n) ") == "y"
+          push_list.push(*conflicts)
+        end
+        return push_list
+      end
+
+      # creates a temp file with contens and returns it's path
+      # @param contents [String] content for the file
+      # @return [String] file path
+      def create_list_file contents
+        file = Tempfile.new( "list_file" )
+        file.write contents
+        file.close
+
+        return file
+      end
+
+      # @param output [String]
+      # @return [Array]
+      def map_rsync_output_to_list output
+        return output.split("\n").keep_if do |f|
+          f.match( /^(?:<|>)f/ ) && !f.match(/\/$/ )
+        end.map do |f|
+          f.split()[1]
+        end
+      end
+
+      def list_local_files
+        Dir[ File.join( Dir.pwd, root, '**/*' ) ].keep_if do |f|
+          !f.match(/\.manifest$/) && !File.directory?(f)
+        end.map do |f|
+          f.sub( "#{File.join( Dir.pwd, root )}/", "" )
+        end
+      end
+
+      def list_remote_files
+        manifest.files
+      end
+
+  end
+end

data/lib/branchable_cdn_assets/file_manager/checks.rb

@@ -0,0 +1,41 @@
+module BranchableCDNAssets
+  class FileManager
+    module Checks
+
+      def manifest_list_clean_for_master
+        manifests = Dir[ File.join( root, "*.manifest" ) ]
+        if Asgit.current_branch == "master" && manifests.length > 1
+          raise "there should only be a production manifest on the master branch\n" +
+                "consider cleaning up with rake cdn:move_to_production[branch_name]"
+        end
+      end
+
+      def local_file_conflict
+        intersection = list(:both)
+        if !intersection.empty?
+          puts "#{intersection.length} files are duplicated locally and on the remote"
+          unless Shell.get_input("do you want to continue? (y|n)") == "y"
+            puts "the conflicting files: \n#{intersection.join("\n")}"
+            abort
+          end
+        end
+      end
+
+      def ready_for_production
+        if Asgit.current_branch != "master"
+          puts "you shouldn't move to production except from master"
+          abort
+        end
+        if !Asgit.remote_up_to_date?
+          puts "make sure you've pulled all remote changes"
+          abort
+        end
+        if !list(:local).empty?
+          puts "push all your local files and commit first"
+          abort
+        end
+      end
+
+    end
+  end
+end