branchable_cdn_assets 0.5.3 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ab78d952715ccd097239e67bccdd4e0e1a6c7635
-  data.tar.gz: f7a994bd5aee7df2ada620b1641e769946abfb8e
+  metadata.gz: 5d4b5ae8228cc5c14fb3ed1ed06abf2efd1056b4
+  data.tar.gz: 1a3676d5d335a8a329d2196a98ea518bbeb79348
 SHA512:
-  metadata.gz: 1f80a5a192ced43ab482fa31e870e7baf429e6e5f33a6e71be4419c0dc4b5de7bcb26105d6a5b47c04056b1324b4ca08ad7ddef6204868ef6ebef39283d30c68
-  data.tar.gz: 8765796e5a26a58cd1519e0a707889d26d9ea26dea1448cfc53f46e1cc7e394f089a6e106487fd9210da50a3af9d0447f7e698dcfabcb83e7bf5eac9b5ff1a3d
+  metadata.gz: 9dca1c4d503f5f9f3c442118cdf17685f02e74d1eedfb7000fff59d107624240edd3fed79545b300a865c4f64a687bbfbf9d93e46e8ae78450ee5d361bdfd050
+  data.tar.gz: 7c94bfdadfc58cc6d8b3c9648c2dce11dec0aaa1eb28d0a68afc19d219b3af5aade65dc9a2ace355c504fe03cb38a4b9a9e0faf8cfe9d361ec06249a3c5a4a99

data/lib/branchable_cdn_assets/config.rb

@@ -2,12 +2,13 @@ require_relative 'config/environment_attribute_reader'
 
 module BranchableCDNAssets
   class Config
-    extend EnvironmentAttributeReader
+    include EnvironmentAttributeReader
+    extend  EnvironmentAttributeReader::ClassMethods
 
     attr_reader :raw_data, :branch,
                 :production_branch, :cloudfront, :cdn_dir,
-                :env, :environments
-    env_attr_reader :host, :root, :url
+                :dir_permissions, :file_permissions, :rsync_flags
+    env_attr_reader :host, :root, :url, :manage_cloudfront
 
     def initialize data, branch=Asgit.current_branch
       @raw_data = normalize_data data
@@ -16,13 +17,48 @@ module BranchableCDNAssets
       @production_branch = raw_data.fetch :production_branch, 'master'
       @cloudfront        = raw_data.fetch :cloudfront,        {}
       @cdn_dir           = raw_data.fetch :dir,               'cdn'
+      @dir_permissions   = env_attr(:dir_permissions)      || 755
+      @file_permissions  = env_attr(:file_permissions)     || 644
+      @rsync_flags       = env_attr(:rsync_flags)          || '-aviz'
+    end
+
+    def env
+      @env ||= env_for_branch
+    end
+
+    def environments
+      @environments ||= add_env_path_modifications_to_env_data raw_data.fetch(:environments, {})
+    end
+
+    def remotes
+      return @_remotes if @_remotes
 
-      @env          = env_for_branch
-      @environments = add_env_path_modifications_to_env_data raw_data.fetch(:environments, {})
+      @_remotes = environments[env.to_sym].fetch(:remotes, [default_remote]).map do |r|
+        remote = Remote.new
+        default_remote.merge( normalize_data(r) ).each do |k, v|
+          remote.public_send "#{k}=", v
+        end
+        remote
+      end
     end
 
     private
 
+      class Remote < Struct.new(:host, :root, :url,
+                                :dir_permissions, :file_permissions, :rsync_flags)
+      end
+
+      def default_remote
+        {
+          host: host,
+          root: root,
+          url:  url,
+          dir_permissions:  dir_permissions,
+          file_permissions: file_permissions,
+          rsync_flags:      rsync_flags
+        }
+      end
+
       def add_env_path_modifications_to_env_data environments
         Hash[*environments.map do |env,data|
           data = data.dup
@@ -31,14 +67,14 @@ module BranchableCDNAssets
             data[:url]  = File.join( data[:url],  branch )
           end
 
-          [env,data]
+          [env, data]
         end.flatten]
       end
 
       def env_for_branch
         if branch == production_branch
           return :production
-        elsif raw_data.fetch(:environments).has_key?( branch.to_sym )
+        elsif raw_data.fetch(:environments, {}).has_key?( branch.to_sym )
           return branch.to_sym
         else
           return raw_data.fetch(:default_env, 'staging').to_sym

data/lib/branchable_cdn_assets/config/environment_attribute_reader.rb

@@ -2,12 +2,18 @@ module BranchableCDNAssets
   class Config
 
     module EnvironmentAttributeReader
-      def env_attr_reader *keys
-        keys.each do |key|
-          define_method key do
-            environments[env.to_sym].fetch(key) {
-              raise ArgumentError, "No key '#{key.to_s}' exists for '#{env}'"
-            }
+
+      def env_attr key
+        environments.fetch(env.to_sym, {})
+                    .fetch(key, nil)
+      end
+
+      module ClassMethods
+        def env_attr_reader *keys
+          keys.each do |key|
+            define_method key do
+              env_attr(key) || raise( ArgumentError, "No key '#{key.to_s}' exists for '#{env}'" )
+            end
           end
         end
       end

data/lib/branchable_cdn_assets/file_manager.rb

@@ -1,4 +1,5 @@
 require_relative 'file_manager/checks'
+require_relative 'file_manager/permissions'
 
 module BranchableCDNAssets
   class FileManager
@@ -26,35 +27,44 @@ module BranchableCDNAssets
     # @return [Array]
     def list where=:all
       case where
-      when :local then
-        return list_local_files
-      when :remote then
-        return list_remote_files
-      when :both then
-        return list_local_files & list_remote_files
-      else
-        return list_local_files + list_remote_files
+      when :local  then return list_local_files
+      when :remote then return list_remote_files
+      when :both   then return list_local_files & list_remote_files
+      else return list_local_files + list_remote_files
       end
     end
 
+
+    # make cdn dir on the remotes with given permissions
+    def setup
+      setup_remotes
+
+      return nil
+    end
+
+
     # destructive pull, removes references to remote
     # files once they're pulled
     # @return [Array] the pulled files
     def pull!
-      pull_list = pull
+      pull_list = pull_all
       @manifest.remove_files( pull_list )
       @manifest.update_source_file!
+
       return pull_list
     end
 
+
     # destructive push, removes the local versions of
     # the pushed files and adds them to the manifest
     # @return [Array] the pushed files
     def push!
-      setup_remote
+      setup_remotes
 
       push_list = push
-      invalidate( push_list ) if config.env.to_sym == :production
+      if config.env.to_sym == :production && config.manage_cloudfront
+        invalidate( push_list )
+      end
 
       @manifest.merge_files( push_list )
       @manifest.update_source_file!
@@ -62,6 +72,7 @@ module BranchableCDNAssets
       return push_list
     end
 
+
     # remove local files that exist on the remote
     # @return [Array]
     def prune!
@@ -71,9 +82,12 @@ module BranchableCDNAssets
       list(:both).each do |f|
         Shell.run_local "rm #{File.join( root, f )}"
       end
+
       remove_empty_directories
+      return nil
     end
 
+
     # move the current cdn files to the production cdn
     # guarded to only run from prod
     def move_to_production branch
@@ -86,25 +100,127 @@ module BranchableCDNAssets
       push!
     end
 
+
     def remove_empty_directories
       empty_directories.each do |dir|
         Dir.rmdir( dir )
       end
     end
 
+
     def find file
-      return :local if list(:local).include?(file)
-      return File.join( config.url, file ) if list(:remote).include?(file)
+      return :local                      if list(:local).include?(file)
+      return File.join(config.url, file) if list(:remote).include?(file)
 
       unless config.env == :production
         production_config   = Config.new( config.raw_data, config.production_branch )
         production_manifest = Manifest.new( File.join( root, "#{config.production_branch}.manifest" ) )
+
         return File.join( production_config.url, file ) if production_manifest.files.include?(file)
       end
     end
 
+
+    # x files were not on every remote
+    # remote-x (x files):
+    # - file/path
+    # - file/path
+    #
+    # remote-y (x files):
+    # - path/to/file
+    # - path
+    def doctor
+      db    = uniq_files_per_remote
+      count = config.remotes.flat_map { |r| db[r.host].length }.inject(0, :+)
+
+      if count > 0
+        puts "#{count} files were not on every remote"
+        config.remotes.each do |remote|
+          puts "#{remote.host} (#{db[remote.host].length} files)"
+          if db[remote.host].length > 0
+            db[remote.host].each { |f| puts "  + #{f}" }
+          end
+        end
+      else
+        puts "all good, content is the same on all remotes"
+      end
+
+      return nil
+    end
+
+    def heal
+      db       = uniq_files_per_remote
+      all_uniq = []
+      config.remotes.each do |remote|
+        all_uniq.push *db[remote.host]
+        list_file = create_list_file db[remote.host].join("\n")
+        pull_list = map_rsync_output_to_list rsync_files_from_remote(list_file, remote)
+        list_file.unlink
+
+        puts "#{pull_list.length} files pulled from #{remote.host}"
+      end
+
+      config.remotes.each do |remote|
+        list_file = create_list_file ( all_uniq - db[remote.host] ).join("\n")
+        push_list = map_rsync_output_to_list rsync_files_to_remote( list_file, remote )
+        list_file.unlink
+
+        puts "#{push_list.length} files pushed to #{remote.host}"
+      end
+
+      nil
+    end
+
     private
 
+      def pull_all
+        db      = uniq_files_per_remote
+        pulled  = []
+        config.remotes.each_with_index do |remote, idx|
+          if idx == 0
+            file_list = db[remote.host] + db["all"]
+          else
+            file_list = db[remote.host]
+          end
+
+          list_file = create_list_file file_list.join("\n")
+          pull_list = map_rsync_output_to_list rsync_files_from_remote( list_file, remote )
+          list_file.unlink
+
+          pulled.push *pull_list
+          puts "#{pull_list.length} files pulled from #{remote.host}"
+        end
+
+        return pulled
+      end
+
+      def files_per_remote
+        config.remotes.each_with_object({}) do |remote, collection|
+          resp = Shell.run_remote "find #{remote.root} -type f", hostname: remote.host
+          if resp.success?
+            collection[remote.host] = resp.stdout.split("\n")
+                                          .map { |f| f.sub /^#{Regexp.escape(remote.root)}\/?/, '' }
+          else
+            warn "problem pulling file list for #{remote.host}\n#{resp.stderr}"
+          end
+        end
+      end
+
+      def uniq_files_per_remote
+        files_by_remote = files_per_remote
+        files_on_all    = []
+        files_by_remote.each_with_index do |(remote, files), idx|
+          if idx == 0
+            files_on_all = files
+          else
+            files_on_all = files_on_all & files
+          end
+        end
+
+        Hash[ files_by_remote.map { |r, files| [r, (files - files_on_all)] }
+                             .push ["all", files_on_all] ]
+      end
+
       def invalidate files
         to_invalidate = ( files & list(:local) ).map do |f|
           File.join( config.cloudfront[:path_prefix], f )
@@ -115,9 +231,15 @@ module BranchableCDNAssets
       end
 
       # create the root dir on remote
-      def setup_remote
-        resp = Shell.run_remote "mkdir -p #{config.root}", hostname: config.host
-        raise resp.stderr unless resp.success?
+      def setup_remotes
+        config.remotes.each do |remote|
+          resp = Shell.run_remote "mkdir -p -m #{remote.dir_permissions} #{remote.root}", hostname: remote.host
+          unless resp.success?
+            raise "problem setting up #{remote.host}\n#{resp.stderr}"
+          else
+            puts "set up #{remote.root} on #{remote.host}"
+          end
+        end
       end
 
       def empty_directories
@@ -140,20 +262,30 @@ module BranchableCDNAssets
       def push
         list_file = create_list_file( list(:local).join("\n") )
         ensure_local_file_permissions
-        push_list = map_rsync_output_to_list rsync_files_to_remote(list_file)
-        list_file.unlink
 
+        push_list = config.remotes.flat_map do |remote|
+          map_rsync_output_to_list( rsync_files_to_remote(list_file, remote) )
+        end.uniq
+
+        list_file.unlink
         push_list = resolve_conflicts(push_list)
         puts "#{push_list.length} files pushed to #{branch}"
         return push_list
       end
 
-      def rsync_files_to_remote list_file
-        resp = Shell.run_local "rsync -aviz --files-from=#{list_file.path} " +
-                               "-e ssh #{root}/ #{config.host}:#{config.root}"
+      def rsync_files_to_remote list_file, remote
+        dir_perms  = Permissions.new( remote.dir_permissions ).to_chmod
+        file_perms = Permissions.new( remote.file_permissions ).to_chmod
+        file_permissions = "--chmod=" +
+                           "Du=#{dir_perms.user},Dg=#{dir_perms.group},Do=#{dir_perms.others}," +
+                           "Fu=#{file_perms.user},Fg=#{file_perms.group},Fo=#{file_perms.others}"
 
-        raise "rsync failed\n#{resp.stderr}" unless resp.success?
-        return resp.stdout
+
+        resp = Shell.run_local "rsync #{remote.rsync_flags} #{file_permissions} --files-from=#{list_file.path} " +
+                               "-e ssh #{root}/ #{remote.host}:#{remote.root}"
+
+        raise "rsync failed for #{remote.host}\n#{resp.stderr}" unless resp.success?
+        resp.stdout
       end
 
       def ensure_local_file_permissions
@@ -174,9 +306,9 @@ module BranchableCDNAssets
         return pull_list
       end
 
-      def rsync_files_from_remote list_file
-        resp = Shell.run_local "rsync -aviz --files-from=#{list_file.path}" +
-                               " -e ssh #{config.host}:#{config.root} #{root}/"
+      def rsync_files_from_remote list_file, remote
+        resp = Shell.run_local "rsync #{config.rsync_flags} --files-from=#{list_file.path}" +
+                               " -e ssh #{remote.host}:#{remote.root} #{root}/"
 
         raise "rsync failed\n#{resp.stderr}" unless resp.success?
         return resp.stdout

data/lib/branchable_cdn_assets/file_manager/checks.rb

@@ -5,17 +5,21 @@ module BranchableCDNAssets
       def manifest_list_clean_for_master
         manifests = Dir[ File.join( root, "*.manifest" ) ]
         if Asgit.current_branch == "master" && manifests.length > 1
-          raise "there should only be a production manifest on the master branch\n" +
-                "consider cleaning up with rake cdn:move_to_production[branch_name]"
+          raise "the master branch should only have a production manifest\n" +
+                "move branched assets to production with the `move_to_production` task"
         end
       end
 
       def local_file_conflict
         intersection = list(:both)
         if !intersection.empty?
-          puts "#{intersection.length} files are duplicated locally and on the remote"
-          unless Shell.get_input("do you want to continue? (y|n)") == "y"
+          puts "#{intersection.length} files detected locally and in the manifest"
+
+          if Shell.get_input("would you like to list the conflicting files? (y|n) ") == "y"
             puts "the conflicting files: \n#{intersection.join("\n")}"
+          end
+
+          unless Shell.get_input("do you want to continue? (y|n) ") == "y"
             abort
           end
         end
@@ -23,15 +27,18 @@ module BranchableCDNAssets
 
       def ready_for_production
         if Asgit.current_branch != "master"
-          puts "you shouldn't move to production except from master"
+          puts "you shouldn't move to production except from master branch"
           abort
         end
+
         if !Asgit.remote_up_to_date?
-          puts "make sure you've pulled all remote changes"
+          puts "make sure you're repo is in sync with the remote"
           abort
         end
+
         if !list(:local).empty?
-          puts "push all your local files and commit first"
+          puts "assets detected in the local cdn directory\n" +
+               "remove them by running the `prune` task"
           abort
         end
       end

data/lib/branchable_cdn_assets/file_manager/permissions.rb

@@ -0,0 +1,34 @@
+module BranchableCDNAssets
+  class FileManager
+
+    class Permissions
+
+      attr_reader :user, :group, :others
+
+      def initialize int
+        @user, @group, @others = int.to_s.chars.to_a
+      end
+
+      def to_chmod
+        Struct.new(:user, :group, :others)
+              .new( chmod_map(user), chmod_map(group), chmod_map(others) )
+      end
+
+      private
+
+        def chmod_map num
+          {
+            "7" => "rwx",
+            "6" => "rw",
+            "5" => "rx",
+            "4" => "r",
+            "3" => "wx",
+            "2" => "w",
+            "1" => "x",
+            "0" => "-"
+          }.fetch num
+        end
+    end
+
+  end
+end

data/lib/branchable_cdn_assets/manifest.rb

@@ -7,7 +7,7 @@ module BranchableCDNAssets
 
     def initialize source_file
       @source_file = source_file
-      @file_set = File.exists?(source_file) ? read_source_file : Set.new
+      @file_set    = File.exists?(source_file) ? read_source_file : Set.new
     end
 
     # list files set as an Array

data/lib/branchable_cdn_assets/rake_tasks.rb

@@ -15,17 +15,20 @@ module BranchableCDNAssets
     attr_reader :file_manager, :rake_namespace
 
     def initialize namespace, config
-      @file_manager = FileManager.new config
+      @file_manager   = FileManager.new config
       @rake_namespace = namespace
     end
 
     def tasks
       {
-        list:  'list of local files',
+        list:   'list of local files',
         pull!:  'move the current branch\'s remote files to local',
         push!:  'move local files to the current branch\'s remote',
         prune!: 'remove local files with the same name as remote files',
-        move_to_production: 'move named branch files to production cdn'
+        move_to_production: 'move named branch files to production cdn',
+        doctor: 'list file differences between remotes',
+        heal:   'pulls then pushes necessary files to make all remotes lists match',
+        setup:  'setup dir on remotes'
       }
     end
 
@@ -59,7 +62,7 @@ module BranchableCDNAssets
       def register_task task_name, task_desc
         in_namespace do
           desc task_desc
-          task task_name.to_s.sub('!','') do
+          task task_name.to_s.sub('!', '') do
             puts file_manager.public_send( :with_check, task_name )
           end
         end

data/lib/branchable_cdn_assets/version.rb

@@ -1,3 +1,3 @@
 module BranchableCDNAssets
-  VERSION = "0.5.3"
+  VERSION = "0.6.0"
 end

data/spec/lib/branchable_cdn_assets/file_manager_spec.rb

@@ -9,6 +9,13 @@ describe BranchableCDNAssets::FileManager do
       cdn_dir: 'cdn',
       host: 'host',
       root: 'root',
+      manage_cloudfront: true,
+      dir_permissions: 755,
+      file_permissions: 644,
+      rsync_flags: '-aviz',
+      remotes: [
+        BranchableCDNAssets::Config::Remote.new( 'host', 'root', 'url', 755, 644, '-aviz' )
+      ],
       cloudfront: {
         path_prefix: '/prefix'
       }
@@ -140,10 +147,16 @@ describe BranchableCDNAssets::FileManager do
       end
 
       it "returns list of pulled files" do
+        allow( BranchableCDNAssets::Shell ).to receive(:run_remote)
+                                           .with("find root -type f", hostname: "host")
+                                           .and_return(@success_response)
         expect( described_class.new(@config).pull! ).to match_array @file_list
       end
 
       it "removes files & updates manifest" do
+        allow( BranchableCDNAssets::Shell ).to receive(:run_remote)
+                                           .with("find root -type f", hostname: "host")
+                                           .and_return(@success_response)
         expect( @manifest ).to receive(:remove_files).with(@file_list)
         expect( @manifest ).to receive(:update_source_file!)
         described_class.new(@config).pull!
@@ -172,11 +185,15 @@ describe BranchableCDNAssets::FileManager do
 
     context "when setup_remote is successful" do
       before :each do
-        allow( BranchableCDNAssets::Shell ).to receive(:run_remote).with('mkdir -p root', hostname: 'host').and_return(@success_response)
+        allow( BranchableCDNAssets::Shell ).to receive(:run_remote)
+                                           .with('mkdir -p -m 755 root', hostname: 'host')
+                                           .and_return(@success_response)
       end
 
       it "sets up remote" do
-        expect( BranchableCDNAssets::Shell ).to receive(:run_remote).with('mkdir -p root', hostname: 'host').and_return(@success_response)
+        expect( BranchableCDNAssets::Shell ).to receive(:run_remote)
+                                            .with('mkdir -p -m 755 root', hostname: 'host')
+                                            .and_return(@success_response)
         described_class.new(@config).push!
       end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: branchable_cdn_assets
 version: !ruby/object:Gem::Version
-  version: 0.5.3
+  version: 0.6.0
 platform: ruby
 authors:
 - Steven Sloan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-27 00:00:00.000000000 Z
+date: 2015-02-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -93,6 +93,7 @@ files:
 - lib/branchable_cdn_assets/config/environment_attribute_reader.rb
 - lib/branchable_cdn_assets/file_manager.rb
 - lib/branchable_cdn_assets/file_manager/checks.rb
+- lib/branchable_cdn_assets/file_manager/permissions.rb
 - lib/branchable_cdn_assets/manifest.rb
 - lib/branchable_cdn_assets/rake_tasks.rb
 - lib/branchable_cdn_assets/shell.rb
@@ -128,7 +129,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: Helpers for syncing and finding assets accross multiple remotes