engineyard-serverside 2.0.7 → 2.1.0.pre

data/lib/engineyard-serverside/deprecation.rb

@@ -6,10 +6,20 @@ module EY
       $stderr.puts "DEPRECATION WARNING: #{msg}\n\t#{caller(2).first}"
     end
 
+    def self.deprecated_task(receiver, old_task, new_task)
+      if receiver.respond_to?(old_task)
+        deprecation_warning("Task ##{old_task} has been renamed to ##{new_task}.")
+      end
+    end
+
     def self.const_missing(const)
-      if const == :LoggedOutput
+      case const
+      when :LoggedOutput
         EY::Serverside.deprecation_warning("EY::Serverside::LoggedOutput has been deprecated. Use EY::Serverside::Shell::Helpers instead.")
         EY::Serverside::Shell::Helpers
+      when :LockfileParser
+        EY::Serverside.deprecation_warning("EY::Serverside::LockfileParser has been deprecated. Use EY::Serverside::DependencyManager::BundlerLock::Lockfile instead.")
+        EY::Serverside::DependencyManager::BundlerLock::Lockfile
       else
         super
       end

data/lib/engineyard-serverside/paths.rb

@@ -70,6 +70,7 @@ module EY
       def_path :enabled_maintenance_page, [:shared_system,  'maintenance.html']
       def_path :shared_assets,            [:shared,         'assets']
       def_path :bundled_gems,             [:shared,         'bundled_gems']
+      def_path :shared_services_yml,      [:shared_config,  'ey_services_config_deploy.yml']
       def_path :ruby_version,             [:bundled_gems,   'RUBY_VERSION']
       def_path :system_version,           [:bundled_gems,   'SYSTEM_VERSION']
       def_path :latest_revision,          [:latest_release, 'REVISION']
@@ -127,6 +128,11 @@ module EY
         end
       end
 
+      def previous_revision
+        rel = previous_release(active_release)
+        rel && rel.join('REVISION')
+      end
+
       # deploy_root/releases/<latest timestamp>
       def latest_release
         all_releases.last

data/lib/engineyard-serverside/propagator.rb

@@ -67,8 +67,8 @@ module EY
 
       def propagate
         shell.status "Propagating #{About.name_with_version} to #{count_servers(servers)}."
-        servers.run_on_each { |server| shell.logged_system(scp_command(server)) }
-        servers.run_on_each { |server| shell.logged_system(server.command_on_server('sudo sh -l -c', install_command)) }
+        servers.run_on_each(shell) { |server| shell.logged_system(scp_command(server)) }
+        servers.run_on_each(shell) { |server| shell.logged_system(server.command_on_server('sudo sh -l -c', install_command)) }
       end
     end
   end

data/lib/engineyard-serverside/rails_assets.rb

@@ -0,0 +1,152 @@
+require 'engineyard-serverside/rails_assets/strategy'
+require 'forwardable'
+
+module EY
+  module Serverside
+    class RailsAssets
+      extend Forwardable
+
+      def self.detect_and_compile(*args)
+        new(*args).detect_and_compile
+      end
+
+      attr_reader :config, :shell, :runner
+
+      def initialize(config, shell, runner)
+        @config, @shell, @runner = config, shell, runner
+      end
+
+      def_delegators :config,
+        :paths, :asset_dependencies, :asset_roles,
+        :framework_envs, :precompile_assets?, :skip_precompile_assets?,
+        :precompile_unchanged_assets?, :precompile_assets_task
+
+      def detect_and_compile
+        runner.roles asset_roles do
+          if precompile_assets?
+            if precompile_unchanged_assets?
+              shell.status "Precompiling assets without change detection. (precompile_unchanged_assets: true)"
+              run_precompile_assets_task
+            elsif reuse_assets?
+              shell.status "Reusing existing assets. (configured asset_dependencies unchanged from #{previous_revision[0,7]}..#{active_revision[0,7]})"
+              asset_strategy.reuse
+            else
+              shell.status "Precompiling assets. (precompile_assets: true)"
+              run_precompile_assets_task
+            end
+          elsif skip_precompile_assets?
+            shell.status "Skipping asset precompilation. (precompile_assets: false)"
+          elsif !application_rb_path.readable? || !app_assets_path.directory?
+            # Not a Rails app. Ignore assets completely.
+          elsif app_disables_assets?
+            shell.status "Skipping asset precompilation. ('config/application.rb' disables assets.)"
+          elsif paths.public_assets.exist?
+            shell.status "Skipping asset precompilation. ('public/assets' directory already exists.)"
+          else
+            precompile_detected_assets
+          end
+        end
+      end
+
+      def run_precompile_assets_task
+        asset_strategy.prepare do
+          cd   = "cd #{paths.active_release}"
+          task = "PATH=#{paths.binstubs}:$PATH #{framework_envs} rake #{precompile_assets_task} RAILS_GROUPS=assets"
+          runner.run "#{cd} && #{task}"
+        end
+      end
+
+      def previous_revision
+        @previous_revision ||= config.previous_revision
+      end
+
+      def active_revision
+        @active_revision ||= config.active_revision
+      end
+
+      # Note on reusing assets when assets may fail silently:
+      # It's difficult and error prone to reuse assets that may have failed
+      # silently in the previous deploy. If the assets are unchanged during
+      # this deploy, but failed last deploy, we would incorrectly reuse
+      # silentely failed assets. Only reusing when assets are enabled
+      # ensures that existing assets were successful.
+      def reuse_assets?
+        previous_revision &&
+          active_revision &&
+          runner.strategy.same?(previous_revision, active_revision, asset_dependencies)
+      end
+
+      def precompile_detected_assets
+        shell.status "Precompiling assets. ('#{app_assets}' exists, 'public/assets' not found, not disabled in config.)"
+        if !runner.dependency_manager.rails_version
+          shell.warning "Precompiling assets even though Rails was not bundled."
+        end
+
+        run_precompile_assets_task
+
+        shell.warning <<-WARN
+Inferred asset compilation succeeded, but failures may be silently ignored!
+
+ACTION REQUIRED: Add precompile_assets option to ey.yml.
+  precompile_assets: true  # precompile assets when asset changes detected
+        WARN
+      rescue EY::Serverside::RemoteFailure => e
+        # If we are implicitly precompiling, we want to fail non-destructively
+        # because we don't know if the rake task exists or if the user
+        # actually intended for assets to be compiled.
+        if e.to_s =~ /Don't know how to build task '#{precompile_assets_task}'/
+          shell.warning <<-WARN
+Asset precompilation detected but compilation failure ignored!
+Rake task '#{precompile_assets_task}' was not found.
+
+ACTION REQUIRED: Add precompile_assets option to ey.yml.
+  precompile_assets: false # disable assets to avoid this error.
+          WARN
+        else
+          shell.error <<-ERROR
+Asset precompilation detected but compilation failed!
+
+ACTION REQUIRED: Add precompile_assets option to ey.yml.
+  precompile_assets: true  # precompile assets when asset changes detected
+  precompile_assets: false # disable asset compilation.
+          ERROR
+          raise
+        end
+      end
+
+      def app_disables_assets?
+        application_rb_path.open do |fd|
+          fd.grep(/^[^#]*config\.assets\.enabled\s*=\s*(false|nil)/).any?
+        end
+      end
+
+      # This check is very expensive, and has been deemed not worth the time.
+      # Leaving this here in case someone comes up with a faster way.
+      #
+      # Runs 'rake -T' to see if there is an assets:precompile task.
+      def app_has_asset_task?
+        # We just run this locally on the app master; everybody else should
+        # have the same code anyway.
+        task_check = "PATH=#{paths.binstubs}:$PATH #{framework_envs} rake -T #{precompile_assets_task} | grep '#{precompile_assets_task}'"
+        cmd = "cd #{paths.active_release} && #{task_check}"
+        shell.logged_system(cmd).success?
+      end
+
+      def application_rb_path
+        paths.active_release.join('config','application.rb')
+      end
+
+      def app_assets
+        File.join('app','assets')
+      end
+
+      def app_assets_path
+        paths.active_release.join(app_assets)
+      end
+
+      def asset_strategy
+        @asset_strategy ||= RailsAssets::Strategy.fetch(config.asset_strategy, paths, runner)
+      end
+    end
+  end
+end

data/lib/engineyard-serverside/rails_assets/strategy.rb

@@ -0,0 +1,197 @@
+require 'yaml'
+
+module EY
+  module Serverside
+    class RailsAssets
+      module Strategy
+        def self.all
+          {
+            'shared'   => Shared,
+            'cleaning' => Cleaning,
+            'private'  => Private,
+            'shifting' => Shifting,
+          }
+        end
+
+        def self.fetch(name, *args)
+          (all[name.to_s] || Shifting).new(*args)
+        end
+
+
+        # Precompile assets fresh every time. Shared assets are not symlinked
+        # and assets stay with the release that compiled them. The assets of
+        # the previous deploy are symlinked as into the current deploy to
+        # prevent errors during deploy.
+        #
+        # When no assets changes are detected, the deploy uses rsync to copy
+        # the previous release's assets into the current assets directory.
+        class Private
+          attr_reader :paths, :runner
+
+          def initialize(paths, runner)
+            @paths = paths
+            @runner = runner
+          end
+
+          def reuse
+            run("mkdir -p #{paths.public_assets} && rsync -aq #{previous_assets_path}/ #{paths.public_assets}")
+          end
+
+          # ?ink the previous assets into the new public/last_assets/assets
+          # to prevent missing assets during deploy.
+          #
+          # This results in the directory structure:
+          #   deploy_root/current/public/last_assets/assets -> deploy_root/releases/<prev>/public/assets
+          def prepare
+            last = paths.public.join('last_assets')
+            run "mkdir -p #{last} && ln -nfs #{previous_assets_path} #{last.join('assets')}"
+            yield
+          end
+
+          protected
+
+          def run(cmd)
+            runner.run(cmd)
+          end
+
+          def previous_assets_path
+            paths.previous_release(paths.active_release).join('public','assets')
+          end
+        end
+
+        # Basic shared assets.
+        # Precompiled assets go into a single shared assets directory. The
+        # assets directory is never cleaned, so a deploy hook should be used
+        # to clean assets appropriately.
+        #
+        # When no assets changes are detected, shared directory is only
+        # symlinked and precompile task is not run.
+        class Shared
+          attr_reader :paths, :runner
+          def initialize(paths, runner)
+            @paths = paths
+            @runner = runner
+          end
+
+          def reuse
+            run "mkdir -p #{shared_assets_path} && ln -nfs #{shared_assets_path} #{paths.public}"
+          end
+
+          def prepare
+            reuse
+            yield
+          end
+
+          protected
+
+          def run(cmd)
+            runner.run(cmd)
+          end
+
+          def shared_assets_path
+            paths.shared_assets
+          end
+        end
+
+        # Precompiled assets are shared across all deploys like Shared.
+        # Before compiling the active deploying assets, all assets that are not
+        # referenced by the manifest.yml from the previous deploy are removed.
+        # After cleaning, the new assets are compiled over the top. The result
+        # is an assets dir that contains the last assets and the current assets.
+        #
+        # When no assets changes are detected, shared directory is only
+        # symlinked and cleaning and precompile tasks are not run.
+        class Cleaning < Shared
+          def prepare
+            reuse
+            remove_old_assets
+            yield
+          rescue
+            # how do you restore back to the old assets if some have been overwritten?
+            # probably just deploy again I suppose.
+            raise
+          end
+
+          protected
+
+          def remove_old_assets
+            return unless manifest_path.readable?
+
+            Dir.chdir(shared_assets_path)
+
+            all_assets_on_disk = Dir.glob(shared_assets_path.join('**','*.*').to_s) - [manifest_path.to_s]
+            $stderr.puts "all_assets_on_disk #{all_assets_on_disk.inspect}"
+            assets_on_disk     = all_assets_on_disk.reject {|a| a =~ /\.gz$/}
+            $stderr.puts "assets_on_disk #{assets_on_disk.inspect}"
+            assets_in_manifest = YAML.load_file(manifest_path.to_s).values
+            $stderr.puts "assets_in_manifest #{assets_in_manifest.inspect}"
+
+            remove_assets = []
+            (assets_on_disk - assets_in_manifest).each do |asset|
+              remove_assets << "'#{asset}'"
+              remove_assets << "'#{asset}.gz'" if all_assets_on_disk.include?("#{asset}.gz")
+            end
+            run("rm -rf #{remove_assets.join(' ')}")
+          end
+
+          def manifest_path
+            shared_assets_path.join('manifest.yml')
+          end
+        end
+
+        # The default behavior and the one used since the beginning of asset
+        # support in engineyard-serverside. Assets are compiled into a fresh
+        # shared directory. Previous shared assets are shifted to a last_assets
+        # directory to prevent errors during deploy.
+        #
+        # When no assets changes are detected, the two shared directories are
+        # symlinked into the active release without any changes.
+        class Shifting < Shared
+          # link shared/assets and shared/last_assets into public
+          def reuse
+            run "mkdir -p #{shared_assets_path} #{last_assets_path} && #{link_assets}"
+          end
+
+          def prepare
+            shift_existing_assets
+            yield
+          rescue
+            unshift_existing_assets
+            raise
+          end
+
+          protected
+
+          def last_assets_path
+            paths.shared.join('last_assets')
+          end
+
+          # If there are current shared assets, move them under a 'last_assets' directory.
+          #
+          # To support operations like Unicorn's hot reload, it is useful to have
+          # the prior release's assets as well. Otherwise, while a deploy is running,
+          # clients may request stale assets that you just deleted.
+          # Making use of this requires a properly-configured front-end HTTP server.
+          #
+          # Note: This results in the directory structure:
+          #   deploy_root/current/public/assets -> deploy_root/shared/assets
+          #   deploy_root/current/public/last_assets -> deploy_root/shared/last_assets
+          # where last_assets has an assets dir under it.
+          #   deploy_root/shared/last_assets/assets
+          def shift_existing_assets
+            run "rm -rf #{last_assets_path} && mkdir -p #{shared_assets_path} #{last_assets_path} && mv #{shared_assets_path} #{last_assets_path.join('assets')} && mkdir -p #{shared_assets_path} && #{link_assets}"
+          end
+
+          # Restore shared/last_assets to shared/assets and relink them to the app public
+          def unshift_existing_assets
+            run "rm -rf #{shared_assets_path} && mv #{last_assets_path.join('assets')} #{shared_assets_path} && mkdir -p #{last_assets_path} && #{link_assets}"
+          end
+
+          def link_assets
+            "ln -nfs #{shared_assets_path} #{last_assets_path} #{paths.public}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/engineyard-serverside/server.rb

@@ -12,6 +12,11 @@ module EY
         "#{user}@#{hostname}"
       end
 
+      def inspect
+        name_s = name && ":#{name}"
+        "#{hostname}(#{role}#{name_s})"
+      end
+
       def role
         roles.first
       end

data/lib/engineyard-serverside/servers.rb

@@ -64,17 +64,17 @@ module EY
 
       # Run a command on this set of servers.
       def run(shell, cmd, &block)
-        run_on_each do |server|
+        run_on_each(shell) do |server|
           exec_cmd = server.command_on_server('sh -l -c', cmd, &block)
-          shell.logged_system(exec_cmd)
+          shell.logged_system(exec_cmd, server)
         end
       end
 
       # Run a sudo command on this set of servers.
       def sudo(shell, cmd, &block)
-        run_on_each do |server|
+        run_on_each(shell) do |server|
           exec_cmd = server.command_on_server('sudo sh -l -c', cmd, &block)
-          shell.logged_system(exec_cmd)
+          shell.logged_system(exec_cmd, server)
         end
       end
 
@@ -93,12 +93,24 @@ module EY
       # Makes a theard for each server and executes the block,
       # Assumes that the return value of the block is a CommandResult
       # and ensures that all the command results were successful.
-      def run_on_each(&block)
+      def run_on_each(shell, &block)
         results = map_in_parallel(&block)
         failures = results.reject {|result| result.success? }
+
         if failures.any?
-          message = failures.map { |f| f.inspect }.join("\n")
-          raise EY::Serverside::RemoteFailure.new(failures)
+          commands = failures.map { |f| f.command }.uniq
+          servers = failures.map { |f| f.server }.compact.map { |s| s.inspect }
+          outputs = failures.map { |f| f.output }.uniq
+          message = "The following command#{commands.size == 1 ? '' : 's'} failed"
+          if servers.any?
+            message << " on server#{servers.size == 1 ? '' : 's'} [#{servers.join(', ')}]"
+          end
+          message << "\n\n"
+          commands.each do |cmd|
+            message << "$ #{cmd}\n"
+          end
+          message << "\n" << outputs.join("\n\n") << "\n"
+          raise EY::Serverside::RemoteFailure.new(message)
         end
       end