engineyard-serverside 1.6.0.pre5 → 1.6.3

data/lib/engineyard-serverside.rb

@@ -8,7 +8,7 @@ else
 end
 
 $LOAD_PATH.unshift File.expand_path('vendor/thor/lib', File.dirname(__FILE__))
-$LOAD_PATH.unshift File.expand_path('vendor/open4/lib', File.dirname(__FILE__))
+$LOAD_PATH.unshift File.expand_path('vendor/systemu/lib', File.dirname(__FILE__))
 $LOAD_PATH.unshift File.expand_path('vendor/escape/lib', File.dirname(__FILE__))
 $LOAD_PATH.unshift File.expand_path('vendor/json_pure/lib', File.dirname(__FILE__))
 
@@ -26,8 +26,6 @@ require 'engineyard-serverside/cli'
 require 'engineyard-serverside/configuration'
 require 'engineyard-serverside/deprecation'
 require 'engineyard-serverside/future'
-require 'engineyard-serverside/shell'
-
 
 module EY
   module Serverside

data/lib/engineyard-serverside/cli.rb

@@ -1,8 +1,5 @@
 require 'thor'
 require 'pathname'
-require 'engineyard-serverside/deploy'
-require 'engineyard-serverside/shell'
-require 'engineyard-serverside/server'
 
 module EY
   module Serverside
@@ -25,14 +22,6 @@ module EY
                                       :desc     => "Application to deploy",
                                       :aliases  => ["-a"]
 
-      method_option :environment_name,:type     => :string,
-                                      :required => true,
-                                      :desc     => "Environment name"
-
-      method_option :account_name,    :type     => :string,
-                                      :required => true,
-                                      :desc     => "Account name"
-
       method_option :framework_env,   :type     => :string,
                                       :desc     => "Ruby web framework environment",
                                       :aliases  => ["-e"]
@@ -55,13 +44,21 @@ module EY
                                       :desc     => "Instance names, keyed on hostname. e.g. instance1:name1 instance2:name2"
 
       method_option :verbose,         :type     => :boolean,
+                                      :default  => false,
                                       :desc     => "Verbose output",
                                       :aliases  => ["-v"]
 
       desc "deploy", "Deploy code from /data/<app>"
       def deploy(default_task=:deploy)
-        config, shell = init_and_propagate(options, 'deploy')
-        EY::Serverside::Deploy.new(config, shell).send(default_task)
+        config = EY::Serverside::Deploy::Configuration.new(options)
+        load_servers(config)
+
+        EY::Serverside::LoggedOutput.verbose = options[:verbose]
+        EY::Serverside::LoggedOutput.logfile = File.join(ENV['HOME'], "#{options[:app]}-deploy.log")
+
+        invoke :propagate
+
+        EY::Serverside::Deploy.new(config).send(default_task)
       end
 
       method_option :app,           :type     => :string,
@@ -69,14 +66,6 @@ module EY
                                     :desc     => "Which application's hooks to run",
                                     :aliases  => ["-a"]
 
-      method_option :environment_name, :type     => :string,
-                                       :required => true,
-                                       :desc     => "Environment name"
-
-      method_option :account_name,  :type     => :string,
-                                    :required => true,
-                                    :desc     => "Account name"
-
       method_option :release_path,  :type     => :string,
                                     :desc     => "Value for #release_path in hooks (mostly for internal coordination)",
                                     :aliases  => ["-r"]
@@ -95,14 +84,9 @@ module EY
       method_option :current_name,  :type     => :string,
                                     :desc     => "Value for #current_name in hooks"
 
-      method_option :verbose,       :type     => :boolean,
-                                    :desc     => "Verbose output",
-                                    :aliases  => ["-v"]
-
       desc "hook [NAME]", "Run a particular deploy hook"
       def hook(hook_name)
-        config, shell = init(options, "hook-#{hook_name}")
-        EY::Serverside::DeployHook.new(config, shell).run(hook_name)
+        EY::Serverside::DeployHook.new(options).run(hook_name)
       end
 
 
@@ -111,14 +95,6 @@ module EY
                                       :desc     => "Application to deploy",
                                       :aliases  => ["-a"]
 
-      method_option :environment_name,:type     => :string,
-                                      :required => true,
-                                      :desc     => "Environment name"
-
-      method_option :account_name,    :type     => :string,
-                                      :required => true,
-                                      :desc     => "Account name"
-
       method_option :framework_env,   :type     => :string,
                                       :required => true,
                                       :desc     => "Ruby web framework environment",
@@ -139,10 +115,13 @@ module EY
                                       :desc     => "Instance names, keyed on hostname. e.g. instance1:name1 instance2:name2"
 
       method_option :verbose,         :type     => :boolean,
+                                      :default  => false,
                                       :desc     => "Verbose output",
                                       :aliases  => ["-v"]
       desc "integrate", "Integrate other instances into this cluster"
       def integrate
+        EY::Serverside::LoggedOutput.verbose = options[:verbose]
+        EY::Serverside::LoggedOutput.logfile = File.join(ENV['HOME'], "#{options[:app]}-integrate.log")
 
         app_dir = Pathname.new "/data/#{options[:app]}"
         current_app_dir = app_dir + "current"
@@ -154,19 +133,23 @@ module EY
         # we have to deploy the same SHA there as here
         integrate_options[:branch] = (current_app_dir + 'REVISION').read.strip
 
-        config, shell = init_and_propagate(integrate_options, 'integrate')
+        config = EY::Serverside::Deploy::Configuration.new(integrate_options)
+
+        load_servers(config)
+
+        invoke :propagate
 
         EY::Serverside::Server.all.each do |server|
-          server.sync_directory(app_dir) { |cmd| shell.logged_system(cmd) }
+          server.sync_directory app_dir
           # we're just about to recreate this, so it has to be gone
           # first. otherwise, non-idempotent deploy hooks could screw
           # things up, and since we don't control deploy hooks, we must
           # assume the worst.
-          run_on_server(server,"rm -rf #{current_app_dir}")
+          server.run("rm -rf #{current_app_dir}")
         end
 
         # deploy local-ref to other instances into /data/$app/local-current
-        EY::Serverside::Deploy.new(config, shell).cached_deploy
+        EY::Serverside::Deploy.new(config).cached_deploy
       end
 
       method_option :app,             :type     => :string,
@@ -174,14 +157,6 @@ module EY
                                       :desc     => "Application to deploy",
                                       :aliases  => ["-a"]
 
-      method_option :environment_name,:type     => :string,
-                                      :required => true,
-                                      :desc     => "Environment name"
-
-      method_option :account_name,    :type     => :string,
-                                      :required => true,
-                                      :desc     => "Account name"
-
       method_option :stack,           :type     => :string,
                                       :desc     => "Web stack (so we can restart it correctly)"
 
@@ -197,13 +172,20 @@ module EY
                                       :desc     => "Instance names, keyed on hostname. e.g. instance1:name1 instance2:name2"
 
       method_option :verbose,         :type     => :boolean,
+                                      :default  => false,
                                       :desc     => "Verbose output",
                                       :aliases  => ["-v"]
       desc "restart", "Restart app servers, conditionally enabling maintenance page"
       def restart
-        config, shell = init_and_propagate(options, 'restart')
+        EY::Serverside::LoggedOutput.verbose = options[:verbose]
+        EY::Serverside::LoggedOutput.logfile = File.join(ENV['HOME'], "#{options[:app]}-restart.log")
 
-        EY::Serverside::Deploy.new(config, shell).restart_with_maintenance_page
+        config = EY::Serverside::Deploy::Configuration.new(options)
+        load_servers(config)
+
+        invoke :propagate
+
+        EY::Serverside::Deploy.new(config).restart_with_maintenance_page
       end
 
       desc "install_bundler [VERSION]", "Make sure VERSION of bundler is installed (in system ruby)"
@@ -221,10 +203,8 @@ module EY
         end
       end
 
-      private
-
-      # Put the same engineyard-serverside on all the servers (Used to be public but is unused as an actual CLI command now)
-      def propagate(shell)
+      desc "propagate", "Propagate the engineyard-serverside gem to the other instances in the cluster. This will install exactly version #{EY::Serverside::VERSION}."
+      def propagate
         config          = EY::Serverside::Deploy::Configuration.new
         gem_filename    = "engineyard-serverside-#{EY::Serverside::VERSION}.gem"
         local_gem_file  = File.join(Gem.dir, 'cache', gem_filename)
@@ -239,38 +219,23 @@ module EY
           # 0.5.11, and mistakenly thinking 0.5.1 is there
           has_gem_cmd = "#{gem_binary} list engineyard-serverside | grep \"engineyard-serverside\" | egrep -q '#{egrep_escaped_version}[,)]'"
 
-          if !run_on_server(server,has_gem_cmd) # doesn't have this exact version
-            shell.status "Installing engineyard-serverside on #{server.hostname}"
+          if !server.run(has_gem_cmd)  # doesn't have this exact version
+            puts "~> Installing engineyard-serverside on #{server.hostname}"
 
-            shell.logged_system(Escape.shell_command([
+            system(Escape.shell_command([
               'scp', '-i', "#{ENV['HOME']}/.ssh/internal",
               "-o", "StrictHostKeyChecking=no",
               local_gem_file,
              "#{config.user}@#{server.hostname}:#{remote_gem_file}",
             ]))
-            run_on_server(server,"sudo #{gem_binary} install --no-rdoc --no-ri '#{remote_gem_file}'")
+            server.run("sudo #{gem_binary} install --no-rdoc --no-ri '#{remote_gem_file}'")
           end
         end
 
         EY::Serverside::Future.success?(futures)
       end
 
-      def init_and_propagate(*args)
-        config, shell = init(*args)
-        load_servers(config)
-        propagate(shell)
-        [config, shell]
-      end
-
-      def init(options, action)
-        config = EY::Serverside::Deploy::Configuration.new(options)
-        shell  = EY::Serverside::Shell.new(:verbose  => config.verbose, :log_path => File.join(ENV['HOME'], "#{config.app}-#{action}.log"))
-        [config, shell]
-      end
-
-      def run_on_server(server, command)
-        server.run(command) { |cmd| shell.logged_system(cmd) }
-      end
+      private
 
       def load_servers(config)
         EY::Serverside::Server.load_all_from_array(assemble_instance_hashes(config))

data/lib/engineyard-serverside/configuration.rb

@@ -10,50 +10,41 @@ module EY
         "bundle_without" => "test development",
       })
 
+      attr_reader :configuration
+      alias :c :configuration
+
       attr_writer :release_path
 
       def initialize(options={})
         opts = options.dup
         @release_path = opts[:release_path]
         config = JSON.parse(opts.delete("config") || "{}")
-        @configs = [config, opts] # low to high priority
-      end
-
-      def configuration
-        @configuration ||= @configs.inject(DEFAULT_CONFIG) {|low,high| low.merge(high)}
+        @configuration = DEFAULT_CONFIG.merge(config).merge(opts)
       end
-      alias :c :configuration # FIXME: awful, but someone is probably using it :(
 
       # Delegate to the configuration objects
       def method_missing(meth, *args, &blk)
-        configuration.key?(meth.to_s) ? c[meth.to_s] : super
+        c.key?(meth.to_s) ? c[meth.to_s] : super
       end
 
       def respond_to?(meth, include_private=false)
-        configuration.key?(meth.to_s) ? true : super
-      end
-
-      def ey_yml_data=(data)
-        environments = data['environments']
-        if environments && (env_data = environments[environment_name])
-          @configuration = nil # reset cached configuration hash
-          @configs.unshift(env_data) # insert just above default configuration
-          true
-        else
-          false
-        end
+        c.key?(meth.to_s) ? true : super
       end
 
       def [](key)
         if respond_to?(key.to_sym)
           send(key.to_sym)
         else
-          configuration[key]
+          c[key]
         end
       end
 
       def has_key?(key)
-        respond_to?(key.to_sym) || configuration.has_key?(key)
+        if respond_to?(key.to_sym)
+          true
+        else
+          c.has_key?(key)
+        end
       end
 
       def to_json
@@ -64,22 +55,9 @@ module EY
         EY::Serverside.node
       end
 
-      def verbose
-        configuration['verbose']
-      end
-
       def app
         configuration['app'].to_s
       end
-      alias app_name app
-
-      def environment_name
-        configuration['environment_name'].to_s
-      end
-
-      def account_name
-        configuration['account_name'].to_s
-      end
 
       def revision
         IO.read(File.join(latest_release, 'REVISION'))
@@ -117,10 +95,6 @@ module EY
         node['instance_role']
       end
 
-      def current_roles
-        configuration['current_roles'] || []
-      end
-
       def current_role
         current_roles.first
       end

data/lib/engineyard-serverside/deploy.rb

@@ -7,22 +7,22 @@ require 'engineyard-serverside/rails_asset_support'
 module EY
   module Serverside
     class DeployBase < Task
+      include LoggedOutput
       include ::EY::Serverside::RailsAssetSupport
 
       # default task
       def deploy
-        shell.status "Starting deploy at #{shell.start_time.asctime}"
+        debug "Starting deploy at #{Time.now.asctime}"
         update_repository_cache
         cached_deploy
       end
 
       def cached_deploy
-        shell.status "Deploying app from cached copy at #{Time.now.asctime}"
+        debug "Deploying app from cached copy at #{Time.now.asctime}"
         require_custom_tasks
-        load_ey_yml
         push_code
 
-        shell.status "Starting full deploy"
+        info "~> Starting full deploy"
         copy_repository_cache
         check_repository
 
@@ -47,9 +47,9 @@ module EY
         disable_maintenance_page
 
         cleanup_old_releases
-        shell.status "Finished deploy at #{Time.now.asctime}"
+        debug "Finished deploy at #{Time.now.asctime}"
       rescue Exception
-        shell.status "Finished failing to deploy at #{Time.now.asctime}"
+        debug "Finished failing to deploy at #{Time.now.asctime}"
         puts_deploy_failure
         raise
       end
@@ -73,9 +73,9 @@ module EY
 
       def check_repository
         if gemfile?
-          shell.status "Gemfile found."
+          info "~> Gemfile found."
           if lockfile
-            shell.status "Gemfile.lock found."
+            info "~> Gemfile.lock found."
             unless lockfile.any_database_adapter?
               warning <<-WARN
 Gemfile.lock does not contain a recognized database adapter.
@@ -97,7 +97,7 @@ To fix this problem, commit your Gemfile.lock to your repository and redeploy.
             WARN
           end
         else
-          shell.status "No Gemfile. Deploying without bundler support."
+          info "~> No Gemfile. Deploying without bundler support."
         end
       end
 
@@ -163,9 +163,9 @@ To fix this problem, commit your Gemfile.lock to your repository and redeploy.
 
       # task
       def push_code
-        shell.status "Pushing code to all servers"
+        info "~> Pushing code to all servers"
         futures = EY::Serverside::Future.call(EY::Serverside::Server.all) do |server|
-          server.sync_directory(config.repository_cache) { |cmd| shell.logged_system(cmd) }
+          server.sync_directory(config.repository_cache)
         end
         EY::Serverside::Future.success?(futures)
       end
@@ -173,7 +173,7 @@ To fix this problem, commit your Gemfile.lock to your repository and redeploy.
       # task
       def restart
         @restart_failed = true
-        shell.status "Restarting app servers"
+        info "~> Restarting app servers"
         roles :app_master, :app, :solo do
           run(restart_command)
         end
@@ -191,8 +191,12 @@ To fix this problem, commit your Gemfile.lock to your repository and redeploy.
 
       # If we don't have a local version of the ssh wrapper script yet,
       # create it on all the servers that will need it.
+      # TODO - This logic likely fails when people change deploy keys.
       def ssh_executable
-        @path ||= generate_ssh_wrapper
+        roles :app_master, :app, :solo, :util do
+          run(generate_ssh_wrapper)
+        end
+        ssh_wrapper_path
       end
 
       # We specify 'IdentitiesOnly' to avoid failures on systems with > 5 private keys available.
@@ -203,15 +207,14 @@ To fix this problem, commit your Gemfile.lock to your repository and redeploy.
       def generate_ssh_wrapper
         path = ssh_wrapper_path
         identity_file = "~/.ssh/#{c.app}-deploy-key"
-        run <<-SCRIPT
+<<-WRAP
 [[ -x #{path} ]] || cat > #{path} <<'SSH'
 #!/bin/sh
 unset SSH_AUTH_SOCK
-ssh -o 'CheckHostIP no' -o 'StrictHostKeyChecking no' -o 'PasswordAuthentication no' -o 'LogLevel DEBUG' -o 'IdentityFile #{identity_file}' -o 'IdentitiesOnly yes' -o 'UserKnownHostsFile /dev/null' $*
+ssh -o CheckHostIP=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o PasswordAuthentication=no -o LogLevel=DEBUG -o IdentityFile=#{identity_file} -o IdentitiesOnly=yes $*
 SSH
 chmod 0700 #{path}
-        SCRIPT
-        path
+WRAP
       end
 
       def ssh_wrapper_path
@@ -239,7 +242,7 @@ chmod 0700 #{path}
       def clean_release_directory(dir, count = 3)
         @cleanup_failed = true
         ordinal = count.succ.to_s
-        shell.status "Cleaning release directory: #{dir}"
+        info "~> Cleaning release directory: #{dir}"
         sudo "ls -r #{dir} | tail -n +#{ordinal} | xargs -I@ rm -rf #{dir}/@"
         @cleanup_failed = false
       end
@@ -251,15 +254,15 @@ chmod 0700 #{path}
           c.release_path = c.previous_release(rolled_back_release)
 
           revision = File.read(File.join(c.release_path, 'REVISION')).strip
-          shell.status "Rolling back to previous release: #{short_log_message(revision)}"
+          info "~> Rolling back to previous release: #{short_log_message(revision)}"
 
           run_with_callbacks(:symlink)
           sudo "rm -rf #{rolled_back_release}"
           bundle
-          shell.status "Restarting with previous release."
+          info "~> Restarting with previous release."
           with_maintenance_page { run_with_callbacks(:restart) }
         else
-          shell.status "Already at oldest release, nothing to roll back to."
+          info "~> Already at oldest release, nothing to roll back to."
           exit(1)
         end
       end
@@ -270,17 +273,17 @@ chmod 0700 #{path}
         @migrations_reached = true
         roles :app_master, :solo do
           cmd = "cd #{c.release_path} && PATH=#{c.binstubs_path}:$PATH #{c.framework_envs} #{c.migration_command}"
-          shell.status "Migrating: #{cmd}"
+          info "~> Migrating: #{cmd}"
           run(cmd)
         end
       end
 
       # task
       def copy_repository_cache
-        shell.status "Copying to #{c.release_path}"
+        info "~> Copying to #{c.release_path}"
         run("mkdir -p #{c.release_path} #{c.failed_release_dir} && rsync -aq #{c.exclusions} #{c.repository_cache}/ #{c.release_path}")
 
-        shell.status "Ensuring proper ownership."
+        info "~> Ensuring proper ownership."
         sudo("chown -R #{c.user}:#{c.group} #{c.deploy_to}")
       end
 
@@ -296,12 +299,8 @@ chmod 0700 #{path}
         "/usr/local/ey_resin/ruby/bin/ey-services-setup #{config.app}"
       end
 
-      def node_package_manager_command_check
-        "which npm"
-      end
-
       def setup_services
-        shell.status "Setting up external services."
+        info "~> Setting up external services."
         previously_configured_services = parse_configured_services
         begin
           sudo(services_command_check)
@@ -337,24 +336,24 @@ YML
 WRAP
            ["Symlink database.yml", "ln -nfs #{c.shared_path}/config/database.sqlite3.yml #{c.release_path}/config/database.yml"],
           ].each do |what, cmd|
-            shell.status "#{what}"
+            info "~> #{what}"
             run(cmd)
           end
 
           owner = [c.user, c.group].join(':')
-          shell.status "Setting ownership to #{owner}"
+          info "~> Setting ownership to #{owner}"
           sudo "chown -R #{owner} #{c.release_path}"
         end
       end
 
       def symlink_configs(release_to_link=c.release_path)
-        shell.status "Preparing shared resources for release."
+        info "~> Preparing shared resources for release."
         symlink_tasks(release_to_link).each do |what, cmd|
-          shell.substatus what
+          info "~> #{what}"
           run(cmd)
         end
         owner = [c.user, c.group].join(':')
-        shell.status "Setting ownership to #{owner}"
+        info "~> Setting ownership to #{owner}"
         sudo "chown -R #{owner} #{release_to_link}"
       end
 
@@ -377,7 +376,7 @@ WRAP
 
       # task
       def symlink(release_to_link=c.release_path)
-        shell.status "Symlinking code."
+        info "~> Symlinking code."
         run "rm -f #{c.current_path} && ln -nfs #{release_to_link} #{c.current_path} && chown -R #{c.user}:#{c.group} #{c.current_path}"
         @symlink_changed = true
       rescue Exception
@@ -411,14 +410,11 @@ WRAP
       end
 
       def base_callback_command_for(what)
-        cmd =  [serverside_bin, 'hook', what.to_s]
-        cmd << '--app'              << config.app
-        cmd << '--environment-name' << config.environment_name
-        cmd << '--account-name'     << config.account_name
-        cmd << '--release-path'     << config.release_path.to_s
-        cmd << '--framework-env'    << config.environment.to_s
-        cmd << '--verbose' if config.verbose
-        cmd
+        [serverside_bin, 'hook', what.to_s,
+          '--app', config.app,
+          '--release-path', config.release_path.to_s,
+          '--framework-env', c.environment.to_s,
+        ].compact
       end
 
       def serverside_bin
@@ -428,22 +424,21 @@ WRAP
 
       def puts_deploy_failure
         if @cleanup_failed
-          shell.notice "[Relax] Your site is running new code, but clean up of old deploys failed."
+          info "~> [Relax] Your site is running new code, but clean up of old deploys failed."
         elsif @maintenance_up
-          message = "[Attention] Maintenance page still up, consider the following before removing:\n"
-          message << " * Deploy hooks ran. This might cause problems for reverting to old code.\n" if @callbacks_reached
-          message << " * Migrations ran. This might cause problems for reverting to old code.\n" if @migrations_reached
+          info "~> [Attention] Maintenance page still up, consider the following before removing:"
+          info " * Deploy hooks ran. This might cause problems for reverting to old code." if @callbacks_reached
+          info " * Migrations ran. This might cause problems for reverting to old code." if @migrations_reached
           if @symlink_changed
-            message << " * Your new code is symlinked as current.\n"
+            info " * Your new code is symlinked as current."
           else
-            message << " * Your old code is still symlinked as current.\n"
+            info " * Your old code is still symlinked as current."
           end
-          message << " * Application servers failed to restart.\n" if @restart_failed
-          message << "\n"
-          message << "Need help? File a ticket for support.\n"
-          shell.notice message
+          info " * Application servers failed to restart." if @restart_failed
+          info ""
+          info "~> Need help? File a ticket for support."
         else
-          shell.notice "[Relax] Your site is still running old code and nothing destructive has occurred."
+          info "~> [Relax] Your site is still running old code and nothing destructive has occurred."
         end
       end
 
@@ -456,7 +451,7 @@ WRAP
       def with_failed_release_cleanup
         yield
       rescue Exception
-        shell.status "Release #{c.release_path} failed, saving release to #{c.failed_release_dir}."
+        info "~> Release #{c.release_path} failed, saving release to #{c.failed_release_dir}."
         sudo "mv #{c.release_path} #{c.failed_release_dir}"
         raise
       end
@@ -489,7 +484,7 @@ WRAP
 
       def check_ruby_bundler
         if gemfile?
-          shell.status "Bundling gems..."
+          info "~> Bundling gems..."
 
           clean_bundle_on_system_version_change
 
@@ -520,18 +515,14 @@ WRAP
 
       def check_node_npm
         if File.exist?("#{c.release_path}/package.json")
-          unless run(node_package_manager_command_check)
-            abort "*** [Error] package.json detected, but npm was not installed"
-          else
-            shell.status "package.json detected, installing npm packages"
-            run "cd #{c.release_path} && npm install"
-          end
+          info "~> package.json detected, installing npm packages"
+          run "cd #{c.release_path} && npm install"
         end
       end
     end   # DeployBase
 
     class Deploy < DeployBase
-      def self.new(config, shell=nil)
+      def self.new(config)
         # include the correct fetch strategy
         include EY::Serverside::Strategies.const_get(config.strategy)::Helpers
         super