engineyard-serverside 1.5.35.pre.2 → 1.6.0.pre

data/lib/engineyard-serverside.rb

@@ -8,7 +8,7 @@ else
 end
 
 $LOAD_PATH.unshift File.expand_path('vendor/thor/lib', File.dirname(__FILE__))
-$LOAD_PATH.unshift File.expand_path('vendor/systemu/lib', File.dirname(__FILE__))
+$LOAD_PATH.unshift File.expand_path('vendor/open4/lib', File.dirname(__FILE__))
 $LOAD_PATH.unshift File.expand_path('vendor/escape/lib', File.dirname(__FILE__))
 $LOAD_PATH.unshift File.expand_path('vendor/json_pure/lib', File.dirname(__FILE__))
 
@@ -26,6 +26,8 @@ require 'engineyard-serverside/cli'
 require 'engineyard-serverside/configuration'
 require 'engineyard-serverside/deprecation'
 require 'engineyard-serverside/future'
+require 'engineyard-serverside/shell'
+
 
 module EY
   module Serverside

data/lib/engineyard-serverside/cli.rb

@@ -1,5 +1,8 @@
 require 'thor'
 require 'pathname'
+require 'engineyard-serverside/deploy'
+require 'engineyard-serverside/shell'
+require 'engineyard-serverside/server'
 
 module EY
   module Serverside
@@ -22,6 +25,14 @@ module EY
                                       :desc     => "Application to deploy",
                                       :aliases  => ["-a"]
 
+      method_option :environment_name,:type     => :string,
+                                      :required => true,
+                                      :desc     => "Environment name"
+
+      method_option :account_name,    :type     => :string,
+                                      :required => true,
+                                      :desc     => "Account name"
+
       method_option :framework_env,   :type     => :string,
                                       :desc     => "Ruby web framework environment",
                                       :aliases  => ["-e"]
@@ -44,21 +55,13 @@ module EY
                                       :desc     => "Instance names, keyed on hostname. e.g. instance1:name1 instance2:name2"
 
       method_option :verbose,         :type     => :boolean,
-                                      :default  => false,
                                       :desc     => "Verbose output",
                                       :aliases  => ["-v"]
 
       desc "deploy", "Deploy code from /data/<app>"
       def deploy(default_task=:deploy)
-        config = EY::Serverside::Deploy::Configuration.new(options)
-        load_servers(config)
-
-        EY::Serverside::LoggedOutput.verbose = options[:verbose]
-        EY::Serverside::LoggedOutput.logfile = File.join(ENV['HOME'], "#{options[:app]}-deploy.log")
-
-        invoke :propagate
-
-        EY::Serverside::Deploy.new(config).send(default_task)
+        config, shell = init_and_propagate(options, 'deploy')
+        EY::Serverside::Deploy.new(config, shell).send(default_task)
       end
 
       method_option :app,           :type     => :string,
@@ -66,6 +69,14 @@ module EY
                                     :desc     => "Which application's hooks to run",
                                     :aliases  => ["-a"]
 
+      method_option :environment_name, :type     => :string,
+                                       :required => true,
+                                       :desc     => "Environment name"
+
+      method_option :account_name,  :type     => :string,
+                                    :required => true,
+                                    :desc     => "Account name"
+
       method_option :release_path,  :type     => :string,
                                     :desc     => "Value for #release_path in hooks (mostly for internal coordination)",
                                     :aliases  => ["-r"]
@@ -84,9 +95,14 @@ module EY
       method_option :current_name,  :type     => :string,
                                     :desc     => "Value for #current_name in hooks"
 
+      method_option :verbose,       :type     => :boolean,
+                                    :desc     => "Verbose output",
+                                    :aliases  => ["-v"]
+
       desc "hook [NAME]", "Run a particular deploy hook"
       def hook(hook_name)
-        EY::Serverside::DeployHook.new(options).run(hook_name)
+        config, shell = init(options, "hook-#{hook_name}")
+        EY::Serverside::DeployHook.new(config, shell).run(hook_name)
       end
 
 
@@ -95,6 +111,14 @@ module EY
                                       :desc     => "Application to deploy",
                                       :aliases  => ["-a"]
 
+      method_option :environment_name,:type     => :string,
+                                      :required => true,
+                                      :desc     => "Environment name"
+
+      method_option :account_name,    :type     => :string,
+                                      :required => true,
+                                      :desc     => "Account name"
+
       method_option :framework_env,   :type     => :string,
                                       :required => true,
                                       :desc     => "Ruby web framework environment",
@@ -115,13 +139,10 @@ module EY
                                       :desc     => "Instance names, keyed on hostname. e.g. instance1:name1 instance2:name2"
 
       method_option :verbose,         :type     => :boolean,
-                                      :default  => false,
                                       :desc     => "Verbose output",
                                       :aliases  => ["-v"]
       desc "integrate", "Integrate other instances into this cluster"
       def integrate
-        EY::Serverside::LoggedOutput.verbose = options[:verbose]
-        EY::Serverside::LoggedOutput.logfile = File.join(ENV['HOME'], "#{options[:app]}-integrate.log")
 
         app_dir = Pathname.new "/data/#{options[:app]}"
         current_app_dir = app_dir + "current"
@@ -133,23 +154,19 @@ module EY
         # we have to deploy the same SHA there as here
         integrate_options[:branch] = (current_app_dir + 'REVISION').read.strip
 
-        config = EY::Serverside::Deploy::Configuration.new(integrate_options)
-
-        load_servers(config)
-
-        invoke :propagate
+        config, shell = init_and_propagate(integrate_options, 'integrate')
 
         EY::Serverside::Server.all.each do |server|
-          server.sync_directory app_dir
+          server.sync_directory(app_dir) { |cmd| shell.logged_system(cmd) }
           # we're just about to recreate this, so it has to be gone
           # first. otherwise, non-idempotent deploy hooks could screw
           # things up, and since we don't control deploy hooks, we must
           # assume the worst.
-          server.run("rm -rf #{current_app_dir}")
+          run_on_server(server,"rm -rf #{current_app_dir}")
         end
 
         # deploy local-ref to other instances into /data/$app/local-current
-        EY::Serverside::Deploy.new(config).cached_deploy
+        EY::Serverside::Deploy.new(config, shell).cached_deploy
       end
 
       method_option :app,             :type     => :string,
@@ -157,6 +174,14 @@ module EY
                                       :desc     => "Application to deploy",
                                       :aliases  => ["-a"]
 
+      method_option :environment_name,:type     => :string,
+                                      :required => true,
+                                      :desc     => "Environment name"
+
+      method_option :account_name,    :type     => :string,
+                                      :required => true,
+                                      :desc     => "Account name"
+
       method_option :stack,           :type     => :string,
                                       :desc     => "Web stack (so we can restart it correctly)"
 
@@ -172,20 +197,13 @@ module EY
                                       :desc     => "Instance names, keyed on hostname. e.g. instance1:name1 instance2:name2"
 
       method_option :verbose,         :type     => :boolean,
-                                      :default  => false,
                                       :desc     => "Verbose output",
                                       :aliases  => ["-v"]
       desc "restart", "Restart app servers, conditionally enabling maintenance page"
       def restart
-        EY::Serverside::LoggedOutput.verbose = options[:verbose]
-        EY::Serverside::LoggedOutput.logfile = File.join(ENV['HOME'], "#{options[:app]}-restart.log")
+        config, shell = init_and_propagate(options, 'restart')
 
-        config = EY::Serverside::Deploy::Configuration.new(options)
-        load_servers(config)
-
-        invoke :propagate
-
-        EY::Serverside::Deploy.new(config).restart_with_maintenance_page
+        EY::Serverside::Deploy.new(config, shell).restart_with_maintenance_page
       end
 
       desc "install_bundler [VERSION]", "Make sure VERSION of bundler is installed (in system ruby)"
@@ -203,8 +221,10 @@ module EY
         end
       end
 
-      desc "propagate", "Propagate the engineyard-serverside gem to the other instances in the cluster. This will install exactly version #{EY::Serverside::VERSION}."
-      def propagate
+      private
+
+      # Put the same engineyard-serverside on all the servers (Used to be public but is unused as an actual CLI command now)
+      def propagate(shell)
         config          = EY::Serverside::Deploy::Configuration.new
         gem_filename    = "engineyard-serverside-#{EY::Serverside::VERSION}.gem"
         local_gem_file  = File.join(Gem.dir, 'cache', gem_filename)
@@ -219,23 +239,38 @@ module EY
           # 0.5.11, and mistakenly thinking 0.5.1 is there
           has_gem_cmd = "#{gem_binary} list engineyard-serverside | grep \"engineyard-serverside\" | egrep -q '#{egrep_escaped_version}[,)]'"
 
-          if !server.run(has_gem_cmd)  # doesn't have this exact version
-            puts "~> Installing engineyard-serverside on #{server.hostname}"
+          if !run_on_server(server,has_gem_cmd) # doesn't have this exact version
+            shell.status "Installing engineyard-serverside on #{server.hostname}"
 
-            system(Escape.shell_command([
+            shell.logged_system(Escape.shell_command([
               'scp', '-i', "#{ENV['HOME']}/.ssh/internal",
               "-o", "StrictHostKeyChecking=no",
               local_gem_file,
              "#{config.user}@#{server.hostname}:#{remote_gem_file}",
             ]))
-            server.run("sudo #{gem_binary} install --no-rdoc --no-ri '#{remote_gem_file}'")
+            run_on_server(server,"sudo #{gem_binary} install --no-rdoc --no-ri '#{remote_gem_file}'")
           end
         end
 
         EY::Serverside::Future.success?(futures)
       end
 
-      private
+      def init_and_propagate(*args)
+        config, shell = init(*args)
+        load_servers(config)
+        propagate(shell)
+        [config, shell]
+      end
+
+      def init(options, action)
+        config = EY::Serverside::Deploy::Configuration.new(options)
+        shell  = EY::Serverside::Shell.new(:verbose  => config.verbose, :log_path => File.join(ENV['HOME'], "#{config.app}-#{action}.log"))
+        [config, shell]
+      end
+
+      def run_on_server(server, command)
+        server.run(command) { |cmd| shell.logged_system(cmd) }
+      end
 
       def load_servers(config)
         EY::Serverside::Server.load_all_from_array(assemble_instance_hashes(config))

data/lib/engineyard-serverside/configuration.rb

@@ -10,41 +10,50 @@ module EY
         "bundle_without" => "test development",
       })
 
-      attr_reader :configuration
-      alias :c :configuration
-
       attr_writer :release_path
 
       def initialize(options={})
         opts = options.dup
         @release_path = opts[:release_path]
         config = JSON.parse(opts.delete("config") || "{}")
-        @configuration = DEFAULT_CONFIG.merge(config).merge(opts)
+        @configs = [config, opts] # low to high priority
+      end
+
+      def configuration
+        @configuration ||= @configs.inject(DEFAULT_CONFIG) {|low,high| low.merge(high)}
       end
+      alias :c :configuration # FIXME: awful, but someone is probably using it :(
 
       # Delegate to the configuration objects
       def method_missing(meth, *args, &blk)
-        c.key?(meth.to_s) ? c[meth.to_s] : super
+        configuration.key?(meth.to_s) ? c[meth.to_s] : super
       end
 
       def respond_to?(meth, include_private=false)
-        c.key?(meth.to_s) ? true : super
+        configuration.key?(meth.to_s) ? true : super
+      end
+
+      def ey_yml_data=(data)
+        environments = data['environments']
+        if environments && (env_data = environments[environment_name])
+          @configuration = nil # reset cached configuration hash
+          @configs.unshift(env_data) # insert just above default configuration
+          true
+        else
+          false
+        end
       end
 
       def [](key)
         if respond_to?(key.to_sym)
           send(key.to_sym)
         else
-          c[key]
+          configuration[key]
         end
       end
 
       def has_key?(key)
-        if respond_to?(key.to_sym)
-          true
-        else
-          c.has_key?(key)
-        end
+        respond_to?(key.to_sym) || configuration.has_key?(key)
       end
 
       def to_json
@@ -55,9 +64,22 @@ module EY
         EY::Serverside.node
       end
 
+      def verbose
+        configuration['verbose']
+      end
+
       def app
         configuration['app'].to_s
       end
+      alias app_name app
+
+      def environment_name
+        configuration['environment_name'].to_s
+      end
+
+      def account_name
+        configuration['account_name'].to_s
+      end
 
       def revision
         IO.read(File.join(latest_release, 'REVISION'))
@@ -95,6 +117,10 @@ module EY
         node['instance_role']
       end
 
+      def current_roles
+        configuration['current_roles'] || []
+      end
+
       def current_role
         current_roles.first
       end

data/lib/engineyard-serverside/deploy.rb

@@ -7,22 +7,22 @@ require 'engineyard-serverside/rails_asset_support'
 module EY
   module Serverside
     class DeployBase < Task
-      include LoggedOutput
       include ::EY::Serverside::RailsAssetSupport
 
       # default task
       def deploy
-        debug "Starting deploy at #{Time.now.asctime}"
+        shell.status "Starting deploy at #{shell.start_time.asctime}"
         update_repository_cache
         cached_deploy
       end
 
       def cached_deploy
-        debug "Deploying app from cached copy at #{Time.now.asctime}"
+        shell.status "Deploying app from cached copy at #{Time.now.asctime}"
         require_custom_tasks
+        load_ey_yml
         push_code
 
-        info "~> Starting full deploy"
+        shell.status "Starting full deploy"
         copy_repository_cache
         check_repository
 
@@ -47,9 +47,9 @@ module EY
         disable_maintenance_page
 
         cleanup_old_releases
-        debug "Finished deploy at #{Time.now.asctime}"
+        shell.status "Finished deploy at #{Time.now.asctime}"
       rescue Exception
-        debug "Finished failing to deploy at #{Time.now.asctime}"
+        shell.status "Finished failing to deploy at #{Time.now.asctime}"
         puts_deploy_failure
         raise
       end
@@ -73,9 +73,9 @@ module EY
 
       def check_repository
         if gemfile?
-          info "~> Gemfile found."
+          shell.status "Gemfile found."
           if lockfile
-            info "~> Gemfile.lock found."
+            shell.status "Gemfile.lock found."
             unless lockfile.any_database_adapter?
               warning <<-WARN
 Gemfile.lock does not contain a recognized database adapter.
@@ -97,7 +97,7 @@ To fix this problem, commit your Gemfile.lock to your repository and redeploy.
             WARN
           end
         else
-          info "~> No Gemfile. Deploying without bundler support."
+          shell.status "No Gemfile. Deploying without bundler support."
         end
       end
 
@@ -163,9 +163,9 @@ To fix this problem, commit your Gemfile.lock to your repository and redeploy.
 
       # task
       def push_code
-        info "~> Pushing code to all servers"
+        shell.status "Pushing code to all servers"
         futures = EY::Serverside::Future.call(EY::Serverside::Server.all) do |server|
-          server.sync_directory(config.repository_cache)
+          server.sync_directory(config.repository_cache) { |cmd| shell.logged_system(cmd) }
         end
         EY::Serverside::Future.success?(futures)
       end
@@ -173,7 +173,7 @@ To fix this problem, commit your Gemfile.lock to your repository and redeploy.
       # task
       def restart
         @restart_failed = true
-        info "~> Restarting app servers"
+        shell.status "Restarting app servers"
         roles :app_master, :app, :solo do
           run(restart_command)
         end
@@ -191,12 +191,8 @@ To fix this problem, commit your Gemfile.lock to your repository and redeploy.
 
       # If we don't have a local version of the ssh wrapper script yet,
       # create it on all the servers that will need it.
-      # TODO - This logic likely fails when people change deploy keys.
       def ssh_executable
-        roles :app_master, :app, :solo, :util do
-          run(generate_ssh_wrapper)
-        end
-        ssh_wrapper_path
+        @path ||= generate_ssh_wrapper
       end
 
       # We specify 'IdentitiesOnly' to avoid failures on systems with > 5 private keys available.
@@ -205,12 +201,17 @@ To fix this problem, commit your Gemfile.lock to your repository and redeploy.
       # Learned this at http://lists.mindrot.org/pipermail/openssh-unix-dev/2009-February/027271.html
       # (Thanks Jim L.)
       def generate_ssh_wrapper
+        path = ssh_wrapper_path
         identity_file = "~/.ssh/#{c.app}-deploy-key"
-        %{ echo "#{wrapper_for(identity_file)}" > #{ssh_wrapper_path} && chmod 0700 #{ssh_wrapper_path} }
-      end
-
-      def wrapper_for(identity)
-        "#!/bin/sh\nunset SSH_AUTH_SOCK; ssh -o CheckHostIP=no -o StrictHostKeyChecking=no -o PasswordAuthentication=no -o LogLevel=DEBUG -o IdentityFile=#{identity} -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null $*"
+        run <<-SCRIPT
+[[ -x #{path} ]] || cat > #{path} <<'SSH'
+#!/bin/sh
+unset SSH_AUTH_SOCK
+ssh -o 'CheckHostIP no' -o 'StrictHostKeyChecking no' -o 'PasswordAuthentication no' -o 'LogLevel DEBUG' -o 'IdentityFile #{identity_file}' -o 'IdentitiesOnly yes' -o 'UserKnownHostsFile /dev/null' $*
+SSH
+chmod 0700 #{path}
+        SCRIPT
+        path
       end
 
       def ssh_wrapper_path
@@ -238,7 +239,7 @@ To fix this problem, commit your Gemfile.lock to your repository and redeploy.
       def clean_release_directory(dir, count = 3)
         @cleanup_failed = true
         ordinal = count.succ.to_s
-        info "~> Cleaning release directory: #{dir}"
+        shell.status "Cleaning release directory: #{dir}"
         sudo "ls -r #{dir} | tail -n +#{ordinal} | xargs -I@ rm -rf #{dir}/@"
         @cleanup_failed = false
       end
@@ -250,15 +251,15 @@ To fix this problem, commit your Gemfile.lock to your repository and redeploy.
           c.release_path = c.previous_release(rolled_back_release)
 
           revision = File.read(File.join(c.release_path, 'REVISION')).strip
-          info "~> Rolling back to previous release: #{short_log_message(revision)}"
+          shell.status "Rolling back to previous release: #{short_log_message(revision)}"
 
           run_with_callbacks(:symlink)
           sudo "rm -rf #{rolled_back_release}"
           bundle
-          info "~> Restarting with previous release."
+          shell.status "Restarting with previous release."
           with_maintenance_page { run_with_callbacks(:restart) }
         else
-          info "~> Already at oldest release, nothing to roll back to."
+          shell.status "Already at oldest release, nothing to roll back to."
           exit(1)
         end
       end
@@ -269,17 +270,17 @@ To fix this problem, commit your Gemfile.lock to your repository and redeploy.
         @migrations_reached = true
         roles :app_master, :solo do
           cmd = "cd #{c.release_path} && PATH=#{c.binstubs_path}:$PATH #{c.framework_envs} #{c.migration_command}"
-          info "~> Migrating: #{cmd}"
+          shell.status "Migrating: #{cmd}"
           run(cmd)
         end
       end
 
       # task
       def copy_repository_cache
-        info "~> Copying to #{c.release_path}"
+        shell.status "Copying to #{c.release_path}"
         run("mkdir -p #{c.release_path} #{c.failed_release_dir} && rsync -aq #{c.exclusions} #{c.repository_cache}/ #{c.release_path}")
 
-        info "~> Ensuring proper ownership."
+        shell.status "Ensuring proper ownership."
         sudo("chown -R #{c.user}:#{c.group} #{c.deploy_to}")
       end
 
@@ -295,8 +296,12 @@ To fix this problem, commit your Gemfile.lock to your repository and redeploy.
         "/usr/local/ey_resin/ruby/bin/ey-services-setup #{config.app}"
       end
 
+      def node_package_manager_command_check
+        "which npm"
+      end
+
       def setup_services
-        info "~> Setting up external services."
+        shell.status "Setting up external services."
         previously_configured_services = parse_configured_services
         begin
           sudo(services_command_check)
@@ -332,24 +337,24 @@ YML
 WRAP
            ["Symlink database.yml", "ln -nfs #{c.shared_path}/config/database.sqlite3.yml #{c.release_path}/config/database.yml"],
           ].each do |what, cmd|
-            info "~> #{what}"
+            shell.status "#{what}"
             run(cmd)
           end
 
           owner = [c.user, c.group].join(':')
-          info "~> Setting ownership to #{owner}"
+          shell.status "Setting ownership to #{owner}"
           sudo "chown -R #{owner} #{c.release_path}"
         end
       end
 
       def symlink_configs(release_to_link=c.release_path)
-        info "~> Preparing shared resources for release."
+        shell.status "Preparing shared resources for release."
         symlink_tasks(release_to_link).each do |what, cmd|
-          info "~> #{what}"
+          shell.substatus what
           run(cmd)
         end
         owner = [c.user, c.group].join(':')
-        info "~> Setting ownership to #{owner}"
+        shell.status "Setting ownership to #{owner}"
         sudo "chown -R #{owner} #{release_to_link}"
       end
 
@@ -372,7 +377,7 @@ WRAP
 
       # task
       def symlink(release_to_link=c.release_path)
-        info "~> Symlinking code."
+        shell.status "Symlinking code."
         run "rm -f #{c.current_path} && ln -nfs #{release_to_link} #{c.current_path} && chown -R #{c.user}:#{c.group} #{c.current_path}"
         @symlink_changed = true
       rescue Exception
@@ -382,7 +387,7 @@ WRAP
       end
 
       def callback(what)
-        @callbacks_reached = true
+        @callbacks_reached ||= true
         if File.exist?("#{c.release_path}/deploy/#{what}.rb")
           run Escape.shell_command(base_callback_command_for(what)) do |server, cmd|
             per_instance_args = [
@@ -406,11 +411,14 @@ WRAP
       end
 
       def base_callback_command_for(what)
-        [serverside_bin, 'hook', what.to_s,
-          '--app', config.app,
-          '--release-path', config.release_path.to_s,
-          '--framework-env', c.environment.to_s,
-        ].compact
+        cmd =  [serverside_bin, 'hook', what.to_s]
+        cmd << '--app'              << config.app
+        cmd << '--environment-name' << config.environment_name
+        cmd << '--account-name'     << config.account_name
+        cmd << '--release-path'     << config.release_path.to_s
+        cmd << '--framework-env'    << config.environment.to_s
+        cmd << '--verbose' if config.verbose
+        cmd
       end
 
       def serverside_bin
@@ -420,9 +428,9 @@ WRAP
 
       def puts_deploy_failure
         if @cleanup_failed
-          info "~> [Relax] Your site is running new code, but clean up of old deploys failed."
+          shell.status "[Relax] Your site is running new code, but clean up of old deploys failed."
         elsif @maintenance_up
-          info "~> [Attention] Maintenance page still up, consider the following before removing:"
+          shell.status "[Attention] Maintenance page still up, consider the following before removing:"
           info " * Deploy hooks ran. This might cause problems for reverting to old code." if @callbacks_reached
           info " * Migrations ran. This might cause problems for reverting to old code." if @migrations_reached
           if @symlink_changed
@@ -432,9 +440,9 @@ WRAP
           end
           info " * Application servers failed to restart." if @restart_failed
           info ""
-          info "~> Need help? File a ticket for support."
+          shell.status "Need help? File a ticket for support."
         else
-          info "~> [Relax] Your site is still running old code and nothing destructive has occurred."
+          shell.status "[Relax] Your site is still running old code and nothing destructive has occurred."
         end
       end
 
@@ -447,7 +455,7 @@ WRAP
       def with_failed_release_cleanup
         yield
       rescue Exception
-        info "~> Release #{c.release_path} failed, saving release to #{c.failed_release_dir}."
+        shell.status "Release #{c.release_path} failed, saving release to #{c.failed_release_dir}."
         sudo "mv #{c.release_path} #{c.failed_release_dir}"
         raise
       end
@@ -480,7 +488,7 @@ WRAP
 
       def check_ruby_bundler
         if gemfile?
-          info "~> Bundling gems..."
+          shell.status "Bundling gems..."
 
           clean_bundle_on_system_version_change
 
@@ -511,14 +519,18 @@ WRAP
 
       def check_node_npm
         if File.exist?("#{c.release_path}/package.json")
-          info "~> package.json detected, installing npm packages"
-          run "cd #{c.release_path} && npm install"
+          unless run(node_package_manager_command_check)
+            abort "*** [Error] package.json detected, but npm was not installed"
+          else
+            shell.status "package.json detected, installing npm packages"
+            run "cd #{c.release_path} && npm install"
+          end
         end
       end
     end   # DeployBase
 
     class Deploy < DeployBase
-      def self.new(config)
+      def self.new(config, shell=nil)
         # include the correct fetch strategy
         include EY::Serverside::Strategies.const_get(config.strategy)::Helpers
         super