engineyard-serverside 2.7.0.pre → 2.7.8pre2

data/bin/engineyard-serverside

@@ -8,4 +8,4 @@ $LOAD_PATH.push(File.expand_path("../lib", File.dirname(__FILE__)))
 
 require 'engineyard-serverside'
 
-EY::Serverside::CLI::App.start
+EY::Serverside::CLI.start

data/lib/engineyard-serverside/about.rb

@@ -25,16 +25,17 @@ module EY
       end
 
       def gem_binary
-        gem_bin_path = File.join(Gem.default_bindir, 'gem')
-        if File.exists?("/usr/local/ey_resin/bin/ruby")
-          "/usr/local/ey_resin/bin/ruby -rubygems #{gem_bin_path}"
-        else
-          gem_bin_path
-        end
+        File.join(Gem.default_bindir, 'gem')
       end
 
       def binary
-        File.expand_path("../../../bin/#{gem_name}", __FILE__)
+        resin_path = "/usr/local/ey_resin/ruby/bin/engineyard-serverside"
+        if File.exists?(resin_path)
+          resin_path
+        else
+          #gem relative path causes deploy hook failures if gem version mismatches with other app servers
+          File.expand_path("../../../bin/#{gem_name}", __FILE__)
+        end
       end
 
       def hook_executor

data/lib/engineyard-serverside/cli.rb

@@ -1,11 +1,232 @@
-require 'engineyard-serverside/cli/app'
+require 'thor'
+require 'pathname'
+require 'engineyard-serverside/about'
+require 'engineyard-serverside/deploy'
+require 'engineyard-serverside/shell'
+require 'engineyard-serverside/servers'
+require 'engineyard-serverside/cli_helpers'
 
 module EY
   module Serverside
+    class CLI < Thor
+
+      extend CLIHelpers
+
+      method_option :migrate,         :type     => :string,
+                                      :desc     => "Run migrations with this deploy",
+                                      :aliases  => ["-m"]
+
+      method_option :branch,          :type     => :string,
+                                      :desc     => "Git ref to deploy, defaults to master. May be a branch, a tag, or a SHA",
+                                      :aliases  => %w[-b --ref --tag]
+
+      method_option :repo,            :type     => :string,
+                                      :desc     => "Remote repo to deploy",
+                                      :aliases  => ["-r"]
+
+
+      # Archive source strategy
+      method_option :archive,        :type     => :string,
+                                     :desc     => "Remote URI for archive to download and unzip"
+
+      # Git source strategy
+      method_option :git,            :type     => :string,
+                                     :desc     => "Remote git repo to deploy"
+
+      method_option :clean,          :type     => :boolean,
+                                     :desc     => "Run deploy without relying on existing files"
+
+
+      account_app_env_options
+      config_option
+      framework_env_option
+      instances_options
+      stack_option
+      verbose_option
+
+      desc "deploy", "Deploy code to /data/<app>"
+      def deploy(default_task=:deploy)
+        init_and_propagate(options, default_task.to_s) do |servers, config, shell|
+          EY::Serverside::Deploy.new(servers, config, shell).send(default_task)
+        end
+      end
+
+      account_app_env_options
+      config_option
+      instances_options
+      verbose_option
+      desc "enable_maintenance", "Enable maintenance page (disables web access)"
+      def enable_maintenance
+        init_and_propagate(options, 'enable_maintenance') do |servers, config, shell|
+          EY::Serverside::Maintenance.new(servers, config, shell).manually_enable
+        end
+      end
+
+      account_app_env_options
+      config_option
+      instances_options
+      verbose_option
+      desc "maintenance_status", "Maintenance status"
+      def maintenance_status
+        init(options, "maintenance-status") do |servers, config, shell|
+          EY::Serverside::Maintenance.new(servers, config, shell).status
+        end
+      end
+
+      account_app_env_options
+      config_option
+      instances_options
+      verbose_option
+      desc "disable_maintenance", "Disable maintenance page (enables web access)"
+      def disable_maintenance
+        init_and_propagate(options, 'disable_maintenance') do |servers, config, shell|
+          EY::Serverside::Maintenance.new(servers, config, shell).manually_disable
+        end
+      end
+
+      method_option :release_path,  :type     => :string,
+                                    :desc     => "Value for #release_path in hooks (mostly for internal coordination)",
+                                    :aliases  => ["-r"]
+
+      method_option :current_roles, :type     => :array,
+                                    :desc     => "Value for #current_roles in hooks"
+
+      method_option :current_name,  :type     => :string,
+                                    :desc     => "Value for #current_name in hooks"
+      account_app_env_options
+      config_option
+      framework_env_option
+      verbose_option
+      desc "hook [NAME]", "Run a particular deploy hook"
+      def hook(hook_name)
+        init(options, "hook-#{hook_name}") do |servers, config, shell|
+          EY::Serverside::DeployHook.new(config, shell, hook_name).call
+        end
+      end
+
+      method_option :ignore_existing, :type     => :boolean,
+                                      :desc     => "When syncing /data/app directory, don't overwrite destination files"
+      account_app_env_options
+      config_option
+      framework_env_option
+      instances_options
+      stack_option
+      verbose_option
+      desc "integrate", "Integrate other instances into this cluster"
+      def integrate
+        app_dir = Pathname.new "/data/#{options[:app]}"
+        current_app_dir = app_dir.join("current")
+
+        # so that we deploy to the same place there that we have here
+        integrate_options = options.dup
+        integrate_options[:release_path] = current_app_dir.realpath.to_s
+
+        # we have to deploy the same SHA there as here
+        integrate_options[:branch] = current_app_dir.join('REVISION').read.strip
+
+        # always rebundle gems on integrate to make sure the instance comes up correctly.
+        integrate_options[:clean] = true
+
+        logname = "integrate-#{options[:instances].join('-')}".gsub(/[^-.\w]/,'')
+
+        init_and_propagate(integrate_options, logname) do |servers, config, shell|
+
+          # We have to rsync the entire app dir, so we need all the permissions to be correct!
+          chown_command = %|find #{app_dir} \\( -not -user #{config.user} -or -not -group #{config.group} \\) -exec chown -h #{config.user}:#{config.group} "{}" +|
+          shell.logged_system("sudo sh -l -c '#{chown_command}'", servers.detect {|s| s.local?})
+
+          servers.run_for_each! do |server|
+            chown = server.command_on_server('sudo sh -l -c', chown_command)
+            sync  = server.sync_directory_command(app_dir, options[:ignore_existing])
+            clean = server.command_on_server('sh -l -c', "rm -rf #{current_app_dir}")
+            "(#{chown}) && (#{sync}) && (#{clean})"
+          end
+
+          # deploy local-ref to other instances into /data/$app/local-current
+          EY::Serverside::Deploy.new(servers, config, shell).cached_deploy
+        end
+      end
+
+      account_app_env_options
+      instances_options
+      stack_option
+      verbose_option
+      desc "restart", "Restart app servers, conditionally enabling maintenance page"
+      def restart
+        options = self.options.dup
+        options[:release_path] = Pathname.new("/data/#{options[:app]}/current").realpath.to_s
+
+        init_and_propagate(options, 'restart') do |servers, config, shell|
+          EY::Serverside::Deploy.new(servers, config, shell).restart_with_maintenance_page
+        end
+      end
+
+      private
+
+      def init_and_propagate(*args)
+        init(*args) do |servers, config, shell|
+          propagate(servers, shell)
+          yield servers, config, shell
+        end
+      end
+
+      def init(options, action)
+        config = EY::Serverside::Deploy::Configuration.new(options)
+        shell  = EY::Serverside::Shell.new(
+          :verbose  => config.verbose,
+          :log_path => File.join(ENV['HOME'], "#{config.app}-#{action}.log")
+        )
+        shell.debug "Initializing #{About.name_with_version}."
+        servers = load_servers(config, shell)
+        begin
+          yield servers, config, shell
+        rescue EY::Serverside::RemoteFailure => e
+          shell.fatal e.message
+          raise
+        rescue Exception => e
+          shell.fatal "#{e.backtrace[0]}: #{e.message} (#{e.class})"
+          raise
+        end
+      end
+
+      def propagate(servers, shell)
+        shell.status "Verifying and propagating #{About.name_with_version} to all servers."
+
+        gem_binary = File.join(Gem.default_bindir, 'gem')
+        remote_gem_file = File.join(Dir.tmpdir, About.gem_filename)
+
+        # the [,)] is to stop us from looking for e.g. 0.5.1, seeing
+        # 0.5.11, and mistakenly thinking 0.5.1 is there
+        check_command = %{#{gem_binary} list #{About.gem_name} | grep "#{About.gem_name}" | egrep -q "#{About.version.gsub(/\./, '\.')}[,)]"}
+        install_command = "#{gem_binary} install --no-rdoc --no-ri '#{remote_gem_file}'"
+
+        servers.remote.run_for_each! do |server|
+          check   = server.command_on_server('sh -l -c', check_command)
+          scp     = server.scp_command(About.gem_file, remote_gem_file)
+          install = server.command_on_server('sudo sh -l -c', install_command)
+
+          "(#{check}) || ((#{scp}) && (#{install}))"
+        end
+      end
+
+      def load_servers(config, shell)
+        EY::Serverside::Servers.from_hashes(assemble_instance_hashes(config), shell)
+      end
+
+      def assemble_instance_hashes(config)
+        if options[:instances]
+          options[:instances].collect { |hostname|
+            { :hostname => hostname,
+              :roles => options[:instance_roles][hostname].to_s.split(','),
+              :name => options[:instance_names][hostname],
+              :user => config.user,
+            }
+          }
+        else
+          []
+        end
+      end
 
-    # CLI is a namespace that encompasses all concepts around command-line
-    # interactions with EY::Serverside
-    module CLI
     end
   end
 end

data/lib/engineyard-serverside/cli_helpers.rb

@@ -0,0 +1,53 @@
+module EY
+  module Serverside
+    module CLIHelpers
+      def account_app_env_options
+        method_option :app,              :type     => :string,
+                                         :required => true,
+                                         :desc     => "Application to deploy",
+                                         :aliases  => %w[-a --app-name]
+        method_option :environment_name, :type     => :string,
+                                         :required => true,
+                                         :desc     => "Environment name"
+        method_option :account_name,     :type     => :string,
+                                         :required => true,
+                                         :desc     => "Account name"
+      end
+
+      def framework_env_option
+        method_option :framework_env,    :type     => :string,
+                                         :required => true,
+                                         :desc     => "Ruby web framework environment",
+                                         :aliases  => ["-e"]
+      end
+
+      def stack_option
+        method_option :stack,            :type     => :string,
+                                         :desc     => "Web stack (so we can restart it correctly)"
+      end
+
+
+      def config_option
+        method_option :config,           :type     => :string,
+                                         :desc     => "Additional configuration"
+      end
+
+      def instances_options
+        method_option :instances,        :type     => :array,
+                                         :desc     => "Hostnames of instances to deploy to, e.g. --instances localhost app1 app2"
+        method_option :instance_roles,   :type     => :hash,
+                                         :default  => {},
+                                         :desc     => "Roles of instances, keyed on hostname, comma-separated. e.g. instance1:app_master,etc instance2:db,memcached ..."
+        method_option :instance_names,   :type     => :hash,
+                                         :default  => {},
+                                         :desc     => "Instance names, keyed on hostname. e.g. instance1:name1 instance2:name2"
+      end
+
+      def verbose_option
+        method_option :verbose,          :type     => :boolean,
+                                         :desc     => "Verbose output",
+                                         :aliases  => ["-v"]
+      end
+    end
+  end
+end

data/lib/engineyard-serverside/configuration.rb

@@ -101,7 +101,6 @@ module EY
       def_option(:group)                  { user }
       def_option :services_check_command, "which /usr/local/ey_resin/ruby/bin/ey-services-setup >/dev/null 2>&1"
       def_option(:services_setup_command) { "/usr/local/ey_resin/ruby/bin/ey-services-setup #{app}" }
-      def_option :ignore_ey_config_warning, false
 
       DEFAULT_KEEP_RELEASES = 3
 

data/lib/engineyard-serverside/dependency_manager/bundler.rb

@@ -4,19 +4,12 @@ module EY
   module Serverside
     class DependencyManager
       class Bundler < Base
-        DEFAULT_VERSION = "1.14.6"
-
-        # Number of Bundler workers (--jobs)
-        JOBS_NUMBER = 4
+        DEFAULT_VERSION = "1.7.9"
 
         def self.default_version
           DEFAULT_VERSION
         end
 
-        def self.jobs_number
-          JOBS_NUMBER
-        end
-
         def detected?
           gemfile?
         end
@@ -66,7 +59,7 @@ To fix this problem, commit your Gemfile.lock to your repository and redeploy.
         end
 
         def show_ey_config_instructions
-          if lockfile && !lockfile.has_ey_config? && !config.ignore_ey_config_warning
+          if lockfile && !lockfile.has_ey_config?
             shell.warning "Gemfile.lock does not contain ey_config. Add gem 'ey_config' to your Gemfile to access service configurations through EY::Config."
           end
         end
@@ -94,7 +87,7 @@ To fix this problem, commit your Gemfile.lock to your repository and redeploy.
           # the [,)] is to stop us from looking for e.g. 0.9.2, seeing
           # 0.9.22, and mistakenly thinking 0.9.2 is there
           has_gem_cmd = %{gem list bundler | grep "bundler " | egrep -q "#{egrep_escaped_version}[,)]"}
-          install_cmd = %{gem install bundler -q -v "#{bundler_version}"}
+          install_cmd = %{gem install bundler -q --no-rdoc --no-ri -v "#{bundler_version}"}
           sudo "#{clean_environment} && #{has_gem_cmd} || #{install_cmd}"
         end
 
@@ -110,11 +103,6 @@ To fix this problem, commit your Gemfile.lock to your repository and redeploy.
             "--path",     "#{paths.bundled_gems}",
             "--binstubs", "#{paths.binstubs}",
           ]
-
-          # Install gems in parallel (Bundler.1.5+ only)
-          # @see EYPP-58
-          options += ["--jobs", self.class.jobs_number] if check_version(bundler_version, ">= 1.5")
-
           options += ["--deployment"] if lockfile
           options += config.extra_bundle_install_options
           options
@@ -132,9 +120,10 @@ To fix this problem, commit your Gemfile.lock to your repository and redeploy.
             shell.substatus "Clean bundle forced (--clean)"
             run clean_bundle
           else
+            check_ruby   = "#{config.ruby_version_command} | diff - #{paths.ruby_version} >/dev/null 2>&1"
             check_system = "#{config.system_version_command} | diff - #{paths.system_version} >/dev/null 2>&1"
             shell.substatus "Checking for system version changes"
-            run "#{check_system} || #{clean_bundle}"
+            run "#{check_ruby} && #{check_system} || #{clean_bundle}"
           end
         end
 
@@ -151,24 +140,9 @@ To fix this problem, commit your Gemfile.lock to your repository and redeploy.
           end
         end
 
-        ##
-        # Checks that given version is satisfies give requirement.
-        #
-        # @param [String] version
-        # @param [String] requirement
-        # @return [true, false]
-        #
-        def check_version(version, requirement)
-          Gem::Requirement.new( requirement ).satisfied_by? Gem::Version.new( version )
-        end
-
         class Lockfile
           attr_reader :bundler_version
 
-          METADATA     = 'METADATA'.freeze
-          BUNDLED      = 'BUNDLED WITH'.freeze
-          DEPENDENCIES = 'DEPENDENCIES'.freeze
-
           def initialize(lockfile_contents, default = EY::Serverside::DependencyManager::Bundler.default_version)
             @contents = lockfile_contents
             @default = default
@@ -206,8 +180,9 @@ To fix this problem, commit your Gemfile.lock to your repository and redeploy.
           end
 
           def parse
-            @bundler_version = parse_from_metadata || parse_from_bundled_with || parse_from_dependencies
-            raise("Malformed or pre bundler-1.0.0 Gemfile.lock: #{@contents[0,50]}...") unless @bundler_version
+            parse_from_metadata ||
+              parse_from_dependencies ||
+              raise("Malformed or pre bundler-1.0.0 Gemfile.lock: #{@contents[0,50]}...")
           end
 
           def slice_section(header)
@@ -220,28 +195,25 @@ To fix this problem, commit your Gemfile.lock to your repository and redeploy.
           end
 
           def parse_from_metadata
-            section = slice_section(METADATA)
-            return if section.empty?
-
-            result = section.scan(/^\s*version:\s*(.*)$/).first
-            result ? result.first : @default
-          end
+            section = slice_section('METADATA')
 
-          def parse_from_bundled_with
-            section = slice_section(BUNDLED)
-            return if section.empty?
+            if section.empty?
+              return nil
+            end
 
-            result = section.scan(/^#{BUNDLED}\s*(.*)$/).first
-            result ? result.first : @default
+            result = section.scan(/^\s*version:\s*(.*)$/).first
+            @bundler_version = result ? result.first : @default
           end
 
           def dependencies_section
-            @dependencies_section ||= slice_section(DEPENDENCIES)
+            @dependencies_section ||= slice_section('DEPENDENCIES')
           end
 
           def parse_from_dependencies
             section = dependencies_section
-            return if section.empty?
+            if section.empty?
+              return nil
+            end
 
             result = scan_gem('bundler', section)
             bundler_version = result ? result.last : nil