engineyard-serverside 2.0.7 → 2.1.0.pre

data/lib/engineyard-serverside/dependency_manager/bundler_lock.rb

@@ -0,0 +1,155 @@
+require 'engineyard-serverside/dependency_manager/bundler'
+
+module EY
+  module Serverside
+    module DependencyManager
+      class BundlerLock < Bundler
+        def detected?
+          gemfile? && lockfile?
+        end
+
+        def check
+          shell.status "Gemfile and Gemfile.lock found"
+
+          if config.check_database_adapter? && !lockfile.any_database_adapter?
+            shell.warning <<-WARN
+Gemfile.lock does not contain a recognized database adapter.
+A database-adapter gem such as pg, mysql2, mysql, or do_mysql was expected.
+This can prevent applications that use PostgreSQL or MySQL from booting.
+
+To fix, add any needed adapter to your Gemfile, bundle, commit, and redeploy.
+Applications not using PostgreSQL or MySQL can safely ignore this warning by
+adding ignore_database_adapter_warning: true to the application's ey.yml file
+under this environment's name and adding the file to your repository.
+            WARN
+          end
+        end
+
+        def uses_sqlite3?
+          lockfile.uses_sqlite3?
+        end
+
+        def check_ey_config
+          unless lockfile.has_ey_config?
+            shell.warning "Gemfile.lock does not contain ey_config.\nAdd gem 'ey_config' to get access to service configuration through EY::Config."
+          end
+        end
+
+        def rails_version
+          lockfile.rails_version
+        end
+
+        private
+
+        def lockfile_path
+          paths.gemfile_lock
+        end
+
+        def lockfile
+          @lockfile ||= Lockfile.new(lockfile_path.read, self.class.default_version)
+        end
+
+        # deployment mode is not supported without a Gemfile.lock
+        def bundle_install_options
+          super + ["--deployment"]
+        end
+
+        def bundler_version
+          @bundler_version ||= lockfile.bundler_version || super
+        end
+
+        class Lockfile
+          attr_reader :bundler_version
+
+          def initialize(lockfile_contents, default = EY::Serverside::DependencyManager::Bundler.default_version)
+            @contents = lockfile_contents
+            @default = default
+            @default_gem_version = Gem::Version.new(@default)
+            parse
+          end
+
+          def has_ey_config?
+            !!@contents.index(/^\s+ey_config\s\([^\)]+\)$/)
+          end
+
+          def rails_version
+            section = dependencies_section
+            if section.empty?
+              return nil
+            end
+            result = scan_gem('rails', section)
+            result ? result.last : nil
+          end
+
+          def any_database_adapter?
+            any_ruby_adapter = %w[mysql2 mysql do_mysql pg do_postgres].any? do |type|
+              @contents.index(/^\s+#{type}\s\([^\)]+\)$/)
+            end
+
+            any_jruby_adapter = %w[mysql postgresql postgres].any? do |type|
+              @contents.index(/^\s+jdbc-#{type}\s\([^\)]+\)$/) || @contents.index(/^\s+activerecord-jdbc#{type}-adapter\s\([^\)]+\)$/)
+            end
+
+            any_ruby_adapter || any_jruby_adapter
+          end
+
+          def uses_sqlite3?
+            !any_database_adapter? && @contents.index(/^\s+sqlite3\s\([^\)]+\)$/)
+          end
+
+          def parse
+            parse_from_metadata ||
+              parse_from_dependencies ||
+              raise("Malformed or pre bundler-1.0.0 Gemfile.lock: #{@contents[0,50]}...")
+          end
+
+          def slice_section(header)
+            if start = @contents.index(/^#{header}/)
+              finish = @contents.index(/(^\S|\Z)/, start + header.length)
+              @contents.slice(start..finish)
+            else
+              ""
+            end
+          end
+
+          def parse_from_metadata
+            section = slice_section('METADATA')
+
+            if section.empty?
+              return nil
+            end
+
+            result = section.scan(/^\s*version:\s*(.*)$/).first
+            @bundler_version = result ? result.first : @default
+          end
+
+          def dependencies_section
+            @dependencies_section ||= slice_section('DEPENDENCIES')
+          end
+
+          def parse_from_dependencies
+            section = dependencies_section
+            if section.empty?
+              return nil
+            end
+
+            result = scan_gem('bundler', section)
+            bundler_version = result ? result.last : nil
+            version_qualifier = result ? result.first : nil
+            @bundler_version = fetch_version(version_qualifier, bundler_version)
+          end
+
+          def fetch_version(operator, version)
+            return version || @default unless operator && version
+            req = Gem::Requirement.new(["#{operator} #{version}"])
+            req.satisfied_by?(@default_gem_version) ? @default : version
+          end
+
+          def scan_gem(gem, dep_section)
+            dep_section.scan(/^\s*#{Regexp.escape(gem)}\s*\((>?=|~>)\s*([^,\)]+)/).first
+          end
+        end
+      end
+    end
+  end
+end

data/lib/engineyard-serverside/dependency_manager/legacy_helpers.rb

@@ -0,0 +1,24 @@
+module EY
+  module Serverside
+    module DependencyManager
+      module LegacyHelpers
+        [
+          :gemfile?,
+          :bundler_config,
+          :lockfile,
+          :check_ruby_bundler,
+          :clean_bundle_on_system_version_change,
+          :write_system_version,
+          :check_node_npm,
+          :clean_environment,
+        ].each do |meth|
+          define_method(meth) do |*a|
+            if dependency_manager.respond_to?(meth)
+              dependency_manager.send(meth)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/engineyard-serverside/dependency_manager/npm.rb

@@ -0,0 +1,16 @@
+module EY
+  module Serverside
+    module DependencyManager
+      class Npm < Base
+        def detected?
+          paths.package_json.exist?
+        end
+
+        def install
+          shell.status "Installing npm packages (package.json detected)"
+          run "cd #{paths.active_release} && npm install"
+        end
+      end
+    end
+  end
+end

data/lib/engineyard-serverside/deploy.rb

@@ -1,14 +1,15 @@
-# stolen wholesale from capistrano, thanks Jamis!
+# Inspired by capistrano, thanks Jamis!
 require 'base64'
-require 'fileutils'
 require 'multi_json'
-require 'engineyard-serverside/rails_asset_support'
+require 'engineyard-serverside/rails_assets'
 require 'engineyard-serverside/maintenance'
+require 'engineyard-serverside/dependency_manager'
+require 'engineyard-serverside/dependency_manager/legacy_helpers'
 
 module EY
   module Serverside
     class DeployBase < Task
-      include ::EY::Serverside::RailsAssetSupport
+      include ::EY::Serverside::DependencyManager::LegacyHelpers
 
       # default task
       def deploy
@@ -31,7 +32,6 @@ module EY
           create_revision_file
           run_with_callbacks(:bundle)
           setup_services
-          check_for_ey_config
           symlink_configs
           setup_sqlite3_if_necessary
           run_with_callbacks(:compile_assets) # defined in RailsAssetSupport
@@ -72,60 +72,12 @@ module EY
         strategy.short_log_message(revision)
       end
 
-      def parse_configured_services
-        result = YAML.load_file "#{paths.shared_config}/ey_services_config_deploy.yml"
-        return {} unless result.is_a?(Hash)
-        result
-      rescue
-        {}
-      end
-
-      def check_for_ey_config
-        if gemfile? && lockfile
-          configured_services = parse_configured_services
-          if !configured_services.empty? && !lockfile.has_ey_config?
-            shell.warning "Gemfile.lock does not contain ey_config. Add it to get EY::Config access to: #{configured_services.keys.join(', ')}."
-          end
-        end
-      end
-
-      # The nodatabase.yml file is dropped by server configuration when there is
-      # no database in the cluster.
-      def has_database?
-        paths.shared_config.join('database.yml').exist? &&
-          !paths.shared_config.join('nodatabase.yml').exist?
+      def check_repository
+        check_dependencies
       end
 
-      def check_repository
-        if gemfile?
-          shell.status "Gemfile found."
-          if lockfile
-            shell.status "Gemfile.lock found."
-            if !config.ignore_database_adapter_warning? && has_database? && !lockfile.any_database_adapter?
-              shell.warning <<-WARN
-Gemfile.lock does not contain a recognized database adapter.
-A database-adapter gem such as pg, mysql2, mysql, or do_mysql was expected.
-This can prevent applications that use PostgreSQL or MySQL from booting.
-
-To fix, add any needed adapter to your Gemfile, bundle, commit, and redeploy.
-Applications not using PostgreSQL or MySQL can safely ignore this warning by
-adding ignore_database_adapter_warning: true to the application's ey.yml file
-under this environment's name and adding the file to your repository.
-              WARN
-            end
-          else
-            shell.warning <<-WARN
-Gemfile.lock is missing!
-You can get different versions of gems in production than what you tested with.
-You can get different versions of gems on every deployment even if your Gemfile hasn't changed.
-Deploying will take longer.
-
-To fix this problem, commit your Gemfile.lock to your repository and redeploy.
-            WARN
-          end
-        else
-          shell.status "No Gemfile. Deploying without bundler support."
-        end
+      def check_dependencies
+        dependency_manager.check
       end
 
       def restart_with_maintenance_page
@@ -153,7 +105,7 @@ To fix this problem, commit your Gemfile.lock to your repository and redeploy.
       # task
       def push_code
         shell.status "Pushing code to all servers"
-        servers.remote.run_on_each do |server|
+        servers.remote.run_on_each(shell) do |server|
           cmd = server.sync_directory_command(paths.repository_cache)
           shell.logged_system(cmd)
         end
@@ -173,9 +125,8 @@ To fix this problem, commit your Gemfile.lock to your repository and redeploy.
         %{LANG="en_US.UTF-8" /engineyard/bin/app_#{config.app} deploy}
       end
 
-      # GIT_SSH needs to be defined in the environment for customers with private bundler repos in their Gemfile.
-      def clean_environment
-        %Q[export GIT_SSH="#{ssh_executable}" && export LANG="en_US.UTF-8" && unset RUBYOPT BUNDLE_PATH BUNDLE_FROZEN BUNDLE_WITHOUT BUNDLE_BIN BUNDLE_GEMFILE]
+      def ensure_git_ssh_wrapper
+        ENV['GIT_SSH'] = ssh_executable.to_s
       end
 
       # create ssh wrapper on all servers
@@ -206,18 +157,17 @@ chmod 0700 #{path}
         SCRIPT
       end
 
-      # task
       def bundle
-        roles :app_master, :app, :solo, :util do
-          check_ruby_bundler
-          check_node_npm
-        end
+        install_dependencies
+      end
+
+      def install_dependencies
+        dependency_manager.install
       end
 
       # task
       def cleanup_old_releases
         clean_release_directory(paths.releases)
-        clean_release_directory(paths.releases_failed)
       end
 
       # Remove all but the most-recent +count+ releases from the specified
@@ -272,39 +222,31 @@ chmod 0700 #{path}
       def copy_repository_cache
         shell.status "Copying to #{paths.active_release}"
         exclusions = Array(config.copy_exclude).map { |e| %|--exclude="#{e}"| }.join(' ')
-        run("mkdir -p #{paths.active_release} #{paths.releases_failed} #{paths.shared_config} && rsync -aq #{exclusions} #{paths.repository_cache}/ #{paths.active_release}")
+        run("mkdir -p #{paths.active_release} #{paths.shared_config} && rsync -aq #{exclusions} #{paths.repository_cache}/ #{paths.active_release}")
 
         shell.status "Ensuring proper ownership."
-        sudo("chown -R #{config.user}:#{config.group} #{paths.active_release} #{paths.releases_failed}")
+        ensure_ownership(paths.active_release)
       end
 
       def create_revision_file
         run create_revision_file_command
       end
 
-      def services_command_check
-        "which /usr/local/ey_resin/ruby/bin/ey-services-setup >/dev/null 2>&1"
-      end
-
-      def services_setup_command
-        "/usr/local/ey_resin/ruby/bin/ey-services-setup #{config.app}"
-      end
-
       def setup_services
         shell.status "Setting up external services."
-        previously_configured_services = parse_configured_services
+        previously_configured_services = config.configured_services
 
         begin
-          sudo(services_command_check)
-        rescue StandardError => e
+          sudo(config.services_check_command)
+        rescue EY::Serverside::RemoteFailure
           shell.info "Could not setup services. Upgrade your environment to get services configuration."
           return
         end
 
         begin
-          sudo(services_setup_command)
-        rescue StandardError => e
-          unless previously_configured_services.empty?
+          sudo(config.services_setup_command)
+        rescue EY::Serverside::RemoteFailure => e
+          if previously_configured_services
             shell.warning <<-WARNING
 External services configuration not updated. Using previous version.
 Deploy again if your services configuration appears incomplete or out of date.
@@ -312,32 +254,40 @@ Deploy again if your services configuration appears incomplete or out of date.
             WARNING
           end
         end
+
+        if services = config.configured_services
+          shell.status "Services configured: #{services.join(', ')}"
+          dependency_manager.check_ey_config
+        end
       end
 
       def setup_sqlite3_if_necessary
-        if gemfile? && lockfile && lockfile.uses_sqlite3?
-          [
-           ["Create databases directory if needed", "mkdir -p #{paths.shared}/databases"],
-           ["Creating SQLite database if needed", "touch #{paths.shared}/databases/#{config.framework_env}.sqlite3"],
-           ["Create config directory if needed", "mkdir -p #{paths.active_release_config}"],
-           ["Generating SQLite config", <<-WRAP],
+        if dependency_manager.uses_sqlite3?
+          shell.status "Setting up SQLite3 Database for compatibility with application's chosen adapter"
+          shell.warning "SQLite3 cannot persist across servers. Please upgrade to a supported database."
+
+          shell.substatus "Create databases directory if needed"
+          run "mkdir -p #{paths.shared}/databases"
+
+          shell.substatus "Create SQLite database if needed"
+          run "touch #{paths.shared}/databases/#{config.framework_env}.sqlite3"
+
+          shell.substatus "Create config directory if needed"
+          run "mkdir -p #{paths.active_release_config}"
+
+          shell.substatus "Generating SQLite config"
+          run <<-WRAP
 cat > #{paths.shared_config}/database.sqlite3.yml<<'YML'
 #{config.framework_env}:
-  adapter: sqlite3
-  database: #{paths.shared}/databases/#{config.framework_env}.sqlite3
-  pool: 5
-  timeout: 5000
+adapter: sqlite3
+database: #{paths.shared}/databases/#{config.framework_env}.sqlite3
+pool: 5
+timeout: 5000
 YML
-WRAP
-           ["Symlink database.yml", "ln -nfs #{paths.shared_config}/database.sqlite3.yml #{paths.active_release_config}/database.yml"],
-          ].each do |what, cmd|
-            shell.status "#{what}"
-            run(cmd)
-          end
+          WRAP
 
-          owner = [config.user, config.group].join(':')
-          shell.status "Setting ownership to #{owner}"
-          sudo "chown -R #{owner} #{paths.active_release}"
+          shell.substatus "Symlink database.yml"
+          run "ln -nfs #{paths.shared_config}/database.sqlite3.yml #{paths.active_release_config}/database.yml"
         end
       end
 
@@ -347,9 +297,6 @@ WRAP
           shell.substatus what
           run(cmd)
         end
-        owner = [config.user, config.group].join(':')
-        shell.status "Setting ownership to #{owner}"
-        sudo "chown -R #{owner} #{paths.active_release}"
       end
 
       def symlink_tasks
@@ -372,14 +319,31 @@ WRAP
       # task
       def symlink
         shell.status "Symlinking code."
-        run "rm -f #{paths.current} && ln -nfs #{paths.active_release} #{paths.current} && find #{paths.current} -not -user #{config.user} -or -not -group #{config.group} -exec chown #{config.user}:#{config.group} {} +"
+        run move_symlink(paths.active_release, paths.current, "deploying")
+        ensure_ownership(paths.current)
         @symlink_changed = true
       rescue Exception
-        sudo "rm -f #{paths.current} && ln -nfs #{paths.previous_release(paths.active_release)} #{paths.current} && chown -R #{config.user}:#{config.group} #{paths.current}"
+        sudo move_symlink(paths.previous_release(paths.active_release), paths.current, "reverting")
+        ensure_ownership(paths.current)
         @symlink_changed = false
         raise
       end
 
+      def ensure_ownership(*targets)
+        sudo "find #{targets.join(' ')} -not -user #{config.user} -or -not -group #{config.group} -exec chown #{config.user}:#{config.group} {} +"
+      end
+
+      # Move a symlink as atomically as we can.
+      #
+      # mv -T renames 'next' to 'current' instead of moving 'next' to current/next'
+      # mv -T isn't available on OS X and maybe elsewhere, so fallback to rm && ln
+      def move_symlink(source, link, name)
+        next_link = link.dirname.join(name)
+        mv_t  = "ln -nfs #{source} #{next_link} && mv -T #{next_link} #{link} >/dev/null 2>&1"
+        rm_ln = "rm -rf #{next_link} #{link} && ln -nfs #{source} #{link}"
+        "((#{mv_t}) || (#{rm_ln}))"
+      end
+
       def callback(what)
         @callbacks_reached ||= true
         if paths.deploy_hook(what).exist?
@@ -402,7 +366,7 @@ WRAP
       # Rollback doesn't know about the repository location (nor
       # should it need to), but it would like to use #short_log_message.
       def strategy
-        ENV['GIT_SSH'] = ssh_executable.to_s
+        ensure_git_ssh_wrapper
         @strategy ||= config.strategy_class.new(
           shell,
           :verbose          => config.verbose,
@@ -412,10 +376,7 @@ WRAP
           :ref              => config[:branch]
         )
       end
-
-      def gemfile?
-        paths.gemfile.exist?
-      end
+      public :strategy
 
       def base_callback_command_for(what)
         cmd =  [About.binary, 'hook', what.to_s]
@@ -453,82 +414,29 @@ WRAP
         end
       end
 
-      def maintenance
-        @maintenance ||= Maintenance.new(servers, config, shell)
-      end
-
       def with_failed_release_cleanup
         yield
-      rescue Exception
+      rescue Exception => e
         shell.status "Release #{paths.active_release} failed, saving release to #{paths.releases_failed}."
-        sudo "mv #{paths.active_release} #{paths.releases_failed}"
+        run "mkdir -p #{paths.releases_failed}"
+        ensure_ownership(paths.active_release, paths.releases_failed)
+        run "mv #{paths.active_release} #{paths.releases_failed}"
         clean_release_directory(paths.releases_failed)
-        raise
-      end
-
-      def bundler_config
-        version = LockfileParser.default_version
-        options = [
-          "--gemfile #{paths.gemfile}",
-          "--path #{paths.bundled_gems}",
-          "--binstubs #{paths.binstubs}",
-          "--without #{config.bundle_without}"
-        ]
-
-        if lockfile
-          version = lockfile.bundler_version
-          options.unshift('--deployment') # deployment mode is not supported without a Gemfile.lock
-        end
-
-        return [version, options.join(' ')]
-      end
-
-      def lockfile
-        lockfile_path = paths.gemfile_lock
-        if lockfile_path.exist?
-          @lockfile_parser ||= LockfileParser.new(lockfile_path.read)
-        else
-          nil
-        end
-      end
-
-      def check_ruby_bundler
-        if gemfile?
-          shell.status "Bundling gems..."
-
-          clean_bundle_on_system_version_change
-
-          bundler_version, install_switches = bundler_config
-          sudo "#{clean_environment} && #{About.binary} install_bundler #{bundler_version}"
-          run  "#{clean_environment} && cd #{paths.active_release} && ruby -S bundle _#{bundler_version}_ install #{install_switches}"
-
-          write_system_version
-        end
+        raise e
       end
 
-      def clean_bundle_on_system_version_change
-        # diff exits with 0 for same and 1/2 for different/file not found.
-        check_ruby   = "#{config.ruby_version_command} | diff - #{paths.ruby_version} >/dev/null 2>&1"
-        check_system = "#{config.system_version_command} | diff - #{paths.system_version} >/dev/null 2>&1"
-        say_cleaning = "echo 'New deploy or system version change detected, cleaning bundled gems.'"
-        clean_bundle = "rm -Rf #{paths.bundled_gems}"
-
-        run "#{check_ruby} && #{check_system} || (#{say_cleaning} && #{clean_bundle})"
+      def maintenance
+        @maintenance ||= Maintenance.new(servers, config, shell)
       end
 
-      def write_system_version
-        store_ruby_version   = "#{config.ruby_version_command} > #{paths.ruby_version}"
-        store_system_version = "#{config.system_version_command} > #{paths.system_version}"
-
-        run "mkdir -p #{paths.bundled_gems} && chown #{config.user}:#{config.group} #{paths.bundled_gems}"
-        run "#{store_ruby_version} && #{store_system_version}"
+      def dependency_manager
+        ensure_git_ssh_wrapper
+        @dependency_manager ||= DependencyManager.detect(servers, config, shell, self)
       end
+      public :dependency_manager # FIXME
 
-      def check_node_npm
-        if paths.package_json.exist?
-          shell.info "~> package.json detected, installing npm packages"
-          run "cd #{paths.active_release} && npm install"
-        end
+      def compile_assets
+        RailsAssets.detect_and_compile(config, shell, self)
       end
     end   # DeployBase