engineyard-serverside 1.2.0

data/LICENSE

@@ -0,0 +1,19 @@
+ Copyright (c) 2010 Engine Yard, Inc
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.

data/bin/engineyard-serverside

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+begin
+  require 'rubygems'
+rescue LoadError
+end
+
+$LOAD_PATH.push(File.expand_path("../lib", File.dirname(__FILE__)))
+require 'engineyard-serverside'
+
+EY::CLI.start

data/lib/engineyard-serverside.rb

@@ -0,0 +1,49 @@
+$LOAD_PATH.push(File.expand_path("engineyard-serverside", File.dirname(__FILE__)))
+$LOAD_PATH.unshift File.expand_path('vendor/thor/lib', File.dirname(__FILE__))
+$LOAD_PATH.unshift File.expand_path('vendor/open4/lib', File.dirname(__FILE__))
+$LOAD_PATH.unshift File.expand_path('vendor/escape/lib', File.dirname(__FILE__))
+$LOAD_PATH.unshift File.expand_path('vendor/json_pure/lib', File.dirname(__FILE__))
+$LOAD_PATH.unshift File.expand_path('vendor/dataflow', File.dirname(__FILE__))
+
+require 'escape'
+require 'json'
+require 'dataflow'
+
+require 'version'
+require 'strategies/git'
+require 'task'
+require 'server'
+require 'deploy'
+require 'deploy_hook'
+require 'lockfile_parser'
+require 'bundle_installer'
+require 'cli'
+require 'configuration'
+
+module EY
+  def self.node
+    @node ||= deep_indifferentize(JSON.parse(dna_json))
+  end
+
+  def self.dna_json
+    @dna_json ||= `sudo cat /etc/chef/dna.json`
+  end
+
+  RemoteFailure = Class.new StandardError
+
+  private
+  def self.deep_indifferentize(thing)
+    if thing.kind_of?(Hash)
+      indifferent_hash = Thor::CoreExt::HashWithIndifferentAccess.new
+      thing.each do |k, v|
+        indifferent_hash[k] = deep_indifferentize(v)
+      end
+      indifferent_hash
+    elsif thing.kind_of?(Array)
+      thing.map {|x| deep_indifferentize(x)}
+    else
+      thing
+    end
+  end
+
+end

data/lib/engineyard-serverside/bundle_installer.rb

@@ -0,0 +1,4 @@
+module EY
+  class BundleInstaller < Struct.new(:version, :options)
+  end
+end

data/lib/engineyard-serverside/cli.rb

@@ -0,0 +1,146 @@
+require 'thor'
+
+module EY
+  class CLI < Thor
+    include Dataflow
+
+    def self.start(*)
+      super
+    rescue RemoteFailure
+      exit(1)
+    end
+
+    method_option :migrate,       :type     => :string,
+                                  :desc     => "Run migrations with this deploy",
+                                  :aliases  => ["-m"]
+
+    method_option :branch,        :type     => :string,
+                                  :desc     => "Git ref to deploy, defaults to master. May be a branch, a tag, or a SHA",
+                                  :aliases  => %w[-b --ref --tag]
+
+    method_option :repo,          :type     => :string,
+                                  :desc     => "Remote repo to deploy",
+                                  :aliases  => ["-r"]
+
+    method_option :app,           :type     => :string,
+                                  :required => true,
+                                  :desc     => "Application to deploy",
+                                  :aliases  => ["-a"]
+
+    method_option :framework_env, :type     => :string,
+                                  :required => true,
+                                  :desc     => "Ruby web framework environment",
+                                  :aliases  => ["-e"]
+
+    method_option :config,        :type     => :string,
+                                  :desc     => "Additional configuration"
+
+    method_option :stack,         :type     => :string,
+                                  :desc     => "Web stack (so we can restart it correctly)"
+
+    method_option :instances,     :type     => :array,
+                                  :desc     => "Instances in cluster"
+
+    method_option :verbose,       :type     => :boolean,
+                                  :default  => false,
+                                  :desc     => "Verbose output",
+                                  :aliases  => ["-v"]
+
+    desc "deploy", "Deploy code from /data/<app>"
+    def deploy(default_task=:deploy)
+      EY::Server.all = parse_instances(options[:instances])
+      EY::LoggedOutput.verbose = options[:verbose]
+      EY::LoggedOutput.logfile = File.join(ENV['HOME'], "#{options[:app]}-deploy.log")
+
+      invoke :propagate
+      EY::Deploy.run(options.merge("default_task" => default_task))
+    end
+
+    method_option :app,           :type     => :string,
+                                  :required => true,
+                                  :desc     => "Which application's hooks to run",
+                                  :aliases  => ["-a"]
+
+    method_option :release_path,  :type     => :string,
+                                  :desc     => "Value for #release_path in hooks (mostly for internal coordination)",
+                                  :aliases  => ["-r"]
+
+    method_option :current_role,  :type     => :string,
+                                  :desc     => "Value for #current_role in hooks"
+
+    method_option :framework_env, :type     => :string,
+                                  :required => true,
+                                  :desc     => "Ruby web framework environment",
+                                  :aliases  => ["-e"]
+
+    method_option :config,        :type     => :string,
+                                  :desc     => "Additional configuration"
+
+    method_option :current_name,  :type     => :string,
+                                  :desc     => "Value for #current_name in hooks"
+
+    desc "hook [NAME]", "Run a particular deploy hook"
+    def hook(hook_name)
+      EY::DeployHook.new(options).run(hook_name)
+    end
+
+    desc "install_bundler [VERSION]", "Make sure VERSION of bundler is installed (in system ruby)"
+    def install_bundler(version)
+      egrep_escaped_version = version.gsub(/\./, '\.')
+      # the grep "bundler " is so that gems like bundler08 don't get
+      # their versions considered too
+      #
+      # the [,$] is to stop us from looking for e.g. 0.9.2, seeing
+      # 0.9.22, and mistakenly thinking 0.9.2 is there
+      has_bundler_cmd = "gem list bundler | grep \"bundler \" | egrep -q '#{egrep_escaped_version}[,)]'"
+
+      unless system(has_bundler_cmd)
+        system("gem install bundler -q --no-rdoc --no-ri -v '#{version}'")
+      end
+    end
+
+    desc "propagate", "Propagate the engineyard-serverside gem to the other instances in the cluster. This will install exactly version #{VERSION} and remove other versions if found."
+    def propagate
+      config          = EY::Deploy::Configuration.new
+      gem_filename    = "engineyard-serverside-#{VERSION}.gem"
+      local_gem_file  = File.join(Gem.dir, 'cache', gem_filename)
+      remote_gem_file = File.join(Dir.tmpdir, gem_filename)
+      gem_binary      = File.join(Gem.default_bindir, 'gem')
+
+      EY::Server.config = config
+
+      barrier(*(EY::Server.all.find_all do |server|
+        !server.local?            # of course this machine has it
+      end.map do |server|
+        need_later do
+          egrep_escaped_version = VERSION.gsub(/\./, '\.')
+          # the [,)] is to stop us from looking for e.g. 0.5.1, seeing
+          # 0.5.11, and mistakenly thinking 0.5.1 is there
+          has_gem_cmd = "#{gem_binary} list engineyard-serverside | grep \"engineyard-serverside\" | egrep -q '#{egrep_escaped_version}[,)]'"
+
+          if !server.run(has_gem_cmd)  # doesn't have this exact version
+            puts "~> Installing engineyard-serverside on #{server.hostname}"
+
+            system(Escape.shell_command([
+              'scp', '-i', "#{ENV['HOME']}/.ssh/internal",
+              "-o", "StrictHostKeyChecking=no",
+              local_gem_file,
+             "#{config.user}@#{server.hostname}:#{remote_gem_file}",
+            ]))
+            server.run("sudo #{gem_binary} install --no-rdoc --no-ri '#{remote_gem_file}'")
+          end
+        end
+      end))
+    end
+
+    private
+
+    def parse_instances(instance_strings)
+      instance_strings.map do |s|
+        tuple = s.split(/,/)
+        {:hostname => tuple[0], :role => tuple[1], :name => tuple[2]}
+      end
+    end
+
+  end
+end

data/lib/engineyard-serverside/configuration.rb

@@ -0,0 +1,130 @@
+require 'json'
+require 'thor'
+
+module EY
+  class Deploy::Configuration
+    DEFAULT_CONFIG = Thor::CoreExt::HashWithIndifferentAccess.new({
+      "branch"       => "master",
+      "strategy"     => "Git",
+    })
+
+    attr_reader :configuration
+    alias :c :configuration
+
+    attr_writer :release_path
+
+    def initialize(opts={})
+      @release_path = opts[:release_path]
+      config = JSON.parse(opts["config"] || "{}")
+      @configuration = DEFAULT_CONFIG.merge(config).merge(opts)
+    end
+
+    # Delegate to the configuration objects
+    def method_missing(meth, *args, &blk)
+      c.key?(meth.to_s) ? c[meth.to_s] : super
+    end
+
+    def respond_to?(meth, include_private=false)
+      c.key?(meth.to_s) ? true : super
+    end
+
+    def [](key)
+      if respond_to?(key.to_sym)
+        send(key.to_sym)
+      else
+        c[key]
+      end
+    end
+
+    def has_key?(key)
+      if respond_to?(key.to_sym)
+        true
+      else
+        c.has_key?(key)
+      end
+    end
+
+    def node
+      EY.node
+    end
+
+    def revision
+      IO.read(File.join(latest_release, 'REVISION'))
+    end
+
+    def repository_cache
+      configuration['repository_cache'] || File.join(deploy_to, "/shared/cached-copy")
+    end
+
+    def deploy_to
+      configuration['deploy_to'] || "/data/#{app}"
+    end
+
+    def migrate?
+      !!configuration['migrate']
+    end
+
+    def migration_command
+      configuration['migrate'] == "migrate" ? DEFAULT_CONFIG["migrate"] : configuration['migrate']
+    end
+
+    def user
+      ENV['USER']
+    end
+    alias :group :user
+
+    def role
+      node['instance_role']
+    end
+
+    def copy_exclude
+      @copy_exclude ||= Array(configuration.fetch("copy_exclude", []))
+    end
+
+    def environment
+      configuration['framework_env']
+    end
+
+    def latest_release
+      all_releases.last
+    end
+
+    def previous_release(current=latest_release)
+      index = all_releases.index(current)
+      all_releases[index-1]
+    end
+
+    def oldest_release
+      all_releases.first
+    end
+
+    def all_releases
+      Dir.glob("#{release_dir}/*").sort
+    end
+
+    def framework_envs
+      "RAILS_ENV=#{environment} RACK_ENV=#{environment} MERB_ENV=#{environment}"
+    end
+
+    def current_path
+      File.join(deploy_to, "current")
+    end
+
+    def shared_path
+      File.join(deploy_to, "shared")
+    end
+
+    def release_dir
+      File.join(deploy_to, "releases")
+    end
+
+    def release_path
+      @release_path ||= File.join(release_dir, Time.now.utc.strftime("%Y%m%d%H%M%S"))
+    end
+
+    def exclusions
+      copy_exclude.map { |e| %|--exclude="#{e}"| }.join(' ')
+    end
+
+  end
+end

data/lib/engineyard-serverside/default_maintenance_page.html

@@ -0,0 +1,29 @@
+<!DOCTYPE html >
+<html>
+<head>
+  <title>Undergoing Maintenance</title>
+
+  <style type="text/css">
+    body{font-size: 75%;font-family: Helvetica,Arial,sans-serif;color:#333;background:#FDFDFD;}
+    .container{font-size:1.1em;width:700px;text-align:center;margin:80px auto;}
+    .message{background:#FFF;border:1px solid #EEE;padding:30px;}
+    h1{margin:0 0 0.5em;font-weight:normal;}
+    p{margin:0;line-height:1.75em;}
+    .foot{color:#999;margin-top:30px}
+    .foot a{color:#666;text-decoration:none}
+  </style>
+</head>
+
+<body>
+  <div class="container">
+    <div class="message">
+      <h1>This site is currently undergoing some maintenance.</h1>
+      <p>Customize this maintenance page by updating the file located at: /public/maintenance.html</p>
+    </div>
+    <div class="foot">
+      Hosted by <a href="http://www.engineyard.com">Engine Yard</a>
+    </div>
+  </div>
+</body>
+
+</html>

data/lib/engineyard-serverside/deploy.rb

@@ -0,0 +1,321 @@
+# stolen wholesale from capistrano, thanks Jamis!
+require 'base64'
+require 'fileutils'
+require 'json'
+
+module EY
+  class DeployBase < Task
+    include LoggedOutput
+
+    # default task
+    def deploy
+      debug "Starting deploy at #{Time.now.asctime}"
+      update_repository_cache
+      require_custom_tasks
+      push_code
+
+      info "~> Starting full deploy"
+      copy_repository_cache
+
+      with_failed_release_cleanup do
+        create_revision_file
+        bundle
+        symlink_configs
+        conditionally_enable_maintenance_page
+        run_with_callbacks(:migrate)
+        callback(:before_symlink)
+        symlink
+      end
+
+      callback(:after_symlink)
+      run_with_callbacks(:restart)
+      disable_maintenance_page
+
+      cleanup_old_releases
+      debug "Finished deploy at #{Time.now.asctime}"
+    rescue Exception
+      debug "Finished failing to deploy at #{Time.now.asctime}"
+      puts_deploy_failure
+      raise
+    end
+
+    def enable_maintenance_page
+      maintenance_page_candidates = [
+        "public/maintenance.html.custom",
+        "public/maintenance.html.tmp",
+        "public/maintenance.html",
+        "public/system/maintenance.html.default",
+      ].map do |file|
+        File.join(c.latest_release, file)
+      end
+
+      # this one is guaranteed to exist
+      maintenance_page_candidates <<  File.expand_path(
+        "default_maintenance_page.html",
+        File.dirname(__FILE__)
+        )
+
+      # put in the maintenance page
+      maintenance_file = maintenance_page_candidates.detect do |file|
+        File.exists?(file)
+      end
+
+      @maintenance_up = true
+      roles :app_master, :app, :solo do
+        visible_maint_page = File.join(c.shared_path, "system", "maintenance.html")
+        run "cp '#{maintenance_file}' '#{visible_maint_page}'"
+      end
+    end
+
+    def conditionally_enable_maintenance_page
+      if c.migrate? || c.stack == "nginx_mongrel"
+        enable_maintenance_page
+      end
+    end
+
+    def disable_maintenance_page
+      @maintenance_up = false
+      roles :app_master, :app, :solo do
+        run "rm -f #{File.join(c.shared_path, "system", "maintenance.html")}"
+      end
+    end
+
+    def run_with_callbacks(task, *args)
+      callback(:"before_#{task}")
+      send(task, *args)
+      callback(:"after_#{task}")
+    end
+
+    # task
+    def push_code
+      info "~> Pushing code to all servers"
+      barrier *(EY::Server.all.map do |server|
+        need_later { server.push_code }
+      end)
+    end
+
+    # task
+    def restart
+      @restart_failed = true
+      info "~> Restarting app servers"
+      roles :app_master, :app, :solo do
+        restart_command = case c.stack
+        when "nginx_unicorn"
+          pidfile = "/var/run/engineyard/unicorn_#{c.app}.pid"
+          condition = "[ -e #{pidfile} ] && [ ! -d /proc/`cat #{pidfile}` ]"
+          run("if #{condition}; then rm -f #{pidfile}; fi")
+          run("/engineyard/bin/app_#{c.app} deploy")
+        when "nginx_mongrel"
+          sudo("monit restart all -g #{c.app}")
+        when "nginx_passenger"
+          run("touch #{c.current_path}/tmp/restart.txt")
+        else
+          raise "Unknown stack #{c.stack}; restart failed!"
+        end
+      end
+      @restart_failed = false
+    end
+
+    # task
+    def bundle
+      if File.exist?("#{c.release_path}/Gemfile")
+        info "~> Gemfile detected, bundling gems"
+        lockfile = File.join(c.release_path, "Gemfile.lock")
+
+        bundler_installer = if File.exist?(lockfile)
+                              get_bundler_installer(lockfile)
+                            else
+                              warn_about_missing_lockfile
+                              BundleInstaller.new(DEFAULT_09_BUNDLER, "--without=development --without=test")
+                            end
+
+        sudo "#{$0} _#{VERSION}_ install_bundler #{bundler_installer.version}"
+
+        run "cd #{c.release_path} && bundle _#{bundler_installer.version}_ install #{bundler_installer.options}"
+      end
+    end
+
+    # task
+    def cleanup_old_releases
+      @cleanup_failed = true
+      info "~> Cleaning up old releases"
+      sudo "ls #{c.release_dir} | head -n -3 | xargs -I{} rm -rf #{c.release_dir}/{}"
+      @cleanup_failed = false
+    end
+
+    # task
+    def rollback
+      if c.all_releases.size > 1
+        c.release_path = c.previous_release
+
+        revision = File.read(File.join(c.release_path, 'REVISION')).strip
+        info "~> Rolling back to previous release: #{short_log_message(revision)}"
+
+        run_with_callbacks(:symlink, c.previous_release)
+        cleanup_current_release
+        bundle
+        info "~> Restarting with previous release"
+        with_maintenance_page { run_with_callbacks(:restart) }
+      else
+        info "~> Already at oldest release, nothing to roll back to"
+        exit(1)
+      end
+    end
+
+    # task
+    def migrate
+      return unless c.migrate?
+      @migrations_reached = true
+      roles :app_master, :solo do
+        cmd = "cd #{c.release_path} && #{c.framework_envs} #{c.migration_command}"
+        info "~> Migrating: #{cmd}"
+        run(cmd)
+      end
+    end
+
+    # task
+    def copy_repository_cache
+      info "~> Copying to #{c.release_path}"
+      run("mkdir -p #{c.release_path} && rsync -aq #{c.exclusions} #{c.repository_cache}/ #{c.release_path}")
+
+      info "~> Ensuring proper ownership"
+      sudo("chown -R #{c.user}:#{c.group} #{c.deploy_to}")
+    end
+
+    def create_revision_file
+      run create_revision_file_command
+    end
+
+    def symlink_configs(release_to_link=c.release_path)
+      info "~> Symlinking configs"
+      [ "chmod -R g+w #{release_to_link}",
+        "rm -rf #{release_to_link}/log #{release_to_link}/public/system #{release_to_link}/tmp/pids",
+        "mkdir -p #{release_to_link}/tmp",
+        "ln -nfs #{c.shared_path}/log #{release_to_link}/log",
+        "mkdir -p #{release_to_link}/public",
+        "mkdir -p #{release_to_link}/config",
+        "ln -nfs #{c.shared_path}/system #{release_to_link}/public/system",
+        "ln -nfs #{c.shared_path}/pids #{release_to_link}/tmp/pids",
+        "ln -nfs #{c.shared_path}/config/database.yml #{release_to_link}/config/database.yml",
+        "ln -nfs #{c.shared_path}/config/mongrel_cluster.yml #{release_to_link}/config/mongrel_cluster.yml",
+      ].each do |cmd|
+        run cmd
+      end
+
+      sudo "chown -R #{c.user}:#{c.group} #{release_to_link}"
+      run "if [ -f \"#{c.shared_path}/config/newrelic.yml\" ]; then ln -nfs #{c.shared_path}/config/newrelic.yml #{release_to_link}/config/newrelic.yml; fi"
+    end
+
+    # task
+    def symlink(release_to_link=c.release_path)
+      info "~> Symlinking code"
+      run "rm -f #{c.current_path} && ln -nfs #{release_to_link} #{c.current_path} && chown -R #{c.user}:#{c.group} #{c.current_path}"
+      @symlink_changed = true
+    rescue Exception
+      sudo "rm -f #{c.current_path} && ln -nfs #{c.previous_release(release_to_link)} #{c.current_path} && chown -R #{c.user}:#{c.group} #{c.current_path}"
+      @symlink_changed = false
+      raise
+    end
+
+    def callback(what)
+      @callbacks_reached ||= true
+      if File.exist?("#{c.release_path}/deploy/#{what}.rb")
+        eydeploy_path = $0   # invoke others just like we were invoked
+        run "#{eydeploy_path} _#{VERSION}_ hook '#{what}' --app '#{config.app}' --release-path #{config.release_path}" do |server, cmd|
+          cmd << " --framework-env '#{c.environment}'"
+          cmd << " --current-role '#{server.role}'"
+          cmd << " --current-name '#{server.name}'" if server.name
+          cmd << " --config '#{c[:config]}'" if c.has_key?(:config)
+          cmd
+        end
+      end
+    end
+
+    protected
+
+    def puts_deploy_failure
+      if @cleanup_failed
+        info "~> [Relax] Your site is running new code, but cleaning up old deploys failed"
+      elsif @maintenance_up
+        info "~> [Attention] Maintenance page still up, consider the following before removing:"
+        info " * any deploy hooks ran, be careful if they were destructive" if @callbacks_reached
+        info " * any migrations ran, be careful if they were destructive" if @migrations_reached
+        if @symlink_changed
+          info " * your new code is symlinked as current"
+        else
+          info " * your old code is still symlinked as current"
+        end
+        info " * application servers failed to restart" if @restart_failed
+      else
+        info "~> [Relax] Your site is still running old code and nothing destructive could have occurred"
+      end
+    end
+
+    def with_maintenance_page
+      conditionally_enable_maintenance_page
+      yield if block_given?
+      disable_maintenance_page
+    end
+
+    def with_failed_release_cleanup
+      yield
+    rescue Exception
+      cleanup_current_release
+      raise
+    end
+
+    def cleanup_current_release
+      sudo "rm -rf #{c.release_path}"
+    end
+
+    DEFAULT_09_BUNDLER = '0.9.26'
+    DEFAULT_10_BUNDLER = '1.0.0.rc.3'
+
+    def warn_about_missing_lockfile
+      info "!>"
+      info "!> WARNING: Gemfile.lock is missing!"
+      info "!> You can get different gems in production than what you tested with."
+      info "!> You can get different gems on every deployment even if your Gemfile hasn't changed."
+      info "!> Deploying may take a long time."
+      info "!>"
+      info "!> Fix this by running \"git add Gemfile.lock; git commit\" and deploying again."
+      info "!> If you don't have a Gemfile.lock, run \"bundle lock\" to create one."
+      info "!>"
+      info "!> This deployment will use bundler #{DEFAULT_09_BUNDLER} to run 'bundle install'."
+      info "!>"
+    end
+
+    def get_bundler_installer(lockfile)
+      parser = LockfileParser.new(File.read(lockfile))
+      case parser.lockfile_version
+      when :bundler09
+        BundleInstaller.new(
+          parser.bundler_version || DEFAULT_09_BUNDLER,
+          "--without=development --without=test")
+      when :bundler10
+        BundleInstaller.new(
+          parser.bundler_version || DEFAULT_10_BUNDLER,
+          "--deployment --path #{c.shared_path}/bundled_gems --without development test"
+          )
+      else
+        raise "Unknown lockfile version #{parser.lockfile_version}"
+      end
+    end
+    public :get_bundler_installer
+
+  end   # DeployBase
+
+  class Deploy < DeployBase
+    def self.new(opts={})
+      # include the correct fetch strategy
+      include EY::Strategies.const_get(opts.strategy)::Helpers
+      super
+    end
+
+    def self.run(opts={})
+      conf = EY::Deploy::Configuration.new(opts)
+      EY::Server.config = conf
+      new(conf).send(opts["default_task"])
+    end
+  end
+end