ol-whisk_deploy 0.6.25

data/Rakefile

@@ -0,0 +1,105 @@
+require 'rake'
+require 'rake/testtask'
+
+desc 'Default: run unit tests.'
+task :default => :test
+
+desc 'Test whiskey_disk'
+task :test do
+  files = Dir['spec/**/*_spec.rb'].join(" ")
+  system("bacon #{files}")
+end
+
+namespace :integration do
+  def say(mesg)
+    STDERR.puts mesg
+  end
+
+  def vagrant_path
+    File.expand_path(File.join(File.dirname(__FILE__), 'scenarios', 'setup', 'vagrant'))
+  end
+
+  def root_path
+    File.expand_path(File.dirname(__FILE__))
+  end
+
+  def pidfile
+    File.join(vagrant_path, 'pids', 'git-daemon.pid')
+  end
+
+  def start_git_daemon
+    stop_git_daemon
+    say "Starting git daemon..."
+    run(root_path, "git daemon --base-path=#{root_path}/scenarios/git_repositories/ --reuseaddr --verbose --detach --pid-file=#{pidfile}")
+  end
+
+  def stop_git_daemon
+    return unless File.exists?(pidfile)
+    pid = File.read(pidfile).chomp
+    return if pid == ''
+    say "Stopping git daemon..."
+    run(root_path, "kill #{pid}")
+  end
+
+  def start_vm
+    say "Bringing up vagrant vm..."
+    run(vagrant_path, 'vagrant up')
+    copy_ssh_config
+  end
+
+  def stop_vm
+    say "Shutting down vagrant vm..."
+    run(vagrant_path, 'vagrant halt')
+  end
+
+  def copy_ssh_config
+    say "Capturing vagrant ssh_config data..."
+    run(vagrant_path, "vagrant ssh_config > #{vagrant_path}/ssh_config")
+  end
+
+  def run(path, cmd)
+    Dir.chdir(path)
+    say "running: #{cmd} [cwd: #{Dir.pwd}]"
+    system(cmd)
+  end
+
+  desc 'Start a vagrant VM and git-daemon server to support running integration specs'
+  task :up do
+    start_vm
+    start_git_daemon
+  end
+
+  desc 'Shut down integration vagrant VM and git-daemon server'
+  task :down do
+    stop_git_daemon
+    stop_vm
+  end
+
+  desc 'Completely remove the vagrant VM files used by the integration spec suite'
+  task :destroy do
+    stop_git_daemon
+    stop_vm
+    run(vagrant_path, 'vagrant destroy')
+  end
+end
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name = "whisk_deploy"
+    gemspec.summary = "embarrassingly fast deployments."
+    gemspec.description = "Opinionated gem for doing fast git-based server deployments."
+    gemspec.email = "jesse@techno-geeks.org"
+    gemspec.homepage = "http://github.com/jesseadams/whiskey"
+    gemspec.authors = ["Rick Bradley", "Jesse R. Adams"]
+    gemspec.add_dependency('rake')
+
+    # I've decided that the integration spec shizzle shouldn't go into the gem
+    rejected = %w(scenarios spec/integration)
+    gemspec.files.reject {|f| rejected.include?(f) }
+    gemspec.test_files.reject {|f| rejected.include?(f) }
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  # if you get here, you need Jeweler installed to do packaging and gem installation, yo.
+end

data/VERSION

@@ -0,0 +1 @@
+0.6.25

data/WHY.txt

@@ -0,0 +1,45 @@
+
+Why?
+----
+
+The idea here is inspired by github's cleaned up deployment scripts, and mislav's git-deploy project.  Only, we gave up on capistrano a long time ago, and after a few years of doing deployments on a few dozen projects, we realized that we really have very little variation on how we do things.  That is, we can afford to have a very opinionated tool to do our deployments.
+
+Here are the features/constraints we're envisioning:
+
+ - We need some sort of very basic "run this on a remote server" functionality.  We've been using vlad for years now and this would suffice:  it does what we're looking for and is much much smaller than capistrano.  If we don't load the included recipes it's basically a fancy ruby ssh wrapper.
+ 
+ - Setup should mostly just do a very fast remote git checkout in the right place.  Deployment should very quickly update that checkout.
+ 
+ - We have been considering a move towards tracking configuration data across our projects as a separate concern.  So, if we can have a private repo that stores per-project and per-environment (here I mean staging vs. production, etc.) configuration files, and have our deployments overlay those files quickly, that would be ideal.  I'm talking about hoptoad configs, database.yml files, AWS cert files, GeoKit API keys, etc., etc., etc.
+
+ - We should be able to use the same "setup == clone" + "deploy == reset" technique to manage the per-project/per-environment config files.
+ 
+ - Using rsync on the remote to those overlay config files on the deployed project would be a fast way to get them in place.
+ 
+ - Get rid of a bunch of annoying symlinks and symlink-hoops-to-jump-through.
+ 
+ - Get rid of a bunch of space (yeah yeah disk is cheap, but copying isn't) on the disk devoted to umpteen "releases".
+ 
+ - Obviously reduce deployment time by doing less, ssh-ing less, and taking less time to do whatever.
+ 
+ - should be rake based, and should provide a bare minimum of tasks -- like deploy:setup, deploy:now, and maybe a deploy:refresh_config_files.
+ 
+ - While a very basic task or few would run after setup or after deployment (e.g., rake db:migrate if migrations were changed, or touch tmp/restart.txt if the web server needs a restart; see git-deploy for more examples), we should be able to declare optional rake tasks (e.g., "deploy:staging:post_deploy") and have them run on this project if they are declared.
+ 
+ - Should work with projects that aren't remotely ruby.
+ 
+ - Should be loadable as a gem, meaning that it doesn't need to live in your project's space.  (see also non-ruby projects)
+ 
+ - Should be able to use a non-ruby config, preferably yaml, for information for all environments.  That could be stored in <project>/config/deploy.yml and saved with the project.  Even if this is just shoved into vlad 'set' commands, it's still an improvement:  we don't need ruby in the config file because we're opinionated.
+ 
+ - should be able to override settings for an environment locally by declaring a <project>/config/deploy-<environment>.yml.  Ideal for testing out deployments to different servers (or deploying locally).  This also makes it possible to .gitignore your local settings, so everyone can have their config repos in different places.
+ 
+ - should make it easier to do local development (e.g., on a laptop) by being able to overlay config files using the same rake tasks as used for remote deployments, just not running the functionality remotely.
+ 
+ - dropping in a project Rakefile can add post-deploy / post-setup hooks transparently.
+ 
+ - actually have meaningful error messages, unlike anything that ever seems to happen with cap or vlad.  :-/
+
+ - build this spec-first (whenever possible) so that there's a useful test suite.
+ 
+ - M$ windows hasn't been a priority for me for over a decade, not starting now.

data/bin/wd

@@ -0,0 +1,61 @@
+#!/usr/bin/env ruby 
+
+require 'optparse'
+require 'whiskey_disk/rake'
+
+$0 = "#{$0} setup|deploy"  # jesus, this is a hack.
+
+options = {}
+op = OptionParser.new do |opts|
+  opts.on('-t=TARGET', '--to=TARGET', "deployment target") do |target| 
+    options[:target] = target
+  end
+  
+  opts.on('-p=TARGET', '--path=TARGET', "configuration path") do |path|
+    options[:path] = path
+  end
+    
+  opts.on('-o=DOMAIN', '--only=DOMAIN', "limit deployment to this domain") do |domain|
+    options[:only] = domain
+  end
+    
+  opts.on('-c', '--check', "do a staleness check before deploying") do |path|
+    options[:check] = 'true'
+  end
+
+  opts.on('-d', '--debug', "turn on debug mode (ssh -v and rake --trace)") do
+    options[:debug] = 'true'
+  end
+    
+  opts.on('--version', 'show current version') do
+    puts File.read(File.expand_path(File.join(File.dirname(__FILE__), '..', 'VERSION')))
+    exit 0
+  end
+  
+  opts.on_tail('-h', '--help', 'show this message') do
+    abort opts.to_s
+  end
+end
+
+begin
+  rest = op.parse(ARGV) 
+rescue 
+  abort op.to_s 
+end
+
+abort op.to_s unless options[:target]
+abort op.to_s unless rest and rest.size == 1
+command = rest.first
+abort op.to_s unless ['deploy', 'setup'].include?(command)
+
+ENV['to'] = options[:target]
+ENV['path'] = options[:path]
+ENV['only'] = options[:only]
+ENV['check'] = options[:check]
+ENV['debug'] = options[:debug]
+
+if command == 'deploy'
+  Rake::Task['deploy:now'].invoke
+else
+  Rake::Task['deploy:setup'].invoke
+end

data/bin/wd_role

@@ -0,0 +1,42 @@
+#!/usr/bin/env ruby 
+require 'whiskey_disk/helpers'
+
+# simple command-line script to detect whether this deployment target
+# is in a specified whiskey_disk role.  sets exit status appropriately
+#
+# useful for conditionalizing shell scripting based on roles.
+
+role = ARGV.shift
+exit(1) unless role?(role)
+
+__END__
+
+( this example session presumes you `gem install rockhands` first... )
+
+$ export WD_ROLES='app:db'
+$ wd_role web && rock || shocker
+     .-.     
+   .-.U|     
+   |U| | .-. 
+   | | |_|U| 
+   | | | | | 
+  /|     ` |
+ | |       | 
+ |         | 
+  \        / 
+  |       |  
+  |       |  
+             
+$ wd_role app && rock || shocker
+   .-.       
+   |U|       
+   | |   .-. 
+   | |-._|U| 
+   | | | | | 
+  /|     ` | 
+ | |       | 
+ |         | 
+          / 
+  |       |  
+  |       |  
+

data/examples/deploy-configs.yml

@@ -0,0 +1,13 @@
+production:
+  domain: "ogc@hoenir.websages.com"
+  deploy_to: "/tmp/test-deployment"
+  deploy_config_to: "/tmp/test-config"
+  repository: "git@git.ogtastic.com:whiskey_disk.git"
+  config_repository: "git@git.ogtastic.com:ogc-config.git"
+staging:
+  domain: "ogc@hoenir.websages.com"
+  deploy_to: "/tmp/test-deployment"
+  deploy_config_to: "/tmp/test-config"
+  repository: "git@git.ogtastic.com:whiskey_disk.git"
+  config_repository: "git@git.ogtastic.com:ogc-config.git"
+  config_target: "local"

data/examples/deploy-local.yml

@@ -0,0 +1,4 @@
+local:
+  deploy_to: "/Users/rick/deploy/whiskey_disk/local"
+  repository: "/Users/rick/git/whiskey_disk"
+  branch: "develop"

data/examples/deploy-multiple-remotes.yml

@@ -0,0 +1,26 @@
+multi:
+  domain:
+  - "ogc@hoenir.websages.com"
+  - "ogc@nerthus.websages.com"
+  deploy_to: "/tmp/test-deployment/"
+  repository: "git@git.ogtastic.com:whiskey_disk.git"
+  branch: "develop"
+roles:
+  domain:
+  - name: "ogc@hoenir.websages.com"
+    roles:
+    - web
+    - app
+  - name: "ogc@nerthus.websages.com"
+    roles: db 
+  deploy_to: "/tmp/test-deployment/"
+  repository: "git@git.ogtastic.com:whiskey_disk.git"
+  branch: "feature/support-domain-roles"
+  post_deploy_script: "/tmp/role-checker.sh"
+badmulti:
+  domain:
+  - "ogc@hoenir.websages.com"
+  - "ogc@bogus.example.com"
+  deploy_to: "/tmp/test-deployment/"
+  repository: "git@git.ogtastic.com:whiskey_disk.git"
+  branch: "develop"

data/examples/deploy-staging.yml

@@ -0,0 +1,8 @@
+staging:
+  domain: "user@www.example.com"
+  deploy_to: "/var/www/suparsite.com/"
+  repository: "git://github.com/clarkkent/suparsite.git"
+  config_repository: "git@github.com:clarkkent/suparconfig.git"
+  deploy_config_to: "/var/cache/git/suparconfig"
+  rake_env:
+    RAILS_ENV: 'development'

data/examples/deploy.rake

@@ -0,0 +1,11 @@
+namespace :deploy do
+  namespace :staging do
+    task :post_setup do
+      puts "This is my local post_setup hook."
+    end
+
+    task :post_deploy do
+      puts "This is my local post_deploy hook."
+    end
+  end
+end

data/examples/deploy.yml

@@ -0,0 +1,16 @@
+staging:
+  domain: "user@www.example.com"
+  deploy_to: "/var/www/suparsite.com/"
+  repository: "git://github.com/clarkkent/suparsite.git"
+  config_repository: "git@github.com:clarkkent/suparconfig.git"
+  deploy_config_to: "/var/cache/git/suparconfig"
+  branch: "production"
+  rake_env:
+    RAILS_ENV: "production"
+local:
+  repository: "git://github.com/clarkkent/suparsite.git"
+  config_repository: "git@github.com:clarkkent/suparconfig.git"
+  deploy_to: "/Users/clark/git/suparsite"
+  deploy_config_to: "/Users/clark/git/suparconfig"
+  rake_env:
+    RAILS_ENV: "production"

data/init.rb

@@ -0,0 +1 @@
+require File.expand_path(File.join(File.dirname(__FILE__), 'lib', 'whiskey_disk'))

data/install.rb

@@ -0,0 +1,5 @@
+def readme_contents
+  IO.read(File.expand_path(File.join(File.dirname(__FILE__), 'README.markdown')))
+end
+
+puts readme_contents

data/lib/whiskey_disk.rb

@@ -0,0 +1,327 @@
+require File.expand_path(File.join(File.dirname(__FILE__), 'whiskey_disk', 'config'))
+
+class WhiskeyDisk
+  attr_writer :configuration, :config
+  attr_reader :results
+  
+  def initialize(options = {})
+    @staleness_checks = true if options[:staleness_checks]
+  end
+  
+  def buffer
+    @buffer ||= []
+  end
+  
+  def config
+    @config ||= WhiskeyDisk::Config.new
+  end
+  
+  def configuration
+    @configuration ||= config.fetch
+  end
+  
+  def debugging?
+    config.debug?
+  end
+  
+  def setting(key)
+    configuration[key.to_s]
+  end
+  
+  def check_staleness?
+    config.check_staleness?
+  end
+  
+  def staleness_checks_enabled?
+    !!@staleness_checks
+  end    
+
+  def enqueue(command)
+    buffer << command
+  end
+  
+  def remote?(domain)
+    return false unless domain
+    return false if domain == 'local'
+    limit = config.domain_limit 
+    return false if limit and domain_limit_match?(domain, limit)
+
+    true
+  end
+  
+  def has_config_repo?
+    ! (setting(:config_repository).nil? or setting(:config_repository) == '')
+  end
+  
+  def project_name_specified?
+    setting(:project) != 'unnamed_project'
+  end
+
+  def branch
+    (setting(:branch) and setting(:branch) != '') ? setting(:branch) : 'master'
+  end
+  
+  def config_branch
+    (setting(:config_branch) and setting(:config_branch) != '') ? setting(:config_branch) : 'master'
+  end
+  
+  def env_vars
+    return '' unless setting(:rake_env)
+    setting(:rake_env).keys.inject('') do |buffer,k| 
+      buffer += "#{k}='#{setting(:rake_env)[k]}' "
+      buffer
+    end
+  end
+  
+  def parent_path(path)
+    File.split(path).first
+  end
+  
+  def tail_path(path)
+    File.split(path).last
+  end
+  
+  def needs(*keys)
+    keys.each do |key|
+      raise "No value for '#{key}' declared in configuration files [#{config.configuration_file}]" unless setting(key)
+    end
+  end
+
+  def apply_staleness_check(commands)
+    needs(:deploy_to, :repository)
+    
+    check = "cd #{setting(:deploy_to)}; " +
+            "ml=\`git log -1 --pretty=format:%H\`; " +
+            "mr=\`git ls-remote #{setting(:repository)} refs/heads/#{branch}\`; "
+    
+    if setting(:deploy_config_to)
+      check += "cd #{setting(:deploy_config_to)}; " +
+               "cl=\`git log -1 --pretty=format:%H\`; " +
+               "cr=\`git ls-remote #{setting(:config_repository)} refs/heads/#{config_branch}\`; "
+    end
+    
+    check += "if [[ $ml != ${mr%%\t*} ]] " +
+             (setting(:deploy_config_to) ? "|| [[ $cl != ${cr%%\t*} ]]" : '') +
+             "; then #{commands}; else echo \"No changes to deploy.\"; fi"
+  end
+  
+  def join_commands
+    buffer.collect {|c| "{ #{c} ; }"}.join(' && ')
+  end
+  
+  def bundle
+    return '' if buffer.empty?
+    (staleness_checks_enabled? and check_staleness?) ? apply_staleness_check(join_commands) : join_commands
+  end
+
+  def domain_limit_match?(domain, limit)
+    domain.sub(%r{^.*@}, '') == limit
+  end
+  
+  def domain_of_interest?(domain)
+    return true unless limit = config.domain_limit
+    domain_limit_match?(domain, limit)
+  end
+  
+  def encode_roles(roles)
+    return '' unless roles and !roles.empty?
+    "export WD_ROLES='#{roles.join(':')}'; "
+  end
+
+  def build_command(domain, cmd)
+    "#{'set -x; ' if debugging?}" + encode_roles(domain['roles']) + cmd
+  end
+
+  def rake_command
+    (setting(:rake_command) and setting(:rake_command) != '') ? setting(:rake_command) : 'rake'
+  end
+  
+  def run(domain, cmd)
+    ssh(domain, cmd)
+  end
+
+  def ssh(domain, cmd)
+    args = []
+    args << domain['name']
+    args << '-v' if debugging?
+    args += domain['ssh_options'] if domain['ssh_options']
+    args << build_command(domain, cmd)
+
+    puts "Running: ssh #{args.join(' ')}" if debugging?
+    system('ssh', *args)
+  end
+  
+  def shell(domain, cmd)
+    puts "Running command locally: [#{cmd}]" if debugging?
+    system('bash', '-c', build_command(domain, cmd))
+  end
+  
+  def flush
+    needs(:domain)
+    setting(:domain).each do |domain|
+      next unless domain_of_interest?(domain['name'])
+      puts "Deploying #{domain['name']}..."
+      status = remote?(domain['name']) ? run(domain, bundle) : shell(domain, bundle)
+      record_result(domain['name'], status)
+    end
+  end
+  
+  def record_result(domain, status)
+    @results ||= []
+    @results << { 'domain' => domain, 'status' => status }
+  end
+
+  def summarize_results(results)
+    successes = failures = 0
+    results.each do |result|
+      puts "#{result['domain']} => #{result['status'] ? 'succeeded' : 'failed'}."
+      if result['status']
+        successes += 1 
+      else
+        failures += 1
+      end
+    end
+    [successes + failures, successes, failures]
+  end
+  
+  def summarize
+    puts "\nResults:"
+    if results and not results.empty?
+      total, successes, failures = summarize_results(results)
+      puts "Total: #{total} deployment#{total == 1 ? '' : 's'}, " +
+        "#{successes} success#{successes == 1 ? '' : 'es'}, " +
+        "#{failures} failure#{failures == 1 ? '' : 's'}."
+    else
+      puts "No deployments to report."
+    end       
+  end
+  
+  def success?
+    return true if !results or results.empty?
+    results.all? {|result| result['status'] }
+  end
+  
+  def if_file_present(path, cmd)
+    "if [ -e #{path} ]; then #{cmd}; fi"
+  end
+  
+  def if_task_defined(task, cmd)
+    %Q(rakep=`#{env_vars} #{rake_command} -P` && if [[ `echo "${rakep}" | grep #{task}` != "" ]]; then #{cmd}; fi )
+  end
+  
+  def safe_branch_checkout(path, my_branch)
+    %Q(cd #{path} && git checkout -b #{my_branch} origin/#{my_branch} || git checkout #{my_branch} origin/#{my_branch} || git checkout #{my_branch})
+  end
+  
+  def clone_repository(repo, path, my_branch)
+    enqueue "cd #{parent_path(path)}"
+    enqueue("if [ -e #{path} ]; then echo 'Repository already cloned to [#{path}].  Skipping.'; " +
+            "else git clone #{repo} #{tail_path(path)} && #{safe_branch_checkout(path, my_branch)}; fi")
+  end
+ 
+  def refresh_checkout(path, repo_branch)
+    enqueue "cd #{path}"
+    enqueue "git fetch origin +refs/heads/#{repo_branch}:refs/remotes/origin/#{repo_branch} #{'&>/dev/null' unless debugging?}"
+    enqueue "git checkout --force #{repo_branch} #{'&>/dev/null' unless debugging?}"
+    enqueue "git reset --hard origin/#{repo_branch} #{'&>/dev/null' unless debugging?}"
+  end
+
+  def run_rake_task(path, task_name)
+    enqueue "echo Running rake #{task_name}..."
+    enqueue "cd #{path}"
+    enqueue(if_file_present("#{setting(:deploy_to)}/Rakefile", 
+      if_task_defined(task_name, "#{env_vars} #{rake_command} #{'--trace' if debugging?} #{task_name} to=#{setting(:environment)}")))
+  end
+  
+  def build_path(path)
+    return path if path =~ %r{^/}
+    File.join(setting(:deploy_to), path)
+  end
+
+  def run_script(script)
+    return unless script
+    enqueue(%Q<cd #{setting(:deploy_to)}; echo "Running post script..."; #{env_vars} bash #{'-x' if debugging?} #{build_path(script)}>)
+  end
+
+  def ensure_main_parent_path_is_present
+    needs(:deploy_to)
+    enqueue "mkdir -p #{parent_path(setting(:deploy_to))}"
+  end
+  
+  def ensure_config_parent_path_is_present
+    needs(:deploy_config_to)
+    enqueue "mkdir -p #{parent_path(setting(:deploy_config_to))}"
+  end
+
+  def checkout_main_repository
+    needs(:deploy_to, :repository)
+    clone_repository(setting(:repository), setting(:deploy_to), branch)
+  end
+  
+  def checkout_configuration_repository
+    needs(:deploy_config_to, :config_repository)
+    clone_repository(setting(:config_repository), setting(:deploy_config_to), config_branch)
+  end
+  
+  def snapshot_git_revision
+    needs(:deploy_to)
+    enqueue "cd #{setting(:deploy_to)}"
+    enqueue %Q{ml=\`git log -1 --pretty=format:%H\`}
+  end
+  
+  def initialize_git_changes
+    needs(:deploy_to)
+    enqueue "rm -f #{setting(:deploy_to)}/.whiskey_disk_git_changes"
+    snapshot_git_revision
+  end
+  
+  def initialize_rsync_changes
+    needs(:deploy_to)
+    enqueue "rm -f #{setting(:deploy_to)}/.whiskey_disk_rsync_changes"
+  end
+  
+  def initialize_all_changes
+    needs(:deploy_to)
+    initialize_git_changes
+    initialize_rsync_changes
+  end
+  
+  def capture_git_changes
+    needs(:deploy_to)
+    enqueue "git diff --name-only ${ml}..HEAD > #{setting(:deploy_to)}/.whiskey_disk_git_changes"
+  end
+  
+  def update_main_repository_checkout
+    needs(:deploy_to)
+    initialize_git_changes
+    refresh_checkout(setting(:deploy_to), branch)
+    capture_git_changes
+  end
+  
+  def update_configuration_repository_checkout
+    needs(:deploy_config_to)
+    initialize_rsync_changes
+    refresh_checkout(setting(:deploy_config_to), config_branch)
+  end
+  
+  def refresh_configuration
+    needs(:deploy_to, :deploy_config_to)
+    raise "Must specify project name when using a configuration repository." unless project_name_specified?
+    enqueue "echo Rsyncing configuration..."
+    enqueue("rsync -a#{'v --progress' if debugging?} " + '--log-format="%t [%p] %i %n" ' +
+            "#{setting(:deploy_config_to)}/#{setting(:project)}/#{setting(:config_target)}/ #{setting(:deploy_to)}/ " + 
+            "> #{setting(:deploy_to)}/.whiskey_disk_rsync_changes")
+  end
+  
+  def run_post_setup_hooks
+    needs(:deploy_to)
+    run_script(setting(:post_setup_script))
+    run_rake_task(setting(:deploy_to), "deploy:post_setup")
+  end
+  
+  def run_post_deploy_hooks
+    needs(:deploy_to)
+    run_script(setting(:post_deploy_script))
+    run_rake_task(setting(:deploy_to), "deploy:post_deploy")
+  end
+end