k3_capistrano 1.0.0

data/lib/k3_capistrano/capistrano.rb

@@ -0,0 +1,132 @@
+# This is the main file that you should require. It loads everything except k3_capistrano/test_request.
+
+require 'k3_capistrano/version'
+require 'bundler/capistrano'
+require 'capistrano/ext/multistage'  # Note: This currently requires the capistrano-ext gem
+require 'capistrano_colors'
+
+#===================================================================================================
+Capistrano::Configuration::Namespaces::Namespace.class_eval do
+  def capture(*args)
+    parent.capture *args
+  end
+end
+
+Capistrano::Configuration.instance(:must_exist).load do
+
+#===================================================================================================
+
+set :scm,         'git'
+_cset :scm_verbose, false
+#set :git_shallow_clone, 1
+
+current_branch = `git symbolic-ref HEAD | sed s#^refs/heads/##`.chomp
+#current_branch = `git branch 2> /dev/null | sed -e '/^[^*]/d' -e 's/* \(.*\)/\1/'`.chomp
+_cset :branch, ENV['branch'] || current_branch
+puts "Warning: Deploying branch '#{branch}'" unless branch == 'master'
+
+# Deploying to a '/u/' directory seems silly. Set a more reasonable default here.
+if deploy_to.start_with? '/u/'
+  #_cset :deploy_to,  '~'
+  set :deploy_to,     "/apps/#{application}"
+end
+
+#set :deploy_via, :remote_cache
+_cset :deploy_via, :export
+_cset :rails_env,  'production'
+
+_cset :user, application
+_cset :use_sudo, false
+
+# We can always use this for the application user, even if user is unset.
+# Question: Where is app_user used?
+#_cset :app_user, user
+
+# Skip bundling cucumber and other non-production environments
+_cset :bundle_without, [:development, :test, :cucumber]
+
+# Get rid of annoying zlib error message at end of deploy.  I can't imagine compression is helping much...
+ssh_options[:compression] = "none"
+
+# Enable this if you run into problems with the bundle command hanging
+default_run_options[:pty] = true
+
+set :skip, ENV['skip'] ? ENV['skip'].split(',') : []
+
+#===================================================================================================
+# Stages
+
+_cset :stages, %w(staging production)
+_cset :default_stage, "staging"
+
+#===================================================================================================
+require 'rvm/capistrano'
+
+set :rvm_type, :system
+
+#===================================================================================================
+# Triggers
+
+before 'deploy',             'git:check_if_needs_push'
+before 'deploy:update_code', 'db:read_remote_my_cnf'
+before 'deploy:update_code', 'db:database_yml:setup'
+after  'deploy:update_code', 'deploy:ensure_db_is_set_up'
+after  'deploy:update_code', 'deploy:migrate' unless skip.include?('migrate') || ARGV.include?('deploy:update_code')
+
+#===================================================================================================
+# Problem: release_path is set to a new path each time you run cap, and usually that path doesn't exist.
+# It only exists after doing a deploy.
+# TODO: Can we detect whether it exists? Like this: capture("ls -1 #{release_path} #{current_path}").split.first
+# (Possibly not because release_or_current_path might get called too early and thus release_path would never exist by that point in the deploy.)
+if ARGV.include? 'deploy'
+  # If deploying, use release_path, which *will* become current_path, but will not be symlinked from current until the deploy is done
+  set :release_or_current_path, -> { release_path }
+else
+  # If not deploying, use current_path, which is the path of the last/current deploy. release_path wouldn't even exist or make sense in this case.
+  set :release_or_current_path, -> { current_path }
+end
+#puts %(release_or_current_path=#{(release_or_current_path).inspect})
+
+#===================================================================================================
+# Common tasks
+
+_cset :http_server,   'unicorn'    # Valid options: 'unicorn', 'passenger'
+require 'k3_capistrano/unicorn'
+
+require 'k3_capistrano/ensure_db_is_set_up'
+require 'k3_capistrano/db'
+require 'k3_capistrano/db_backups'
+require 'k3_capistrano/git'
+require 'k3_capistrano/ssh'
+require 'k3_capistrano/logs'
+require 'k3_capistrano/asset_pipeline'
+require 'k3_capistrano/test_results'
+
+task :uname do
+  run "uname -a"
+end
+
+#===================================================================================================
+
+def random_password
+  alphanumerics = [('0'..'9'),('A'..'Z'),('a'..'z')].map {|range| range.to_a}.flatten
+  (0...25).map { alphanumerics[Kernel.rand(alphanumerics.size)] }.join
+end
+
+def exec_verbose(command)
+  puts command
+  exec command
+end
+
+def system_verbose(command)
+  puts command
+  system command
+end
+
+def beep
+  STDOUT.print "#{7.chr}"
+end
+
+#===================================================================================================
+
+end

data/lib/k3_capistrano/chef.rb

@@ -0,0 +1,41 @@
+# Chef integration
+# Reuses as much configuration data about the app from the app data bag as possible, so that the
+# app's configuration is only defined once, in one canonical location.
+
+gem 'capistrano-ext'
+gem 'tylerrick-chef'
+gem 'capistrano-chef'
+
+Capistrano::Configuration.instance(:must_exist).load do
+
+require 'chef'
+ENV['chef_home'] or abort("Please set the chef_home environment variable to point to your copy of K3's deployment/kitchen/")
+Chef::Config[:data_bag_path] = ENV['chef_home'] + '/data_bags'
+Chef::Config[:solo] = true
+require 'capistrano/chef'
+set_from_data_bag
+
+# Note: If you require 'rvm/capistrano', make sure you do that *after* requiring
+# 'k3_capistrano/chef' because the 'rvm' namespace that 'rvm/capistrano' creates would shadow the
+# 'rvm' variables that set_from_data_bag would create.
+
+# Some variables from the data bag have different options for staging/production, so we set the
+# corresponding cap variables here but since we know whether we're on staging/production, we only
+# grab the appropriate value for the server we're deploying to instead of the full hash...
+set :rails_env_from_data_bag, rails_env; unset :rails_env
+_cset :domain,   ->{ server_name[stage.to_s]             or server_name             or raise("server_name should have been set from app data bag but was not defined") }
+set :database,   ->{ databases[stage.to_s]                                          or raise("databases have been set from app data bag but was not defined") }
+_cset :rails_env,->{ rails_env_from_data_bag[stage.to_s] or rails_env_from_data_bag or raise("rails_env have been set from app data bag but was not defined") }
+
+# If we wanted to set the revision/branch from the data_bag as well, we could do something like this...
+#set :revision_from_data_bag, revision; unset :revision
+#set :revision, ->{ revision_from_data_bag[stage.to_s] or revision_from_data_bag or raise("revision was not defined") }
+
+set :rvm_type, :system
+rvm['ruby_string'] or fail("rvm['ruby_string'] was nil. Please make sure you aren't requiring 'rvm/capistrano' before 'k3/capistrano'. rvm=#{rvm.inspect}")
+set :rvm_ruby_string, rvm['ruby_string']
+set :user, owner
+
+set :repository, "#{customer}@git.k3integrations.com:#{application}.git"
+
+end

data/lib/k3_capistrano/db.rb

@@ -0,0 +1,72 @@
+require 'tempfile'
+
+Capistrano::Configuration.instance(:must_exist).load do
+
+_cset :database, {
+  'adapter'   => "mysql2",
+  'host'      => "localhost",
+  'database'  => application,
+  'username'  => application,
+  'reconnect' => "true",
+  'encoding'  => "utf8",
+}
+
+namespace :db do
+  # Adapted from deploy:migrate task in capistrano-2.12.0/lib/capistrano/recipes/deploy.rb
+  desc <<-DESC
+    Run the db:reset rake task. By default, it runs this in most recently \
+    deployed version of the app. However, you can specify a different release \
+    via the migrate_target variable, which must be one of :latest (for the \
+    default behavior), or :current (for the release indicated by the \
+    `current' symlink). Strings will work for those values instead of symbols, \
+    too. You can also specify additional environment variables to pass to rake \
+    via the migrate_env variable. Finally, you can specify the full path to the \
+    rake executable by setting the rake variable. The defaults are:
+
+    (Note: We should actually be using `rake db:setup` instead of `rake db:reset` but `rake db:setup`
+    currently fails with "db/schema.rb doesn't exist yet" error due to
+    https://github.com/rails/rails/issues/4772. The main difference is that `rake db:reset` does a drop
+    first.)
+
+
+      set :rake,           "rake"
+      set :rails_env,      "production"
+      set :migrate_env,    ""
+      set :migrate_target, :latest
+  DESC
+  task :setup, :roles => :db, :only => { :primary => true } do
+    rake = fetch(:rake, "rake")
+    rails_env = fetch(:rails_env, "production")
+    migrate_env = fetch(:migrate_env, "")
+    migrate_target = fetch(:migrate_target, :latest)
+
+    directory = case migrate_target.to_sym
+      when :current then current_path
+      when :latest  then latest_release
+      else raise ArgumentError, "unknown migration target #{migrate_target.inspect}"
+      end
+
+    run "cd #{directory} && #{rake} RAILS_ENV=#{rails_env} #{migrate_env} db:reset"
+  end
+
+  task :read_remote_my_cnf, :roles => :app do
+    path = "#{deploy_to}/.my.cnf"
+    raw = capture("cat #{path}")
+    config = nil
+    Tempfile.new('remote_my_cnf').tap do |file|
+      file.write raw
+      file.close
+      config = ParseConfig.new(file.path)
+    end
+    (config.params['client']['password'] rescue nil) or raise "Couldn't find password in #{path}"
+    set :my_cnf, config.params
+  end
+
+  task :list_tables, on_error: :continue do
+    tables = capture(%(echo "show tables" | mysql #{database['database']})).cleanlines
+    #puts %(tables=#{tables.to_a.inspect})
+    tables
+  end
+end
+
+end

data/lib/k3_capistrano/db_backups.rb

@@ -0,0 +1,207 @@
+# Database backup/restore
+
+Capistrano::Configuration.instance(:must_exist).load do
+
+_cset :db_backups_dir, ->{ "#{shared_path}/db_backups" }
+
+# Lets us expand ~'s so sftp won't complain
+set :home_dir, -> { capture('echo $HOME') }
+
+namespace :backup do
+
+  desc "Back up the database"
+  task :create, :roles => :db, :only => { :primary => true }, on_error: :continue do
+    run "#{shared_path}/script/back_up_db"
+    backup.locate_last
+  end
+
+  task :list do
+    run("ls -t #{db_backups_dir}")
+  end
+  task :_list do
+    capture("ls -1t #{db_backups_dir}")
+  end
+
+  desc "Locate the latest backup"
+  task :locate_last do
+    last = "#{db_backups_dir}/" + backup._list.cleanlines.grep(/.*\.sql/).first
+    backup_details = capture("ls -alh #{last}")
+    puts ' '*6 + "Last backup is: #{last}"
+    puts ' '*6 + "#{backup_details}"
+    last
+  end
+
+  desc "Download the latest backup"
+  task :download do
+    set :which_backup, backup.locate_last unless exists?(:which_backup)
+    which_backup = capture("echo #{which_backup()}").chomp # expand ~'s so sftp won't complain
+    get which_backup, "#{local_dir}/#{File.basename(which_backup)}"
+  end
+
+  desc <<-End
+    Upload a database backup file to the remote host. Specify filename with file=.
+  End
+  task :upload_backup, :roles => :db do
+    file = ENV['file'] or raise(ArgumentError, "file must be specified")
+    upload file db_backups_dir.gsub('~', home_dir)
+  end
+
+  desc <<-End
+    Restore a database backup file.
+    The backup to restore must be specified by the which_backup variable and must already exist on the server.
+  End
+  task :restore, :roles => :db do
+    fetch(:which_backup) or raise(ArgumentError, "which_backup must be specified")
+    run "mysql #{application} < #{which_backup}"
+  end
+end
+
+#_cset :local_dir, ->{ "#{File.dirname(__FILE__)}/../" + "db/backups/production" }
+#namespace :backup do
+#  # Tasks for downloading a database snapshot from production and loading into localhost DB.
+#  namespace :local do
+#
+#    desc "Create a new backup/snapshot of production database and populate dev database with it."
+#    task :from_new_backup, :roles => :db do
+#      puts ' '*6 + 'Current path: ' + current_path
+#      backup.create
+#      local.from_latest_backup
+#    end
+#
+#    desc "Populate the database with contents of your latest backup on the server."
+#    task :from_latest_backup, :roles => :db do
+#      set :which_backup, backup.locate_last
+#      from_backup
+#    end
+#
+#    desc "Populate the database with contents of a backup (database dump)."
+#    task :from_backup, :roles => :db do
+#      local_backup_file = Dir["#{local_dir}/#{File.basename(which_backup)}"].to_a.first
+#      puts "Populating local database from '#{local_backup_file}'..."
+#      if local_backup_file
+#        # Note: rails db seems to ignore file input specified with <
+#        #command = "rails db -p < #{local_backup_file}"
+#        # so I just made a ./script/dbconsole file instead which invokes mysql client directly.
+#        command = "./script/dbconsole < #{local_backup_file}"
+#        puts command; system command
+#      else
+#        backup.download
+#        # Try again now (recursive) that we've downloaded the file
+#        local.from_backup
+#      end
+#    end
+#  end
+#end
+
+#---------------------------------------------------------------------------------------------------
+# Taps (simple database import/export app)
+_cset :local_rails_env, ->{ ENV['RAILS_ENV'] || 'development' }
+
+=begin
+Ideas for improvement:
+
+Consider using https://github.com/hsume2/cap-taffy, which tries to solve the same problem.
+They really do pretty much the same thing in a pretty similar way, but cap-taffy is a bit more advanced.
+
+Advantages of cap-taffy:
+* It starts a server and client with a single command (cap db:pull)
+
+Advantages of current system:
+* Zero-configuration. Reads all the connection info it needs from chef data bag, database.yml, etc.
+  cap-taffy appears to read database.yml for you, but does not parse it with eRuby and doesn't know about chef. 
+* uses bundle exec
+
+Idea: Fork cap-taff. Refactor it to allow us to reuse most of it and simply pass connection string
+(built from chef vars) and let cap-taffy handle the rest. It looks like we should be able to set
+@remote_database_url somehow.
+
+See https://github.com/hsume2/cap-taffy/blob/master/lib/cap-taffy/db.rb for the meat of cap-taffy.
+=end
+namespace :db do
+  desc <<-End
+    Start up a taps server on the server to let you pull the data from production database over to another host (such as localhost, for debugging with production data)
+
+    This uses your database config from your app's data bag item and password from .my.cnf to connect to the database.
+
+    You need to add 'taps' gem to your Gemfile and deploy your app first.
+
+    Then run:
+    [window 1] cap production db:taps_server
+    [window 2] cap production db:pull
+
+    You can also pull from one server to another:
+    [window 1] cap production db:taps_server
+    [window 2] cap staging db:pull_from_production
+
+    See https://github.com/ricardochimal/taps for more info.
+  End
+  task :taps_server, roles: :db do
+    # TODO: write pw to tmp/
+    http_user, http_password = application, random_password
+    File.open('tmp/taps_password', 'w') {|f| f.puts http_password }
+    run "cd #{current_path} && bundle exec taps server #{database['adapter']}://#{my_cnf['client']['user']}:#{my_cnf['client']['password']}@localhost/#{application}?encoding=#{database['encoding']} #{http_user} #{http_password}"
+  end
+
+  desc <<-End
+    Pull data from remote database to your local app database on localhost.
+
+    You'll need to start the taps server first with 'cap {stage} db:taps_server' and leave that running in another window.
+
+    This will connect to the host named by the domain variable (via an ssh tunnel since port 5000 is unlikely to be open), which depends on whether you run 'cap staging db:pull' or 'cap production db:pull'. This is where it will 'pull' the data from.
+
+    It will connect to your local app database using the connection info in your local config/database.yml. This is where the data gets pulled *to*.
+  End
+  task :pull do
+    http_user, http_password = application, ENV['password'] || File.read('tmp/taps_password').chomp
+    http_password.to_s.length > 1 or raise("could not read password")
+    ActiveRecord::Base.configurations = YAML::load(Erubis::Eruby.new(File.read('config/database.yml')).result)
+    configs = ActiveRecord::Base.configurations
+    database = configs[local_rails_env]
+    fork do
+      puts "Creating ssh tunnel..."
+      exec_verbose "ssh -N -L5000:127.0.0.1:5000 #{domain}"
+    end
+    system_verbose "bundle exec taps pull #{database['adapter']}://#{database['username']}:#{database['password']}@localhost/#{database['database']}?encoding=#{database['encoding']} " +
+      "http://#{http_user}:#{http_password}@localhost:5000"
+    puts "Done. Ctrl-C to exit ssh."
+    Process.wait
+  end
+
+  desc <<-End
+    Pull data from production database to staging database.
+
+    Usage: cap staging db:pull_to_staging
+  End
+  task :pull_to_staging do
+    production_domain = server_name['production']
+
+    http_user, http_password = application, ENV['password'] || File.read('tmp/taps_password').chomp
+    http_password.to_s.length > 1 or raise("could not read password")
+    remote_database_yml = capture("cat #{current_path}/config/database.yml")
+
+    ActiveRecord::Base.configurations = YAML::load(Erubis::Eruby.new(remote_database_yml).result)
+    configs = ActiveRecord::Base.configurations
+    database = configs[rails_env]
+
+    fork do
+      puts "Creating ssh tunnel..."
+      #run "ssh -N -L5000:127.0.0.1:5000 #{production_domain}"
+      exec_verbose "ssh #{domain} 'ssh -N -L5000:127.0.0.1:5000 #{production_domain}'"
+      # Note: If you get this error "Host key verification failed." then manually ssh to staging and
+      # from there to production (answer yes when prompted) so that the host key gets saved.
+    end
+    sleep 3  # TODO: read ssh output until it's ready
+    run "cd #{current_path} && bundle exec taps pull #{database['adapter']}://#{database['username']}:#{database['password']}@localhost/#{database['database']}?encoding=#{database['encoding']} " +
+      "http://#{http_user}:#{http_password}@localhost:5000"
+    puts "Done. Ctrl-C to exit ssh."
+    Process.wait
+    # For some reason the ssh processes seem to stick around, so just to make sure we clean up,
+    # let's try to kill them.
+    run 'killall ssh'
+  end
+end
+
+before 'db:taps_server', 'db:read_remote_my_cnf'
+before 'deploy:migrate', 'backup:create'
+
+end