k3_capistrano 1.0.0

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in k3_capistrano.gemspec
+gemspec

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/Readme.md

@@ -0,0 +1,86 @@
+k3_capistrano
+=============
+
+The k3_capistrano gem sets some reasonable defaults so that your app's config/deploy* files can be
+very minimal and ideally only need to include any app-specific configuration.
+
+It also adds some additional deploy tasks such as deploy:test_request (see below for a list).
+
+Getting started
+===============
+
+Add k3_capistrano to your Gemfile and add something like this to your config/deploy.rb:
+
+    set :application,   'app_name'
+    set :repository,    "user@server:#{application}.git"
+
+    require 'k3_capistrano/capistrano'
+
+Add your staging/production -server-specific config in config/deploy/staging.rb /
+config/deploy/production.rb, respectively. For example:
+
+    set :domain,        'my.domain.org'
+
+To deploy, you just do the usual:
+
+    cap deploy  # defaults to staging
+
+or to deploy to production:
+
+    cap production deploy
+
+If your app needs to, it can override any of the defaults, for example these:
+
+    set :deploy_to,     "/apps/#{application}"
+    set :branch,        'master'     # The git branch to deploy. Could also be set in config/deploy/#{stage}.rb
+
+Integrating with Chef
+=====================
+
+If you are using Chef and would like capistrano to simply use the configuration that you've already
+defined for this app using Chef, you can follow this alternate instructions to avoid duplicating
+that configuration data in two places...
+
+Set the chef_home environment variable to point to your copy of K3's deployment/kitchen/.
+
+Just add a line like this to your .bashrc and source your .bashrc:
+
+    export chef_home=/path/to/deployment/kitchen/
+
+Add k3_capistrano to your Gemfile and add something like this to your config/deploy.rb:
+
+    set :application,   'app_name'
+    require 'k3_capistrano/chef'
+    require 'k3_capistrano/capistrano'
+
+
+Tasks provided by k3_capistrano
+===============================
+
+cap backup:create                     # Back up the database
+cap backup:download                   # Download the latest backup
+cap backup:locate_last                # Locate the latest backup
+cap backup:restore                    # Restore a database backup file.
+cap backup:upload_backup              # Upload a database backup file to the ...
+cap db:database_yml:setup             # Creates the database.yml file under t...
+cap db:pull                           # Pull data from remote database to you...
+cap db:pull_to_staging                # Pull data from production database to...
+cap db:setup                          # Run the db:setup rake task.
+cap db:taps_server                    # This uses your database config from y...
+cap deploy:test_request               # Does a test request to the web site s...
+cap git:check_if_needs_push           # Checks to see if your branch is ahead...
+cap git:push                          # Pushes your local commits to the git ...
+cap logs:tail                         # tail all 4 app log files to help you ...
+cap logs:watch                        # tail -f the production.log file
+
+Skipping steps
+==============
+
+This deploy script (and its before/after hooks) will take care of doing everything, including
+running db:migrate and precompiling your assets.
+
+Since some of those steps aren't necessary for every signle deploy and can make your deploy take
+longer than it needs to, you can skip some of those steps by setting the skip environment variable
+when you run cap deploy. For example, to skip the migrate and assets steps, just run this:
+
+    skip=assets,migrate cap production deploy

data/k3_capistrano.gemspec

@@ -0,0 +1,30 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "k3_capistrano/version"
+
+Gem::Specification.new do |s|
+  s.name        = "k3_capistrano"
+  s.version     = K3Capistrano.version
+  s.authors     = ["Tyler Rick"]
+  s.email       = ["tyler@k3integrations.com"]
+  s.homepage    = ""
+  s.summary     = %q{Common Capistrano tasks and defaults used by K3 Integrations}
+  s.description = s.summary
+
+  s.rubyforge_project = "k3_capistrano"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_runtime_dependency 'parseconfig'
+  s.add_runtime_dependency 'capistrano-ext'
+  s.add_runtime_dependency 'capistrano_colors'
+  s.add_runtime_dependency 'facets'
+  s.add_runtime_dependency 'rvm-capistrano'
+  s.add_runtime_dependency 'colored'
+
+  #s.add_runtime_dependency 'tylerrick-chef'
+  #s.add_runtime_dependency 'capistrano-chef'
+end

data/lib/capistrano_database_yml.rb

@@ -0,0 +1,142 @@
+# 
+# = Capistrano database.yml task
+#
+# Provides a couple of tasks for creating the database.yml 
+# configuration file dynamically when deploy:setup is run.
+#
+# Category::    Capistrano
+# Package::     Database
+# Author::      Simone Carletti <weppos@weppos.net>
+# Copyright::   2007-2010 The Authors
+# License::     MIT License
+# Link::        http://www.simonecarletti.com/
+# Source::      http://gist.github.com/2769
+#
+#
+# == Requirements
+#
+# This extension requires the original <tt>config/database.yml</tt> to be excluded
+# from source code checkout. You can easily accomplish this by renaming
+# the file (for example to database.example.yml) and appending <tt>database.yml</tt>
+# value to your SCM ignore list.
+#
+#   # Example for Subversion
+#
+#   $ svn mv config/database.yml config/database.example.yml
+#   $ svn propset svn:ignore 'database.yml' config
+#
+#   # Example for Git
+#
+#   $ git mv config/database.yml config/database.example.yml
+#   $ echo 'config/database.yml' >> .gitignore 
+#
+# 
+# == Usage
+# 
+# Include this file in your <tt>deploy.rb</tt> configuration file.
+# Assuming you saved this recipe as capistrano_database_yml.rb:
+# 
+#   require "capistrano_database_yml"
+# 
+# Now, when <tt>deploy:setup</tt> is called, this script will automatically
+# create the <tt>database.yml</tt> file in the shared folder.
+# Each time you run a deploy, this script will also create a symlink
+# from your application <tt>config/database.yml</tt> pointing to the shared configuration file. 
+# 
+# == Custom template
+# 
+# By default, this script creates an exact copy of the default
+# <tt>database.yml</tt> file shipped with a new Rails 2.x application.
+# If you want to overwrite the default template, simply create a custom Erb template
+# called <tt>database.yml.erb</tt> and save it into <tt>config/deploy</tt> folder.
+# 
+# Although the name of the file can't be changed, you can customize the directory
+# where it is stored defining a variable called <tt>:template_dir</tt>.
+# 
+#   # store your custom template at foo/bar/database.yml.erb
+#   set :template_dir, "foo/bar"
+# 
+#   # example of database template
+#   
+#   base: &base
+#     adapter: sqlite3
+#     timeout: 5000
+#   development:
+#     database: #{shared_path}/db/development.sqlite3
+#     <<: *base
+#   test:
+#     database: #{shared_path}/db/test.sqlite3
+#     <<: *base
+#   production:
+#     adapter: mysql
+#     database: #{application}_production
+#     username: #{user}
+#     password: #{Capistrano::CLI.ui.ask("Enter MySQL database password: ")}
+#     encoding: utf8
+#     timeout: 5000
+#
+# Because this is an Erb template, you can place variables and Ruby scripts
+# within the file.
+# For instance, the template above takes advantage of Capistrano CLI
+# to ask for a MySQL database password instead of hard coding it into the template.
+#
+# === Password prompt
+#
+# For security reasons, in the example below the password is not
+# hard coded (or stored in a variable) but asked on setup.
+# I don't like to store passwords in files under version control
+# because they will live forever in your history.
+# This is why I use the Capistrano::CLI utility.
+#
+
+unless Capistrano::Configuration.respond_to?(:instance)
+  abort "This extension requires Capistrano 2"
+end
+
+require 'pathname'
+
+Capistrano::Configuration.instance.load do
+
+  namespace :db do
+    namespace :database_yml do
+
+      desc <<-DESC
+        Creates the database.yml file under the shared dir on the server.
+        
+        During a deploy, the db:database_yml:symlink task will symlink from config/database.yml to
+        the shared file.
+
+        By default, this task uses the default template found in lib/templates/database.yml.erb of
+        the k3_capistrano gem unless a template called database.yml.erb is found either in your
+        app's config/deploy directory.
+      DESC
+      task :setup, :except => { :no_release => true } do
+        upload_to = "#{shared_path}/config/database.yml"
+
+        default_template = Pathname.new(__FILE__).dirname + 'templates/database.yml.erb'
+
+        override_template = Pathname.new(fetch(:template_dir, "config/deploy")) + 'database.yml.erb'
+        template = override_template.file? ? override_template.read : default_template.read
+
+        config = ERB.new(template).result(binding)
+        puts "Installing this database.yml to #{upload_to}:\n#{config}"
+
+        run "mkdir -p #{shared_path}/db" 
+        run "mkdir -p #{shared_path}/config" 
+        put config, upload_to
+      end
+
+      desc <<-DESC
+        [internal] Updates the symlink for database.yml file to the just deployed release.
+      DESC
+      task :symlink, :except => { :no_release => true } do
+        run "ln -nfs #{shared_path}/config/database.yml #{release_path}/config/database.yml" 
+      end
+
+    end
+  end
+
+  before 'db:database_yml:setup', 'db:read_remote_my_cnf'  # lib/templates/database.yml.erb gets password from my_cnf
+  after "deploy:setup",           "db:database_yml:setup"   unless fetch(:skip_db_setup, false)
+  after "deploy:finalize_update", "db:database_yml:symlink"
+end

data/lib/k3/capistrano.rb

@@ -0,0 +1,2 @@
+# This file is here for backwards compatibility. You should just require the following file instead:
+require 'k3_capistrano/capistrano'

data/lib/k3_capistrano.rb

@@ -0,0 +1,5 @@
+# This file will be automatically required if the k3_capistrano gem is in your
+# Gemfile, so we don't want to put anything Capistrano-dependent here in case
+# they aren't loading/running cap.
+#
+# To actually load this into your config/deploy.rb, you need to explicitly require 'k3_capistrano/capistrano' instead.

data/lib/k3_capistrano/asset_pipeline.rb

@@ -0,0 +1,316 @@
+Capistrano::Configuration.instance(:must_exist).load do
+
+# From Rails 3.1 and later, we should assume they want to use the asset pipeline. Can be disabled by setting to false.
+_cset :asset_pipeline, true
+
+if asset_pipeline
+#===================================================================================================
+# Standard recipe
+
+load 'deploy/assets'
+# The assets recipes that are included with capistrano work like this:
+# 1. deploy:assets:symlink creates a symlink from #{latest_release}/public/assets to #{shared_path}/assets
+# 2. Runs rake assets:precompile on the server
+#
+# We will make use of #1, but override #2 as needed to also allow assets to precompiled locally on
+# the developer's localhost.
+
+#===================================================================================================
+# More libraries and requires
+
+require 'pp'
+require 'facets/class/to_proc'
+require 'facets/string/cleanlines'
+require 'colored'
+
+# We can just get the digests from the manifest.yml instead of recomputing them!
+# Also, when I had it recomputing the digest of every file, it would give false positives for a
+# bunch of .gz files. Apparently, gzipping the same file twice can produce a file with a different
+# digest each time? Anyway, if the source file that the .gz file was produced to hasn't changed, we
+# don't care if the .gz file itself is not identical.
+#Pathname.class_eval do
+#  def digest
+#    Digest::MD5.file(@path).hexdigest
+#  end
+#end
+class K3Capistrano::AssetManifest
+  attr_reader :digests
+  def initialize(manifest_file)
+    @manifest_file = manifest_file
+    @digests = YAML.load_file(manifest_file)
+  end
+
+  # Compare to digest_for actionpack-3.2.11/lib/sprockets/helpers/rails_helper.rb
+  # We could just reuse that, if we were okay with it loading the Rails application (in order to get
+  # Rails.application.config.assets)
+  def digest_for!(logical_path)
+    digest_for(logical_path) or (raise "#{logical_path} not found in #{@manifest_file}" if logical_path =~ /\.js$|\.css$/)
+  end
+
+  def digest_for(logical_path)
+    #puts %(@digests[#{logical_path}]=#{(@digests[logical_path]).inspect})
+    @digests[logical_path]
+  end
+
+  def to_s
+    @manifest_file.to_s
+  end
+end
+
+#===================================================================================================
+# Variables
+
+_cset :precompile_assets_locally, false
+# The following options only apply if precompile_assets_locally is true:
+(
+  # Set skip to include 'local_precompile' to skip running assets:precompile locally during this deploy.
+  # You may want to do this to speed up the deploy if you know that your local assets directory is
+  # already current -- for example, if you just deployed to staging (and did the precompile during
+  # that deploy) and now you are ready to deploy the exact same code to production.
+
+  # Can also set to :previous_local
+  #_cset :compare_to_manifest_from, :remote
+
+  # If true, keeps a local public/assets; if false, renames it immediately after running precompile
+  # It is recommended to set this to false, because if a public/assets dir exists in development mode,
+  # changes to your assets files will *not* take effect until you run precompile, whereas if there is
+  # *no* public/assets dir, changes will be compiled automatically as needed and take immediately
+  # after changing them.
+  set :keep_public_assets_dir, false
+
+  # TODO: How many backups of public/assets to keep
+  set :local_asset_dirs_to_keep, 5
+
+  # Set to false if you don't want it to show the list of changed assets during a deploy.
+  set :list_changed_assets_during_deploy, true
+)
+
+#===================================================================================================
+namespace :deploy do
+namespace :assets do
+
+  # If you already know that no changes have been made to your assets since your last deploy, you
+  # can speed up the deploy by several minutes by skipping the precompilation step and just reusing
+  # the assets in shared/assets that were used by the previous deploy.
+  # 
+  # Usage: skip=assets cap deploy
+  #
+  if skip.include?('assets')
+    task :precompile, :roles => :app, :except => { :no_release => false } do
+      # Do nothing. 
+      puts "Warning: Skipping the 'deploy:assets:precompile' task. Make sure the assets in shared/assets are current."
+    end
+
+  else
+    if precompile_assets_locally
+      # This strategy precompiles your assets locally and then copies the files to the remote server.
+      # This way you can deploy to multiple servers in a row without having to recompile the assets each time,
+      # and you can redeploy to the same server without recompiling as long as no assets have changed.
+      # You must set :precompile_assets_locally, true if you want to opt in to using this.
+      #
+      # Overview:
+      # * Downloads current manifest from shared/assets/manifest.yml on server.
+      # * For each file in local public/assets dir (or most recent timestamped dir):
+      #   * If local file not present in currently deployed manifest, upload it
+      #   * If local file has different digest, upload it
+      # Do incremental update instead of rsyncing all
+
+      # List all asset dirs (dirs matching public/assets.*), sorted by filename, except always returning
+      # public/assets last if it exists.
+      def local_asset_dirs
+        (
+          Dir["public/assets.*"].map(&Pathname).sort +
+          [Pathname.new('public/assets')]
+        ).select(&:directory?).select(&:exist?).to_a
+      end
+
+      task :list_local_asset_dirs do
+        puts local_asset_dirs
+      end
+
+      def changed_from(previous_file_list, previous_manifest, verbose = false)
+        current_dir = local_asset_dirs.last or raise "no local public/assets* dirs exist"
+        current_manifest = K3Capistrano::AssetManifest.new(current_dir + 'manifest.yml')
+
+        if verbose
+          puts "Listing changes between '#{previous_manifest}' (previous) and '#{current_manifest}' (latest local precompile, at '#{current_dir}')..."
+          puts "Legend: " + "doesn't exist (in dir of '#{previous_manifest}')".cyan + ' | ' + "exists but has changed since '#{previous_manifest}'".yellow
+        end
+
+        descendants = []; current_dir.find {|_| descendants << _ if _.file? }
+        descendants.select do |file_in_current|
+          base_name = file_in_current.relative_path_from(current_dir).to_s.gsub(%r<^../>, '')
+          logical_path = base_name.gsub(/-\w{32}/, '').gsub(/\.gz$/, '')
+          #file_in_previous = previous_dir + base_name if previous_dir
+
+          # Only if the file *exists* in and is in the manifest can we skip it.
+          # This is a safeguard against the case where the manifest.yml file gets uploaded but the
+          # files themselves haven't yet.
+
+          file_in_previous = previous_file_list.include?(base_name) && previous_manifest.digest_for(logical_path)
+          changed =
+            if file_in_previous #&& file_in_previous.exist?
+              if previous_manifest.digest_for!(logical_path) == current_manifest.digest_for!(logical_path)
+                color = :green
+                false
+              else
+                color = :yellow
+                true
+              end
+            else
+              color = :cyan
+              true
+            end
+          puts file_in_current.to_s.send(color) if verbose && changed
+          changed
+        end.map do |_|
+          _.relative_path_from(current_dir).to_s.gsub(%r<^../>, '')
+        end
+      end
+
+      def changed_since_previous_local_precompile(verbose = false)
+        # TODO: It is also possible for there to be no previous assets dir, in which case it should
+        # list *all* files in latest_dir as being new files.
+        previous_dir = local_asset_dirs[-2]
+        previous_file_list = []; previous_dir.find {|_| previous_file_list << _ if _.file? }
+        latest_dir = local_asset_dirs.last
+        puts "Listing changes between '#{previous_dir}' and latest local precompile, '#{latest_dir}'..." if verbose
+        previous_manifest = K3Capistrano::AssetManifest.new(previous_dir + 'manifest.yml')
+        changed_from(previous_file_list, previous_manifest, verbose)
+      end
+
+      def remote_file_list
+        capture("cd #{shared_path}/assets && find . -type f").
+          cleanlines.map {|_| _.gsub(%r<^\./>, '') }
+      end
+
+      def changed_since_last_deploy(verbose = false)
+        remote_manifest = 'tmp/manifest_from_last_deploy.yml'
+        get "#{shared_path}/assets/manifest.yml", remote_manifest, via: :scp
+        remote_manifest = K3Capistrano::AssetManifest.new(remote_manifest)
+        remote_file_list = remote_file_list()
+        Pathname('tmp/asset_file_from_last_deploy.txt').tap do |file|
+          puts %(Saving list of files from server at: #{file})
+          file.open('w') {|_| _.puts remote_file_list.join("\n" )}
+        end
+        puts "Listing changes between latest deploy and latest local precompile..." if verbose
+        changed_from(remote_file_list, remote_manifest, verbose)
+      end
+
+      def changed_assets(verbose = false)
+       #case compare_to_manifest_from
+       #when :remote
+          changed_since_last_deploy(verbose)
+       #when :previous_local
+       #  changed_since_previous_local_precompile(verbose)
+       #else
+       #  raise "unrecognized value for compare_to_manifest_from: #{compare_to_manifest_from.inspect}"
+       #end
+      end
+
+      task :list_changed_since_previous_local_precompile do
+        puts changed_since_previous_local_precompile(verbose = true)
+      end
+
+      task :list_changed_assets do
+        changed_assets(verbose = true)
+      end
+
+      task :list_changed_since_last_deploy do
+        changed_since_last_deploy(verbose = true)
+      end
+
+      task :push_changed_assets do
+        current_dir = local_asset_dirs.last or raise "no local public/assets* dirs exist"
+        changed_files = changed_assets(verbose = list_changed_assets_during_deploy)
+
+        #remote_host_and_path = "#{user}@#{domain}:#{release_or_current_path.gsub('~/', './')}/public/assets/"
+        remote_host_and_path = "#{user}@#{domain}:#{shared_path}/assets/"
+
+        changed_files_in_root, changed_files_in_subdir = changed_files.partition {|_|
+          Pathname.new(_).dirname.to_s == '.'
+        }
+        if changed_files_in_root.any?
+          # TODO: shell escape arguments
+          command = "time scp " +
+                    "#{changed_files_in_root.map {|_| "#{current_dir}/#{_}" }.join(' ')} " +
+                    "#{remote_host_and_path}"
+          #command = "rsync -a --itemize-changes --recursive --compress --times #{changed_files} #{remote_host_and_path}"
+          run_locally command
+
+          # See also this version from https://groups.google.com/forum/#!topic/capistrano/cuOeI-aNLfo
+          #      servers = find_servers_for_task(current_task) 
+          #      port_option = port ? " -e 'ssh -p #{port}' " : '' 
+          #      servers.each do |server| 
+          #        run_locally("rsync --recursive --times --rsh=ssh --compress --human-readable #{port_option} --progress public/assets #{user} @#{server}:#{shared_path}") 
+          #      end 
+        end
+
+        remote_dir_list = remote_file_list().map {|_| Pathname(_).dirname }.uniq
+        changed_files_in_subdir.each do |_|
+          # mkdir subdir if it doesn't exist on server. (Check in remote_file_list so we don't
+          # have to hit the server to check each file individually.)
+          dir = Pathname(_).dirname
+          unless remote_dir_list.include? dir
+            remote_dir_list << dir
+            run "mkdir -p #{shared_path}/assets/#{dir}"
+          end
+
+          #put "#{current_dir}/#{_}", "#{shared_path}/assets/#{_}", via: :scp
+          command = "scp #{"#{current_dir}/#{_}"} #{remote_host_and_path}#{_}"
+          run_locally command
+        end
+      end
+
+      def skip_local_precompile?
+        skip.include?('local_precompile')
+      end
+
+      def local_asset_current?
+        # TODO: Compare time of last commit to app/assets or vendor/assets with time of last local
+        # precompile. Return true if there have been any changes since the last precompile that
+        # wouldn't be included in the last precompile.
+        #
+        #local_asset_dirs.any?
+        skip_local_precompile?
+      end
+
+      task :precompile_locally do
+        run_locally "time bundle exec rake assets:precompile RAILS_ENV=production RAILS_GROUPS=assets"
+        after_local_precompile
+      end
+
+      task :after_local_precompile do
+        unless keep_public_assets_dir
+          latest_dir = "public/assets.#{Time.now.strftime("%Y-%m-%dT%H.%M.%S")}"
+          # Move them out of the way locally, so that in development it won't try to use (possibly stale) assets from public/assets instead of dynamically generating them
+          run_locally "mv public/assets #{latest_dir}"
+        end
+      end
+
+      task :precompile, :roles => :app, :except => { :no_release => false } do
+        if local_asset_current?
+          # We'll just reuse the assets that we've already previously precompiled locally.
+          #latest_dir = local_asset_dirs.last or raise "local_asset_dirs empty"
+          puts "Using existing local asset dir: #{local_asset_dirs.last}"
+        else
+          precompile_locally
+        end
+        #Pathname(latest_dir).exist? or raise "local assets dir #{latest_dir.inspect} does not exist"
+
+        push_changed_assets
+
+        # This is so that the deploy will be aborted if the previous steps failed to produce
+        # asset files in the target directory on the server.
+        run "ls #{release_or_current_path}/public/assets/*.js"
+      end
+
+    else
+      # Just use the standard recipe supplied with capistrano in 'deploy/assets'
+    end
+  end
+
+end
+end
+end # if asset_pipeline
+end