etsy-deployinator 1.0.1 → 1.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cd2670b5a14fac9a1b92c16876c4d361f5d6f49a
-  data.tar.gz: a23addf61909a7feb8b9f56bdfbaff2faf30fcef
+  metadata.gz: 974ad4c20edcc72fab530b9d6292a1d81598b3ac
+  data.tar.gz: 622e28bd20fd84731ce311441a2d292491e210cd
 SHA512:
-  metadata.gz: be256bfe8ee18d7c03d4f92424f37ed98eb6ff2e92d776feafafd351f382bfdb981de5c5766a4a25d8819b4e035b447ba25b19a5c91bf6e17c6e4961eb604b29
-  data.tar.gz: 6f638ef633763f9ac4a4f4a2b097cb508031eae0fb2482e14d654991c42375c2942ef50d85b56f6ca9a01d6c6057042e39b1b26f6c4c9f622807343fead188b2
+  metadata.gz: 2a70027a504454016d46d1ee4ff76ebdcea9eaa5950b25700308ab622bf0a7b5c309bf922404f9018569c6a4d309bd9f549c02c57dc13ea987618c644071c930
+  data.tar.gz: 0708eda8da10d0b12085767410969bb9a138871755a768537190c1503806f282f7977df3afe3d9b4e7d815c4c02be5e0b38105b81aeb4038856b0af70978c021

data/.gitignore

@@ -0,0 +1,3 @@
+.bundle/
+vendor/
+Gemfile.lock

data/.travis.yml

@@ -0,0 +1,8 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - 2.1.0
+  - 2.2.0
+before_install:
+  - gem update bundler

data/Gemfile

@@ -1,4 +1,3 @@
 source 'https://rubygems.org'
 
-# Specify your gem's dependencies in deployinator.gemspec
 gemspec

data/README.md

@@ -2,8 +2,8 @@
 _________               ______                _____                 _____
 ______  /_____ ________ ___  /______ _____  _____(_)_______ ______ ___  /_______ ________
 _  __  / _  _ \___  __ \__  / _  __ \__  / / /__  / __  __ \_  __ `/_  __/_  __ \__  ___/
-/ /_/ /  /  __/__  /_/ /_  /  / /_/ /_  /_/ / _  /  _  / / // /_/ / / /_  / /_/ /_  / 
-\__,_/   \___/ _  .___/ /_/   \____/ _\__, /  /_/   /_/ /_/ \__,_/  \__/  \____/ /_/ 
+/ /_/ /  /  __/__  /_/ /_  /  / /_/ /_  /_/ / _  /  _  / / // /_/ / / /_  / /_/ /_  /
+\__,_/   \___/ _  .___/ /_/   \____/ _\__, /  /_/   /_/ /_/ \__,_/  \__/  \____/ /_/
                 /_/                   /____/             Deploy with style!
 </pre>
 
@@ -28,9 +28,9 @@ Deployinator is a deployment framework extracted from Etsy. We've been using it
 ## Stacks
 Deployments are grouped by "stacks". You might have a "web" and "search" stack.
 
-Each of those stacks might have different deployment environments, such as "staging" or "production". 
+Each of those stacks might have different deployment environments, such as "staging" or "production".
 
-You can map a button to each of these environments,  to create multi-stage pushes within each stack. 
+You can map a button to each of these environments,  to create multi-stage pushes within each stack.
 
 ## Installation
 
@@ -55,13 +55,14 @@ you can skip the bundler steps.
 - Run the following command to make deployinator gem's rake tasks available to you:
 
 ```sh
+    $ shopt -s xpg_echo
     $ echo "require 'deployinator'\nload 'deployinator/tasks/initialize.rake' " > Rakefile
 ```
 
 - Create a binstub for the deploy log tailing backend:
 
 ```sh
-    bundle install --binstubs deployinator
+    bundle binstub etsy-deployinator
 ```
 
 - Initialize the project directory by running the following command replacing ___Company___ with the name of your company/organization. This must start with a capital letter.
@@ -116,14 +117,14 @@ underscores but if you forget the rake task will convert from camelcase for you.
     $ bundle exec shotgun --host localhost -p 7777 config.ru
 ```
 
-- You will probably want a robust server like apache to handle production traffic. 
+- You will probably want a robust server like apache to handle production traffic.
 
 - The `config/base.rb` file is the base config for the application. Replace all
 occurences of ___test_stack___ with the name you chose above. Also the example
 below uses a git repository of http://github.com/etsy/deployinator, feel free to
 replace this with your specific repository
 
-```ruby                                                                                                         
+```ruby
 Deployinator.app_context['test_stack_config'] = {
    :prod_host               => "localhost",
    :checkout_path           => "/tmp/deployinator_dev/"
@@ -232,7 +233,7 @@ For example you could wrap your capistrano deploy:
     run_cmd %Q{cap deploy}
 
 
-#### log_and_stream 
+#### log_and_stream
 
 Output information to the log file, and the streaming output handler.
 The real time output console renders HTML so you should use markup here.
@@ -241,7 +242,7 @@ The real time output console renders HTML so you should use markup here.
 
 #### log_and_shout
 
-Output an announcement message with build related information.  
+Output an announcement message with build related information.
 Also includes hooks for Email.
 
     log_and_shout({
@@ -348,8 +349,20 @@ To set these simple override the methods in your view class. For example:
 This can be done on a global layout that extends the gem's default layout or on
 a stack by stack basis in their own view.
 
+### Maintenance mode
+Deployinator has a setting for maintenance mode, which is mostly useful if you
+have major changes that affect all stacks and you want to make sure no deploys
+are going on while you make the change. In order to enable it, you have to set
+`Deployinator.maintenance_mode = true`. This will make all pages for anyone
+not in `Deployinator.admin_groups` go to `/maintenance`. As a Deployinator
+admin you can then still deploy stacks and use the app.
+
+On the maintenance page Deployinator will show the value of
+`Deployinator.maintenance_contact` as the place to get help in case you need
+any or are confused about maintenance mode.
+
 ## Hacking on the gem
-If you find issues with the gem, or would like to play around with it, you can check it out from git and start hacking on it. 
+If you find issues with the gem, or would like to play around with it, you can check it out from git and start hacking on it.
 First tell bundler to use your local copy instead by running:
 ```sh
     $ bundle config local.deployinator /path/to/DeployinatorGem
@@ -359,6 +372,30 @@ Next, on every code change, you can install from the checked out gem by running
     $ bundle install --no-deployment && bundle install --deployment
 ```
 
+### Stats dashboard
+The `/stats` page pulls from `log/deployinator.log` to show a graph of deployments per day for each stack over time. By default, it shows all stacks. To blacklist or whitelist certain stacks, update `config/base.rb` with:
+```rb
+  Deployinator.stats_included_stacks = ['my_whitelisted_stack', 'another_whitelisted_stack']
+  Deployinator.stats_ignored_stacks = ['my_stack_to_ignore', 'another_stack_to_ignore']
+  Deployinator.stats_extra_grep = 'Production deploy' # filter out log entries matching this string
+```
+
+Whitelisting stacks or applying a custom extra grep can help speed up graph rendering when you have a large log file.
+
+If at some point you change the name of a stack, you can group the old log entries with the new by adding the following to `config/base.rb`:
+
+```rb
+ Deployinator.stats_renamed_stacks = [
+   {
+     :previous_stack => {
+       :stack => [ "old_stack_name" ]
+     },
+     :new_name => "new_stack_name"
+   }
+ ]
+```
+
+
 ### Contributing
 
 1. Fork it

data/Rakefile

@@ -1,4 +1,3 @@
-
 require 'rake/testtask'
 require 'rdoc/task'
 require 'bundler/gem_tasks'

data/bin/deployinator-tailer.rb

@@ -54,8 +54,8 @@ EM.run do
         if stack && @@channels.key?(stack)
           @@channels[stack].unsubscribe(session_id)
           @@channels_count[stack] -= 1
-          if @@channels_count[stack] == 0
-            @@tailers[stack].close
+          if @@channels_count[stack] == 0 && @@tailers.has_key?(stack)
+            @@tailers[stack].close 
             @@tailers.delete(stack)
           end
         end

data/deployinator.gemspec

@@ -10,22 +10,29 @@ Gem::Specification.new do |gem|
   gem.email         = ["jpaul@etsy.com"]
   gem.description   = %q{Deployinator as a Gem}
   gem.summary       = %q{Rewrite of deployinator to be a gem}
-  gem.homepage      = ""
+  gem.homepage      = "http://github.com/etsy/Deployinator"
+  gem.licenses      = ["MIT"]
 
   gem.files         = `git ls-files`.split($/)
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
+  gem.required_ruby_version = '>= 1.9.3'
 
-  gem.add_development_dependency "mocha"
+  gem.add_development_dependency "mocha", "~> 0.14"
+  gem.add_development_dependency "test-unit", ">= 3"
+  gem.add_development_dependency "rdoc", "~> 4.2"
 
-  gem.add_runtime_dependency "rake"
-  gem.add_runtime_dependency "json"
+  gem.add_runtime_dependency "rake", "~> 10", ">= 10.3.2"
+  gem.add_runtime_dependency "json", "~> 1.8"
   gem.add_runtime_dependency "mustache", "~> 0.99"
-  gem.add_runtime_dependency "pony"
-  gem.add_runtime_dependency "tlsmail"
-  gem.add_runtime_dependency "eventmachine"
-  gem.add_runtime_dependency "eventmachine-tail"
-  gem.add_runtime_dependency "em-websocket"
-  gem.add_runtime_dependency "sinatra", ">=1.4.3"
+  # pony pulls in mail >2.0. mail 3.0 needs ruby 2 or greater. to maintain ruby 1.9 compat we're pinning mail here
+  gem.add_runtime_dependency "mail", "2.6.3"
+  gem.add_runtime_dependency "pony", "~> 1.5"
+  gem.add_runtime_dependency "tlsmail", "~> 0.0"
+  gem.add_runtime_dependency "eventmachine", "~> 1.0", ">= 1.0.4"
+  gem.add_runtime_dependency "eventmachine-tail", "~> 0.6", ">= 0.6.5"
+  gem.add_runtime_dependency "em-websocket", "~> 0.5", ">= 0.5.1"
+  gem.add_runtime_dependency "sinatra", "~> 1.4", ">=1.4.3"
+  gem.add_runtime_dependency "celluloid", "~> 0.16"
 end

data/lib/deployinator/app.rb

@@ -12,6 +12,8 @@ require 'deployinator/views/log'
 require 'deployinator/views/run_logs'
 require 'deployinator/views/log_table'
 require 'deployinator/views/deploys_status'
+require 'deployinator/views/stats'
+require 'deployinator/views/maintenance'
 
 module Deployinator
   class DeployinatorApp < Sinatra::Base
@@ -41,12 +43,32 @@ module Deployinator
       register_plugins(nil)
       init(env)
       @disabled_override = params[:override].nil? ? false : true
+      pass if [
+        "/maintenance",
+        "/css/maintenance.css",
+        "/images/maintenance.gif",
+        "/static/css/style.css"
+      ].include? request.path_info
+      if Deployinator.maintenance_mode == true && !is_admin?
+        redirect "/maintenance"
+      end
+    end
+
+    before do
     end
 
     get '/' do
         mustache Deployinator::Views::Index
     end
 
+    get '/stats' do
+        mustache Deployinator::Views::Stats
+    end
+
+    get '/maintenance' do
+        mustache Deployinator::Views::Maintenance
+    end
+
     get '/run_logs/view/:log' do
       template = open("#{File.dirname(__FILE__)}/templates/stream.mustache").read
       template.gsub!("{{ yield }}", "{{{yield}}}")
@@ -95,17 +117,6 @@ module Deployinator
       git_head_rev(params[:stack]).chomp
     end
 
-    get '/:thing' do
-      @stack = params[:thing]
-      @params = params
-      register_plugins(@stack)
-      begin
-        mustache @stack
-      rescue Errno::ENOENT
-        pass
-      end
-    end
-
     get '/:stack/remove-lock' do
       stack = params[:stack]
       unlock_pushes(stack) if can_remove_stack_lock?
@@ -129,6 +140,10 @@ module Deployinator
       send_file "#{File.dirname(__FILE__)}/static/css/highlight.css"
     end
 
+    get '/css/maintenance.css' do
+      send_file "#{File.dirname(__FILE__)}/static/css/maintenance.css"
+    end
+
     get '/js/flot/jquery.flot.min.js' do
       send_file "#{File.dirname(__FILE__)}/static/js/flot/jquery.flot.min.js"
     end
@@ -149,6 +164,14 @@ module Deployinator
       send_file "#{File.dirname(__FILE__)}/static/js/jquery.timed_bar.js"
     end
 
+    get '/js/stats_load.js' do
+      send_file "#{File.dirname(__FILE__)}/static/js/stats_load.js"
+    end
+
+    get '/images/maintenance.gif' do
+      send_file "#{File.dirname(__FILE__)}/static/images/maintenance.gif"
+    end
+
     get '/deploys_status' do
         mustache Deployinator::Views::DeploysStatus
     end
@@ -172,6 +195,7 @@ module Deployinator
     # the background.
     post '/deploys/?' do
       params[:username] = @username
+      params[:groups] = @groups
       params[:block] = Proc.new { |line| foo = line }
       deploy_running = is_deploy_active?(params[:stack], params[:stage])
 
@@ -180,7 +204,7 @@ module Deployinator
         return 403
       end
 
-      fork {
+      pid = fork {
         Signal.trap("HUP") { exit }
         Deployinator.setup_logging
         $0 = get_deploy_process_title(params[:stack], params[:stage])
@@ -188,6 +212,7 @@ module Deployinator
         d = controller.new
         d.run(params)
       }
+      Process.detach(pid)
       200
     end
 
@@ -200,5 +225,21 @@ module Deployinator
 
       200
     end
+
+    get '/:thing' do
+      @stack = params[:thing]
+
+      unless Deployinator.get_stacks.include?(@stack)
+        raise "No such stack #{@stack}"
+      end
+
+      @params = params
+      register_plugins(@stack)
+      begin
+        mustache @stack
+      rescue Errno::ENOENT
+        pass
+      end
+    end
   end
 end

data/lib/deployinator/controller.rb

@@ -29,6 +29,7 @@ module Deployinator
       @deploy_start_time = Time.now.to_i
       @start_time = Time.now.to_i
       @username = args[:username]
+      @groups = args[:groups]
       @host     = `hostname -s`
       @stack = args[:stack]
       @method = args[:method]
@@ -37,7 +38,7 @@ module Deployinator
 
       # This gets the runlog output on the console; is used by log_and_stream
       @block = args[:block] || Proc.new do |output|
-        $stdout.write output.gsub!(/(<[^>]*>)|\n|\t/s) {" "}
+        $stdout.write output.gsub!(/(<[^>]*>)|\n|\t/m) {" "}
         $stdout.write "\n"
       end
     end
@@ -102,7 +103,11 @@ module Deployinator
       deploy_instance = deploy_class.new(options)
       deploy_instance.register_plugins(options[:stack])
 
-      deploy_instance.lock_pushes(options[:stack], options[:username], options[:method])
+      locked = deploy_instance.lock_pushes(options[:stack], options[:username], options[:method])
+
+      unless locked
+        return deploy_instance
+      end
 
       @start_time = Time.now
       deploy_instance.log_and_stream "Push started at #{@start_time.to_i}\n"
@@ -115,7 +120,7 @@ module Deployinator
         state = deploy_instance.send(options[:method], options)
       rescue Exception => e
         deploy_instance.log_error("There was an exception during this deploy. Aborted!", e)
-        deploy_instance.raise_event(:deploy_error)
+        deploy_instance.raise_event(:deploy_error, {:exception => e})
       end
 
       if state.nil? || !state.is_a?(Hash)

data/lib/deployinator/helpers/concurrency.rb

@@ -0,0 +1,70 @@
+require 'celluloid'
+require 'celluloid/autostart'
+
+module Deployinator
+  module Helpers
+    module ConcurrencyHelpers
+      extend Celluloid
+      # Hash of future objects that have been instantiated so far
+      @@futures = {}
+      # Public: run block of code in parallel
+      #
+      # Returns Handle to future object created
+      def run_parallel(name, &block)
+        name = name.to_sym
+        if reference_taken? name
+          raise DuplicateReferenceError, "Name #{name} already taken for future."
+        end
+        log_and_stream '</br>Queueing  execution of future: ' + name.to_s + '</br>'
+        @@futures[name] = Celluloid::Future.new do
+          # Set filename for thread
+          runlog_filename(name)
+          # setting up separate logger
+          log_and_stream '</br>Starting execution of future: ' + name.to_s + '</br>'
+          block.call
+        end
+        @@futures[name]
+      end
+
+
+
+      
+      # Public: check if the reference name for future is taken
+      #
+      # Returns boolean 
+      def reference_taken?(name)
+        return @@futures.has_key?(name)
+      end
+      
+      # Public: returns the value of the code block execution
+      # This also sends all the logged data in stream back to main 
+      # log file and removes the temporary log file created for the thread
+      # Returns the return value of the last line executed in block 
+      def get_value(future, timeout=nil)
+        if @filename
+          file_path = "#{RUN_LOG_PATH}" + runlog_thread_filename(future)
+          return_value = @@futures[future.to_sym].value(timeout)
+          log_and_stream File.read(file_path) 
+          File.delete(file_path) if File.exists?(file_path)
+          return_value
+        else
+          @@futures[future.to_sym].value 
+        end
+      end
+      
+      # Public: returns the values of the specified futures
+      #
+      # Returns hash of values
+      def get_values(*futures)
+        value_hash = {}
+        futures.each do |future|
+          value_hash[future] = get_value(future)
+        end
+        value_hash
+      end
+
+      class DuplicateReferenceError < StandardError
+      end
+    end
+  end
+end

data/lib/deployinator/helpers/dsh.rb

@@ -5,14 +5,19 @@ module Deployinator
         @dsh_fanout || 30
       end
 
-      def run_dsh(groups, cmd, &block)
+      def run_dsh(groups, cmd, only_stdout=true, &block)
         groups = [groups] unless groups.is_a?(Array)
         dsh_groups = groups.map {|group| "-g #{group} "}.join("")
-        run_cmd(%Q{ssh #{Deployinator.default_user}@#{Deployinator.deploy_host} dsh #{dsh_groups} -r ssh -F #{dsh_fanout} "#{cmd}"}, &block)[:stdout]
+        cmd_return = run_cmd(%Q{ssh #{Deployinator.default_user}@#{Deployinator.deploy_host} dsh #{dsh_groups} -r ssh -F #{dsh_fanout} "#{cmd}"}, &block)
+        if only_stdout
+          cmd_return[:stdout]
+        else
+          cmd_return
+        end
       end
 
       # run dsh against a given host or array of hosts
-      def run_dsh_hosts(hosts, cmd, extra_opts='', &block)
+      def run_dsh_hosts(hosts, cmd, extra_opts='', only_stdout=true, &block)
         hosts = [hosts] unless hosts.is_a?(Array)
         if extra_opts.length > 0
           run_cmd %Q{ssh #{Deployinator.default_user}@#{Deployinator.deploy_host} 'dsh -m #{hosts.join(',')} -r ssh -F #{dsh_fanout} #{extra_opts} -- "#{cmd}"'}, &block
@@ -21,9 +26,14 @@ module Deployinator
         end
       end
 
-      def run_dsh_extra(dsh_group, cmd, extra_opts, &block)
+      def run_dsh_extra(dsh_group, cmd, extra_opts, only_stdout=true, &block)
         # runs dsh to a single group with extra args to dsh
-        run_cmd(%Q{ssh #{Deployinator.default_user}@#{Deployinator.deploy_host} dsh -g #{dsh_group} -r ssh #{extra_opts} -F #{dsh_fanout} "#{cmd}"}, &block)[:stdout]
+        cmd_return = run_cmd(%Q{ssh #{Deployinator.default_user}@#{Deployinator.deploy_host} dsh -g #{dsh_group} -r ssh #{extra_opts} -F #{dsh_fanout} "#{cmd}"}, &block)
+        if only_stdout
+          cmd_return[:stdout]
+        else
+          cmd_return
+        end
       end
 
       def hosts_for(group)

data/lib/deployinator/helpers/git.rb

@@ -93,10 +93,16 @@ module Deployinator
       # branch    - the branch to checkout after the fetch
       #
       # Returns nothing
-      def git_freshen_clone(stack, extra_cmd="", path=nil, branch="master")
+      def git_freshen_clone(stack, extra_cmd="", path=nil, branch="master", force_checkout=false)
         path ||= git_checkout_path(checkout_root, stack)
-        cmd = "cd #{path} && git fetch --quiet origin +refs/heads/#{branch}:refs/remotes/origin/#{branch} && git reset --hard origin/#{branch} 2>&1"
-        cmd = build_git_cmd(cmd, extra_cmd)
+        cmd = [
+          "cd #{path}",
+          "git fetch --quiet origin +refs/heads/#{branch}:refs/remotes/origin/#{branch}",
+          "git reset --hard origin/#{branch} 2>&1",
+          "git checkout #{'--force' if force_checkout} #{branch} 2>&1",
+        ]
+        cmd << "git reset --hard origin/#{branch} 2>&1"
+        cmd = build_git_cmd(cmd.join(" && "), extra_cmd)
         run_cmd cmd
         yield "#{path}" if block_given?
       end
@@ -117,15 +123,15 @@ module Deployinator
       # read_write    - boolean; True means clone the repo read/write
       #
       # Returns stdout of the respective git command.
-      def git_freshen_or_clone(stack, extra_cmd, checkout_root, branch="master", read_write=false)
+      def git_freshen_or_clone(stack, extra_cmd, checkout_root, branch="master", read_write=false, protocol="git", force_checkout=false)
         path = git_checkout_path(checkout_root, stack)
         is_git = is_git_repo(path, extra_cmd)
         if is_git == :true
           log_and_stream "</br>Refreshing repo #{stack} at #{path}</br>"
-          git_freshen_clone(stack, extra_cmd, path, branch)
+          git_freshen_clone(stack, extra_cmd, path, branch, force_checkout)
         elsif is_git == :missing
           log_and_stream "</br>Cloning branch #{branch} of #{stack} repo into #{path}</br>"
-          git_clone(stack, git_url(stack, "git", read_write), extra_cmd, checkout_root, branch)
+          git_clone(stack, git_url(stack, protocol, read_write), extra_cmd, checkout_root, branch)
         else
           log_and_stream "</br><span class=\"stderr\">The path for #{stack} at #{path} exists but is not a git repo.</span></br>"
         end
@@ -188,7 +194,7 @@ module Deployinator
 
         unless head_rev
           head_rev = get_git_head_rev(stack, branch)
-          File.open(filename, 'w') {|f| f.write(head_rev) }
+          write_to_cache(filename, head_rev)
         end
 
         return head_rev
@@ -329,11 +335,12 @@ module Deployinator
       #   ssh_cmd: string ssh cmd to get to a host where you've got this repo checked out
       #   extra: string any extra cmds like cd that you need to do on the remote host to get to your checkout
       #   quiet: boolean - if true we make no additional output and just return the files
+      #   diff_filter: string to pass to git to make it only show certain types of changes (added/removed)
       #
       # Returns:
       #   Array of files names changed between these revs
-      def git_show_changed_files(rev1, rev2, ssh_cmd, extra=nil, quiet=false)
-        cmd = %Q{git log --name-only --pretty=oneline --full-index #{rev1}..#{rev2} | grep -vE '^[0-9a-f]{40} ' | sort | uniq}
+      def git_show_changed_files(rev1, rev2, ssh_cmd, extra=nil, quiet=false, diff_filter="")
+        cmd = %Q{git log --name-only --pretty=oneline --full-index #{rev1}..#{rev2} --diff-filter=#{diff_filter} | grep -vE '^[0-9a-f]{40} ' | sort | uniq}
         extra = "#{extra} &&" unless extra.nil?
         if quiet
           list_of_touched_files = %x{#{ssh_cmd} "#{extra} #{cmd}"}