shipit-engine 0.10.0 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: af8d2464cc17caa2786a2a0a9500583b5ed0c7f3
-  data.tar.gz: e00c5b97da7d17dfc23e449630f158e724a5910f
+  metadata.gz: c4329e9e481f6b7065a401570ebd011d1ea4fb19
+  data.tar.gz: ef736fcc3a561d2db82c35795092388abbbfc90a
 SHA512:
-  metadata.gz: 4bff19eadd58b4c9e0bc84c7facd5aaf4dfdc85000c19e1ba50754b7d011a4133ad620e92fef5c6863709643a299c31ecdbcb94634fcd23a48ee137b1fc2fd48
-  data.tar.gz: 9a9ef4184bac10c36545cb0c14105d6c7600a1c528bfa087295542f7d3cf01c1f435ed748ee543bf0cd451bc2707662b85d757496eae2d138bcd0a72ba228c0b
+  metadata.gz: e79a3a01a5d2c71d48dc0bc0ec8ae032721046cccecb5680c876b82c0c246da5ce75a4812ee24afe9aeeaca06d5749637bffb6cb2f9b9aa374fe96f2c790cbff
+  data.tar.gz: 60bde57fcfbd57ff17bc6d0d64785b81bf2e2cfd9ee69d5c9849911a5a6936ceccd9ac80fb2a50bb10254a9789084b26f1095acc43b7c7593ebcd19255d93554

data/README.md

@@ -33,7 +33,7 @@ This guide aims to help you [set up](#installation-and-setup), [use](#using-ship
 * [Format and content of shipit.yml](#configuring-shipit)
 * [Format and content of secrets.yml](#configuring-secrets)
 * [Script parameters](#script-parameters)
-* [Free samples](#sample-file)
+* [Free samples](/examples/shipit.yml)
 
 * * *
 
@@ -45,8 +45,8 @@ This guide aims to help you [set up](#installation-and-setup), [use](#using-ship
 
 Shipit provides you with a Rails template. To bootstrap your Shipit installation:
 
-1. If you don't have Rails installed, run this command: `gem install rails`
-2. Run this command:  `rails new shipit -m https://raw.githubusercontent.com/Shopify/shipit-engine/master/template.rb`
+1. If you don't have Rails installed, run this command: `gem install rails -v 4.2.6`
+2. Run this command:  `rails _4.2.6_ new shipit -m https://raw.githubusercontent.com/Shopify/shipit-engine/master/template.rb`
 3. Enter your **Client ID**, **Client Secret**, and **GitHub API access token** when prompted. These can be found on your application's GitHub page.
 4. To setup the database, run this command: `rake db:setup`
 
@@ -250,6 +250,18 @@ deploy:
 ```
 <br>
 
+**<code>deploy.max_commits</code>** allow to set a limit to the number of commits being shipped per deploys.
+
+Human users will be warned that they are not respecting the recommandation, but allowed to continue.
+
+For example:
+
+```yaml
+deploy:
+  max_commits: 5
+```
+<br>
+
 **<code>rollback.override</code>** contains an array of the shell commands required to rollback the application to a previous state. Shipit will try to infer it from the repository structure, but you can change the default inference. This key defaults to disabled unless Capistrano is detected.
 
 For example:
@@ -360,7 +372,7 @@ You can create custom tasks that users execute directly from a stack's overview
 tasks:
   restart:
     action: "Restart Application"
-    description: "Sometimes needed if you the application to restart but don't want to ship any new code."
+    description: "Sometimes needed if you want the application to restart but don't want to ship any new code."
     steps:
       - ssh deploy@myserver.example.com 'touch myapp/restart.txt'
 ```
@@ -382,7 +394,7 @@ Tasks like deploys can prompt for user defined environment variables:
 tasks:
   restart:
     action: "Restart Application"
-    description: "Sometimes needed if you the application to restart but don't want to ship any new code."
+    description: "Sometimes needed if you want the application to restart but don't want to ship any new code."
     steps:
       - ssh deploy@myserver.example.com 'touch myapp/restart.txt'
     variables:
@@ -444,7 +456,7 @@ The settings in the `secrets.yml` file relate to the ways that GitHub connects w
 For example:
 
 ```yml
-development:
+production:
   secret_key_base: s3cr3t # This needs to be a very long, fully random
 ```
 <br>
@@ -462,7 +474,7 @@ After you change the list of time you have to invoke `bin/rake teams:fetch` in p
 For example:
 
 ```yml
-development:
+production:
   github_oauth:
     id: (your application's Client ID)
     secret: (your application's Client Secret)
@@ -479,7 +491,7 @@ If you specify an `access_token`, you don't need a `login` and `password`. The o
 For example:
 
 ```yml
-development:
+production:
   github_api:
     access_token: 10da65c687f6degaf5475ce12a980d5vd8c44d2a
 ```
@@ -489,7 +501,7 @@ development:
 
 For example:
 ```yml
-development:
+production:
   host: 'http://localhost:3000'
 ```
 <br>
@@ -499,7 +511,7 @@ development:
 For example:
 
 ```yml
-development:
+production:
   redis_url: "redis://127.0.0.1:6379/7"
 ```
 
@@ -509,11 +521,21 @@ If you use GitHub Enterprise, you must also specify the `github_domain`.
 
 For example:
 ```yml
-development:
+production:
   github_domain: "github.example.com"
 
 ```
 
+<br>
+
+**`commands_inactivity_timeout`** is the duration after which shipit will terminate a command if no ouput was received. Default is `300` (5 minutes).
+
+For example:
+```yml
+production:
+  commands_inactivity_timeout: 900 # 15 minutes
+```
+
 <h2 id="script-parameters">Script parameters</h2>
 
 Your deploy scripts have access to the following environment variables:

data/app/assets/javascripts/shipit.js.coffee

@@ -20,8 +20,14 @@
 $(document).on 'click', '.disabled, .btn--disabled', (event) ->
   event.preventDefault()
 
+$(document).on 'click', '.enable-notifications .banner__dismiss', (event) ->
+  $(event.target).closest('.banner').addClass('hidden')
+  localStorage.setItem("dismissed-enable-notifications", true)
+
 jQuery ->
-  $notificationNotice = $('.notifications')
+  if(localStorage.getItem("dismissed-enable-notifications"))
+    return
+  $notificationNotice = $('.enable-notifications')
 
   if $.notifyCheck() == $.NOTIFY_NOT_ALLOWED
     $button = $notificationNotice.find('button')

data/app/assets/stylesheets/_base/_buttons.scss

@@ -22,6 +22,9 @@
   &.btn--disabled {
     box-shadow: 0 0 0 1px rgba(#000, 0.15);
   }
+  &.btn--warning {
+    background-color: $yellow;
+  }
 }
 
 .btn--disabled {

data/app/assets/stylesheets/_base/_colors.scss

@@ -11,7 +11,7 @@ $red: #F39494;
 $bright-red: #F73B3B;
 $light-red: #ffd9d6;
 $yellow: #FFC66C;
-$dark_yellow: #CEA61B;
+$dark-yellow: #CEA61B;
 $orange: #FFAD4C;
 $light-orange: #ffebcc;
 $slate: #2E343A;

data/app/assets/stylesheets/_base/_status-items.scss

@@ -95,7 +95,7 @@
   }
 
   .status-item--pending & {
-    color: $dark_yellow;
+    color: $dark-yellow;
   }
 
   .status-item--error & {

data/app/assets/stylesheets/_pages/_deploy.scss

@@ -201,7 +201,7 @@
 
   &[data-status="scheduled"],
   &[data-status="running"] {
-    border-color: $dark_yellow;
+    border-color: $dark-yellow;
   }
 }
 

data/app/assets/stylesheets/_structure/_main.scss

@@ -60,7 +60,7 @@ pre {
   &.nowrap {
     white-space: pre;
     margin-top: -.25rem;
-    margin-bottom: -3rem;
+    margin-bottom: 0rem;
   }
 }
 

data/app/controllers/shipit/stacks_controller.rb

@@ -17,11 +17,7 @@ module Shipit
       return if flash.empty? && !stale?(last_modified: @stack.updated_at)
 
       @tasks = @stack.tasks.order(id: :desc).preload(:since_commit, :until_commit, :user).limit(10)
-      @commits = @stack.commits.reachable.preload(:author, :statuses).order(id: :desc)
-      if deployed_commit = @stack.last_deployed_commit
-        @commits = @commits.where('id > ?', deployed_commit.id)
-      end
-      @commits = @commits.to_a
+      @commits = @stack.undeployed_commits { |scope| scope.preload(:author, :statuses) }
     end
 
     def lookup

data/app/helpers/shipit/shipit_helper.rb

@@ -19,9 +19,7 @@ module Shipit
     end
 
     def include_plugins(stack)
-      stack.plugins.flat_map do |plugin, config|
-        plugin_tags(plugin, config)
-      end.join.html_safe
+      safe_join(stack.plugins.flat_map { |plugin, config| plugin_tags(plugin, config) })
     end
 
     def plugin_tags(plugin, config)
@@ -33,7 +31,7 @@ module Shipit
     end
 
     def missing_github_oauth_message
-      (<<-MESSAGE).html_safe
+      <<-MESSAGE.html_safe
         Shipit requires a GitHub application to authenticate users.
         If you haven't created an application on GitHub yet, you can do so in the
         #{link_to 'Settings', Shipit.github_url('/settings/applications/new'), target: '_blank'}
@@ -42,21 +40,21 @@ module Shipit
     end
 
     def missing_github_oauth_id_message
-      (<<-MESSAGE).html_safe
+      <<-MESSAGE.html_safe
         Copy the Client ID from your GitHub application,
         and paste it into the secrets.yml file under <code>github_oauth.id</code>.
        MESSAGE
     end
 
     def missing_github_oauth_secret_message
-      (<<-MESSAGE).html_safe
+      <<-MESSAGE.html_safe
         Copy the Client Secret from your GitHub application,
         and paste it into the secrets.yml file under <code>github_oauth.secret</code>.
        MESSAGE
     end
 
     def missing_github_api_credentials_message
-      (<<-MESSAGE).html_safe
+      <<-MESSAGE.html_safe
         Shipit needs API access to GitHub. You can
         #{link_to 'create an access token', Shipit.github_url('/settings/tokens'), target: '_blank'}
         with the following permissions:
@@ -66,14 +64,14 @@ module Shipit
     end
 
     def missing_redis_url_message
-      (<<-MESSAGE).html_safe
+      <<-MESSAGE.html_safe
         Shipit needs a Redis server. Please configure the Redis URL in the secrets.yml file of your app,
         under the key <code>redis_url</code>.
       MESSAGE
     end
 
     def missing_host_message
-      (<<-MESSAGE).html_safe
+      <<-MESSAGE.html_safe
         Shipit needs the host of the application before generating links in background jobs.
         Add the host name to the secrets.yml file, under the <code>host</code> key.
       MESSAGE

data/app/helpers/shipit/stacks_helper.rb

@@ -2,33 +2,34 @@ module Shipit
   module StacksHelper
     COMMIT_TITLE_LENGTH = 79
 
-    def redeploy_button(commit)
-      url = new_stack_deploy_path(@stack, sha: commit.sha)
+    def redeploy_button(deployed_commit)
+      commit = UndeployedCommit.new(deployed_commit, 0)
+      url = new_stack_deploy_path(commit.stack, sha: commit.sha)
       classes = %W(btn btn--primary deploy-action #{commit.state})
 
       unless commit.stack.deployable?
-        classes.push(ignore_lock? ? 'btn--warning' : 'btn--disabled')
+        classes.push(bypass_safeties? ? 'btn--warning' : 'btn--disabled')
       end
 
-      caption = 'Redeploy'
-      caption = 'Locked' if commit.stack.locked? && !ignore_lock?
-      caption = 'Deploy in progress...' if commit.stack.active_task?
-
-      link_to(caption, url, class: classes)
+      link_to(t("redeploy_button.caption.#{commit.redeploy_state(bypass_safeties?)}"), url, class: classes)
     end
 
-    def ignore_lock?
+    def bypass_safeties?
       params[:force].present?
     end
 
     def deploy_button(commit)
-      url = new_stack_deploy_path(@stack, sha: commit.sha)
+      url = new_stack_deploy_path(commit.stack, sha: commit.sha)
       classes = %W(btn btn--primary deploy-action #{commit.state})
-      if deploy_button_disabled?(commit)
-        classes.push(params[:force].present? ? 'btn--warning' : 'btn--disabled')
+      data = {}
+      if commit.deploy_disallowed?
+        classes.push(bypass_safeties? ? 'btn--warning' : 'btn--disabled')
+      elsif commit.deploy_discouraged?
+        classes.push('btn--warning')
+        data[:tooltip] = t('deploy_button.hint.max_commits', maximum: commit.stack.maximum_commits_per_deploy)
       end
 
-      link_to(deploy_button_caption(commit), url, class: classes)
+      link_to(t("deploy_button.caption.#{commit.deploy_state(bypass_safeties?)}"), url, class: classes, data: data)
     end
 
     def github_change_url(commit)
@@ -59,20 +60,5 @@ module Shipit
     def render_raw_commit_id_link(commit)
       link_to(commit.short_sha, github_commit_url(commit), target: '_blank', class: 'number')
     end
-
-    private
-
-    def deploy_button_disabled?(commit)
-      !commit.deployable? || !commit.stack.deployable?
-    end
-
-    def deploy_button_caption(commit)
-      state = commit.status.state
-      state = 'locked' if commit.stack.locked? && !ignore_lock?
-      if commit.deployable?
-        state = commit.stack.active_task? ? 'deploying' : 'enabled'
-      end
-      t("deploy_button.caption.#{state}")
-    end
   end
 end

data/app/jobs/shipit/continuous_delivery_job.rb

@@ -6,7 +6,7 @@ module Shipit
 
     def perform(stack)
       return unless stack.continuous_deployment?
-      stack.trigger_continuous_deploy
+      stack.trigger_continuous_delivery
     end
   end
 end

data/app/jobs/shipit/update_estimated_deploy_duration.rb

@@ -0,0 +1,9 @@
+module Shipit
+  class UpdateEstimatedDeployDurationJob < BackgroundJob
+    queue_as :default
+
+    def perform(stack)
+      stack.update_estimated_deploy_duration!
+    end
+  end
+end

data/app/models/shipit/commit_checks.rb

@@ -1,7 +1,7 @@
 module Shipit
   class CommitChecks
     OUTPUT_TTL = 10.minutes.to_i
-    FINAL_STATUSES = %w(failed error success)
+    FINAL_STATUSES = %w(failed error success).freeze
 
     def initialize(commit)
       @commit = commit

data/app/models/shipit/deploy.rb

@@ -18,7 +18,7 @@ module Shipit
         'success' => 'success',
         'error' => 'error',
         'aborted' => 'error',
-      }
+      }.freeze
 
       def append_status(task_status)
         if github_status = GITHUB_STATUSES[task_status]

data/app/models/shipit/deploy_spec/bundler_discovery.rb

@@ -1,7 +1,7 @@
 module Shipit
   class DeploySpec
     module BundlerDiscovery
-      DEFAULT_BUNDLER_WITHOUT = %w(default production development test staging benchmark debug)
+      DEFAULT_BUNDLER_WITHOUT = %w(default production development test staging benchmark debug).freeze
 
       def discover_dependencies_steps
         discover_bundler || super

data/app/models/shipit/deploy_spec/file_system.rb

@@ -44,7 +44,12 @@ module Shipit
           },
           'plugins' => plugins,
           'dependencies' => {'override' => dependencies_steps},
-          'deploy' => {'override' => deploy_steps, 'variables' => deploy_variables.map(&:to_h)},
+          'deploy' => {
+            'override' => deploy_steps,
+            'variables' => deploy_variables.map(&:to_h),
+            'max_commits' => maximum_commits_per_deploy,
+            'interval' => pause_between_deploys,
+          },
           'rollback' => {'override' => rollback_steps},
           'fetch' => fetch_deployed_revision_steps,
           'tasks' => cacheable_tasks,

data/app/models/shipit/deploy_spec.rb

@@ -63,6 +63,14 @@ module Shipit
     end
     alias_method :dependencies_steps!, :dependencies_steps
 
+    def maximum_commits_per_deploy
+      config('deploy', 'max_commits')
+    end
+
+    def pause_between_deploys
+      Duration.parse(config('deploy', 'interval') { 0 })
+    end
+
     def deploy_steps
       around_steps('deploy') do
         config('deploy', 'override') { discover_deploy_steps }
@@ -143,6 +151,10 @@ module Shipit
       config('machine', 'cleanup') { true }
     end
 
+    def links
+      config('links') { {} }
+    end
+
     private
 
     def around_steps(section)

data/app/models/shipit/duration.rb

@@ -1,5 +1,13 @@
 module Shipit
-  class Duration
+  class Duration < ActiveSupport::Duration
+    FORMAT = /
+      \A
+      (?<days>\d+d)?
+      (?<hours>\d+h)?
+      (?<minutes>\d+m)?
+      (?<seconds>\d+s?)?
+      \z
+    /x
     UNITS = {
       's' => :seconds,
       'm' => :minutes,
@@ -7,21 +15,33 @@ module Shipit
       'd' => :days,
     }.freeze
 
-    def initialize(seconds)
-      @seconds = seconds
+    class << self
+      def parse(value)
+        unless match = FORMAT.match(value.to_s)
+          raise ArgumentError, "not a duration: #{value.inspect}"
+        end
+        parts = []
+        UNITS.values.each do |unit|
+          if value = match[unit]
+            parts << [unit, value.to_i]
+          end
+        end
+
+        time = ::Time.current
+        new(time.advance(parts.to_h) - time, parts)
+      end
     end
 
-    def to_i
-      @seconds.to_i
+    def initialize(value, parts = [[:seconds, value]])
+      super
     end
 
     def to_s
-      seconds = to_i
-      days, seconds = seconds.divmod(1.day.to_i)
+      days, seconds_left = value.divmod(1.day.to_i)
       if days > 0
-        "#{days}d#{Time.at(seconds).utc.strftime('%Hh%Mm%Ss')}"
+        "#{days}d#{Time.at(seconds_left).utc.strftime('%Hh%Mm%Ss')}"
       else
-        Time.at(seconds).utc.strftime('%Hh%Mm%Ss')[/[^0a-z]\w+/] || '0s'
+        Time.at(value).utc.strftime('%Hh%Mm%Ss')[/[^0a-z]\w+/] || '0s'
       end
     end
   end

data/app/models/shipit/github_hook.rb

@@ -45,8 +45,6 @@ module Shipit
       create_hook!
     end
 
-    private
-
     def endpoint_url
       raise NotImplementedError.new('Subclasses must implement a `endpoint_url` method')
     end

data/app/models/shipit/stack.rb

@@ -21,7 +21,7 @@ module Shipit
     REPO_OWNER_MAX_SIZE = 39
     REPO_NAME_MAX_SIZE = 100
     ENVIRONMENT_MAX_SIZE = 50
-    REQUIRED_HOOKS = %i(push status)
+    REQUIRED_HOOKS = %i(push status).freeze
 
     has_many :commits, dependent: :destroy
     has_many :tasks, dependent: :destroy
@@ -62,13 +62,19 @@ module Shipit
     validates :lock_reason, length: {maximum: 4096}
 
     serialize :cached_deploy_spec, DeploySpec
-    delegate :find_task_definition, :supports_rollback?,
+    delegate :find_task_definition, :supports_rollback?, :links,
              :supports_fetch_deployed_revision?, to: :cached_deploy_spec, allow_nil: true
 
     def self.refresh_deployed_revisions
       find_each.select(&:supports_fetch_deployed_revision?).each(&:async_refresh_deployed_revision)
     end
 
+    def self.schedule_continuous_delivery
+      where(continuous_deployment: true).find_each do |stack|
+        ContinuousDeliveryJob.perform_later(stack)
+      end
+    end
+
     def undeployed_commits?
       undeployed_commits_count > 0
     end
@@ -103,11 +109,25 @@ module Shipit
       deploy
     end
 
-    def trigger_continuous_deploy
+    def trigger_continuous_delivery
       return unless deployable?
-      if commit = last_deployable_commit
+      return if deployed_too_recently?
+
+      if commit = next_commit_to_deploy
         return if commit.deployed?
-        trigger_deploy(commit, commit.committer)
+        trigger_deploy(commit, Shipit.user)
+      end
+    end
+
+    def next_commit_to_deploy
+      commits_to_deploy = commits.order(id: :asc).newer_than(last_deployed_commit).reachable.preload(:statuses)
+      commits_to_deploy = commits_to_deploy.limit(maximum_commits_per_deploy) if maximum_commits_per_deploy
+      commits_to_deploy.to_a.reverse.find(&:deployable?)
+    end
+
+    def deployed_too_recently?
+      if task = last_active_task
+        task.ended_at? && (task.ended_at + pause_between_deploys).future?
       end
     end
 
@@ -144,6 +164,8 @@ module Shipit
         'locked'
       else
         significant_statuses = undeployed_commits.map(&:significant_status)
+        significant_statuses << last_deployed_commit.significant_status unless last_deployed_commit.blank?
+
         last_finalized_status = significant_statuses.reject { |s| %w(pending unknown).include?(s.state) }.first
         last_finalized_status.try!(:simple_state) || 'pending'
       end
@@ -155,13 +177,19 @@ module Shipit
     end
 
     def undeployed_commits
-      commits.reachable.newer_than(last_deployed_commit).order(id: :desc)
+      scope = commits.reachable.newer_than(last_deployed_commit).order(id: :asc)
+      yield scope if block_given?
+      scope.map.with_index { |c, i| UndeployedCommit.new(c, i) }.reverse
     end
 
     def last_successful_deploy
       deploys_and_rollbacks.success.order(created_at: :desc).first
     end
 
+    def last_active_task
+      tasks.exclusive.last
+    end
+
     def last_deployed_commit
       if deploy = last_successful_deploy
         deploy.until_commit
@@ -170,10 +198,6 @@ module Shipit
       end
     end
 
-    def last_deployable_commit
-      commits.order(id: :desc).newer_than(last_deployed_commit).reachable.preload(:statuses).to_a.find(&:deployable?)
-    end
-
     def filter_visible_statuses(statuses)
       statuses.reject { |s| hidden_statuses.include?(s.context) }
     end
@@ -288,7 +312,8 @@ module Shipit
     end
 
     delegate :plugins, :task_definitions, :hidden_statuses, :required_statuses, :soft_failing_statuses,
-             :deploy_variables, :filter_task_envs, :filter_deploy_envs, to: :cached_deploy_spec
+             :deploy_variables, :filter_task_envs, :filter_deploy_envs, :maximum_commits_per_deploy,
+             :pause_between_deploys, to: :cached_deploy_spec
 
     def monitoring?
       monitoring.present?
@@ -356,6 +381,18 @@ module Shipit
       super
     end
 
+    def async_update_estimated_deploy_duration
+      UpdateEstimatedDeployDurationJob.perform_later(self)
+    end
+
+    def update_estimated_deploy_duration!
+      update!(estimated_deploy_duration: Stat.p90(recent_deploys_durations) || 1)
+    end
+
+    def recent_deploys_durations
+      tasks.where(type: 'Shipit::Deploy').success.order(id: :desc).limit(100).durations
+    end
+
     private
 
     def clear_cache

data/app/models/shipit/task.rb

@@ -13,7 +13,7 @@ module Shipit
     belongs_to :until_commit, class_name: 'Commit'
     belongs_to :since_commit, class_name: 'Commit'
 
-    has_many :chunks, -> { order(:id) }, class_name: 'OutputChunk', dependent: :destroy
+    has_many :chunks, -> { order(:id) }, class_name: 'OutputChunk', dependent: :delete_all
 
     serialize :definition, TaskDefinition
     serialize :env, Hash
@@ -28,6 +28,12 @@ module Shipit
     after_save :record_status_change
     after_commit :emit_hooks
 
+    class << self
+      def durations
+        pluck(:started_at, :ended_at).select { |s, e| s && e }.map { |s, e| e - s }
+      end
+    end
+
     state_machine :status, initial: :pending do
       before_transition any => :running do |task|
         task.started_at ||= Time.now.utc
@@ -45,6 +51,10 @@ module Shipit
         task.update!(confirmations: 0)
       end
 
+      after_transition any => :success do |task|
+        task.async_update_estimated_deploy_duration
+      end
+
       event :run do
         transition pending: :running
       end
@@ -101,7 +111,8 @@ module Shipit
       error!
     end
 
-    delegate :acquire_git_cache_lock, :async_refresh_deployed_revision, to: :stack
+    delegate :acquire_git_cache_lock, :async_refresh_deployed_revision, :async_update_estimated_deploy_duration,
+             to: :stack
 
     delegate :checklist, to: :definition