shipit-engine 0.27.1 → 0.28.0

data/app/models/shipit/deploy.rb

@@ -36,9 +36,30 @@ module Shipit
     after_create :create_commit_deployments
     after_create :update_release_status
     after_commit :broadcast_update
+    after_commit :update_latest_deployed_ref, on: :update
 
     delegate :broadcast_update, :filter_deploy_envs, to: :stack
 
+    def self.newer_than(deploy)
+      return all unless deploy
+      where('id > ?', deploy.try(:id) || deploy)
+    end
+
+    def self.older_than(deploy)
+      return all unless deploy
+      where('id < ?', deploy.try(:id) || deploy)
+    end
+
+    def self.since(deploy)
+      return all unless deploy
+      where('id >= ?', deploy.try(:id) || deploy)
+    end
+
+    def self.until(deploy)
+      return all unless deploy
+      where('id <= ?', deploy.try(:id) || deploy)
+    end
+
     def build_rollback(user = nil, env: nil, force: false)
       Rollback.new(
         user_id: user&.id,
@@ -65,16 +86,19 @@ module Shipit
       rollback
     end
 
-    # Rolls the stack back to the **previous** deploy
+    # Rolls the stack back to the most recent **previous** successful deploy
     def trigger_revert(force: false)
+      previous_successful_commit = commit_to_rollback_to
+
       rollback = Rollback.create!(
         user_id: user_id,
         stack_id: stack_id,
         parent_id: id,
         since_commit: until_commit,
-        until_commit: since_commit,
+        until_commit: previous_successful_commit,
         allow_concurrency: force,
       )
+
       rollback.enqueue
       lock_reason = "A rollback for #{until_commit.sha} has been triggered. " \
         "Please make sure the reason for the rollback has been addressed before deploying again."
@@ -257,5 +281,10 @@ module Shipit
     def update_last_deploy_time
       stack.update(last_deployed_at: ended_at)
     end
+
+    def update_latest_deployed_ref
+      return unless previous_changes.include?(:status)
+      stack.update_latest_deployed_ref if previous_changes[:status].last == 'success'
+    end
   end
 end

data/app/models/shipit/deploy_spec/lerna_discovery.rb

@@ -3,6 +3,8 @@ require 'json'
 module Shipit
   class DeploySpec
     module LernaDiscovery
+      LATEST_MAJOR_VERSION = Gem::Version.new('3.0.0')
+
       def discover_dependencies_steps
         discover_lerna_json || super
       end
@@ -21,15 +23,21 @@ module Shipit
 
       def discover_lerna_checklist
         if lerna?
+          command = if lerna_lerna >= LATEST_MAJOR_VERSION
+            'lerna version'
+          else
+            %(
+              lerna publish --skip-npm
+              && git add -A
+              && git push --follow-tags
+            )
+          end
+
           [%(
             <strong>Don't forget version and tag before publishing!</strong>
             You can do this with:<br/>
-            <pre>
-            lerna publish --skip-npm
-            && git add -A
-            && git push --follow-tags
-            </pre>
-          )]
+            <pre>#{command}</pre>
+           )]
         end
       end
 
@@ -41,9 +49,16 @@ module Shipit
         file('lerna.json')
       end
 
+      def lerna_config
+        @_lerna_config ||= JSON.parse(lerna_json.read)
+      end
+
+      def lerna_lerna
+        Gem::Version.new(lerna_config['lerna'])
+      end
+
       def lerna_version
-        lerna_config = lerna_json.read
-        JSON.parse(lerna_config)['version']
+        lerna_config['version']
       end
 
       def discover_lerna_packages
@@ -68,18 +83,27 @@ module Shipit
 
       def publish_fixed_version_packages
         check_tags = 'assert-lerna-fixed-version-tag'
-        # `yarn publish` requires user input, so always use npm.
         version = lerna_version
-        publish = %W(
-          node_modules/.bin/lerna publish
-          --yes
-          --skip-git
-          --repo-version #{version}
-          --force-publish=*
-          --npm-tag #{dist_tag(version)}
-          --npm-client=npm
-          --skip-npm=false
-        ).join(" ")
+        publish = if lerna_lerna >= LATEST_MAJOR_VERSION
+          %W(
+            node_modules/.bin/lerna publish
+            from-git
+            --yes
+            --dist-tag #{dist_tag(version)}
+          ).join(" ")
+        else
+          # `yarn publish` requires user input, so always use npm.
+          %W(
+            node_modules/.bin/lerna publish
+            --yes
+            --skip-git
+            --repo-version #{version}
+            --force-publish=*
+            --npm-tag #{dist_tag(version)}
+            --npm-client=npm
+            --skip-npm=false
+          ).join(" ")
+        end
 
         [check_tags, publish]
       end

data/app/models/shipit/deploy_spec/pypi_discovery.rb

@@ -29,7 +29,11 @@ module Shipit
       end
 
       def publish_egg
-        ["assert-egg-version-tag #{setup_dot_py}", 'python setup.py register sdist upload']
+        [
+          "assert-egg-version-tag #{setup_dot_py}",
+          'python setup.py register sdist',
+          'twine upload dist/*',
+        ]
       end
     end
   end

data/app/models/shipit/rollback.rb

@@ -36,9 +36,11 @@ module Shipit
 
     def update_release_status
       return unless stack.release_status?
-      return unless status == 'pending'
 
-      deploy.report_faulty!(description: "A rollback of #{stack.to_param} was triggered")
+      # When we rollback to a certain revision, assume that all later deploys were faulty
+      stack.deploys.newer_than(deploy.id).until(stack.last_completed_deploy.id).to_a.each do |deploy|
+        deploy.report_faulty!(description: "A rollback of #{stack.to_param} was triggered")
+      end
     end
 
     def lock_reverted_commits

data/app/models/shipit/stack.rb

@@ -13,6 +13,10 @@ module Shipit
         ''
       end
 
+      def short_sha
+        ''
+      end
+
       def blank?
         true
       end
@@ -124,10 +128,11 @@ module Shipit
       )
     end
 
-    def trigger_deploy(*args)
-      deploy = build_deploy(*args)
+    def trigger_deploy(*args, **kwargs)
+      run_now = kwargs.delete(:run_now)
+      deploy = build_deploy(*args, **kwargs)
       deploy.save!
-      deploy.enqueue
+      run_now ? deploy.run_now! : deploy.enqueue
       continuous_delivery_resumed!
       deploy
     end
@@ -182,6 +187,12 @@ module Shipit
     end
 
     def async_refresh_deployed_revision
+      async_refresh_deployed_revision!
+    rescue => error
+      logger.warn "Failed to dispatch FetchDeployedRevisionJob: [#{error.class.name}] #{error.message}"
+    end
+
+    def async_refresh_deployed_revision!
       FetchDeployedRevisionJob.perform_later(self)
     end
 
@@ -234,33 +245,57 @@ module Shipit
     end
 
     def lock_reverted_commits!
-      commits_to_lock = []
-
       backlog = undeployed_commits.to_a
+      affected_rows = 0
+
       until backlog.empty?
         backlog = backlog.drop_while { |c| !c.revert? }
-        if revert = backlog.shift
-          commits_to_lock += backlog.reverse.drop_while { |c| !revert.revert_of?(c) }
-        end
+        revert = backlog.shift
+        next if revert.nil?
+
+        commits_to_lock = backlog.reverse.drop_while { |c| !revert.revert_of?(c) }
+        next if commits_to_lock.empty?
+
+        affected_rows += commits
+                         .where(id: commits_to_lock.map(&:id).uniq)
+                         .lock_all(revert.author)
       end
 
-      unless commits_to_lock.empty?
-        if commits.where(id: commits_to_lock.map(&:id).uniq).update_all(locked: true) > 1
-          touch
-        end
+      touch if affected_rows > 1
+    end
+
+    def next_expected_commit_to_deploy(commits: nil)
+      commits ||= undeployed_commits do |scope|
+        scope.preload(:statuses, :check_runs)
       end
+
+      commits_to_deploy = commits.reject(&:active?)
+      if maximum_commits_per_deploy
+        commits_to_deploy = commits_to_deploy.reverse.slice(0, maximum_commits_per_deploy).reverse
+      end
+      commits_to_deploy.find(&:deployable?)
     end
 
     def undeployed_commits
       scope = commits.reachable.newer_than(last_deployed_commit).order(id: :asc)
-      yield scope if block_given?
-      scope.map.with_index { |c, i| UndeployedCommit.new(c, i) }.reverse
+
+      scope = yield scope if block_given?
+
+      scope.to_a.reverse
     end
 
     def last_completed_deploy
       deploys_and_rollbacks.last_completed
     end
 
+    def last_successful_deploy_commit
+      deploys_and_rollbacks.last_successful&.until_commit
+    end
+
+    def previous_successful_deploy(deploy_id)
+      deploys_and_rollbacks.success.where("id < ?", deploy_id).last
+    end
+
     def last_active_task
       tasks.exclusive.last
     end
@@ -269,6 +304,10 @@ module Shipit
       last_completed_deploy&.until_commit || NoDeployedCommit
     end
 
+    def previous_successful_deploy_commit(deploy_id)
+      previous_successful_deploy(deploy_id)&.until_commit || NoDeployedCommit
+    end
+
     def deployable?
       !locked? && !active_task?
     end
@@ -379,6 +418,15 @@ module Shipit
       [repo_owner, repo_name, environment].join('/')
     end
 
+    def self.run_deploy_in_foreground(stack:, revision:)
+      stack = Shipit::Stack.from_param!(stack)
+      until_commit = stack.commits.where(sha: revision).limit(1).first
+      env = stack.cached_deploy_spec.default_deploy_env
+      current_user = Shipit::CommandLineUser.new
+
+      stack.trigger_deploy(until_commit, current_user, env: env, force: true, run_now: true)
+    end
+
     def self.from_param!(param)
       repo_owner, repo_name, environment = param.split('/')
       where(
@@ -414,6 +462,10 @@ module Shipit
       update(undeployed_commits_count: undeployed_commits)
     end
 
+    def update_latest_deployed_ref
+      UpdateGithubLastDeployedRefJob.perform_later(self)
+    end
+
     def broadcast_update
       Pubsubstub.publish(
         "stack.#{id}",
@@ -502,7 +554,12 @@ module Shipit
 
     def emit_lock_hooks
       return unless previous_changes.include?('lock_reason')
-      Hook.emit(:lock, self, locked: locked?, stack: self)
+
+      lock_details = if previous_changes['lock_reason'].last.blank?
+        {from: previous_changes['locked_since'].first, until: Time.zone.now}
+      end
+
+      Hook.emit(:lock, self, locked: locked?, lock_details: lock_details, stack: self)
     end
 
     def emit_added_hooks

data/app/models/shipit/task.rb

@@ -47,6 +47,10 @@ module Shipit
         completed.last
       end
 
+      def last_successful
+        success.last
+      end
+
       def current
         active.exclusive.last
       end
@@ -176,7 +180,13 @@ module Shipit
       PerformTaskJob.perform_later(self)
     end
 
+    def run_now!
+      raise "only persisted jobs can be run" unless persisted?
+      PerformTaskJob.perform_now(self)
+    end
+
     def write(text)
+      log_output(text)
       chunks.create!(text: text)
     end
 
@@ -307,6 +317,16 @@ module Shipit
       Shipit::Engine.routes.url_helpers.stack_task_url(stack, self)
     end
 
+    def commit_to_rollback_to
+      previous_deployed_commit = stack.previous_successful_deploy_commit(id)
+
+      if previous_deployed_commit == Shipit::Stack::NoDeployedCommit
+        since_commit
+      else
+        previous_deployed_commit
+      end
+    end
+
     private
 
     def prevent_concurrency
@@ -320,5 +340,15 @@ module Shipit
     def abort_key
       "#{status_key}:aborting"
     end
+
+    def log_output(text)
+      output_line_buffer.buffer(text) do |line|
+        Shipit.task_logger.info("[#{stack.repo_name}##{id}] #{line}")
+      end
+    end
+
+    def output_line_buffer
+      @output_line_buffer ||= LineBuffer.new
+    end
   end
 end

data/app/models/shipit/undeployed_commit.rb

@@ -2,9 +2,10 @@ module Shipit
   class UndeployedCommit < DelegateClass(Commit)
     attr_reader :index
 
-    def initialize(commit, index)
+    def initialize(commit, index:, next_expected_commit_to_deploy: nil)
       super(commit)
       @index = index
+      @next_expected_commit_to_deploy = next_expected_commit_to_deploy
     end
 
     def deploy_state(bypass_safeties = false)
@@ -38,6 +39,14 @@ module Shipit
       stack.maximum_commits_per_deploy && index >= stack.maximum_commits_per_deploy
     end
 
+    def expected_to_be_deployed?
+      return false if @next_expected_commit_to_deploy.nil?
+      return false unless stack.continuous_deployment
+      return false if active?
+
+      id <= @next_expected_commit_to_deploy.id
+    end
+
     def blocked?
       return @blocked if defined?(@blocked)
       @blocked = super

data/app/serializers/shipit/command_line_user_serializer.rb

@@ -0,0 +1,4 @@
+module Shipit
+  class CommandLineUserSerializer < UserSerializer
+  end
+end

data/app/views/layouts/shipit.html.erb

@@ -1,7 +1,11 @@
 <!DOCTYPE html>
 <html lang="<%= I18n.locale %>" data-controller="<%= controller_name %>" data-action="<%= action_name %>">
 <head>
-  <title><%= [Shipit.app_name, @stack.try!(:repo_name)].compact.join(' - ') %></title>
+  <% if @stack %>
+    <title><%= Shipit.app_name %> - <%= @stack.repo_name %>/<%= @stack.environment %></title>
+  <% else %>
+  <title><%= Shipit.app_name %></title>
+  <% end %>
   <meta name="viewport" content="width=device-width, initial-scale=1">
   <%= favicon_link_tag %>
   <%= stylesheet_link_tag :shipit, media: 'all' %>

data/app/views/shipit/commits/_commit.html.erb

@@ -1,5 +1,5 @@
 <li class="commit <%= 'locked' if commit.locked? %>" id="commit-<%= commit.id %>">
-  <% cache commit do %>
+  <% cache [commit, commit.expected_to_be_deployed?] do %>
     <% cache commit.author do %>
       <%= render 'shipit/shared/author', author: commit.author %>
     <% end %>
@@ -17,12 +17,17 @@
       <p class="commit-meta">
         <%= timeago_tag(commit.committed_at, force: true) %>
       </p>
+      <% if commit.expected_to_be_deployed? %>
+        <p class="commit-meta">
+	  <span class="scheduled">expected to be deployed next</span>
+        </p>
+      <% end %>
     </div>
     <div class="commit-lock" >
       <%= link_to stack_commit_path(commit.stack, commit), class: 'action-lock-commit', data: {tooltip: t('commit.lock')} do %>
         <i class="icon icon--lock"></i>
       <% end %>
-      <%= link_to stack_commit_path(commit.stack, commit), class: 'action-unlock-commit', data: {tooltip: t('commit.unlock'), confirm: t('commit.confirm_unlock')} do %>
+      <%= link_to stack_commit_path(commit.stack, commit), class: 'action-unlock-commit', data: {tooltip: unlock_commit_tooltip(commit), confirm: t('commit.confirm_unlock')} do %>
         <i class="icon icon--lock"></i>
       <% end %>
     </div>