shipit-engine 0.29.0 → 0.30.0

data/app/helpers/shipit/stacks_helper.rb

@@ -86,5 +86,9 @@ module Shipit
         t('commit.unlock')
       end
     end
+
+    def positive_negative_class(value)
+      value.to_f >= 0 ? 'positive' : 'negative'
+    end
   end
 end

data/app/jobs/shipit/background_job/unique.rb

@@ -4,6 +4,8 @@ module Shipit
       extend ActiveSupport::Concern
       DEFAULT_TIMEOUT = 10
 
+      ConcurrentJobError = Class.new(StandardError)
+
       included do
         around_perform { |job, block| job.acquire_lock(&block) }
         cattr_accessor :lock_timeout
@@ -19,7 +21,7 @@ module Shipit
         )
         mutex.lock(&block)
       rescue Redis::Lock::LockTimeout
-        raise unless self.class.drop_duplicate_jobs?
+        raise ConcurrentJobError unless self.class.drop_duplicate_jobs?
       end
 
       def lock_key(*args)

data/app/jobs/shipit/continuous_delivery_job.rb

@@ -3,6 +3,7 @@ module Shipit
     include BackgroundJob::Unique
 
     queue_as :default
+    on_duplicate :drop
 
     def perform(stack)
       return unless stack.continuous_deployment?

data/app/jobs/shipit/destroy_stack_job.rb

@@ -17,7 +17,8 @@ module Shipit
     def perform(stack)
       Shipit::ApiClient.where(stack_id: stack.id).delete_all
       commits_ids = Shipit::Commit.where(stack_id: stack.id).pluck(:id)
-      commit_deployments_ids = Shipit::CommitDeployment.where(commit_id: commits_ids).pluck(:id)
+      tasks_ids = Shipit::Task.where(stack_id: stack.id).pluck(:id)
+      commit_deployments_ids = Shipit::CommitDeployment.where(task_id: tasks_ids).pluck(:id)
       Shipit::CommitDeploymentStatus.where(commit_deployment_id: commit_deployments_ids).delete_all
       Shipit::CommitDeployment.where(id: commit_deployments_ids).delete_all
       Shipit::Status.where(commit_id: commits_ids).delete_all
@@ -25,7 +26,6 @@ module Shipit
       Shipit::GithubHook.where(stack_id: stack.id).destroy_all
       Shipit::Hook.where(stack_id: stack.id).delete_all
       Shipit::PullRequest.where(stack_id: stack.id).delete_all
-      tasks_ids = Shipit::Task.where(stack_id: stack.id).pluck(:id)
       tasks_ids.each_slice(100) do |ids|
         Shipit::OutputChunk.where(task_id: ids).delete_all
         Shipit::Task.where(id: ids).delete_all

data/app/models/shipit/commit.rb

@@ -2,6 +2,8 @@ module Shipit
   class Commit < ActiveRecord::Base
     include DeferredTouch
 
+    RECENT_COMMIT_THRESHOLD = 10.seconds
+
     AmbiguousRevision = Class.new(StandardError)
 
     belongs_to :stack
@@ -102,6 +104,14 @@ module Shipit
       )
     end
 
+    def message=(message)
+      limit = self.class.columns_hash['message'].limit
+      if limit && message && message.size > limit
+        message = message.slice(0, limit)
+      end
+      super(message)
+    end
+
     def reload(*)
       @status = nil
       super
@@ -126,7 +136,9 @@ module Shipit
     end
 
     def refresh_statuses!
-      github_statuses = stack.handle_github_redirections { Shipit.github.api.statuses(github_repo_name, sha) }
+      github_statuses = stack.handle_github_redirections do
+        Shipit.github.api.statuses(github_repo_name, sha, per_page: 100)
+      end
       github_statuses.each do |status|
         create_status_from_github!(status)
       end
@@ -177,13 +189,7 @@ module Shipit
     def active?
       return false unless stack.active_task?
 
-      active_task = stack.active_task
-
-      if active_task.since_commit == active_task.until_commit
-        id == active_task.since_commit.id
-      else
-        id > active_task.since_commit.id && id <= active_task.until_commit.id
-      end
+      stack.active_task.includes_commit?(self)
     end
 
     def deployable?
@@ -242,7 +248,9 @@ module Shipit
 
     def schedule_continuous_delivery
       return unless deployable? && stack.continuous_deployment? && stack.deployable?
-      ContinuousDeliveryJob.perform_later(stack)
+      # This buffer is to allow for statuses and checks to be refreshed before evaluating if the commit is deployable
+      # - e.g. if the commit was fast-forwarded with already passing CI.
+      ContinuousDeliveryJob.set(wait: RECENT_COMMIT_THRESHOLD).perform_later(stack)
     end
 
     def github_commit
@@ -311,6 +319,10 @@ module Shipit
       update!(locked: false, lock_author: nil)
     end
 
+    def recently_pushed?
+      created_at > RECENT_COMMIT_THRESHOLD.ago
+    end
+
     private
 
     def message_parser

data/app/models/shipit/commit_deployment.rb

@@ -1,6 +1,5 @@
 module Shipit
   class CommitDeployment < ActiveRecord::Base
-    belongs_to :commit
     belongs_to :task
     has_many :statuses, dependent: :destroy, class_name: 'CommitDeploymentStatus'
 
@@ -9,11 +8,10 @@ module Shipit
     delegate :stack, :author, to: :task
 
     def create_on_github!
-      return unless commit.pull_request?
-
       create_deployment_on_github!
       statuses.order(id: :asc).each(&:create_on_github!)
-    rescue Octokit::NotFound, Octokit::Forbidden
+    rescue Octokit::NotFound, Octokit::Forbidden => error
+      Rails.logger.warn("Got #{error.class.name} creating deployment or statuses: #{error.message}")
       # If no one can create the deployment we can only give up
     end
 
@@ -34,25 +32,31 @@ module Shipit
       update!(github_id: response.id, api_url: response.url)
     end
 
-    def pull_request_head
-      pull_request = Shipit.github.api.pull_request(stack.github_repo_name, commit.pull_request_number)
-      pull_request.head.sha
-    end
-
     def schedule_create_on_github
       CreateOnGithubJob.perform_later(self)
     end
 
+    def short_sha
+      sha[0..9]
+    end
+
     private
 
     def create_deployment_on_github(client)
       client.create_deployment(
         stack.github_repo_name,
-        pull_request_head,
+        sha,
         auto_merge: false,
         required_contexts: [],
-        description: "Via Shipit",
+        description: "Via #{Shipit.app_name}",
         environment: stack.environment,
+        payload: {
+          shipit: {
+            task_id: task.id,
+            from_sha: task.since_commit.sha,
+            to_sha: task.until_commit.sha,
+          },
+        },
       )
     end
   end

data/app/models/shipit/commit_deployment_status.rb

@@ -1,5 +1,7 @@
 module Shipit
   class CommitDeploymentStatus < ActiveRecord::Base
+    DESCRIPTION_CHARACTER_LIMIT_ON_GITHUB = 140
+
     belongs_to :commit_deployment
 
     after_commit :schedule_create_on_github, on: :create
@@ -25,7 +27,7 @@ module Shipit
     def description
       I18n.t(
         "deployment_description.#{task_type}.#{status}",
-        sha: task.until_commit.sha,
+        sha: task.until_commit.short_sha,
         author: task.author.login,
         stack: stack.to_param,
       )
@@ -45,8 +47,10 @@ module Shipit
       client.create_deployment_status(
         commit_deployment.api_url,
         status,
+        accept: 'application/vnd.github.flash-preview+json',
         target_url: url_helpers.stack_deploy_url(stack, task),
-        description: description,
+        description: description.truncate(DESCRIPTION_CHARACTER_LIMIT_ON_GITHUB),
+        environment_url: stack.deploy_url,
       )
     end
 

data/app/models/shipit/deploy.rb

@@ -14,20 +14,41 @@ module Shipit
       after_transition any => any, do: :update_last_deploy_time
     end
 
+    belongs_to :until_commit, class_name: 'Commit', required: true, inverse_of: :deploys
+    belongs_to :since_commit, class_name: 'Commit', required: true, inverse_of: :deploys
     has_many :commit_deployments, dependent: :destroy, inverse_of: :task, foreign_key: :task_id do
       GITHUB_STATUSES = {
         'pending' => 'pending',
+        'running' => 'in_progress',
         'failed' => 'failure',
+        'timedout' => 'failure',
         'success' => 'success',
+        'faulty' => 'error',
         'error' => 'error',
         'aborted' => 'error',
       }.freeze
 
       def append_status(task_status)
         if github_status = GITHUB_STATUSES[task_status]
-          each do |deployment|
+          # Deployments and statuses are created async, we reload the association to ensure we update all instances
+          reload.each do |deployment|
+            Rails.logger.info(
+              "Creating #{github_status} deploy status for deployment #{deployment.id}. "\
+              "Commit: #{deployment.sha}, Github id: #{deployment.github_id}, "\
+              "Repo: #{deployment.stack.repo_name}, Environment: #{deployment.stack.environment}, "\
+              "API Url: #{deployment.api_url}.",
+            )
             deployment.statuses.create!(status: github_status)
           end
+        else
+          each do |deployment|
+            Rails.logger.warn(
+              "No GitHub status for task status #{task_status}. "\
+              "Commit: #{deployment.sha}, Github id: #{deployment.github_id}, "\
+              "Repo: #{deployment.stack.repo_name}, Environment: #{deployment.stack.environment}, "\
+              "API Url: #{deployment.api_url}.",
+            )
+          end
         end
       end
     end
@@ -215,12 +236,36 @@ module Shipit
       end
     end
 
+    def update_commit_deployments
+      commit_deployments.append_status(status)
+    end
+
     private
 
     def create_commit_deployments
-      commits.each do |commit|
-        commit_deployments.create!(commit: commit)
+      # Create one deployment for the head of the batch
+      commit_deployments.create!(sha: until_commit.sha)
+
+      # Create one for each pull request in the batch, to give feedback on the PR timeline
+      commits.select(&:pull_request?).each do |commit|
+        if (pull_request_head = pull_request_head_for_commit(commit))
+          commit_deployments.create!(sha: pull_request_head)
+        end
       end
+
+      # Immediately update to publish the status to the commit deployments
+      update_commit_deployments
+    rescue Octokit::ClientError => error
+      Rails.logger.warn("Got #{error.class.name} (#{error.message}) when creating CommitDeployments for Deploy##{id}")
+    end
+
+    def pull_request_head_for_commit(commit)
+      pull_request = Shipit.github.api.pull_request(commit.stack.github_repo_name, commit.pull_request_number)
+      pull_request.head.sha
+    rescue Octokit::ClientError => error
+      pr_ref = "#{commit.stack.github_repo_name}##{commit.pull_request_number}"
+      Rails.logger.warn("Got #{error.class.name} (#{error.message}) when loading pull request #{pr_ref}")
+      nil
     end
 
     def update_release_status
@@ -245,10 +290,6 @@ module Shipit
       end
     end
 
-    def update_commit_deployments
-      commit_deployments.append_status(status)
-    end
-
     def trigger_revert_if_required
       return unless rollback_once_aborted?
       return unless supports_rollback?

data/app/models/shipit/deploy_stats.rb

@@ -0,0 +1,57 @@
+module Shipit
+  class DeployStats
+    delegate :empty?, to: :@deploys
+
+    def initialize(deploys)
+      @deploys = deploys
+      @durations = @deploys.map { |d| d.duration.value }.compact
+    end
+
+    def count
+      @deploys.length
+    end
+
+    def average_duration
+      return if empty?
+      @durations.sum / @durations.length.to_f
+    end
+
+    def max_duration
+      @durations.max
+    end
+
+    def min_duration
+      @durations.min
+    end
+
+    def median_duration
+      return if @durations.empty?
+      (sorted_durations[(@durations.length - 1) / 2] + sorted_durations[@durations.length / 2]) / 2.0
+    end
+
+    def success_rate
+      return if empty?
+      (@deploys.count(&:success?) / @deploys.length.to_f) * 100
+    end
+
+    def compare(compare_stats)
+      {
+        count: percent_change(compare_stats.count, count),
+        average_duration: percent_change(compare_stats.average_duration, average_duration),
+        median_duration: percent_change(compare_stats.median_duration, median_duration),
+      }
+    end
+
+    protected
+
+    def sorted_durations
+      @sorted ||= @durations.sort
+    end
+
+    def percent_change(from, to)
+      return if to.nil? || from.nil?
+      return to * 100 if from.zero?
+      ((to - from) / from.to_f) * 100
+    end
+  end
+end

data/app/models/shipit/repository.rb

@@ -0,0 +1,38 @@
+module Shipit
+  class Repository < ApplicationRecord
+    OWNER_MAX_SIZE = 39
+    private_constant :OWNER_MAX_SIZE
+
+    NAME_MAX_SIZE = 100
+    private_constant :NAME_MAX_SIZE
+
+    validates :name, uniqueness: {scope: %i(owner), case_sensitive: false,
+                                  message: 'cannot be used more than once'}
+    validates :owner, :name, presence: true, ascii_only: true
+    validates :owner, format: {with: /\A[a-z0-9_\-\.]+\z/}, length: {maximum: OWNER_MAX_SIZE}
+    validates :name, format: {with: /\A[a-z0-9_\-\.]+\z/}, length: {maximum: NAME_MAX_SIZE}
+
+    has_many :stacks, dependent: :destroy
+
+    def self.from_github_repo_name(github_repo_name)
+      repo_owner, repo_name = github_repo_name.downcase.split('/')
+      find_by(owner: repo_owner, name: repo_name)
+    end
+
+    def name=(n)
+      super(n&.downcase)
+    end
+
+    def owner=(o)
+      super(o&.downcase)
+    end
+
+    def http_url
+      Shipit.github.url("#{owner}/#{name}")
+    end
+
+    def git_url
+      "https://#{Shipit.github.domain}/#{owner}/#{name}.git"
+    end
+  end
+end

data/app/models/shipit/stack.rb

@@ -22,8 +22,6 @@ module Shipit
       end
     end
 
-    REPO_OWNER_MAX_SIZE = 39
-    REPO_NAME_MAX_SIZE = 100
     ENVIRONMENT_MAX_SIZE = 50
     REQUIRED_HOOKS = %i(push status).freeze
 
@@ -40,6 +38,14 @@ module Shipit
     has_many :hooks, dependent: :destroy
     has_many :api_clients, dependent: :destroy
     belongs_to :lock_author, class_name: :User, optional: true
+    belongs_to :repository
+    validates_associated :repository
+
+    scope :not_archived, -> { where(archived_since: nil) }
+
+    def repository
+      super || build_repository
+    end
 
     def lock_author(*)
       super || AnonymousUser.new
@@ -49,11 +55,6 @@ module Shipit
       super(user&.logged_in? ? user : nil)
     end
 
-    def self.repo(full_name)
-      repo_owner, repo_name = full_name.downcase.split('/')
-      where(repo_owner: repo_owner, repo_name: repo_name)
-    end
-
     before_validation :update_defaults
     before_destroy :clear_local_files
     before_save :set_locked_since
@@ -66,11 +67,10 @@ module Shipit
     after_commit :sync_github, on: :create
     after_commit :schedule_merges_if_necessary, on: :update
 
-    validates :repo_name, uniqueness: {scope: %i(repo_owner environment), case_sensitive: false,
-                                       message: 'cannot be used more than once with this environment'}
-    validates :repo_owner, :repo_name, :environment, presence: true, ascii_only: true
-    validates :repo_owner, format: {with: /\A[a-z0-9_\-\.]+\z/}, length: {maximum: REPO_OWNER_MAX_SIZE}
-    validates :repo_name, format: {with: /\A[a-z0-9_\-\.]+\z/}, length: {maximum: REPO_NAME_MAX_SIZE}
+    validates :repository, uniqueness: {
+      scope: %i(environment), case_sensitive: false,
+      message: 'cannot be used more than once with this environment. Check archived stacks.'
+    }
     validates :environment, format: {with: /\A[a-z0-9\-_\:]+\z/}, length: {maximum: ENVIRONMENT_MAX_SIZE}
     validates :deploy_url, format: {with: URI.regexp(%w(http https ssh))}, allow_blank: true
 
@@ -157,7 +157,8 @@ module Shipit
         return
       end
 
-      if commit.deploy_failed? || (checks? && !EphemeralCommitChecks.new(commit).run.success?)
+      if commit.deploy_failed? || (checks? && !EphemeralCommitChecks.new(commit).run.success?) ||
+         commit.recently_pushed?
         continuous_delivery_delayed!
         return
       end
@@ -320,21 +321,12 @@ module Shipit
       cached_deploy_spec&.pull_request_merge_method || Shipit.default_merge_method
     end
 
-    def repo_name=(name)
-      super(name&.downcase)
-    end
-
-    def repo_owner=(name)
-      super(name&.downcase)
-    end
-
-    def repo_http_url
-      Shipit.github.url("#{repo_owner}/#{repo_name}")
-    end
-
-    def repo_git_url
-      "https://#{Shipit.github.domain}/#{repo_owner}/#{repo_name}.git"
-    end
+    delegate :name=, to: :repository, prefix: :repo
+    delegate :name, to: :repository, prefix: :repo
+    delegate :owner=, to: :repository, prefix: :repo
+    delegate :owner, to: :repository, prefix: :repo
+    delegate :http_url, to: :repository, prefix: :repo
+    delegate :git_url, to: :repository, prefix: :repo
 
     def base_path
       Rails.root.join('data', 'stacks', repo_owner, repo_name, environment)
@@ -389,7 +381,7 @@ module Shipit
       if resource.try(:message) == 'Moved Permanently'
         resource = Shipit.github.api.get(resource.url)
       end
-      update!(repo_owner: resource.owner.login, repo_name: resource.name)
+      repository.update!(owner: resource.owner.login, name: resource.name)
     end
 
     def active_task?
@@ -414,6 +406,18 @@ module Shipit
       update!(lock_reason: nil, lock_author: nil, locked_since: nil)
     end
 
+    def archived?
+      archived_since.present?
+    end
+
+    def archive!(user)
+      update!(archived_since: Time.now, lock_reason: "Archived", lock_author: user)
+    end
+
+    def unarchive!
+      update!(archived_since: nil, lock_reason: nil, lock_author: nil, locked_since: nil)
+    end
+
     def to_param
       [repo_owner, repo_name, environment].join('/')
     end
@@ -429,11 +433,14 @@ module Shipit
 
     def self.from_param!(param)
       repo_owner, repo_name, environment = param.split('/')
-      where(
-        repo_owner: repo_owner.downcase,
-        repo_name: repo_name.downcase,
-        environment: environment,
-      ).first!
+      includes(:repository)
+        .where(
+          repositories: {
+            owner: repo_owner.downcase,
+            name: repo_name.downcase,
+          },
+          environment: environment,
+        ).first!
     end
 
     delegate :plugins, :task_definitions, :hidden_statuses, :required_statuses, :soft_failing_statuses,