shipit-engine 0.31.0 → 0.32.0

data/app/models/shipit/pull_request.rb

@@ -1,8 +1,9 @@
+# frozen_string_literal: true
 module Shipit
   class PullRequest < ApplicationRecord
     include DeferredTouch
 
-    MERGE_REQUEST_FIELD = 'Merge-Requested-By'.freeze
+    MERGE_REQUEST_FIELD = 'Merge-Requested-By'
 
     WAITING_STATUSES = %w(fetching pending).freeze
     QUEUED_STATUSES = %w(pending revalidating).freeze
@@ -45,7 +46,7 @@ module Shipit
 
     deferred_touch stack: :updated_at
 
-    validates :number, presence: true, uniqueness: {scope: :stack_id}
+    validates :number, presence: true, uniqueness: { scope: :stack_id }
 
     scope :waiting, -> { where(merge_status: WAITING_STATUSES) }
     scope :pending, -> { where(merge_status: 'pending') }
@@ -176,10 +177,10 @@ module Shipit
         # branch was already deleted somehow
       end
       complete!
-      return true
+      true
     rescue Octokit::MethodNotAllowed # merge conflict
       reject!('merge_conflict')
-      return false
+      false
     rescue Octokit::Conflict # shas didn't match, PR was updated.
       raise NotReady
     end
@@ -289,7 +290,7 @@ module Shipit
 
     def find_or_create_commit_from_github_by_sha!(sha, attributes)
       if commit = stack.commits.by_sha(sha)
-        return commit
+        commit
       else
         github_commit = Shipit.github.api.commit(stack.github_repo_name, sha)
         stack.commits.create_from_github!(github_commit, attributes)

data/app/models/shipit/record.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+module Shipit
+  class Record < ActiveRecord::Base
+    self.abstract_class = true
+
+    class << self
+      def serializer_class
+        if defined? @serializer_class
+          @serializer_class
+        else
+          @serializer_class = "#{name}Serializer".safe_constantize
+        end
+      end
+    end
+
+    delegate :serializer_class, to: :class
+  end
+end

data/app/models/shipit/release_status.rb

@@ -1,5 +1,6 @@
+# frozen_string_literal: true
 module Shipit
-  class ReleaseStatus < ActiveRecord::Base
+  class ReleaseStatus < Record
     MAX_DESCRIPTION_LENGTH = 140
     include DeferredTouch
 
@@ -13,7 +14,7 @@ module Shipit
     scope :to_be_created, -> { where(github_id: nil).order(id: :asc) }
 
     STATES = %w(pending success failure error).freeze
-    validates :state, presence: true, inclusion: {in: STATES}
+    validates :state, presence: true, inclusion: { in: STATES }
 
     def create_status_on_github!
       return true if github_id?

data/app/models/shipit/repository.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class Repository < ApplicationRecord
     OWNER_MAX_SIZE = 39
@@ -6,11 +7,11 @@ module Shipit
     NAME_MAX_SIZE = 100
     private_constant :NAME_MAX_SIZE
 
-    validates :name, uniqueness: {scope: %i(owner), case_sensitive: false,
-                                  message: 'cannot be used more than once'}
+    validates :name, uniqueness: { scope: %i(owner), case_sensitive: false,
+                                   message: 'cannot be used more than once' }
     validates :owner, :name, presence: true, ascii_only: true
-    validates :owner, format: {with: /\A[a-z0-9_\-\.]+\z/}, length: {maximum: OWNER_MAX_SIZE}
-    validates :name, format: {with: /\A[a-z0-9_\-\.]+\z/}, length: {maximum: NAME_MAX_SIZE}
+    validates :owner, format: { with: /\A[a-z0-9_\-\.]+\z/ }, length: { maximum: OWNER_MAX_SIZE }
+    validates :name, format: { with: /\A[a-z0-9_\-\.]+\z/ }, length: { maximum: NAME_MAX_SIZE }
 
     has_many :stacks, dependent: :destroy
 
@@ -28,7 +29,11 @@ module Shipit
     end
 
     def http_url
-      Shipit.github.url("#{owner}/#{name}")
+      Shipit.github.url(full_name)
+    end
+
+    def full_name
+      "#{owner}/#{name}"
     end
 
     def git_url

data/app/models/shipit/rollback.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class Rollback < Deploy
     belongs_to :deploy, foreign_key: :parent_id, inverse_of: false

data/app/models/shipit/stack.rb

@@ -1,7 +1,8 @@
+# frozen_string_literal: true
 require 'fileutils'
 
 module Shipit
-  class Stack < ActiveRecord::Base
+  class Stack < Record
     module NoDeployedCommit
       extend self
 
@@ -43,6 +44,8 @@ module Shipit
 
     scope :not_archived, -> { where(archived_since: nil) }
 
+    default_scope { preload(:repository) }
+
     def env
       {
         'ENVIRONMENT' => environment,
@@ -82,10 +85,10 @@ module Shipit
       scope: %i(environment), case_sensitive: false,
       message: 'cannot be used more than once with this environment. Check archived stacks.'
     }
-    validates :environment, format: {with: /\A[a-z0-9\-_\:]+\z/}, length: {maximum: ENVIRONMENT_MAX_SIZE}
-    validates :deploy_url, format: {with: URI.regexp(%w(http https ssh))}, allow_blank: true
+    validates :environment, format: { with: /\A[a-z0-9\-_\:]+\z/ }, length: { maximum: ENVIRONMENT_MAX_SIZE }
+    validates :deploy_url, format: { with: URI.regexp(%w(http https ssh)) }, allow_blank: true
 
-    validates :lock_reason, length: {maximum: 4096}
+    validates :lock_reason, length: { maximum: 4096 }
 
     serialize :cached_deploy_spec, DeploySpec
     delegate :find_task_definition, :supports_rollback?, :release_status?, :release_status_delay,
@@ -140,9 +143,24 @@ module Shipit
     end
 
     def trigger_deploy(*args, **kwargs)
+      if changed?
+        # If this is the first deploy since the spec changed it's possible the record will be dirty here, meaning we
+        # cant lock. In this one case persist the changes, otherwise log a warning and let the lock raise, so we
+        # can debug what's going on here. We don't expect anything other than the deploy spec to dirty the model
+        # instance, because of how that field is serialised.
+        if changes.keys == ['cached_deploy_spec']
+          save!
+        else
+          Rails.logger.warning("#{changes.keys} field(s) were unexpectedly modified on stack #{id} while deploying")
+        end
+      end
+
       run_now = kwargs.delete(:run_now)
-      deploy = build_deploy(*args, **kwargs)
-      deploy.save!
+      deploy = with_lock do
+        deploy = build_deploy(*args, **kwargs)
+        deploy.save!
+        deploy
+      end
       run_now ? deploy.run_now! : deploy.enqueue
       continuous_delivery_resumed!
       deploy
@@ -161,15 +179,16 @@ module Shipit
     end
 
     def trigger_continuous_delivery
+      return if cached_deploy_spec.blank?
+
       commit = next_commit_to_deploy
 
-      if !deployable? || deployed_too_recently? || commit.nil? || commit.deployed?
+      if should_resume_continuous_delivery?(commit)
         continuous_delivery_resumed!
         return
       end
 
-      if commit.deploy_failed? || (checks? && !EphemeralCommitChecks.new(commit).run.success?) ||
-         commit.recently_pushed?
+      if should_delay_continuous_delivery?(commit)
         continuous_delivery_delayed!
         return
       end
@@ -201,7 +220,7 @@ module Shipit
     def async_refresh_deployed_revision
       async_refresh_deployed_revision!
     rescue => error
-      logger.warn "Failed to dispatch FetchDeployedRevisionJob: [#{error.class.name}] #{error.message}"
+      logger.warn("Failed to dispatch FetchDeployedRevisionJob: [#{error.class.name}] #{error.message}")
     end
 
     def async_refresh_deployed_revision!
@@ -269,8 +288,8 @@ module Shipit
         next if commits_to_lock.empty?
 
         affected_rows += commits
-                         .where(id: commits_to_lock.map(&:id).uniq)
-                         .lock_all(revert.author)
+          .where(id: commits_to_lock.map(&:id).uniq)
+          .lock_all(revert.author)
       end
 
       touch if affected_rows > 1
@@ -409,7 +428,7 @@ module Shipit
     end
 
     def lock(reason, user)
-      params = {lock_reason: reason, lock_author: user}
+      params = { lock_reason: reason, lock_author: user }
       update!(params)
     end
 
@@ -481,13 +500,15 @@ module Shipit
     end
 
     def update_latest_deployed_ref
-      UpdateGithubLastDeployedRefJob.perform_later(self)
+      if Shipit.update_latest_deployed_ref
+        UpdateGithubLastDeployedRefJob.perform_later(self)
+      end
     end
 
     def broadcast_update
       Pubsubstub.publish(
         "stack.#{id}",
-        {id: id, updated_at: updated_at}.to_json,
+        { id: id, updated_at: updated_at }.to_json,
         name: 'update',
       )
     end
@@ -581,7 +602,7 @@ module Shipit
       return unless previous_changes.include?('lock_reason')
 
       lock_details = if previous_changes['lock_reason'].last.blank?
-        {from: previous_changes['locked_since'].first, until: Time.zone.now}
+        { from: previous_changes['locked_since'].first, until: Time.zone.now }
       end
 
       Hook.emit(:lock, self, locked: locked?, lock_details: lock_details, stack: self)
@@ -607,5 +628,18 @@ module Shipit
     def ci_enabled_cache_key
       "stacks:#{id}:ci_enabled"
     end
+
+    def should_resume_continuous_delivery?(commit)
+      !deployable? ||
+        deployed_too_recently? ||
+        commit.nil? ||
+        commit.deployed?
+    end
+
+    def should_delay_continuous_delivery?(commit)
+      commit.deploy_failed? ||
+        (checks? && !EphemeralCommitChecks.new(commit).run.success?) ||
+        commit.recently_pushed?
+    end
   end
 end

data/app/models/shipit/status.rb

@@ -1,5 +1,6 @@
+# frozen_string_literal: true
 module Shipit
-  class Status < ActiveRecord::Base
+  class Status < Record
     include Common
     include DeferredTouch
 
@@ -11,7 +12,7 @@ module Shipit
 
     deferred_touch commit: :updated_at
 
-    validates :state, inclusion: {in: STATES, allow_blank: true}, presence: true
+    validates :state, inclusion: { in: STATES, allow_blank: true }, presence: true
 
     after_create :enable_ci_on_stack
     after_commit :schedule_continuous_delivery, :broadcast_update, on: :create

data/app/models/shipit/status/common.rb

@@ -1,24 +1,25 @@
+# frozen_string_literal: true
 module Shipit
   class Status
     module Common
       def unknown?
-        state == 'unknown'.freeze
+        state == 'unknown'
       end
 
       def pending?
-        state == 'pending'.freeze
+        state == 'pending'
       end
 
       def success?
-        state == 'success'.freeze
+        state == 'success'
       end
 
       def error?
-        state == 'error'.freeze
+        state == 'error'
       end
 
       def failure?
-        state == 'failure'.freeze
+        state == 'failure'
       end
 
       def missing?
@@ -30,7 +31,7 @@ module Shipit
       end
 
       def simple_state
-        state == 'error'.freeze ? 'failure'.freeze : state
+        state == 'error' ? 'failure' : state
       end
 
       def allowed_to_fail?

data/app/models/shipit/status/group.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class Status
     class Group

data/app/models/shipit/status/missing.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class Status
     class Missing
@@ -15,7 +16,7 @@ module Shipit
       end
 
       def state
-        'pending'.freeze
+        'pending'
       end
 
       def missing?

data/app/models/shipit/status/unknown.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class Status
     class Unknown
@@ -21,7 +22,7 @@ module Shipit
       end
 
       def state
-        'unknown'.freeze
+        'unknown'
       end
 
       def missing?

data/app/models/shipit/task.rb

@@ -1,13 +1,17 @@
+# frozen_string_literal: true
 module Shipit
-  class Task < ActiveRecord::Base
+  class Task < Record
     include DeferredTouch
 
     ConcurrentTaskRunning = Class.new(StandardError)
 
-    PRESENCE_CHECK_TIMEOUT = 15
+    PRESENCE_CHECK_TIMEOUT = 30
     ACTIVE_STATUSES = %w(pending running aborting).freeze
     COMPLETED_STATUSES = %w(success flapping faulty validating).freeze
     UNSUCCESSFUL_STATUSES = %w(error failed aborted flapping timedout faulty).freeze
+    OUTPUT_SIZE_LIMIT = 16.megabytes # A MySQL mediumblob
+    HUMAN_READABLE_OUTPUT_LIMIT = ActionController::Base.helpers.number_to_human_size(OUTPUT_SIZE_LIMIT)
+    OUTPUT_TRUNCATED_MESSAGE = "Output exceeded the limit of #{HUMAN_READABLE_OUTPUT_LIMIT} and was truncated\n"
 
     attr_accessor :pid
 
@@ -19,6 +23,8 @@ module Shipit
     belongs_to :until_commit, class_name: 'Commit', required: false
     belongs_to :since_commit, class_name: 'Commit', required: false
 
+    belongs_to :rollback_once_aborted_to, class_name: 'Task', optional: true
+
     deferred_touch stack: :updated_at
 
     has_many :chunks, -> { order(:id) }, class_name: 'OutputChunk', dependent: :delete_all, inverse_of: :task
@@ -198,7 +204,15 @@ module Shipit
       if rolled_up?
         output
       else
-        chunks.pluck(:text).join
+        blob = chunks.pluck(:text).join
+
+        if blob.size > OUTPUT_SIZE_LIMIT
+          Rails.logger.warn("Task #{id} output exceeds limit of #{HUMAN_READABLE_OUTPUT_LIMIT}, and will be truncated.")
+          blob = blob.last(OUTPUT_SIZE_LIMIT - OUTPUT_TRUNCATED_MESSAGE.size)
+          blob = OUTPUT_TRUNCATED_MESSAGE + blob
+        end
+
+        blob
       end
     end
 
@@ -280,8 +294,12 @@ module Shipit
       end
     end
 
-    def abort!(rollback_once_aborted: false, aborted_by:)
-      update!(rollback_once_aborted: rollback_once_aborted, aborted_by_id: aborted_by.id)
+    def abort!(rollback_once_aborted: false, rollback_once_aborted_to: nil, aborted_by:)
+      update!(
+        rollback_once_aborted: rollback_once_aborted,
+        rollback_once_aborted_to: rollback_once_aborted_to,
+        aborted_by_id: aborted_by.id
+      )
 
       if alive?
         aborting
@@ -349,6 +367,21 @@ module Shipit
       end
     end
 
+    def self.recently_created_at
+      5.minutes.ago
+    end
+
+    ZOMBIE_STATES = %w(running aborting).freeze
+    private_constant :ZOMBIE_STATES
+    def self.zombies
+      where(status: ZOMBIE_STATES)
+        .where(
+          "created_at <= :recently",
+          recently: recently_created_at,
+        )
+        .reject(&:alive?)
+    end
+
     private
 
     def prevent_concurrency

data/app/models/shipit/task_definition.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class TaskDefinition
     NotFound = Class.new(StandardError)

data/app/models/shipit/team.rb

@@ -1,5 +1,6 @@
+# frozen_string_literal: true
 module Shipit
-  class Team < ActiveRecord::Base
+  class Team < Record
     REQUIRED_HOOKS = %i(membership).freeze
 
     has_many :memberships