shipit-engine 0.28.1 → 0.33.0

data/app/models/shipit/review_stack.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+module Shipit
+  class ReviewStack < Shipit::Stack
+    def self.clear_stale_caches
+      Shipit::ReviewStack.where(
+        "archived_since > :earliest AND archived_since < :latest",
+        earliest: 1.day.ago,
+        latest: 1.hour.ago
+      ).each do |review_stack|
+        Shipit::ClearGitCacheJob.perform_later(review_stack)
+      end
+    end
+
+    def self.delete_old_deployment_directories
+      Shipit::Deploy.not_active.where(
+        "created_at > :earliest AND updated_at < :latest",
+        earliest: 1.day.ago,
+        latest: 1.hour.ago
+      ).find_each do |deploy|
+        Shipit::Commands.for(deploy).clear_working_directory
+      end
+    end
+
+    has_one :pull_request, foreign_key: :stack_id
+
+    after_commit :emit_added_hooks, on: :create
+    after_commit :emit_updated_hooks, on: :update
+    after_commit :emit_removed_hooks, on: :destroy
+
+    state_machine :provision_status, initial: :deprovisioned do
+      state :provisioned
+      state :provisioning
+      state :deprovisioning
+      state :deprovisioned
+
+      event :provision do
+        transition deprovisioned: :provisioning
+      end
+
+      event :provision_success do
+        transition provisioning: :provisioned
+      end
+
+      event :provision_failure do
+        transition provisioning: :deprovisioned
+      end
+
+      event :deprovision do
+        transition provisioned: :deprovisioning
+      end
+
+      event :deprovision_success do
+        transition deprovisioning: :deprovisioned
+      end
+
+      event :deprovision_failure do
+        transition deprovisioning: :provisioned
+      end
+
+      after_transition deprovisioned: :provisioning do |stack, _|
+        stack.provisioner.up
+      end
+
+      after_transition provisioned: :deprovisioning do |stack, _|
+        stack.provisioner.down
+      end
+    end
+
+    def env
+      return super unless pull_request.present?
+
+      super
+        .merge(
+          pull_request
+            .labels
+            .each_with_object({}) { |label_name, labels| labels[label_name.upcase] = "true" }
+        )
+    end
+
+    def provisioner
+      provisioner_class.new(self)
+    end
+
+    def provisioner_class
+      ProvisioningHandler.fetch(provisioning_handler_name)
+    end
+
+    def enqueue_for_provisioning
+      return if awaiting_provision
+      update!(awaiting_provision: true)
+    end
+
+    def remove_from_provisioning_queue
+      return unless awaiting_provision
+      update!(awaiting_provision: false)
+    end
+
+    def to_partial_path
+      "shipit/stacks/stack"
+    end
+
+    def emit_added_hooks
+      Hook.emit(:review_stack, self, action: :added, review_stack: self)
+    end
+
+    def emit_updated_hooks
+      changed = !(previous_changes.keys - %w(updated_at)).empty?
+      Hook.emit(:review_stack, self, action: :updated, review_stack: self) if changed
+    end
+
+    def emit_removed_hooks
+      Hook.emit(:review_stack, self, action: :removed, review_stack: self)
+    end
+  end
+end

data/app/models/shipit/review_stack_provisioning_queue.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Shipit
+  class ReviewStackProvisioningQueue
+    def self.work
+      new.work
+    end
+
+    def self.add(stack)
+      stack.enqueue_for_provisioning
+    end
+
+    def self.queued_stacks
+      new.queued_stacks
+    end
+
+    def work
+      queued_stacks.find_each(&method(:provision))
+    end
+
+    def queued_stacks
+      @queued_stacks ||= Shipit::ReviewStack
+        .with_provision_status(:deprovisioned)
+        .where(awaiting_provision: true)
+    end
+
+    private
+
+    def provision(stack)
+      if stack.provisioner.provision?
+        stack.provision
+      else
+        Rails.logger.info(
+          "Putting review ReviewStack<#{stack.id}> back into the provisioning queue - #provision? was falsey."
+        )
+      end
+    end
+  end
+end

data/app/models/shipit/rollback.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class Rollback < Deploy
     belongs_to :deploy, foreign_key: :parent_id, inverse_of: false

data/app/models/shipit/stack.rb

@@ -1,7 +1,8 @@
+# frozen_string_literal: true
 require 'fileutils'
 
 module Shipit
-  class Stack < ActiveRecord::Base
+  class Stack < Record
     module NoDeployedCommit
       extend self
 
@@ -22,13 +23,11 @@ module Shipit
       end
     end
 
-    REPO_OWNER_MAX_SIZE = 39
-    REPO_NAME_MAX_SIZE = 100
     ENVIRONMENT_MAX_SIZE = 50
     REQUIRED_HOOKS = %i(push status).freeze
 
     has_many :commits, dependent: :destroy
-    has_many :pull_requests, dependent: :destroy
+    has_many :merge_requests, dependent: :destroy
     has_many :tasks, dependent: :destroy
     has_many :deploys
     has_many :rollbacks
@@ -40,6 +39,30 @@ module Shipit
     has_many :hooks, dependent: :destroy
     has_many :api_clients, dependent: :destroy
     belongs_to :lock_author, class_name: :User, optional: true
+    belongs_to :repository
+    validates_associated :repository
+
+    scope :not_archived, -> { where(archived_since: nil) }
+
+    include DeferredTouch
+    deferred_touch repository: :updated_at
+
+    default_scope { preload(:repository) }
+
+    def env
+      {
+        'ENVIRONMENT' => environment,
+        'LAST_DEPLOYED_SHA' => last_deployed_commit.sha,
+        'GITHUB_REPO_OWNER' => repository.owner,
+        'GITHUB_REPO_NAME' => repository.name,
+        'DEPLOY_URL' => deploy_url,
+        'BRANCH' => branch,
+      }
+    end
+
+    def repository
+      super || build_repository
+    end
 
     def lock_author(*)
       super || AnonymousUser.new
@@ -49,11 +72,6 @@ module Shipit
       super(user&.logged_in? ? user : nil)
     end
 
-    def self.repo(full_name)
-      repo_owner, repo_name = full_name.downcase.split('/')
-      where(repo_owner: repo_owner, repo_name: repo_name)
-    end
-
     before_validation :update_defaults
     before_destroy :clear_local_files
     before_save :set_locked_since
@@ -66,19 +84,27 @@ module Shipit
     after_commit :sync_github, on: :create
     after_commit :schedule_merges_if_necessary, on: :update
 
-    validates :repo_name, uniqueness: {scope: %i(repo_owner environment),
-                                       message: 'cannot be used more than once with this environment'}
-    validates :repo_owner, :repo_name, :environment, presence: true, ascii_only: true
-    validates :repo_owner, format: {with: /\A[a-z0-9_\-\.]+\z/}, length: {maximum: REPO_OWNER_MAX_SIZE}
-    validates :repo_name, format: {with: /\A[a-z0-9_\-\.]+\z/}, length: {maximum: REPO_NAME_MAX_SIZE}
-    validates :environment, format: {with: /\A[a-z0-9\-_\:]+\z/}, length: {maximum: ENVIRONMENT_MAX_SIZE}
-    validates :deploy_url, format: {with: URI.regexp(%w(http https ssh))}, allow_blank: true
+    validates :repository, uniqueness: {
+      scope: %i(environment), case_sensitive: false,
+      message: 'cannot be used more than once with this environment. Check archived stacks.'
+    }
+    validates :environment, format: { with: /\A[a-z0-9\-_\:]+\z/ }, length: { maximum: ENVIRONMENT_MAX_SIZE }
+    validates :deploy_url, format: { with: URI.regexp(%w(http https ssh)) }, allow_blank: true
 
-    validates :lock_reason, length: {maximum: 4096}
+    validates :lock_reason, length: { maximum: 4096 }
 
     serialize :cached_deploy_spec, DeploySpec
-    delegate :find_task_definition, :supports_rollback?, :links, :release_status?, :release_status_delay,
-             :release_status_context, :supports_fetch_deployed_revision?, to: :cached_deploy_spec, allow_nil: true
+    delegate(
+      :provisioning_handler_name,
+      :find_task_definition,
+      :release_status?,
+      :release_status_context,
+      :release_status_delay,
+      :supports_fetch_deployed_revision?,
+      :supports_rollback?,
+      to: :cached_deploy_spec,
+      allow_nil: true
+    )
 
     def self.refresh_deployed_revisions
       find_each.select(&:supports_fetch_deployed_revision?).each(&:async_refresh_deployed_revision)
@@ -125,13 +151,29 @@ module Shipit
         env: filter_deploy_envs(env&.to_h || {}),
         allow_concurrency: force,
         ignored_safeties: force || !until_commit.deployable?,
+        max_retries: retries_on_deploy,
       )
     end
 
     def trigger_deploy(*args, **kwargs)
+      if changed?
+        # If this is the first deploy since the spec changed it's possible the record will be dirty here, meaning we
+        # cant lock. In this one case persist the changes, otherwise log a warning and let the lock raise, so we
+        # can debug what's going on here. We don't expect anything other than the deploy spec to dirty the model
+        # instance, because of how that field is serialised.
+        if changes.keys == ['cached_deploy_spec']
+          save!
+        else
+          Rails.logger.warning("#{changes.keys} field(s) were unexpectedly modified on stack #{id} while deploying")
+        end
+      end
+
       run_now = kwargs.delete(:run_now)
-      deploy = build_deploy(*args, **kwargs)
-      deploy.save!
+      deploy = with_lock do
+        deploy = build_deploy(*args, **kwargs)
+        deploy.save!
+        deploy
+      end
       run_now ? deploy.run_now! : deploy.enqueue
       continuous_delivery_resumed!
       deploy
@@ -150,14 +192,16 @@ module Shipit
     end
 
     def trigger_continuous_delivery
+      return if cached_deploy_spec.blank?
+
       commit = next_commit_to_deploy
 
-      if !deployable? || deployed_too_recently? || commit.nil? || commit.deployed?
+      if should_resume_continuous_delivery?(commit)
         continuous_delivery_resumed!
         return
       end
 
-      if commit.deploy_failed? || (checks? && !EphemeralCommitChecks.new(commit).run.success?)
+      if should_delay_continuous_delivery?(commit)
         continuous_delivery_delayed!
         return
       end
@@ -169,7 +213,7 @@ module Shipit
     end
 
     def schedule_merges
-      MergePullRequestsJob.perform_later(self)
+      ProcessMergeRequestsJob.perform_later(self)
     end
 
     def next_commit_to_deploy
@@ -189,7 +233,7 @@ module Shipit
     def async_refresh_deployed_revision
       async_refresh_deployed_revision!
     rescue => error
-      logger.warn "Failed to dispatch FetchDeployedRevisionJob: [#{error.class.name}] #{error.message}"
+      logger.warn("Failed to dispatch FetchDeployedRevisionJob: [#{error.class.name}] #{error.message}")
     end
 
     def async_refresh_deployed_revision!
@@ -257,8 +301,8 @@ module Shipit
         next if commits_to_lock.empty?
 
         affected_rows += commits
-                         .where(id: commits_to_lock.map(&:id).uniq)
-                         .lock_all(revert.author)
+          .where(id: commits_to_lock.map(&:id).uniq)
+          .lock_all(revert.author)
       end
 
       touch if affected_rows > 1
@@ -309,7 +353,7 @@ module Shipit
     end
 
     def deployable?
-      !locked? && !active_task?
+      !locked? && !active_task? && !awaiting_provision?
     end
 
     def allows_merges?
@@ -317,24 +361,15 @@ module Shipit
     end
 
     def merge_method
-      cached_deploy_spec&.pull_request_merge_method || Shipit.default_merge_method
-    end
-
-    def repo_name=(name)
-      super(name&.downcase)
-    end
-
-    def repo_owner=(name)
-      super(name&.downcase)
+      cached_deploy_spec&.merge_request_merge_method || Shipit.default_merge_method
     end
 
-    def repo_http_url
-      Shipit.github.url("#{repo_owner}/#{repo_name}")
-    end
-
-    def repo_git_url
-      "https://#{Shipit.github.domain}/#{repo_owner}/#{repo_name}.git"
-    end
+    delegate :name=, to: :repository, prefix: :repo
+    delegate :name, to: :repository, prefix: :repo
+    delegate :owner=, to: :repository, prefix: :repo
+    delegate :owner, to: :repository, prefix: :repo
+    delegate :http_url, to: :repository, prefix: :repo
+    delegate :git_url, to: :repository, prefix: :repo
 
     def base_path
       Rails.root.join('data', 'stacks', repo_owner, repo_name, environment)
@@ -354,15 +389,15 @@ module Shipit
 
     def clear_git_cache!
       tmp_path = "#{git_path}-#{SecureRandom.hex}"
+      return unless File.exist?(git_path)
       acquire_git_cache_lock do
-        return unless File.exist?(git_path)
         File.rename(git_path, tmp_path)
       end
       FileUtils.rm_rf(tmp_path)
     end
 
     def github_repo_name
-      [repo_owner, repo_name].join('/')
+      repository.github_repo_name
     end
 
     def github_commits
@@ -389,7 +424,7 @@ module Shipit
       if resource.try(:message) == 'Moved Permanently'
         resource = Shipit.github.api.get(resource.url)
       end
-      update!(repo_owner: resource.owner.login, repo_name: resource.name)
+      repository.update!(owner: resource.owner.login, name: resource.name)
     end
 
     def active_task?
@@ -406,7 +441,7 @@ module Shipit
     end
 
     def lock(reason, user)
-      params = {lock_reason: reason, lock_author: user}
+      params = { lock_reason: reason, lock_author: user }
       update!(params)
     end
 
@@ -414,6 +449,18 @@ module Shipit
       update!(lock_reason: nil, lock_author: nil, locked_since: nil)
     end
 
+    def archived?
+      archived_since.present?
+    end
+
+    def archive!(user)
+      update!(archived_since: Time.now, lock_reason: "Archived", lock_author: user)
+    end
+
+    def unarchive!
+      update!(archived_since: nil, lock_reason: nil, lock_author: nil, locked_since: nil)
+    end
+
     def to_param
       [repo_owner, repo_name, environment].join('/')
     end
@@ -429,16 +476,20 @@ module Shipit
 
     def self.from_param!(param)
       repo_owner, repo_name, environment = param.split('/')
-      where(
-        repo_owner: repo_owner.downcase,
-        repo_name: repo_name.downcase,
-        environment: environment,
-      ).first!
+      includes(:repository)
+        .where(
+          repositories: {
+            owner: repo_owner.downcase,
+            name: repo_name.downcase,
+          },
+          environment: environment,
+        ).first!
     end
 
     delegate :plugins, :task_definitions, :hidden_statuses, :required_statuses, :soft_failing_statuses,
              :blocking_statuses, :deploy_variables, :filter_task_envs, :filter_deploy_envs,
-             :maximum_commits_per_deploy, :pause_between_deploys, to: :cached_deploy_spec
+             :maximum_commits_per_deploy, :pause_between_deploys, :retries_on_deploy, :retries_on_rollback,
+             to: :cached_deploy_spec
 
     def monitoring?
       monitoring.present?
@@ -463,13 +514,15 @@ module Shipit
     end
 
     def update_latest_deployed_ref
-      UpdateGithubLastDeployedRefJob.perform_later(self)
+      if Shipit.update_latest_deployed_ref
+        UpdateGithubLastDeployedRefJob.perform_later(self)
+      end
     end
 
     def broadcast_update
       Pubsubstub.publish(
         "stack.#{id}",
-        {id: id, updated_at: updated_at}.to_json,
+        { id: id, updated_at: updated_at }.to_json,
         name: 'update',
       )
     end
@@ -521,6 +574,13 @@ module Shipit
       GithubSyncJob.perform_later(stack_id: id)
     end
 
+    def links
+      links_spec = cached_deploy_spec&.links || {}
+      context = EnvironmentVariables.with(env)
+
+      links_spec.transform_values { |url| context.interpolate(url) }
+    end
+
     private
 
     def clear_cache
@@ -556,7 +616,7 @@ module Shipit
       return unless previous_changes.include?('lock_reason')
 
       lock_details = if previous_changes['lock_reason'].last.blank?
-        {from: previous_changes['locked_since'].first, until: Time.zone.now}
+        { from: previous_changes['locked_since'].first, until: Time.zone.now }
       end
 
       Hook.emit(:lock, self, locked: locked?, lock_details: lock_details, stack: self)
@@ -582,5 +642,18 @@ module Shipit
     def ci_enabled_cache_key
       "stacks:#{id}:ci_enabled"
     end
+
+    def should_resume_continuous_delivery?(commit)
+      !deployable? ||
+        deployed_too_recently? ||
+        commit.nil? ||
+        commit.deployed?
+    end
+
+    def should_delay_continuous_delivery?(commit)
+      commit.deploy_failed? ||
+        (checks? && !EphemeralCommitChecks.new(commit).run.success?) ||
+        commit.recently_pushed?
+    end
   end
 end