shipit-engine 0.32.0 → 0.35.1

data/app/models/shipit/repository.rb

@@ -1,5 +1,35 @@
 # frozen_string_literal: true
 module Shipit
+  class NullRepository
+    def id
+      nil
+    end
+
+    def stacks
+      Shipit::Stack.none
+    end
+
+    def review_stacks
+      Shipit::ReviewStack.none
+    end
+
+    def review_stacks_enabled
+      false
+    end
+
+    def provisioning_behavior_allow_all?
+      false
+    end
+
+    def provisioning_behavior_allow_with_label?
+      false
+    end
+
+    def provisioning_behavior_prevent_with_label?
+      false
+    end
+  end
+
   class Repository < ApplicationRecord
     OWNER_MAX_SIZE = 39
     private_constant :OWNER_MAX_SIZE
@@ -8,12 +38,16 @@ module Shipit
     private_constant :NAME_MAX_SIZE
 
     validates :name, uniqueness: { scope: %i(owner), case_sensitive: false,
-                                   message: 'cannot be used more than once' }
+                                   message: 'cannot be used more than once', }
     validates :owner, :name, presence: true, ascii_only: true
     validates :owner, format: { with: /\A[a-z0-9_\-\.]+\z/ }, length: { maximum: OWNER_MAX_SIZE }
     validates :name, format: { with: /\A[a-z0-9_\-\.]+\z/ }, length: { maximum: NAME_MAX_SIZE }
 
     has_many :stacks, dependent: :destroy
+    has_many :review_stacks, dependent: :destroy
+
+    PROVISIONING_BEHAVIORS = %w(allow_all allow_with_label prevent_with_label).freeze
+    enum provisioning_behavior: PROVISIONING_BEHAVIORS.zip(PROVISIONING_BEHAVIORS).to_h, _prefix: :provisioning_behavior
 
     def self.from_github_repo_name(github_repo_name)
       repo_owner, repo_name = github_repo_name.downcase.split('/')
@@ -28,8 +62,12 @@ module Shipit
       super(o&.downcase)
     end
 
+    def github_repo_name
+      [owner, name].join('/')
+    end
+
     def http_url
-      Shipit.github.url(full_name)
+      github_app.url(full_name)
     end
 
     def full_name
@@ -37,7 +75,29 @@ module Shipit
     end
 
     def git_url
-      "https://#{Shipit.github.domain}/#{owner}/#{name}.git"
+      "https://#{github_app.domain}/#{owner}/#{name}.git"
+    end
+
+    def schedule_for_destroy!
+      DestroyRepositoryJob.perform_later(self)
+    end
+
+    def to_param
+      github_repo_name
+    end
+
+    def self.from_param!(param)
+      repo_owner, repo_name = param.split('/')
+      where(
+        owner: repo_owner.downcase,
+        name: repo_name.downcase,
+      ).first!
+    end
+
+    protected
+
+    def github_app
+      Shipit.github(organization: owner)
     end
   end
 end

data/app/models/shipit/review_stack.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+module Shipit
+  class ReviewStack < Shipit::Stack
+    def self.clear_stale_caches
+      Shipit::ReviewStack.where(
+        "archived_since > :earliest AND archived_since < :latest",
+        earliest: 1.day.ago,
+        latest: 1.hour.ago
+      ).find_each do |review_stack|
+        review_stack.clear_local_files
+      end
+    end
+
+    def self.delete_old_deployment_directories
+      Shipit::Deploy.not_active.where(
+        "created_at > :earliest AND updated_at < :latest",
+        earliest: 1.day.ago,
+        latest: 1.hour.ago
+      ).find_each do |deploy|
+        Shipit::Commands.for(deploy).clear_working_directory
+      end
+    end
+
+    def update_latest_deployed_ref
+      # noop: last deployed ref is useless for review stacks
+    end
+
+    model_name.class_eval do
+      def route_key
+        "stacks"
+      end
+
+      def singular_route_key
+        "stack"
+      end
+    end
+
+    has_one :pull_request, foreign_key: :stack_id
+
+    after_commit :emit_added_hooks, on: :create
+    after_commit :emit_updated_hooks, on: :update
+    after_commit :emit_removed_hooks, on: :destroy
+
+    state_machine :provision_status, initial: :deprovisioned do
+      state :provisioned
+      state :provisioning
+      state :deprovisioning
+      state :deprovisioned
+
+      event :provision do
+        transition deprovisioned: :provisioning
+      end
+
+      event :provision_success do
+        transition provisioning: :provisioned
+      end
+
+      event :provision_failure do
+        transition provisioning: :deprovisioned
+      end
+
+      event :deprovision do
+        transition provisioned: :deprovisioning
+      end
+
+      event :deprovision_success do
+        transition deprovisioning: :deprovisioned
+      end
+
+      event :deprovision_failure do
+        transition deprovisioning: :provisioned
+      end
+
+      after_transition deprovisioned: :provisioning do |stack, _|
+        stack.provisioner.up
+      end
+
+      after_transition provisioned: :deprovisioning do |stack, _|
+        stack.provisioner.down
+      end
+    end
+
+    def env
+      return super unless pull_request.present?
+
+      super
+        .merge(
+          pull_request
+            .labels
+            .each_with_object({}) { |label_name, labels| labels[label_name.upcase] = "true" }
+        )
+    end
+
+    def provisioner
+      provisioner_class.new(self)
+    end
+
+    def provisioner_class
+      ProvisioningHandler.fetch(provisioning_handler_name)
+    end
+
+    def enqueue_for_provisioning
+      return if awaiting_provision
+      update!(awaiting_provision: true)
+    end
+
+    def remove_from_provisioning_queue
+      return unless awaiting_provision
+      update!(awaiting_provision: false)
+    end
+
+    def to_partial_path
+      "shipit/stacks/stack"
+    end
+
+    def emit_added_hooks
+      Hook.emit(:review_stack, self, action: :added, review_stack: self)
+    end
+
+    def emit_updated_hooks
+      changed = !(previous_changes.keys - %w(updated_at)).empty?
+      Hook.emit(:review_stack, self, action: :updated, review_stack: self) if changed
+    end
+
+    def emit_removed_hooks
+      Hook.emit(:review_stack, self, action: :removed, review_stack: self)
+    end
+  end
+end

data/app/models/shipit/review_stack_provisioning_queue.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Shipit
+  class ReviewStackProvisioningQueue
+    def self.work
+      new.work
+    end
+
+    def self.add(stack)
+      stack.enqueue_for_provisioning
+    end
+
+    def self.queued_stacks
+      new.queued_stacks
+    end
+
+    def work
+      queued_stacks.find_each(&method(:provision))
+    end
+
+    def queued_stacks
+      @queued_stacks ||= Shipit::ReviewStack
+        .with_provision_status(:deprovisioned)
+        .where(awaiting_provision: true)
+    end
+
+    private
+
+    def provision(stack)
+      if stack.provisioner.provision?
+        stack.provision
+      else
+        Rails.logger.info(
+          "Putting review ReviewStack<#{stack.id}> back into the provisioning queue - #provision? was falsey."
+        )
+      end
+    end
+  end
+end

data/app/models/shipit/rollback.rb

@@ -33,6 +33,10 @@ module Shipit
       'deploys/deploy'
     end
 
+    def report_complete!
+      complete!
+    end
+
     private
 
     def update_release_status
@@ -40,6 +44,7 @@ module Shipit
 
       # When we rollback to a certain revision, assume that all later deploys were faulty
       stack.deploys.newer_than(deploy.id).until(stack.last_completed_deploy.id).to_a.each do |deploy|
+        next if deploy.id == id
         deploy.report_faulty!(description: "A rollback of #{stack.to_param} was triggered")
       end
     end

data/app/models/shipit/stack.rb

@@ -27,14 +27,14 @@ module Shipit
     REQUIRED_HOOKS = %i(push status).freeze
 
     has_many :commits, dependent: :destroy
-    has_many :pull_requests, dependent: :destroy
+    has_many :merge_requests, dependent: :destroy
     has_many :tasks, dependent: :destroy
     has_many :deploys
     has_many :rollbacks
     has_many :deploys_and_rollbacks,
-             -> { where(type: %w(Shipit::Deploy Shipit::Rollback)) },
-             class_name: 'Task',
-             inverse_of: :stack
+      -> { where(type: %w(Shipit::Deploy Shipit::Rollback)) },
+      class_name: 'Task',
+      inverse_of: :stack
     has_many :github_hooks, dependent: :destroy, class_name: 'Shipit::GithubHook::Repo'
     has_many :hooks, dependent: :destroy
     has_many :api_clients, dependent: :destroy
@@ -44,6 +44,9 @@ module Shipit
 
     scope :not_archived, -> { where(archived_since: nil) }
 
+    include DeferredTouch
+    deferred_touch repository: :updated_at
+
     default_scope { preload(:repository) }
 
     def env
@@ -80,19 +83,36 @@ module Shipit
     after_commit :emit_merge_status_hooks, on: :update
     after_commit :sync_github, on: :create
     after_commit :schedule_merges_if_necessary, on: :update
+    after_commit :sync_github_if_necessary, on: :update
+
+    def sync_github_if_necessary
+      if (archived_since_previously_changed? && archived_since.nil?) || branch_previously_changed?
+        sync_github
+      end
+    end
 
     validates :repository, uniqueness: {
       scope: %i(environment), case_sensitive: false,
-      message: 'cannot be used more than once with this environment. Check archived stacks.'
+      message: 'cannot be used more than once with this environment. Check archived stacks.',
     }
     validates :environment, format: { with: /\A[a-z0-9\-_\:]+\z/ }, length: { maximum: ENVIRONMENT_MAX_SIZE }
     validates :deploy_url, format: { with: URI.regexp(%w(http https ssh)) }, allow_blank: true
+    validates :branch, presence: true
 
     validates :lock_reason, length: { maximum: 4096 }
 
     serialize :cached_deploy_spec, DeploySpec
-    delegate :find_task_definition, :supports_rollback?, :release_status?, :release_status_delay,
-             :release_status_context, :supports_fetch_deployed_revision?, to: :cached_deploy_spec, allow_nil: true
+    delegate(
+      :provisioning_handler_name,
+      :find_task_definition,
+      :release_status?,
+      :release_status_context,
+      :release_status_delay,
+      :supports_fetch_deployed_revision?,
+      :supports_rollback?,
+      to: :cached_deploy_spec,
+      allow_nil: true
+    )
 
     def self.refresh_deployed_revisions
       find_each.select(&:supports_fetch_deployed_revision?).each(&:async_refresh_deployed_revision)
@@ -139,6 +159,7 @@ module Shipit
         env: filter_deploy_envs(env&.to_h || {}),
         allow_concurrency: force,
         ignored_safeties: force || !until_commit.deployable?,
+        max_retries: retries_on_deploy,
       )
     end
 
@@ -171,7 +192,7 @@ module Shipit
     end
 
     def continuous_delivery_delayed?
-      continuous_delivery_delayed_since? && continuous_deployment? && checks?
+      continuous_delivery_delayed_since? && continuous_deployment? && (checks? || deployment_checks?)
     end
 
     def continuous_delivery_delayed!
@@ -200,7 +221,7 @@ module Shipit
     end
 
     def schedule_merges
-      MergePullRequestsJob.perform_later(self)
+      ProcessMergeRequestsJob.perform_later(self)
     end
 
     def next_commit_to_deploy
@@ -340,7 +361,7 @@ module Shipit
     end
 
     def deployable?
-      !locked? && !active_task?
+      !locked? && !active_task? && !awaiting_provision? && deployment_checks_passed?
     end
 
     def allows_merges?
@@ -348,7 +369,7 @@ module Shipit
     end
 
     def merge_method
-      cached_deploy_spec&.pull_request_merge_method || Shipit.default_merge_method
+      cached_deploy_spec&.merge_request_merge_method || Shipit.default_merge_method
     end
 
     delegate :name=, to: :repository, prefix: :repo
@@ -359,42 +380,51 @@ module Shipit
     delegate :git_url, to: :repository, prefix: :repo
 
     def base_path
-      Rails.root.join('data', 'stacks', repo_owner, repo_name, environment)
+      @base_path ||= Rails.root.join('data', 'stacks', repo_owner, repo_name, environment)
     end
 
     def deploys_path
-      File.join(base_path, "deploys")
+      @deploys_path ||= base_path.join("deploys")
     end
 
     def git_path
-      File.join(base_path, "git")
+      @git_path ||= base_path.join("git")
     end
 
     def acquire_git_cache_lock(timeout: 15, &block)
-      Flock.new(git_path.to_s + '.lock').lock(timeout: timeout, &block)
+      @git_cache_lock ||= Flock.new(git_path.to_s + '.lock')
+      @git_cache_lock.lock(timeout: timeout, &block)
     end
 
     def clear_git_cache!
       tmp_path = "#{git_path}-#{SecureRandom.hex}"
-      return unless File.exist?(git_path)
+      return unless git_path.exist?
       acquire_git_cache_lock do
-        File.rename(git_path, tmp_path)
+        git_path.rename(tmp_path)
       end
       FileUtils.rm_rf(tmp_path)
     end
 
     def github_repo_name
-      [repo_owner, repo_name].join('/')
+      repository.github_repo_name
     end
 
     def github_commits
       handle_github_redirections do
-        Shipit.github.api.commits(github_repo_name, sha: branch)
+        github_api.commits(github_repo_name, sha: branch)
       end
     rescue Octokit::Conflict
       [] # Repository is empty...
     end
 
+    def github_api
+      github_app.api
+    end
+
+    def github_app
+      Shipit.github(organization: repository.owner)
+    end
+
     def handle_github_redirections
       # https://developer.github.com/v3/#http-redirects
       resource = yield
@@ -407,9 +437,9 @@ module Shipit
     end
 
     def refresh_repository!
-      resource = Shipit.github.api.repo(github_repo_name)
+      resource = github_api.repo(github_repo_name)
       if resource.try(:message) == 'Moved Permanently'
-        resource = Shipit.github.api.get(resource.url)
+        resource = github_api.get(resource.url)
       end
       repository.update!(owner: resource.owner.login, name: resource.name)
     end
@@ -474,8 +504,9 @@ module Shipit
     end
 
     delegate :plugins, :task_definitions, :hidden_statuses, :required_statuses, :soft_failing_statuses,
-             :blocking_statuses, :deploy_variables, :filter_task_envs, :filter_deploy_envs,
-             :maximum_commits_per_deploy, :pause_between_deploys, to: :cached_deploy_spec
+      :blocking_statuses, :deploy_variables, :filter_task_envs, :filter_deploy_envs,
+      :maximum_commits_per_deploy, :pause_between_deploys, :retries_on_deploy, :retries_on_rollback,
+      to: :cached_deploy_spec
 
     def monitoring?
       monitoring.present?
@@ -567,19 +598,31 @@ module Shipit
       links_spec.transform_values { |url| context.interpolate(url) }
     end
 
+    def clear_local_files
+      FileUtils.rm_rf(base_path.to_s)
+    end
+
+    def deployment_checks_passed?
+      return true unless deployment_checks?
+
+      Shipit.deployment_checks.call(self)
+    end
+
     private
 
     def clear_cache
       remove_instance_variable(:@active_task) if defined?(@active_task)
     end
 
-    def clear_local_files
-      FileUtils.rm_rf(base_path.to_s)
-    end
-
     def update_defaults
       self.environment = 'production' if environment.blank?
-      self.branch = 'master' if branch.blank?
+      self.branch = default_branch_name if branch.blank?
+    end
+
+    def default_branch_name
+      Shipit.github.api.repo(github_repo_name).default_branch
+    rescue Octokit::NotFound, Octokit::InvalidRepository
+      nil
     end
 
     def set_locked_since
@@ -593,7 +636,7 @@ module Shipit
     end
 
     def schedule_merges_if_necessary
-      if previous_changes.include?('lock_reason') && previous_changes['lock_reason'].last.blank?
+      if lock_reason_previously_changed? && lock_reason.blank?
         schedule_merges
       end
     end
@@ -630,7 +673,7 @@ module Shipit
     end
 
     def should_resume_continuous_delivery?(commit)
-      !deployable? ||
+      (deployment_checks_passed? && !deployable?) ||
         deployed_too_recently? ||
         commit.nil? ||
         commit.deployed?
@@ -639,7 +682,12 @@ module Shipit
     def should_delay_continuous_delivery?(commit)
       commit.deploy_failed? ||
         (checks? && !EphemeralCommitChecks.new(commit).run.success?) ||
+        !deployment_checks_passed? ||
         commit.recently_pushed?
     end
+
+    def deployment_checks?
+      Shipit.deployment_checks.present?
+    end
   end
 end