shipit-engine 0.32.0 → 0.35.1

data/app/models/shipit/commit_deployment.rb

@@ -20,15 +20,15 @@ module Shipit
       return if github_id?
 
       response = begin
-        create_deployment_on_github(author.github_api)
+        create_deployment_on_github(stack.github_api)
       rescue Octokit::ClientError
-        raise if Shipit.github.api == author.github_api
+        raise if Shipit.github(organization: stack.repository.owner).api == stack.github_api
         # If the deploy author didn't gave us the permission to create the deployment we falback the the main shipit
         # user.
         #
         # Octokit currently raise NotFound, but I'm convinced it should be Forbidden if the user can see the repository.
         # So to be future proof I catch boths.
-        create_deployment_on_github(Shipit.github.api)
+        create_deployment_on_github(stack.github_api)
       end
       update!(github_id: response.id, api_url: response.url)
     end

data/app/models/shipit/commit_deployment_status.rb

@@ -12,15 +12,15 @@ module Shipit
     def create_on_github!
       return if github_id?
       response = begin
-        create_status_on_github(author.github_api)
+        create_status_on_github(stack.github_api)
       rescue Octokit::ClientError
-        raise if Shipit.github.api == author.github_api
+        raise if Shipit.github(organization: stack.repository.owner).api == stack.github_api
         # If the deploy author didn't gave us the permission to create the deployment we falback the the main shipit
         # user.
         #
         # Octokit currently raise NotFound, but I'm convinced it should be Forbidden if the user can see the repository.
         # So to be future proof I catch boths.
-        create_status_on_github(Shipit.github.api)
+        create_status_on_github(stack.github_api)
       end
       update!(github_id: response.id, api_url: response.url)
     end

data/app/models/shipit/deploy.rb

@@ -15,8 +15,8 @@ module Shipit
       after_transition any => any, do: :update_last_deploy_time
     end
 
-    belongs_to :until_commit, class_name: 'Commit', required: true, inverse_of: :deploys
-    belongs_to :since_commit, class_name: 'Commit', required: true, inverse_of: :deploys
+    belongs_to :until_commit, class_name: 'Commit', required: true
+    belongs_to :since_commit, class_name: 'Commit', required: true
     has_many :commit_deployments, dependent: :destroy, inverse_of: :task, foreign_key: :task_id do
       GITHUB_STATUSES = {
         'pending' => 'pending',
@@ -92,18 +92,21 @@ module Shipit
         env: env&.to_h || {},
         allow_concurrency: force,
         ignored_safeties: force,
+        max_retries: stack.retries_on_rollback,
       )
     end
 
     # Rolls the stack back to this deploy
-    def trigger_rollback(user = AnonymousUser.new, env: nil, force: false)
+    def trigger_rollback(user = AnonymousUser.new, env: nil, force: false, lock: true)
       rollback = build_rollback(user, env: env, force: force)
       rollback.save!
       rollback.enqueue
 
-      lock_reason = "A rollback for #{rollback.since_commit.sha} has been triggered. " \
-        "Please make sure the reason for the rollback has been addressed before deploying again."
-      stack.update!(lock_reason: lock_reason, lock_author_id: user.id)
+      if lock
+        lock_reason = "A rollback for #{rollback.since_commit.sha} has been triggered. " \
+          "Please make sure the reason for the rollback has been addressed before deploying again."
+        stack.update!(lock_reason: lock_reason, lock_author_id: user.id)
+      end
 
       rollback
     end
@@ -208,7 +211,7 @@ module Shipit
     end
 
     def report_complete!
-      if stack.release_status? && stack.release_status_delay.positive?
+      if stack.release_status? && !stack.release_status_delay.zero?
         enter_validation!
       else
         super
@@ -268,10 +271,13 @@ module Shipit
       when 'aborted', 'aborting'
         append_release_status('failure', "The deploy on #{stack.environment} was canceled")
       when 'validating'
-        if stack.release_status_delay.positive?
-          append_release_status('pending', "The deploy on #{stack.environment} succeeded")
-          MarkDeployHealthyJob.set(wait: stack.release_status_delay).perform_later(self)
-        end
+        append_release_status(
+          'pending',
+          "The deploy on #{stack.environment} succeeded"
+        ) unless stack.release_status_delay.zero?
+
+        MarkDeployHealthyJob.set(wait: stack.release_status_delay)
+          .perform_later(self) if stack.release_status_delay.positive?
       when 'success'
         if stack.release_status_delay.zero?
           append_release_status('success', "The deploy on #{stack.environment} succeeded")

data/app/models/shipit/deploy_spec/file_system.rb

@@ -13,6 +13,7 @@ module Shipit
       def initialize(app_dir, env)
         @app_dir = Pathname(app_dir)
         @env = env
+        super(nil)
       end
 
       def cacheable
@@ -32,10 +33,10 @@ module Shipit
       def cacheable_config
         (config || {}).deep_merge(
           'merge' => {
-            'require' => pull_request_required_statuses,
-            'ignore' => pull_request_ignored_statuses,
-            'revalidate_after' => revalidate_pull_requests_after&.to_i,
-            'method' => pull_request_merge_method,
+            'require' => merge_request_required_statuses,
+            'ignore' => merge_request_ignored_statuses,
+            'revalidate_after' => revalidate_merge_requests_after&.to_i,
+            'method' => merge_request_merge_method,
             'max_divergence' => {
               'commits' => max_divergence_commits&.to_i,
               'age' => max_divergence_age&.to_i,
@@ -63,13 +64,18 @@ module Shipit
             'delay' => release_status_delay,
           },
           'dependencies' => { 'override' => dependencies_steps },
+          'provision' => { 'handler_name' => provisioning_handler_name },
           'deploy' => {
             'override' => deploy_steps,
             'variables' => deploy_variables.map(&:to_h),
             'max_commits' => maximum_commits_per_deploy,
             'interval' => pause_between_deploys,
+            'retries' => retries_on_deploy,
+          },
+          'rollback' => {
+            'override' => rollback_steps,
+            'retries' => retries_on_rollback,
           },
-          'rollback' => { 'override' => rollback_steps },
           'fetch' => fetch_deployed_revision_steps,
           'tasks' => cacheable_tasks,
         )

data/app/models/shipit/deploy_spec/lerna_discovery.rb

@@ -76,10 +76,18 @@ module Shipit
       end
 
       def publish_independent_packages
-        [
-          'assert-lerna-independent-version-tags',
-          'publish-lerna-independent-packages',
-        ]
+        command = if lerna_lerna >= LATEST_MAJOR_VERSION
+          [
+            'assert-lerna-independent-version-tags',
+            'publish-lerna-independent-packages',
+          ]
+        else
+          [
+            'assert-lerna-independent-version-tags',
+            'publish-lerna-independent-packages-legacy',
+          ]
+        end
+        command
       end
 
       def publish_fixed_version_packages

data/app/models/shipit/deploy_spec.rb

@@ -104,6 +104,10 @@ module Shipit
       Duration.parse(config('deploy', 'interval') { 0 })
     end
 
+    def provisioning_handler_name
+      config('provision', 'handler_name')
+    end
+
     def deploy_steps
       around_steps('deploy') do
         config('deploy', 'override') { discover_deploy_steps }
@@ -122,6 +126,10 @@ module Shipit
       deploy_variables.map { |v| [v.name, v.default] }.to_h
     end
 
+    def retries_on_deploy
+      config('deploy', 'retries') { nil }
+    end
+
     def rollback_steps
       around_steps('rollback') do
         config('rollback', 'override') { discover_rollback_steps }
@@ -132,6 +140,10 @@ module Shipit
       rollback_steps || cant_detect!(:rollback)
     end
 
+    def retries_on_rollback
+      config('rollback', 'retries') { nil }
+    end
+
     def fetch_deployed_revision_steps
       config('fetch') || discover_fetch_deployed_revision_steps
     end
@@ -179,12 +191,12 @@ module Shipit
       Array.wrap(config('ci', 'blocking'))
     end
 
-    def pull_request_merge_method
+    def merge_request_merge_method
       method = config('merge', 'method')
       method if %w(merge rebase squash).include?(method)
     end
 
-    def pull_request_required_statuses
+    def merge_request_required_statuses
       if config('merge', 'require') || config('merge', 'ignore')
         Array.wrap(config('merge', 'require'))
       else
@@ -192,7 +204,7 @@ module Shipit
       end
     end
 
-    def pull_request_ignored_statuses
+    def merge_request_ignored_statuses
       if config('merge', 'require') || config('merge', 'ignore')
         Array.wrap(config('merge', 'ignore')) + [release_status_context].compact
       else
@@ -200,7 +212,7 @@ module Shipit
       end
     end
 
-    def revalidate_pull_requests_after
+    def revalidate_merge_requests_after
       if timeout = config('merge', 'revalidate_after')
         begin
           Duration.parse(timeout)

data/app/models/shipit/duration.rb

@@ -20,6 +20,8 @@ module Shipit
 
     class << self
       def parse(value)
+        return new(-1) if value.to_s == "-1"
+
         unless match = FORMAT.match(value.to_s)
           raise ParseError, "not a duration: #{value.inspect}"
         end

data/app/models/shipit/hook.rb

@@ -1,12 +1,28 @@
 # frozen_string_literal: true
 module Shipit
   class Hook < Record
+    class DeliverySigner
+      attr_reader :secret
+
+      ALGORITHM = 'sha256'
+
+      def initialize(secret)
+        @secret = secret
+      end
+
+      def sign(payload)
+        hmac = OpenSSL::HMAC.hexdigest(ALGORITHM, secret, payload)
+        "#{ALGORITHM}=#{hmac}"
+      end
+    end
+
     class DeliverySpec
-      def initialize(event:, url:, content_type:, payload:)
+      def initialize(event:, url:, content_type:, payload:, secret:)
         @event = event
         @url = url
         @content_type = content_type
         @payload = payload
+        @secret = secret
       end
 
       def send!
@@ -15,7 +31,7 @@ module Shipit
 
       private
 
-      attr_reader :event, :url, :content_type, :payload
+      attr_reader :event, :url, :content_type, :payload, :secret
 
       def http
         Faraday::Connection.new do |connection|
@@ -29,9 +45,16 @@ module Shipit
           'User-Agent' => 'Shipit Webhook',
           'Content-Type' => content_type,
           'X-Shipit-Event' => event,
+          'X-Shipit-Signature' => signature,
           'Accept' => '*/*',
         }
       end
+
+      def signature
+        return nil if secret.blank?
+
+        DeliverySigner.new(secret).sign(payload)
+      end
     end
 
     default_scope { order :id }
@@ -45,6 +68,7 @@ module Shipit
 
     EVENTS = %w(
       stack
+      review_stack
       task
       deploy
       rollback
@@ -53,6 +77,7 @@ module Shipit
       deployable_status
       merge_status
       merge
+      pull_request
     ).freeze
 
     belongs_to :stack, required: false
@@ -117,6 +142,7 @@ module Shipit
         url: delivery_url,
         content_type: CONTENT_TYPES[content_type],
         payload: serialize_payload(payload),
+        secret: secret,
       )
     end
 

data/app/models/shipit/merge_request.rb

@@ -0,0 +1,304 @@
+# frozen_string_literal: true
+module Shipit
+  class MergeRequest < ApplicationRecord
+    include DeferredTouch
+
+    MERGE_REQUEST_FIELD = 'Merge-Requested-By'
+
+    WAITING_STATUSES = %w(fetching pending).freeze
+    QUEUED_STATUSES = %w(pending revalidating).freeze
+    REJECTION_REASONS = %w(ci_missing ci_failing merge_conflict requires_rebase).freeze
+    InvalidTransition = Class.new(StandardError)
+    NotReady = Class.new(StandardError)
+
+    class StatusChecker < Status::Group
+      def initialize(commit, statuses, deploy_spec)
+        @deploy_spec = deploy_spec
+        super(commit, statuses)
+      end
+
+      private
+
+      attr_reader :deploy_spec
+
+      def reject_hidden(statuses)
+        statuses.reject { |s| ignored_statuses.include?(s.context) }
+      end
+
+      def reject_allowed_to_fail(statuses)
+        statuses.reject { |s| ignored_statuses.include?(s.context) }
+      end
+
+      def ignored_statuses
+        deploy_spec&.merge_request_ignored_statuses || []
+      end
+
+      def required_statuses
+        deploy_spec&.merge_request_required_statuses || []
+      end
+    end
+
+    belongs_to :stack
+    belongs_to :head, class_name: 'Shipit::Commit', optional: true
+    belongs_to :base_commit, class_name: 'Shipit::Commit', optional: true
+    belongs_to :merge_requested_by, class_name: 'Shipit::User', optional: true
+    has_one :merge_commit, class_name: 'Shipit::Commit'
+
+    deferred_touch stack: :updated_at
+
+    validates :number, presence: true, uniqueness: { scope: :stack_id }
+
+    scope :waiting, -> { where(merge_status: WAITING_STATUSES) }
+    scope :pending, -> { where(merge_status: 'pending') }
+    scope :to_be_merged, -> { pending.order(merge_requested_at: :asc) }
+    scope :queued, -> { where(merge_status: QUEUED_STATUSES).order(merge_requested_at: :asc) }
+
+    after_save :record_merge_status_change
+    after_commit :emit_hooks
+
+    state_machine :merge_status, initial: :fetching do
+      state :fetching
+      state :pending
+      state :rejected
+      state :canceled
+      state :merged
+      state :revalidating
+
+      event :fetched do
+        transition fetching: :pending
+      end
+
+      event :reject do
+        transition pending: :rejected
+      end
+
+      event :revalidate do
+        transition pending: :revalidating
+      end
+
+      event :cancel do
+        transition any => :canceled
+      end
+
+      event :complete do
+        transition pending: :merged
+      end
+
+      event :retry do
+        transition %i(rejected canceled revalidating) => :pending
+      end
+
+      before_transition rejected: any do |pr|
+        pr.rejection_reason = nil
+      end
+
+      before_transition %i(fetching rejected canceled) => :pending do |pr|
+        pr.merge_requested_at = Time.now.utc
+      end
+
+      before_transition any => :pending do |pr|
+        pr.revalidated_at = Time.now.utc
+      end
+
+      before_transition %i(pending) => :merged do |pr|
+        Stack.increment_counter(:undeployed_commits_count, pr.stack_id)
+      end
+    end
+
+    def self.schedule_merges
+      Shipit::Stack.where(merge_queue_enabled: true).find_each(&:schedule_merges)
+    end
+
+    def self.extract_number(stack, number_or_url)
+      org = stack.repository.owner
+      case number_or_url
+      when /\A#?(\d+)\z/
+        $1.to_i
+      when %r{\Ahttps://#{Regexp.escape(Shipit.github(organization: org).domain)}/([^/]+)/([^/]+)/pull/(\d+)}
+        return unless $1.downcase == stack.repo_owner.downcase
+        return unless $2.downcase == stack.repo_name.downcase
+        $3.to_i
+      end
+    end
+
+    def self.request_merge!(stack, number, user)
+      now = Time.now.utc
+      merge_request = begin
+        create_with(
+          merge_requested_at: now,
+          merge_requested_by: user.presence,
+        ).find_or_create_by!(
+          stack: stack,
+          number: number,
+        )
+      rescue ActiveRecord::RecordNotUnique
+        retry
+      end
+      merge_request.update!(merge_requested_by: user.presence)
+      merge_request.retry! if merge_request.rejected? || merge_request.canceled? || merge_request.revalidating?
+      merge_request.schedule_refresh!
+      merge_request
+    end
+
+    def reject!(reason)
+      unless REJECTION_REASONS.include?(reason)
+        raise ArgumentError, "invalid reason: #{reason.inspect}, must be one of: #{REJECTION_REASONS.inspect}"
+      end
+      self.rejection_reason = reason.presence
+      super()
+      true
+    end
+
+    def reject_unless_mergeable!
+      return reject!('merge_conflict') if merge_conflict?
+      return reject!('ci_missing') if any_status_checks_missing?
+      return reject!('ci_failing') if any_status_checks_failed?
+      return reject!('requires_rebase') if stale?
+      false
+    end
+
+    def merge!
+      raise InvalidTransition unless pending?
+
+      raise NotReady if not_mergeable_yet?
+
+      stack.github_api.merge_pull_request(
+        stack.github_repo_name,
+        number,
+        merge_message,
+        sha: head.sha,
+        commit_message: 'Merged by Shipit',
+        merge_method: stack.merge_method,
+      )
+      begin
+        if stack.github_api.pull_requests(stack.github_repo_name, base: branch).empty?
+          stack.github_api.delete_branch(stack.github_repo_name, branch)
+        end
+      rescue Octokit::UnprocessableEntity
+        # branch was already deleted somehow
+      end
+      complete!
+      true
+    rescue Octokit::MethodNotAllowed # merge conflict
+      reject!('merge_conflict')
+      false
+    rescue Octokit::Conflict # shas didn't match, PR was updated.
+      raise NotReady
+    end
+
+    def all_status_checks_passed?
+      return false unless head
+      StatusChecker.new(head, head.statuses_and_check_runs, stack.cached_deploy_spec).success?
+    end
+
+    def any_status_checks_failed?
+      status = StatusChecker.new(head, head.statuses_and_check_runs, stack.cached_deploy_spec)
+      status.failure? || status.error?
+    end
+
+    def any_status_checks_missing?
+      StatusChecker.new(head, head.statuses_and_check_runs, stack.cached_deploy_spec).missing?
+    end
+
+    def waiting?
+      WAITING_STATUSES.include?(merge_status)
+    end
+
+    def need_revalidation?
+      timeout = stack.cached_deploy_spec&.revalidate_merge_requests_after
+      return false unless timeout
+      (revalidated_at + timeout).past?
+    end
+
+    def merge_conflict?
+      mergeable == false
+    end
+
+    def not_mergeable_yet?
+      mergeable.nil?
+    end
+
+    def schedule_refresh!
+      RefreshMergeRequestJob.perform_later(self)
+    end
+
+    def closed?
+      state == "closed"
+    end
+
+    def merged_upstream?
+      closed? && merged_at
+    end
+
+    def refresh!
+      update!(github_pull_request: stack.github_api.pull_request(stack.github_repo_name, number))
+      head.refresh_statuses!
+      head.refresh_check_runs!
+      fetched! if fetching?
+      @comparison = nil
+    end
+
+    def github_pull_request=(github_pull_request)
+      self.github_id = github_pull_request.id
+      self.api_url = github_pull_request.url
+      self.title = github_pull_request.title
+      self.state = github_pull_request.state
+      self.mergeable = github_pull_request.mergeable
+      self.additions = github_pull_request.additions
+      self.deletions = github_pull_request.deletions
+      self.branch = github_pull_request.head.ref
+      self.head = find_or_create_commit_from_github_by_sha!(github_pull_request.head.sha, detached: true)
+      self.merged_at = github_pull_request.merged_at
+      self.base_ref = github_pull_request.base.ref
+      self.base_commit = find_or_create_commit_from_github_by_sha!(github_pull_request.base.sha, detached: true)
+    end
+
+    def merge_message
+      return title unless merge_requested_by
+      "#{title}\n\n#{MERGE_REQUEST_FIELD}: #{merge_requested_by.login}\n"
+    end
+
+    def stale?
+      return false unless base_commit
+      spec = stack.cached_deploy_spec
+      if max_branch_age = spec.max_divergence_age
+        return true if Time.now.utc - head.committed_at > max_branch_age
+      end
+      if commit_count_limit = spec.max_divergence_commits
+        return true if comparison.behind_by > commit_count_limit
+      end
+      false
+    end
+
+    def comparison
+      @comparison ||= stack.github_api.compare(
+        stack.github_repo_name,
+        base_ref,
+        head.sha,
+      )
+    end
+
+    private
+
+    def record_merge_status_change
+      @merge_status_changed ||= saved_change_to_attribute?(:merge_status)
+    end
+
+    def emit_hooks
+      return unless @merge_status_changed
+      @merge_status_changed = nil
+      Hook.emit('merge', stack, merge_request: self, status: merge_status, stack: stack)
+    end
+
+    def find_or_create_commit_from_github_by_sha!(sha, attributes)
+      if commit = stack.commits.by_sha(sha)
+        commit
+      else
+        github_commit = stack.github_api.commit(stack.github_repo_name, sha)
+        stack.commits.create_from_github!(github_commit, attributes)
+      end
+    rescue ActiveRecord::RecordNotUnique
+      retry
+    end
+  end
+end