shipit-engine 0.32.0 → 0.35.1

data/app/jobs/shipit/perform_task_job.rb

@@ -4,104 +4,10 @@ module Shipit
     queue_as :deploys
 
     def perform(task)
-      @task = task
-      @commands = Commands.for(@task)
-      unless @task.pending?
-        logger.error("Task ##{@task.id} already in `#{@task.status}` state. Aborting.")
-        return
-      end
-      run
-    ensure
-      @commands.clear_working_directory
-    end
-
-    def run
-      @task.ping
-      @task.run!
-      checkout_repository
-      perform_task
-      @task.write("\nCompleted successfully\n")
-      @task.report_complete!
-    rescue Command::TimedOut => error
-      @task.write("\n#{error.message}\n")
-      @task.report_timeout!(error)
-    rescue Command::Error => error
-      @task.write("\n#{error.message}\n")
-      @task.report_failure!(error)
-    rescue StandardError => error
-      @task.report_error!(error)
-    rescue Exception => error
-      @task.report_error!(error)
-      raise
-    end
-
-    def abort!(signal: 'TERM')
-      pid = @task.pid
-      if pid
-        @task.write("$ kill #{pid}\n")
-        Process.kill(signal, pid)
-      else
-        @task.write("Can't abort, no recorded pid, WTF?\n")
-      end
-    rescue SystemCallError => error
-      @task.write("kill: (#{pid}) - #{error.message}\n")
-    end
-
-    def check_for_abort
-      @task.should_abort? do |times_killed|
-        if times_killed > 3
-          abort!(signal: 'KILL')
-        else
-          abort!
-        end
-      end
-    end
-
-    def perform_task
-      capture_all!(@commands.install_dependencies)
-      capture_all!(@commands.perform)
-    end
-
-    def checkout_repository
-      unless @commands.fetched?(@task.until_commit).tap(&:run).success?
-        # acquire_git_cache_lock can take upto 15 seconds
-        # to process. Try to make sure that the job isn't
-        # marked dead while we attempt to acquire the lock.
-        @task.ping
-        @task.acquire_git_cache_lock do
-          @task.ping
-          unless @commands.fetched?(@task.until_commit).tap(&:run).success?
-            capture!(@commands.fetch)
-          end
-        end
-      end
-      capture_all!(@commands.clone)
-      capture!(@commands.checkout(@task.until_commit))
-    end
-
-    def capture_all!(commands)
-      commands.map { |c| capture!(c) }
-    end
-
-    def capture!(command)
-      command.start do
-        @task.ping
-        check_for_abort
-      end
-      @task.write("$ #{command}\npid: #{command.pid}\n")
-      @task.pid = command.pid
-      command.stream! do |line|
-        @task.write(line)
-      end
-      @task.write("\n")
-      command.success?
-    end
-
-    def capture(command)
-      capture!(command)
-      command.success?
-    rescue Command::Error
-      false
+      Shipit
+        .task_execution_strategy
+        .new(task)
+        .execute
     end
   end
 end

data/app/jobs/shipit/process_merge_requests_job.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+module Shipit
+  class ProcessMergeRequestsJob < BackgroundJob
+    include BackgroundJob::Unique
+    on_duplicate :drop
+
+    queue_as :default
+
+    def perform(stack)
+      merge_requests = stack.merge_requests.to_be_merged.to_a
+      merge_requests.each do |merge_request|
+        merge_request.refresh!
+        merge_request.reject_unless_mergeable!
+        merge_request.cancel! if merge_request.closed?
+        merge_request.revalidate! if merge_request.need_revalidation?
+      end
+
+      return false unless stack.allows_merges?
+
+      merge_requests.select(&:pending?).each do |merge_request|
+        merge_request.refresh!
+        next unless merge_request.all_status_checks_passed?
+        begin
+          merge_request.merge!
+        rescue MergeRequest::NotReady
+          ProcessMergeRequestsJob.set(wait: 10.seconds).perform_later(stack)
+          return false
+        end
+      end
+    end
+  end
+end

data/app/jobs/shipit/refresh_merge_request_job.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+module Shipit
+  class RefreshMergeRequestJob < BackgroundJob
+    queue_as :default
+
+    def perform(merge_request)
+      merge_request.refresh!
+      ProcessMergeRequestsJob.perform_later(merge_request.stack)
+    end
+  end
+end

data/app/jobs/shipit/update_github_last_deployed_ref_job.rb

@@ -13,7 +13,7 @@ module Shipit
 
       environment = stack.environment
       stack_ref = create_full_ref(environment)
-      client = Shipit.github.api
+      client = stack.github_api
 
       full_repo_name = stack.github_repo_name
 

data/app/models/shipit/anonymous_user.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 module Shipit
   class AnonymousUser
-    def present?
-      false
+    def blank?
+      true
     end
 
     def email
@@ -35,6 +35,10 @@ module Shipit
       Shipit.authentication_disabled?
     end
 
+    def repositories_contributed_to
+      []
+    end
+
     def stacks_contributed_to
       []
     end
@@ -59,5 +63,9 @@ module Shipit
     def serializer_class
       AnonymousUserSerializer
     end
+
+    def marked_for_destruction?
+      true
+    end
   end
 end

data/app/models/shipit/check_run.rb

@@ -1,10 +1,12 @@
 # frozen_string_literal: true
 module Shipit
   class CheckRun < ApplicationRecord
-    CONCLUSIONS = %w(success failure neutral cancelled timed_out action_required stale).freeze
+    CONCLUSIONS = %w(success failure neutral cancelled timed_out action_required stale skipped).freeze
     include DeferredTouch
     include Status::Common
 
+    CHECK_RUN_REFRESH_DELAY = 5.seconds
+
     belongs_to :stack, required: true
     belongs_to :commit, required: true
 
@@ -19,11 +21,30 @@ module Shipit
         create!(selector.merge(attributes))
       rescue ActiveRecord::RecordNotUnique
         record = find_by!(selector)
+
+        # Checkruns can jump between states and conclusions, and the github timestamps are low precision and unreliable.
+        # Since there's a conflict and the webhook seems older, enqueue a refresh.
+        # Persist the received data anyways, in case it is now the canonical data on GitHub despite the timestamp.
+        if attributes[:conclusion] != record.conclusion && record.newer_than_webhook?(attributes)
+          Rails.logger.warn(
+            "Conflicting stale checkrun received. Checkrun id: #{selector[:github_id]}, Details: #{attributes}"
+          )
+          RefreshCheckRunsJob.set(wait: CHECK_RUN_REFRESH_DELAY).perform_later(commit_id: record.commit_id)
+        end
+
         record.update!(attributes)
         record
       end
 
       def create_or_update_from_github!(stack_id, github_check_run)
+        checkrun_date = parse_newest_date(github_check_run)
+
+        unless checkrun_date
+          Rails.logger.warn("No valid timestamp found in checkrun data. Checkrun id: #{github_check_run.id}.")
+          RefreshCheckRunsJob.set(wait: CHECK_RUN_REFRESH_DELAY).perform_later(stack_id: stack_id)
+          return
+        end
+
         create_or_update_by!(
           selector: {
             github_id: github_check_run.id,
@@ -35,16 +56,27 @@ module Shipit
             title: github_check_run.output.title.to_s.truncate(1_000),
             details_url: github_check_run.details_url,
             html_url: github_check_run.html_url,
+            github_updated_at: checkrun_date,
           },
         )
       end
+
+      def parse_newest_date(github_check_run)
+        started_at = github_check_run.started_at
+        completed_at = github_check_run.completed_at
+
+        started_at_date = Time.parse(started_at.to_s) if started_at
+        completed_at_date = Time.parse(completed_at.to_s) if completed_at
+
+        [started_at_date, completed_at_date].compact.max
+      end
     end
 
     def state
       case conclusion
       when nil, 'action_required'
         'pending'
-      when 'success', 'neutral'
+      when 'success', 'neutral', 'skipped'
         'success'
       when 'failure', 'cancelled', 'stale'
         'failure'
@@ -71,6 +103,10 @@ module Shipit
       'shipit/statuses/status'
     end
 
+    def newer_than_webhook?(webhook_attributes)
+      github_updated_at && github_updated_at >= webhook_attributes[:github_updated_at]
+    end
+
     private
 
     def enable_ci_on_stack

data/app/models/shipit/command_line_user.rb

@@ -35,6 +35,10 @@ module Shipit
       Shipit.authentication_disabled?
     end
 
+    def repositories_contributed_to
+      []
+    end
+
     def stacks_contributed_to
       []
     end

data/app/models/shipit/commit.rb

@@ -8,42 +8,53 @@ module Shipit
     AmbiguousRevision = Class.new(StandardError)
 
     belongs_to :stack
-    has_many :deploys
     has_many :statuses, -> { order(created_at: :desc) }, dependent: :destroy, inverse_of: :commit
     has_many :check_runs, -> { order(created_at: :desc) }, dependent: :destroy, inverse_of: :commit
     has_many :commit_deployments, dependent: :destroy
     has_many :release_statuses, dependent: :destroy
-    belongs_to :pull_request, inverse_of: :merge_commit, optional: true
+    belongs_to :merge_request, inverse_of: :merge_commit, optional: true
 
     deferred_touch stack: :updated_at
 
-    before_create :identify_pull_request
+    before_create :identify_merge_request
     after_commit { broadcast_update }
     after_create { stack.update_undeployed_commits_count }
 
     after_commit :schedule_refresh_statuses!, :schedule_refresh_check_runs!, :schedule_fetch_stats!,
-                 :schedule_continuous_delivery, on: :create
+      :schedule_continuous_delivery, on: :create
 
-    belongs_to :author, class_name: 'User', inverse_of: :authored_commits
-    belongs_to :committer, class_name: 'User', inverse_of: :commits
-    belongs_to :lock_author, class_name: :User, optional: true, inverse_of: false
+    belongs_to :author, class_name: 'User', optional: true, inverse_of: :authored_commits
+    belongs_to :committer, class_name: 'User', optional: true, inverse_of: :commits
+    belongs_to :lock_author, class_name: 'User', optional: true, inverse_of: false
 
     def author
       super || AnonymousUser.new
     end
 
+    def author=(user)
+      super(user.presence)
+    end
+
     def committer
       super || AnonymousUser.new
     end
 
+    def committer=(user)
+      super(user.presence)
+    end
+
     def lock_author
       super || AnonymousUser.new
     end
 
+    def lock_author=(user)
+      super(user.presence)
+    end
+
     scope :reachable, -> { where(detached: false) }
 
     delegate :broadcast_update, :github_repo_name, :hidden_statuses, :required_statuses, :blocking_statuses,
-             :soft_failing_statuses, to: :stack
+      :soft_failing_statuses, to: :stack
 
     def self.newer_than(commit)
       return all unless commit
@@ -113,7 +124,7 @@ module Shipit
 
     def message=(message)
       limit = self.class.columns_hash['message'].limit
-      if limit && message && message.size > limit
+      if limit && message && message.bytesize > limit
         message = message.truncate_bytes(limit)
       end
       super(message)
@@ -144,7 +155,7 @@ module Shipit
 
     def refresh_statuses!
       github_statuses = stack.handle_github_redirections do
-        Shipit.github.api.statuses(github_repo_name, sha, per_page: 100)
+        stack.github_api.statuses(github_repo_name, sha, per_page: 100)
       end
       github_statuses.each do |status|
         create_status_from_github!(status)
@@ -159,7 +170,7 @@ module Shipit
 
     def refresh_check_runs!
       response = stack.handle_github_redirections do
-        Shipit.github.api.check_runs(github_repo_name, sha)
+        stack.github_api.check_runs(github_repo_name, sha)
       end
       response.check_runs.each do |check_run|
         create_or_update_check_run_from_github!(check_run)
@@ -261,7 +272,7 @@ module Shipit
     end
 
     def github_commit
-      @github_commit ||= Shipit.github.api.commit(github_repo_name, sha)
+      @github_commit ||= stack.github_api.commit(github_repo_name, sha)
     end
 
     def schedule_fetch_stats!
@@ -287,13 +298,13 @@ module Shipit
       stack.deploys.unsuccessful.where(until_commit_id: id).any?
     end
 
-    def identify_pull_request
+    def identify_merge_request
       return unless message_parser.pull_request?
-      if pull_request = stack.pull_requests.find_by(number: message_parser.pull_request_number)
-        self.pull_request = pull_request
-        self.pull_request_number = pull_request.number
-        self.pull_request_title = pull_request.title
-        self.author = pull_request.merge_requested_by if pull_request.merge_requested_by
+      if merge_request = stack.merge_requests.find_by(number: message_parser.pull_request_number)
+        self.merge_request = merge_request
+        self.pull_request_number = merge_request.number
+        self.pull_request_title = merge_request.title
+        self.author = merge_request.merge_requested_by if merge_request.merge_requested_by
       end
 
       self.pull_request_number = message_parser.pull_request_number unless self[:pull_request_number]
@@ -301,8 +312,8 @@ module Shipit
     end
 
     def deploy_requested_at
-      if pull_request&.merged?
-        pull_request.merge_requested_at
+      if merge_request&.merged?
+        merge_request.merge_requested_at
       else
         created_at
       end

data/app/models/shipit/commit_checks.rb

@@ -6,17 +6,18 @@ module Shipit
 
     def initialize(commit)
       @commit = commit
+      super(commit)
     end
 
     def synchronize(&block)
-      @lock ||= Redis::Lock.new('lock', redis, expiration: 1, timeout: 2)
+      @lock ||= Redis::Lock.new(key('lock'), Shipit.redis, expiration: 1, timeout: 2)
       @lock.lock(&block)
     end
 
     def schedule
-      return false if redis.get('status').present?
+      return false if Shipit.redis.get(key('status')).present?
       synchronize do
-        return false if redis.get('status').present?
+        return false if Shipit.redis.get(key('status')).present?
 
         initialize_redis_state
       end
@@ -25,34 +26,34 @@ module Shipit
     end
 
     def initialize_redis_state
-      redis.pipelined do
-        redis.set('output', '', ex: OUTPUT_TTL)
-        redis.set('status', 'scheduled', ex: OUTPUT_TTL)
-      end
+      Shipit.redis.set(key('status'), 'scheduled', ex: OUTPUT_TTL)
       @status = 'scheduled'
     end
 
     def status
-      @status ||= redis.get('status')
+      @status ||= Shipit.redis.get(key('status'))
     end
 
     def status=(status)
-      redis.set('status', status)
+      Shipit.redis.set(key('status'), status)
       @status = status
     end
 
     def output(since: 0)
-      redis.getrange('output', since, -1)
+      Shipit.redis.getrange(key('output'), since, -1)
     end
 
     def write(output)
-      redis.append('output', output)
+      Shipit.redis.pipelined do
+        Shipit.redis.append(key('output'), output)
+        Shipit.redis.expire(key('output'), OUTPUT_TTL)
+      end
     end
 
     private
 
-    def redis
-      @redis ||= Shipit.redis("commit:#{commit.id}:checks")
+    def key(key)
+      "commit:#{commit.id}:checks:#{key}"
     end
   end
 end