shipit-engine 0.30.0 → 0.34.0

data/app/jobs/shipit/mark_deploy_healthy_job.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class MarkDeployHealthyJob < BackgroundJob
     include BackgroundJob::Unique

data/app/jobs/shipit/perform_commit_checks_job.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class PerformCommitChecksJob < BackgroundJob
     include BackgroundJob::Unique

data/app/jobs/shipit/perform_task_job.rb

@@ -1,100 +1,13 @@
+# frozen_string_literal: true
 module Shipit
   class PerformTaskJob < BackgroundJob
     queue_as :deploys
 
     def perform(task)
-      @task = task
-      @commands = Commands.for(@task)
-      unless @task.pending?
-        logger.error("Task ##{@task.id} already in `#{@task.status}` state. Aborting.")
-        return
-      end
-      run
-    ensure
-      @commands.clear_working_directory
-    end
-
-    def run
-      @task.run!
-      checkout_repository
-      perform_task
-      @task.write("\nCompleted successfully\n")
-      @task.report_complete!
-    rescue Command::TimedOut => error
-      @task.write("\n#{error.message}\n")
-      @task.report_timeout!(error)
-    rescue Command::Error => error
-      @task.write("\n#{error.message}\n")
-      @task.report_failure!(error)
-    rescue StandardError => error
-      @task.report_error!(error)
-    rescue Exception => error
-      @task.report_error!(error)
-      raise
-    end
-
-    def abort!(signal: 'TERM')
-      pid = @task.pid
-      if pid
-        @task.write("$ kill #{pid}\n")
-        Process.kill(signal, pid)
-      else
-        @task.write("Can't abort, no recorded pid, WTF?\n")
-      end
-    rescue SystemCallError => error
-      @task.write("kill: (#{pid}) - #{error.message}\n")
-    end
-
-    def check_for_abort
-      @task.should_abort? do |times_killed|
-        if times_killed > 3
-          abort!(signal: 'KILL')
-        else
-          abort!
-        end
-      end
-    end
-
-    def perform_task
-      capture_all! @commands.install_dependencies
-      capture_all! @commands.perform
-    end
-
-    def checkout_repository
-      unless @commands.fetched?(@task.until_commit).tap(&:run).success?
-        @task.acquire_git_cache_lock do
-          unless @commands.fetched?(@task.until_commit).tap(&:run).success?
-            capture! @commands.fetch
-          end
-        end
-      end
-      capture_all! @commands.clone
-      capture! @commands.checkout(@task.until_commit)
-    end
-
-    def capture_all!(commands)
-      commands.map { |c| capture!(c) }
-    end
-
-    def capture!(command)
-      command.start do
-        @task.ping
-        check_for_abort
-      end
-      @task.write("$ #{command}\npid: #{command.pid}\n")
-      @task.pid = command.pid
-      command.stream! do |line|
-        @task.write(line)
-      end
-      @task.write("\n")
-      command.success?
-    end
-
-    def capture(command)
-      capture!(command)
-      command.success?
-    rescue Command::Error
-      false
+      Shipit
+        .task_execution_strategy
+        .new(task)
+        .execute
     end
   end
 end

data/app/jobs/shipit/process_merge_requests_job.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+module Shipit
+  class ProcessMergeRequestsJob < BackgroundJob
+    include BackgroundJob::Unique
+    on_duplicate :drop
+
+    queue_as :default
+
+    def perform(stack)
+      merge_requests = stack.merge_requests.to_be_merged.to_a
+      merge_requests.each do |merge_request|
+        merge_request.refresh!
+        merge_request.reject_unless_mergeable!
+        merge_request.cancel! if merge_request.closed?
+        merge_request.revalidate! if merge_request.need_revalidation?
+      end
+
+      return false unless stack.allows_merges?
+
+      merge_requests.select(&:pending?).each do |merge_request|
+        merge_request.refresh!
+        next unless merge_request.all_status_checks_passed?
+        begin
+          merge_request.merge!
+        rescue MergeRequest::NotReady
+          ProcessMergeRequestsJob.set(wait: 10.seconds).perform_later(stack)
+          return false
+        end
+      end
+    end
+  end
+end

data/app/jobs/shipit/purge_old_deliveries_job.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class PurgeOldDeliveriesJob < BackgroundJob
     include BackgroundJob::Unique

data/app/jobs/shipit/reap_dead_tasks_job.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+module Shipit
+  class ReapDeadTasksJob < BackgroundJob
+    include BackgroundJob::Unique
+    queue_as :default
+
+    def perform
+      Rails.logger.info("Reaping #{zombie_tasks.size} running tasks.")
+      zombie_tasks.each do |task|
+        Rails.logger.info("Reaping task #{task.id}: #{task.title}")
+        task.report_dead!
+      end
+    end
+
+    private
+
+    def zombie_tasks
+      @zombie_tasks ||= Task.zombies
+    end
+  end
+end

data/app/jobs/shipit/refresh_check_runs_job.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class RefreshCheckRunsJob < BackgroundJob
     queue_as :default

data/app/jobs/shipit/refresh_github_user_job.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class RefreshGithubUserJob < BackgroundJob
     queue_as :default

data/app/jobs/shipit/refresh_merge_request_job.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+module Shipit
+  class RefreshMergeRequestJob < BackgroundJob
+    queue_as :default
+
+    def perform(merge_request)
+      merge_request.refresh!
+      ProcessMergeRequestsJob.perform_later(merge_request.stack)
+    end
+  end
+end

data/app/jobs/shipit/refresh_statuses_job.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class RefreshStatusesJob < BackgroundJob
     queue_as :default

data/app/jobs/shipit/setup_github_hook_job.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class SetupGithubHookJob < BackgroundJob
     queue_as :default

data/app/jobs/shipit/update_estimated_deploy_duration_job.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class UpdateEstimatedDeployDurationJob < BackgroundJob
     queue_as :default

data/app/jobs/shipit/update_github_last_deployed_ref_job.rb

@@ -1,10 +1,11 @@
+# frozen_string_literal: true
 module Shipit
   class UpdateGithubLastDeployedRefJob < BackgroundJob
     queue_as :default
 
     # We do not prefix 'refs/' because Octokit methods will do this automatically.
-    BRANCH_REF_PREFIX = 'heads'.freeze
-    DEPLOY_PREFIX = 'shipit-deploy'.freeze
+    BRANCH_REF_PREFIX = 'heads'
+    DEPLOY_PREFIX = "#{Shipit.app_name.downcase}-deploy"
 
     def perform(stack)
       stack_sha = stack.last_successful_deploy_commit&.sha
@@ -12,7 +13,7 @@ module Shipit
 
       environment = stack.environment
       stack_ref = create_full_ref(environment)
-      client = Shipit.github.api
+      client = stack.github_api
 
       full_repo_name = stack.github_repo_name
 
@@ -33,7 +34,7 @@ module Shipit
       client.update_ref(repo_name, ref, new_sha)
     rescue Octokit::UnprocessableEntity => e
       error_msg = e.message
-      if error_msg.include? "Reference does not exist"
+      if error_msg.include?("Reference does not exist")
         create_ref(client: client, repo_name: repo_name, ref: ref, sha: new_sha)
       else
         raise

data/app/models/concerns/shipit/deferred_touch.rb

@@ -1,10 +1,11 @@
+# frozen_string_literal: true
 module Shipit
   module DeferredTouch
     extend ActiveSupport::Concern
 
-    SET_KEY = 'shipit:deferred_touches'.freeze
-    TMP_KEY = "#{SET_KEY}:updating".freeze
-    CACHE_KEY = "#{SET_KEY}:scheduled".freeze
+    SET_KEY = 'shipit:deferred_touches'
+    TMP_KEY = "#{SET_KEY}:updating"
+    CACHE_KEY = "#{SET_KEY}:scheduled"
     THROTTLE_TTL = 1.second
 
     included do

data/app/models/shipit/anonymous_user.rb

@@ -1,7 +1,8 @@
+# frozen_string_literal: true
 module Shipit
   class AnonymousUser
-    def present?
-      false
+    def blank?
+      true
     end
 
     def email
@@ -34,6 +35,10 @@ module Shipit
       Shipit.authentication_disabled?
     end
 
+    def repositories_contributed_to
+      []
+    end
+
     def stacks_contributed_to
       []
     end
@@ -54,5 +59,13 @@ module Shipit
     def github_api
       Shipit.github.api
     end
+
+    def serializer_class
+      AnonymousUserSerializer
+    end
+
+    def marked_for_destruction?
+      true
+    end
   end
 end

data/app/models/shipit/api_client.rb

@@ -1,5 +1,6 @@
+# frozen_string_literal: true
 module Shipit
-  class ApiClient < ActiveRecord::Base
+  class ApiClient < Record
     InsufficientPermission = Class.new(StandardError)
 
     belongs_to :creator, class_name: 'User'
@@ -16,7 +17,7 @@ module Shipit
       read:hook
       write:hook
     ).freeze
-    validates :permissions, subset: {of: PERMISSIONS}
+    validates :permissions, subset: { of: PERMISSIONS }
 
     class << self
       def authenticate(token)

data/app/models/shipit/application_record.rb

@@ -1,5 +1,6 @@
+# frozen_string_literal: true
 module Shipit
-  class ApplicationRecord < ActiveRecord::Base
+  class ApplicationRecord < Record
     self.abstract_class = true
   end
 end

data/app/models/shipit/check_run.rb

@@ -1,15 +1,18 @@
+# frozen_string_literal: true
 module Shipit
   class CheckRun < ApplicationRecord
-    CONCLUSIONS = %w(success failure neutral cancelled timed_out action_required).freeze
+    CONCLUSIONS = %w(success failure neutral cancelled timed_out action_required stale skipped).freeze
     include DeferredTouch
     include Status::Common
 
+    CHECK_RUN_REFRESH_DELAY = 5.seconds
+
     belongs_to :stack, required: true
     belongs_to :commit, required: true
 
     deferred_touch commit: :updated_at
 
-    validates :conclusion, inclusion: {in: CONCLUSIONS, allow_nil: true}
+    validates :conclusion, inclusion: { in: CONCLUSIONS, allow_nil: true }
 
     after_create :enable_ci_on_stack
 
@@ -18,11 +21,30 @@ module Shipit
         create!(selector.merge(attributes))
       rescue ActiveRecord::RecordNotUnique
         record = find_by!(selector)
+
+        # Checkruns can jump between states and conclusions, and the github timestamps are low precision and unreliable.
+        # Since there's a conflict and the webhook seems older, enqueue a refresh.
+        # Persist the received data anyways, in case it is now the canonical data on GitHub despite the timestamp.
+        if attributes[:conclusion] != record.conclusion && record.newer_than_webhook?(attributes)
+          Rails.logger.warn(
+            "Conflicting stale checkrun received. Checkrun id: #{selector[:github_id]}, Details: #{attributes}"
+          )
+          RefreshCheckRunsJob.set(wait: CHECK_RUN_REFRESH_DELAY).perform_later(commit_id: record.commit_id)
+        end
+
         record.update!(attributes)
         record
       end
 
       def create_or_update_from_github!(stack_id, github_check_run)
+        checkrun_date = parse_newest_date(github_check_run)
+
+        unless checkrun_date
+          Rails.logger.warn("No valid timestamp found in checkrun data. Checkrun id: #{github_check_run.id}.")
+          RefreshCheckRunsJob.set(wait: CHECK_RUN_REFRESH_DELAY).perform_later(stack_id: stack_id)
+          return
+        end
+
         create_or_update_by!(
           selector: {
             github_id: github_check_run.id,
@@ -34,18 +56,29 @@ module Shipit
             title: github_check_run.output.title.to_s.truncate(1_000),
             details_url: github_check_run.details_url,
             html_url: github_check_run.html_url,
+            github_updated_at: checkrun_date,
           },
         )
       end
+
+      def parse_newest_date(github_check_run)
+        started_at = github_check_run.started_at
+        completed_at = github_check_run.completed_at
+
+        started_at_date = Time.parse(started_at.to_s) if started_at
+        completed_at_date = Time.parse(completed_at.to_s) if completed_at
+
+        [started_at_date, completed_at_date].compact.max
+      end
     end
 
     def state
       case conclusion
       when nil, 'action_required'
         'pending'
-      when 'success', 'neutral'
+      when 'success', 'neutral', 'skipped'
         'success'
-      when 'failure', 'cancelled'
+      when 'failure', 'cancelled', 'stale'
         'failure'
       when 'timed_out'
         'error'
@@ -70,6 +103,10 @@ module Shipit
       'shipit/statuses/status'
     end
 
+    def newer_than_webhook?(webhook_attributes)
+      github_updated_at && github_updated_at >= webhook_attributes[:github_updated_at]
+    end
+
     private
 
     def enable_ci_on_stack

data/app/models/shipit/command_line_user.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class CommandLineUser
     def present?
@@ -34,6 +35,10 @@ module Shipit
       Shipit.authentication_disabled?
     end
 
+    def repositories_contributed_to
+      []
+    end
+
     def stacks_contributed_to
       []
     end