shipit-engine 0.39.0 → 0.40.0

data/app/jobs/shipit/deferred_touch_job.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Shipit
   class DeferredTouchJob < BackgroundJob
     include BackgroundJob::Unique

data/app/jobs/shipit/deliver_hook_job.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Shipit
   class DeliverHookJob < BackgroundJob
     queue_as :hooks

data/app/jobs/shipit/destroy_job.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Shipit
   class DestroyJob < BackgroundJob
     queue_as :default

data/app/jobs/shipit/destroy_repository_job.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Shipit
   class DestroyRepositoryJob < BackgroundJob
     queue_as :default

data/app/jobs/shipit/destroy_stack_job.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Shipit
   class DestroyStackJob < BackgroundJob
     queue_as :default
@@ -16,22 +17,42 @@ module Shipit
     #     +-- chunks
 
     def perform(stack)
-      Shipit::ApiClient.where(stack_id: stack.id).delete_all
-      commits_ids = Shipit::Commit.where(stack_id: stack.id).pluck(:id)
-      tasks_ids = Shipit::Task.where(stack_id: stack.id).pluck(:id)
-      commit_deployments_ids = Shipit::CommitDeployment.where(task_id: tasks_ids).pluck(:id)
-      Shipit::CommitDeploymentStatus.where(commit_deployment_id: commit_deployments_ids).delete_all
-      Shipit::CommitDeployment.where(id: commit_deployments_ids).delete_all
-      Shipit::Status.where(commit_id: commits_ids).delete_all
-      Shipit::Commit.where(id: commits_ids).delete_all
-      Shipit::GithubHook.where(stack_id: stack.id).destroy_all
-      Shipit::Hook.where(stack_id: stack.id).delete_all
-      Shipit::MergeRequest.where(stack_id: stack.id).delete_all
-      tasks_ids.each_slice(100) do |ids|
-        Shipit::OutputChunk.where(task_id: ids).delete_all
-        Shipit::Task.where(id: ids).delete_all
-      end
+      delete(Shipit::ApiClient.where(stack_id: stack.id))
+
+      delete(
+        Shipit::CommitDeploymentStatus
+        .joins(commit_deployment: [:task])
+        .where(commit_deployment: { tasks: { stack_id: stack.id } })
+      )
+
+      delete(Shipit::CommitDeployment.joins(:task).where(task: { stack_id: stack.id }))
+      delete(Shipit::Status.joins(:commit).where(commit: { stack_id: stack.id }))
+      delete(Shipit::GithubHook.where(stack_id: stack.id))
+      delete(Shipit::Hook.where(stack_id: stack.id))
+      delete(Shipit::MergeRequest.where(stack_id: stack.id))
+
+      delete(Shipit::OutputChunk.joins(:task).where(task: { stack_id: stack.id }))
+      delete(Shipit::Task.where(stack_id: stack.id))
+
+      delete(Shipit::Commit.where(stack_id: stack.id))
+
       stack.destroy!
     end
+
+    private
+
+    BATCH_SIZE = 1000
+
+    def delete(relation)
+      if relation.connection.adapter_name.match?(/(mysql|trilogy)/i)
+        while relation.limit(BATCH_SIZE).delete_all == BATCH_SIZE
+          true # loop
+        end
+      else
+        while relation.model.where(id: relation.select(:id).limit(BATCH_SIZE)).delete_all == BATCH_SIZE
+          true # loop
+        end
+      end
+    end
   end
 end

data/app/jobs/shipit/emit_event_job.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Shipit
   class EmitEventJob < BackgroundJob
     queue_as :hooks

data/app/jobs/shipit/fetch_commit_stats_job.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Shipit
   class FetchCommitStatsJob < BackgroundJob
     queue_as :default

data/app/jobs/shipit/fetch_deployed_revision_job.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Shipit
   class FetchDeployedRevisionJob < BackgroundJob
     queue_as :deploys

data/app/jobs/shipit/github_sync_job.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Shipit
   class GithubSyncJob < BackgroundJob
     include BackgroundJob::Unique
@@ -36,13 +37,14 @@ module Shipit
     def fetch_missing_commits(&block)
       commits = []
       github_api = stack&.github_api
-      iterator = Shipit::FirstParentCommitsIterator.new(github_api: github_api, &block)
+      iterator = Shipit::FirstParentCommitsIterator.new(github_api:, &block)
       iterator.each_with_index do |commit, index|
         break if index >= MAX_FETCHED_COMMITS
 
         if shared_parent = lookup_commit(commit.sha)
           return commits, shared_parent
         end
+
         commits.unshift(commit)
       end
       [commits, nil]
@@ -59,7 +61,7 @@ module Shipit
     end
 
     def lookup_commit(sha)
-      stack.commits.find_by(sha: sha)
+      stack.commits.find_by(sha:)
     end
   end
 end

data/app/jobs/shipit/mark_deploy_healthy_job.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Shipit
   class MarkDeployHealthyJob < BackgroundJob
     include BackgroundJob::Unique

data/app/jobs/shipit/perform_commit_checks_job.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Shipit
   class PerformCommitChecksJob < BackgroundJob
     include BackgroundJob::Unique

data/app/jobs/shipit/perform_task_job.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Shipit
   class PerformTaskJob < BackgroundJob
     queue_as :deploys

data/app/jobs/shipit/process_merge_requests_job.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Shipit
   class ProcessMergeRequestsJob < BackgroundJob
     include BackgroundJob::Unique
@@ -20,6 +21,7 @@ module Shipit
       merge_requests.select(&:pending?).each do |merge_request|
         merge_request.refresh!
         next unless merge_request.all_status_checks_passed?
+
         begin
           merge_request.merge!
         rescue MergeRequest::NotReady

data/app/jobs/shipit/purge_old_deliveries_job.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Shipit
   class PurgeOldDeliveriesJob < BackgroundJob
     include BackgroundJob::Unique

data/app/jobs/shipit/reap_dead_tasks_job.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Shipit
   class ReapDeadTasksJob < BackgroundJob
     include BackgroundJob::Unique

data/app/jobs/shipit/refresh_check_runs_job.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Shipit
   class RefreshCheckRunsJob < BackgroundJob
     queue_as :default

data/app/jobs/shipit/refresh_github_user_job.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Shipit
   class RefreshGithubUserJob < BackgroundJob
     queue_as :default

data/app/jobs/shipit/refresh_merge_request_job.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Shipit
   class RefreshMergeRequestJob < BackgroundJob
     queue_as :default

data/app/jobs/shipit/refresh_statuses_job.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Shipit
   class RefreshStatusesJob < BackgroundJob
     queue_as :default

data/app/jobs/shipit/setup_github_hook_job.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Shipit
   class SetupGithubHookJob < BackgroundJob
     queue_as :default

data/app/jobs/shipit/update_estimated_deploy_duration_job.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Shipit
   class UpdateEstimatedDeployDurationJob < BackgroundJob
     queue_as :default

data/app/jobs/shipit/update_github_last_deployed_ref_job.rb

@@ -1,11 +1,12 @@
 # frozen_string_literal: true
+
 module Shipit
   class UpdateGithubLastDeployedRefJob < BackgroundJob
     queue_as :default
 
     # We do not prefix 'refs/' because Octokit methods will do this automatically.
     BRANCH_REF_PREFIX = 'heads'
-    DEPLOY_PREFIX = "#{Shipit.app_name.downcase}-deploy"
+    DEPLOY_PREFIX = "#{Shipit.app_name.downcase}-deploy".freeze
 
     def perform(stack)
       stack_sha = stack.last_successful_deploy_commit&.sha
@@ -17,7 +18,7 @@ module Shipit
 
       full_repo_name = stack.github_repo_name
 
-      update_or_create_ref(client: client, repo_name: full_repo_name, ref: stack_ref, new_sha: stack_sha)
+      update_or_create_ref(client:, repo_name: full_repo_name, ref: stack_ref, new_sha: stack_sha)
     end
 
     private
@@ -34,11 +35,9 @@ module Shipit
       client.update_ref(repo_name, ref, new_sha)
     rescue Octokit::UnprocessableEntity => e
       error_msg = e.message
-      if error_msg.include?("Reference does not exist")
-        create_ref(client: client, repo_name: repo_name, ref: ref, sha: new_sha)
-      else
-        raise
-      end
+      raise unless error_msg.include?("Reference does not exist")
+
+      create_ref(client:, repo_name:, ref:, sha: new_sha)
     end
   end
 end

data/app/models/concerns/shipit/deferred_touch.rb

@@ -1,11 +1,12 @@
 # frozen_string_literal: true
+
 module Shipit
   module DeferredTouch
     extend ActiveSupport::Concern
 
     SET_KEY = 'shipit:deferred_touches'
-    TMP_KEY = "#{SET_KEY}:updating"
-    CACHE_KEY = "#{SET_KEY}:scheduled"
+    TMP_KEY = "#{SET_KEY}:updating".freeze
+    CACHE_KEY = "#{SET_KEY}:scheduled".freeze
     THROTTLE_TTL = 1.second
 
     included do
@@ -39,6 +40,7 @@ module Shipit
       def fetch
         fetch_members do |records|
           return if records.empty?
+
           records = records.each_with_object({}) do |(model, id, attribute), hash|
             attributes = (hash[model] ||= {})
             ids = (attributes[attribute] ||= [])
@@ -78,6 +80,7 @@ module Shipit
 
     def schedule_touches
       return unless self.class.deferred_touches
+
       deferred_touches = self.class.deferred_touches.reject do |m, _fk, _a|
         ActiveRecord::NoTouching.applied_to?(m.constantize)
       end

data/app/models/shipit/anonymous_user.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Shipit
   class AnonymousUser
     def blank?
@@ -21,11 +22,9 @@ module Shipit
       'https://github.com/images/error/octocat_happy.gif'
     end
 
-    def id
-    end
+    def id; end
 
-    def github_id
-    end
+    def github_id; end
 
     def logged_in?
       false
@@ -54,7 +53,7 @@ module Shipit
     def created_at
       Time.at(0).utc
     end
-    alias_method :updated_at, :created_at
+    alias updated_at created_at
 
     def read_attribute_for_serialization(attr)
       public_send(attr)

data/app/models/shipit/api_client.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Shipit
   class ApiClient < Record
     InsufficientPermission = Class.new(StandardError)
@@ -9,14 +10,14 @@ module Shipit
     validates :creator, :name, presence: true
 
     serialize :permissions, coder: Shipit.serialized_column(:permissions, type: Array)
-    PERMISSIONS = %w(
+    PERMISSIONS = %w[
       read:stack
       write:stack
       deploy:stack
       lock:stack
       read:hook
       write:hook
-    ).freeze
+    ].freeze
     validates :permissions, subset: { of: PERMISSIONS }
 
     class << self
@@ -39,6 +40,7 @@ module Shipit
       unless permissions.include?(required_permission)
         raise InsufficientPermission, "This operation requires the `#{required_permission}` permission"
       end
+
       true
     end
   end

data/app/models/shipit/application_record.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Shipit
   class ApplicationRecord < Record
     self.abstract_class = true

data/app/models/shipit/check_run.rb

@@ -1,7 +1,8 @@
 # frozen_string_literal: true
+
 module Shipit
   class CheckRun < ApplicationRecord
-    CONCLUSIONS = %w(success failure neutral cancelled timed_out action_required stale skipped).freeze
+    CONCLUSIONS = %w[success failure neutral cancelled timed_out action_required stale skipped].freeze
     include DeferredTouch
     include Status::Common
 
@@ -41,23 +42,23 @@ module Shipit
 
         unless checkrun_date
           Rails.logger.warn("No valid timestamp found in checkrun data. Checkrun id: #{github_check_run.id}.")
-          RefreshCheckRunsJob.set(wait: CHECK_RUN_REFRESH_DELAY).perform_later(stack_id: stack_id)
+          RefreshCheckRunsJob.set(wait: CHECK_RUN_REFRESH_DELAY).perform_later(stack_id:)
           return
         end
 
         create_or_update_by!(
           selector: {
-            github_id: github_check_run.id,
+            github_id: github_check_run.id
           },
           attributes: {
-            stack_id: stack_id,
+            stack_id:,
             name: github_check_run.name,
             conclusion: github_check_run.conclusion,
             title: github_check_run.output.title.to_s.truncate(1_000),
             details_url: github_check_run.details_url,
             html_url: github_check_run.html_url,
-            github_updated_at: checkrun_date,
-          },
+            github_updated_at: checkrun_date
+          }
         )
       end
 

data/app/models/shipit/command_line_user.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Shipit
   class CommandLineUser
     def present?
@@ -21,11 +22,9 @@ module Shipit
       'https://github.com/images/error/octocat_happy.gif'
     end
 
-    def id
-    end
+    def id; end
 
-    def github_id
-    end
+    def github_id; end
 
     def logged_in?
       false
@@ -50,7 +49,7 @@ module Shipit
     def created_at
       Time.at(0).utc
     end
-    alias_method :updated_at, :created_at
+    alias updated_at created_at
 
     def read_attribute_for_serialization(attr)
       public_send(attr)

data/app/models/shipit/commit.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Shipit
   class Commit < Record
     include DeferredTouch
@@ -21,7 +22,7 @@ module Shipit
     after_create { stack.update_undeployed_commits_count }
 
     after_commit :schedule_refresh_statuses!, :schedule_refresh_check_runs!, :schedule_fetch_stats!,
-      :schedule_continuous_delivery, on: :create
+                 :schedule_continuous_delivery, on: :create
 
     belongs_to :author, class_name: 'User', optional: true, inverse_of: :authored_commits
     belongs_to :committer, class_name: 'User', optional: true, inverse_of: :commits
@@ -54,25 +55,29 @@ module Shipit
     scope :reachable, -> { where(detached: false) }
 
     delegate :broadcast_update, :github_repo_name, :hidden_statuses, :required_statuses, :blocking_statuses,
-      :soft_failing_statuses, to: :stack
+             :soft_failing_statuses, to: :stack
 
     def self.newer_than(commit)
       return all unless commit
+
       where('id > ?', commit.try(:id) || commit)
     end
 
     def self.older_than(commit)
       return all unless commit
+
       where('id < ?', commit.try(:id) || commit)
     end
 
     def self.since(commit)
       return all unless commit
+
       where('id >= ?', commit.try(:id) || commit)
     end
 
     def self.until(commit)
       return all unless commit
+
       where('id <= ?', commit.try(:id) || commit)
     end
 
@@ -85,12 +90,11 @@ module Shipit
     end
 
     def self.by_sha(sha)
-      if sha.to_s.size < 6
-        raise AmbiguousRevision, "Short SHA1 #{sha} is ambiguous (too short)"
-      end
+      raise AmbiguousRevision, "Short SHA1 #{sha} is ambiguous (too short)" if sha.to_s.size < 6
 
       commits = where('sha like ?', "#{sha}%").take(2)
       raise AmbiguousRevision, "Short SHA1 #{sha} is ambiguous (matches multiple commits)" if commits.size > 1
+
       commits.first
     end
 
@@ -107,26 +111,22 @@ module Shipit
       record = new(
         sha: commit.sha,
         message: commit.commit.message,
-        author:  author,
-        committer: committer,
+        author:,
+        committer:,
         committed_at: commit.commit.committer.date,
         authored_at: commit.commit.author.date,
         additions: commit.stats&.additions,
-        deletions: commit.stats&.deletions,
+        deletions: commit.stats&.deletions
       )
 
-      if record.pull_request?
-        record.pull_request_head_sha = commit.parents.last.sha
-      end
+      record.pull_request_head_sha = commit.parents.last.sha if record.pull_request?
 
       record
     end
 
     def message=(message)
       limit = self.class.columns_hash['message'].limit
-      if limit && message && message.bytesize > limit
-        message = message.truncate_bytes(limit)
-      end
+      message = message.truncate_bytes(limit) if limit && message && message.bytesize > limit
       super(message)
     end
 
@@ -168,10 +168,26 @@ module Shipit
       end
     end
 
-    def refresh_check_runs!
+    def paginated_check_runs
       response = stack.handle_github_redirections do
-        stack.github_api.check_runs(github_repo_name, sha)
+        stack.github_api.check_runs(github_repo_name, sha, per_page: 100)
+      end
+
+      return response if stack.github_api.last_response.rels[:next].nil?
+
+      loop do
+        page = stack.handle_github_redirections do
+          stack.github_api.get(stack.github_api.last_response.rels[:next].href)
+        end
+        response.check_runs.concat(page.check_runs)
+        break if stack.github_api.last_response.rels[:next].nil?
       end
+
+      response
+    end
+
+    def refresh_check_runs!
+      response = paginated_check_runs
       response.check_runs.each do |check_run|
         create_or_update_check_run_from_github!(check_run)
       end
@@ -190,11 +206,11 @@ module Shipit
 
       @last_release_status = nil
       release_statuses.create!(
-        stack: stack,
-        user: user,
-        state: state,
-        target_url: target_url,
-        description: description,
+        stack:,
+        user:,
+        state:,
+        target_url:,
+        description:
       )
     end
 
@@ -223,7 +239,7 @@ module Shipit
     end
 
     def children
-      self.class.where(stack_id: stack_id).newer_than(self)
+      self.class.where(stack_id:).newer_than(self)
     end
 
     def detach_children!
@@ -266,6 +282,7 @@ module Shipit
 
     def schedule_continuous_delivery
       return unless deployable? && stack.continuous_deployment? && stack.deployable?
+
       # This buffer is to allow for statuses and checks to be refreshed before evaluating if the commit is deployable
       # - e.g. if the commit was fast-forwarded with already passing CI.
       ContinuousDeliveryJob.set(wait: RECENT_COMMIT_THRESHOLD).perform_later(stack)
@@ -282,7 +299,7 @@ module Shipit
     def fetch_stats!
       update!(
         additions: github_commit.stats&.additions,
-        deletions: github_commit.stats&.deletions,
+        deletions: github_commit.stats&.deletions
       )
     end
 
@@ -300,6 +317,7 @@ module Shipit
 
     def identify_merge_request
       return unless message_parser.pull_request?
+
       if merge_request = stack.merge_requests.find_by(number: message_parser.pull_request_number)
         self.merge_request = merge_request
         self.pull_request_number = merge_request.number
@@ -322,14 +340,14 @@ module Shipit
     def lock(user)
       update!(
         locked: true,
-        lock_author_id: user.id,
+        lock_author_id: user.id
       )
     end
 
     def self.lock_all(user)
       update_all(
         locked: true,
-        lock_author_id: user.id,
+        lock_author_id: user.id
       )
     end
 
@@ -356,19 +374,15 @@ module Shipit
       new_status = status
 
       unless already_deployed
-        payload = { commit: self, stack: stack, status: new_status.state }
-        if previous_status != new_status
-          Hook.emit(:commit_status, stack, payload.merge(commit_status: new_status))
-        end
+        payload = { commit: self, stack:, status: new_status.state }
+        Hook.emit(:commit_status, stack, payload.merge(commit_status: new_status)) if previous_status != new_status
       end
 
       if previous_status.simple_state != new_status.simple_state
         if !already_deployed && (!new_status.pending? || previous_status.unknown?)
           Hook.emit(:deployable_status, stack, payload.merge(deployable_status: new_status))
         end
-        if new_status.pending? || new_status.success?
-          stack.schedule_merges
-        end
+        stack.schedule_merges if new_status.pending? || new_status.success?
       end
       new_status
     end

data/app/models/shipit/commit_checks.rb

@@ -1,8 +1,9 @@
 # frozen_string_literal: true
+
 module Shipit
   class CommitChecks < EphemeralCommitChecks
     OUTPUT_TTL = 10.minutes.to_i
-    FINAL_STATUSES = %w(failed error success).freeze
+    FINAL_STATUSES = %w[failed error success].freeze
 
     def initialize(commit)
       @commit = commit
@@ -16,12 +17,13 @@ module Shipit
 
     def schedule
       return false if Shipit.redis.get(key('status')).present?
+
       synchronize do
         return false if Shipit.redis.get(key('status')).present?
 
         initialize_redis_state
       end
-      PerformCommitChecksJob.perform_later(commit: commit)
+      PerformCommitChecksJob.perform_later(commit:)
       true
     end