shipit-engine 0.33.0 → 0.34.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 47bb069ae71eaf994a57ee8dbe3a409adf62ede12fbb010b18ffb9056cee9e0e
-  data.tar.gz: 2e8399c7f72ed1308de6b0be4715579bb6732f3759b22eb38e2e750bb84439de
+  metadata.gz: e3fd7ab372e13432986f4bd927df32b97c0d4b969193035d9b34fbb0ce80630d
+  data.tar.gz: 5ac2897e8354b9fbae1f37ab192f2eca26cd4160169295a0ac6706e0b023affe
 SHA512:
-  metadata.gz: a0a8986aae58f932aca14da3213df881dabfa1017aa844bfe3f5a61afcabc2f09032eafe4342d8acdb7d4fc50cfe4ec9ad1f05dba9209757ef95cc6aa8c8a142
-  data.tar.gz: 0722dfd987d98c176b4625e23921c5576b1a40e85ed6e4d18208c4a4eff4b8c45ba314055dee3255feb576748431be638e6eb97fda6935c6953b70bfe6f14048
+  metadata.gz: ba96ab58f4a67a0658f02bf7844c8a47bf65526bd4dc4d37560a4bab5a9606a12fde8204cbe4f8829c51578bbd815f99e73ca3e3b81559a9935f7c4e4c45da57
+  data.tar.gz: c20e5c8d2a9ff71c70e332760d46e1e9985624411828f39af00754de2dcf50beec0109b22fc48f8669811c2248f52781638a529f8bc5e5083dff41bc54d9266c

data/README.md

@@ -12,7 +12,7 @@ Shipit is compatible with just about anything that you can deploy using a script
 
 This guide aims to help you [set up](#installation-and-setup), [use](#using-shipit), and [understand](#reference) Shipit.
 
-*Shipit requires a database (MySQL, PostgreSQL or SQLite3), redis, and Ruby 2.1 or superior.*
+*Shipit requires a database (MySQL, PostgreSQL or SQLite3), redis, and Ruby 2.6 or superior.*
 
 * * *
 <h2 id="toc">Table of contents</h2>
@@ -270,7 +270,9 @@ deploy:
 ```
 <br>
 
-**<code>deploy.max_commits</code>** defines the maximum number of commits that should be shipped per deploys. Defaults to `8`.
+**<code>deploy.max_commits</code>** defines the maximum number of commits that should be shipped per deploy. Defaults to `8` if no value is provided.
+
+To disable this limit, you can use use an explicit null value: `max_commits: null`. Continuous Delivery will then deploy any number of commits.
 
 Human users will be warned that they are not respecting the recommendation, but allowed to continue.
 However continuous delivery will respect this limit. If there is no deployable commits in this range, a human intervention will be required.
@@ -292,6 +294,15 @@ deploy:
   interval: 5m
 ```
 
+**<code>deploy.retries</code>** enables retries for a stack, and defines the maximum amount of times that Shipit will retry a deploy that finished with a `failed`, `error` or `timedout` status.
+
+For example, this will retry a deploy twice if it fails.
+
+```yaml
+deploy:
+  retries: 2
+```
+
 **<code>rollback.override</code>** contains an array of the shell commands required to rollback the application to a previous state. Shipit will try to infer it from the repository structure, but you can change the default inference. This key defaults to `disabled` unless Capistrano is detected.
 
 For example:

data/app/assets/stylesheets/_pages/_deploy.scss

@@ -196,8 +196,6 @@
   width: 100%;
   overflow-y: auto;
   overflow-x: hidden;
-  padding-right: 20px;
-  padding-left: 15px;
 }
 
 .commit-checks {

data/app/controllers/shipit/api/ccmenu_controller.rb

@@ -20,7 +20,7 @@ module Shipit
 
       def show
         latest_deploy = stack.deploys_and_rollbacks.last || NoDeploy.new
-        render('shipit/ccmenu/project.xml.builder', formats: [:xml], locals: { stack: stack, deploy: latest_deploy })
+        render('shipit/ccmenu/project', formats: [:xml], locals: { stack: stack, deploy: latest_deploy })
       end
 
       private

data/app/controllers/shipit/api/deploys_controller.rb

@@ -11,11 +11,13 @@ module Shipit
       params do
         requires :sha, String, length: { in: 6..40 }
         accepts :force, Boolean, default: false
+        accepts :require_ci, Boolean, default: false
         accepts :env, Hash, default: {}
       end
       def create
         commit = stack.commits.by_sha(params.sha) || param_error!(:sha, 'Unknown revision')
         param_error!(:force, "Can't deploy a locked stack") if !params.force && stack.locked?
+        param_error!(:require_ci, "Commit is not deployable") if params.require_ci && !commit.deployable?
         deploy = stack.trigger_deploy(commit, current_user, env: params.env, force: params.force)
         render_resource(deploy, status: :accepted)
       end

data/app/controllers/shipit/api/rollbacks_controller.rb

@@ -8,6 +8,7 @@ module Shipit
         requires :sha, String, length: { in: 6..40 }
         accepts :force, Boolean, default: false
         accepts :env, Hash, default: {}
+        accepts :lock, Boolean, default: true
       end
       def create
         commit = stack.commits.by_sha(params.sha) || param_error!(:sha, 'Unknown revision')
@@ -23,7 +24,7 @@ module Shipit
           active_task.abort!(aborted_by: current_user, rollback_once_aborted_to: deploy)
           response = active_task
         else
-          response = deploy.trigger_rollback(current_user, env: deploy_env, force: params.force)
+          response = deploy.trigger_rollback(current_user, env: deploy_env, force: params.force, lock: params.lock)
         end
 
         render_resource(response, status: :accepted)

data/app/controllers/shipit/api/stacks_controller.rb

@@ -40,6 +40,7 @@ module Shipit
       end
 
       params do
+        accepts :branch, String
         accepts :deploy_url, String
         accepts :ignore_ci, Boolean
         accepts :merge_queue_enabled, Boolean

data/app/controllers/shipit/deploys_controller.rb

@@ -55,7 +55,7 @@ module Shipit
     end
 
     def load_stack
-      @stack ||= Stack.from_param!(params[:stack_id]).becomes(Stack)
+      @stack ||= Stack.from_param!(params[:stack_id])
     end
 
     def load_until_commit

data/app/controllers/shipit/stacks_controller.rb

@@ -154,7 +154,7 @@ module Shipit
     end
 
     def create_params
-      params.require(:stack).permit(:environment, :branch, :deploy_url, :ignore_ci)
+      params.require(:stack).permit(:repo_owner, :repo_name, :environment, :branch, :deploy_url, :ignore_ci)
     end
 
     def update_params
@@ -180,7 +180,7 @@ module Shipit
     end
 
     def repository_params
-      params.require(:stack).permit(:repo_owner, :repo_name)
+      params.require(:stack).permit(:repo_owner, :repo_name, :branch, :environment, :deploy_url, :ignore_ci)
     end
   end
 end

data/app/controllers/shipit/tasks_controller.rb

@@ -62,7 +62,7 @@ module Shipit
     def url_for_task
       base_task = @task.is_a?(Deploy) ? @task.becomes(Deploy) : @task
 
-      url_for([base_task.stack.becomes(Stack), base_task])
+      url_for([base_task.stack, base_task])
     end
 
     def task
@@ -70,7 +70,7 @@ module Shipit
     end
 
     def stack
-      @stack ||= Stack.from_param!(params[:stack_id]).becomes(Stack)
+      @stack ||= Stack.from_param!(params[:stack_id])
     end
 
     def task_params

data/app/controllers/shipit/webhooks_controller.rb

@@ -2,7 +2,7 @@
 module Shipit
   class WebhooksController < ActionController::Base
     skip_before_action :verify_authenticity_token, raise: false
-    before_action :check_if_ping, :verify_signature
+    before_action :check_if_ping, :drop_unhandled_event, :verify_signature
 
     respond_to :json
 
@@ -15,8 +15,26 @@ module Shipit
 
     private
 
+    def drop_unhandled_event
+      # Acknowledge, but do nothing
+      head(204) unless Shipit::Webhooks.for_event(event).present?
+    end
+
     def verify_signature
-      head(422) unless Shipit.github.verify_webhook_signature(request.headers['X-Hub-Signature'], request.raw_post)
+      github_app = Shipit.github(organization: repository_owner)
+      verified = github_app.verify_webhook_signature(
+        request.headers['X-Hub-Signature'],
+        request.raw_post
+      )
+      head(422) unless verified
+
+      Rails.logger.info([
+        'WebhookController#verify_signature',
+        "event=#{event}",
+        "repository_owner=#{repository_owner}",
+        "signature=#{request.headers['X-Hub-Signature']}",
+        "status=#{status}",
+      ].join(' '))
     end
 
     def check_if_ping
@@ -27,8 +45,9 @@ module Shipit
       request.headers.fetch('X-Github-Event')
     end
 
-    def stack
-      @stack ||= Stack.find(params[:stack_id])
+    def repository_owner
+      # Fallback to the organization sub-object if repository isn't included in the payload
+      params.dig('repository', 'owner', 'login') || params.dig('organization', 'login')
     end
   end
 end

data/app/helpers/shipit/shipit_helper.rb

@@ -43,7 +43,6 @@ module Shipit
     end
 
     def missing_github_app_message
-      # TODO: Document how to create an app
       <<-MESSAGE.html_safe
         Shipit requires a GitHub App to authenticate users and perform API calls.
       MESSAGE

data/app/jobs/shipit/deliver_hook_job.rb

@@ -4,7 +4,7 @@ module Shipit
     queue_as :hooks
 
     def perform(delivery)
-      delivery = Hook::DeliverySpec.new(delivery) if delivery.is_a?(Hash)
+      delivery = Hook::DeliverySpec.new(**delivery) if delivery.is_a?(Hash)
       delivery.send!
     end
   end

data/app/jobs/shipit/github_sync_job.rb

@@ -3,8 +3,11 @@ module Shipit
   class GithubSyncJob < BackgroundJob
     include BackgroundJob::Unique
 
+    attr_reader :stack
+
     MAX_FETCHED_COMMITS = 25
     queue_as :default
+    on_duplicate :drop
 
     self.timeout = 60
     self.lock_timeout = 20
@@ -13,26 +16,27 @@ module Shipit
       @stack = Stack.find(params[:stack_id])
 
       handle_github_errors do
-        new_commits, shared_parent = fetch_missing_commits { @stack.github_commits }
+        new_commits, shared_parent = fetch_missing_commits { stack.github_commits }
 
-        @stack.transaction do
+        stack.transaction do
           shared_parent&.detach_children!
           appended_commits = new_commits.map do |gh_commit|
             append_commit(gh_commit)
           end
-          @stack.lock_reverted_commits! if appended_commits.any?(&:revert?)
+          stack.lock_reverted_commits! if appended_commits.any?(&:revert?)
         end
       end
-      CacheDeploySpecJob.perform_later(@stack)
+      CacheDeploySpecJob.perform_later(stack)
     end
 
     def append_commit(gh_commit)
-      @stack.commits.create_from_github!(gh_commit)
+      stack.commits.create_from_github!(gh_commit)
     end
 
     def fetch_missing_commits(&block)
       commits = []
-      iterator = Shipit::FirstParentCommitsIterator.new(&block)
+      github_api = stack&.github_api
+      iterator = Shipit::FirstParentCommitsIterator.new(github_api: github_api, &block)
       iterator.each_with_index do |commit, index|
         break if index >= MAX_FETCHED_COMMITS
 
@@ -49,13 +53,13 @@ module Shipit
     def handle_github_errors
       yield
     rescue Octokit::NotFound
-      @stack.mark_as_inaccessible!
+      stack.mark_as_inaccessible!
     else
-      @stack.mark_as_accessible!
+      stack.mark_as_accessible!
     end
 
     def lookup_commit(sha)
-      @stack.commits.find_by(sha: sha)
+      stack.commits.find_by(sha: sha)
     end
   end
 end

data/app/jobs/shipit/update_github_last_deployed_ref_job.rb

@@ -13,7 +13,7 @@ module Shipit
 
       environment = stack.environment
       stack_ref = create_full_ref(environment)
-      client = Shipit.github.api
+      client = stack.github_api
 
       full_repo_name = stack.github_repo_name
 

data/app/models/shipit/anonymous_user.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 module Shipit
   class AnonymousUser
-    def present?
-      false
+    def blank?
+      true
     end
 
     def email
@@ -63,5 +63,9 @@ module Shipit
     def serializer_class
       AnonymousUserSerializer
     end
+
+    def marked_for_destruction?
+      true
+    end
   end
 end

data/app/models/shipit/check_run.rb

@@ -5,6 +5,8 @@ module Shipit
     include DeferredTouch
     include Status::Common
 
+    CHECK_RUN_REFRESH_DELAY = 5.seconds
+
     belongs_to :stack, required: true
     belongs_to :commit, required: true
 
@@ -19,11 +21,30 @@ module Shipit
         create!(selector.merge(attributes))
       rescue ActiveRecord::RecordNotUnique
         record = find_by!(selector)
+
+        # Checkruns can jump between states and conclusions, and the github timestamps are low precision and unreliable.
+        # Since there's a conflict and the webhook seems older, enqueue a refresh.
+        # Persist the received data anyways, in case it is now the canonical data on GitHub despite the timestamp.
+        if attributes[:conclusion] != record.conclusion && record.newer_than_webhook?(attributes)
+          Rails.logger.warn(
+            "Conflicting stale checkrun received. Checkrun id: #{selector[:github_id]}, Details: #{attributes}"
+          )
+          RefreshCheckRunsJob.set(wait: CHECK_RUN_REFRESH_DELAY).perform_later(commit_id: record.commit_id)
+        end
+
         record.update!(attributes)
         record
       end
 
       def create_or_update_from_github!(stack_id, github_check_run)
+        checkrun_date = parse_newest_date(github_check_run)
+
+        unless checkrun_date
+          Rails.logger.warn("No valid timestamp found in checkrun data. Checkrun id: #{github_check_run.id}.")
+          RefreshCheckRunsJob.set(wait: CHECK_RUN_REFRESH_DELAY).perform_later(stack_id: stack_id)
+          return
+        end
+
         create_or_update_by!(
           selector: {
             github_id: github_check_run.id,
@@ -35,9 +56,20 @@ module Shipit
             title: github_check_run.output.title.to_s.truncate(1_000),
             details_url: github_check_run.details_url,
             html_url: github_check_run.html_url,
+            github_updated_at: checkrun_date,
           },
         )
       end
+
+      def parse_newest_date(github_check_run)
+        started_at = github_check_run.started_at
+        completed_at = github_check_run.completed_at
+
+        started_at_date = Time.parse(started_at.to_s) if started_at
+        completed_at_date = Time.parse(completed_at.to_s) if completed_at
+
+        [started_at_date, completed_at_date].compact.max
+      end
     end
 
     def state
@@ -71,6 +103,10 @@ module Shipit
       'shipit/statuses/status'
     end
 
+    def newer_than_webhook?(webhook_attributes)
+      github_updated_at && github_updated_at >= webhook_attributes[:github_updated_at]
+    end
+
     private
 
     def enable_ci_on_stack

data/app/models/shipit/commit.rb

@@ -8,7 +8,6 @@ module Shipit
     AmbiguousRevision = Class.new(StandardError)
 
     belongs_to :stack
-    has_many :deploys
     has_many :statuses, -> { order(created_at: :desc) }, dependent: :destroy, inverse_of: :commit
     has_many :check_runs, -> { order(created_at: :desc) }, dependent: :destroy, inverse_of: :commit
     has_many :commit_deployments, dependent: :destroy
@@ -22,28 +21,40 @@ module Shipit
     after_create { stack.update_undeployed_commits_count }
 
     after_commit :schedule_refresh_statuses!, :schedule_refresh_check_runs!, :schedule_fetch_stats!,
-                 :schedule_continuous_delivery, on: :create
+      :schedule_continuous_delivery, on: :create
 
-    belongs_to :author, class_name: 'User', inverse_of: :authored_commits
-    belongs_to :committer, class_name: 'User', inverse_of: :commits
-    belongs_to :lock_author, class_name: :User, optional: true, inverse_of: false
+    belongs_to :author, class_name: 'User', optional: true, inverse_of: :authored_commits
+    belongs_to :committer, class_name: 'User', optional: true, inverse_of: :commits
+    belongs_to :lock_author, class_name: 'User', optional: true, inverse_of: false
 
     def author
       super || AnonymousUser.new
     end
 
+    def author=(user)
+      super(user.presence)
+    end
+
     def committer
       super || AnonymousUser.new
     end
 
+    def committer=(user)
+      super(user.presence)
+    end
+
     def lock_author
       super || AnonymousUser.new
     end
 
+    def lock_author=(user)
+      super(user.presence)
+    end
+
     scope :reachable, -> { where(detached: false) }
 
     delegate :broadcast_update, :github_repo_name, :hidden_statuses, :required_statuses, :blocking_statuses,
-             :soft_failing_statuses, to: :stack
+      :soft_failing_statuses, to: :stack
 
     def self.newer_than(commit)
       return all unless commit
@@ -144,7 +155,7 @@ module Shipit
 
     def refresh_statuses!
       github_statuses = stack.handle_github_redirections do
-        Shipit.github.api.statuses(github_repo_name, sha, per_page: 100)
+        stack.github_api.statuses(github_repo_name, sha, per_page: 100)
       end
       github_statuses.each do |status|
         create_status_from_github!(status)
@@ -159,7 +170,7 @@ module Shipit
 
     def refresh_check_runs!
       response = stack.handle_github_redirections do
-        Shipit.github.api.check_runs(github_repo_name, sha)
+        stack.github_api.check_runs(github_repo_name, sha)
       end
       response.check_runs.each do |check_run|
         create_or_update_check_run_from_github!(check_run)
@@ -261,7 +272,7 @@ module Shipit
     end
 
     def github_commit
-      @github_commit ||= Shipit.github.api.commit(github_repo_name, sha)
+      @github_commit ||= stack.github_api.commit(github_repo_name, sha)
     end
 
     def schedule_fetch_stats!