shipit-engine 0.38.0 → 0.40.0

data/lib/shipit/github_http_cache_middleware.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Shipit
   class GitHubHTTPCacheMiddleware < Faraday::Middleware
     def call(request_env)

data/lib/shipit/null_serializer.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Shipit
   module NullSerializer
     extend self

data/lib/shipit/octokit_check_runs.rb

@@ -1,9 +1,8 @@
 # frozen_string_literal: true
+
 module OctokitCheckRuns
   def check_runs(repo, sha, options = {})
-    paginate("#{Octokit::Repository.path(repo)}/commits/#{sha}/check-runs", options.reverse_merge(
-      accept: 'application/vnd.github.antiope-preview+json',
-    ))
+    paginate("#{Octokit::Repository.path(repo)}/commits/#{sha}/check-runs", options)
   end
 end
 

data/lib/shipit/octokit_iterator.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Shipit
   class OctokitIterator
     include Enumerable
@@ -20,6 +21,7 @@ module Shipit
 
         response.data.each(&block)
         return unless response.rels[:next]
+
         response = response.rels[:next].get
       end
     end

data/lib/shipit/paginator.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Shipit
   class Paginator
     def initialize(resources, controller, order: { id: :desc }, max_page_size: 100, default_page_size: 30)

data/lib/shipit/rollback_commands.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Shipit
   class RollbackCommands < DeployCommands
     def steps
@@ -7,7 +8,7 @@ module Shipit
 
     def env
       super.merge(
-        'ROLLBACK' => '1',
+        'ROLLBACK' => '1'
       )
     end
   end

data/lib/shipit/same_site_cookie_middleware.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Shipit
   class SameSiteCookieMiddleware
     COOKIE_SEPARATOR = "\n"

data/lib/shipit/simple_message_verifier.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Shipit
   class SimpleMessageVerifier < ActiveSupport::MessageVerifier
     def initialize(secret, **options)

data/lib/shipit/stack_commands.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
-# rubocop:disable Lint/MissingSuper
+
+# rubocop:disable Lint/MissingCopEnableDirective, Lint/MissingSuper
 require 'pathname'
 require 'fileutils'
 
@@ -16,26 +17,26 @@ module Shipit
     def fetch_commit(commit)
       create_directories
       if valid_git_repository?(@stack.git_path)
-        git('fetch', 'origin', '--quiet', '--tags', commit.sha, env: env, chdir: @stack.git_path)
+        git('fetch', 'origin', *quiet_git_arg, '--tags', '--force', commit.sha, env:, chdir: @stack.git_path)
       else
         @stack.clear_git_cache!
-        git_clone(@stack.repo_git_url, @stack.git_path, branch: @stack.branch, env: env, chdir: @stack.deploys_path)
+        git_clone(@stack.repo_git_url, @stack.git_path, branch: @stack.branch, env:, chdir: @stack.deploys_path)
       end
     end
 
     def fetch
       create_directories
       if valid_git_repository?(@stack.git_path)
-        git('fetch', 'origin', '--quiet', '--tags', @stack.branch, env: env, chdir: @stack.git_path)
+        git('fetch', 'origin', *quiet_git_arg, '--tags', '--force', @stack.branch, env:, chdir: @stack.git_path)
       else
         @stack.clear_git_cache!
-        git_clone(@stack.repo_git_url, @stack.git_path, branch: @stack.branch, env: env, chdir: @stack.deploys_path)
+        git_clone(@stack.repo_git_url, @stack.git_path, branch: @stack.branch, env:, chdir: @stack.deploys_path)
       end
     end
 
     def fetched?(commit)
       if valid_git_repository?(@stack.git_path)
-        git('rev-parse', '--quiet', '--verify', "#{commit.sha}^{commit}", env: env, chdir: @stack.git_path)
+        git('rev-parse', *quiet_git_arg, '--verify', "#{commit.sha}^{commit}", env:, chdir: @stack.git_path)
       else
         # When the stack's git cache is not valid, the commit is
         # NOT fetched. To keep the interface of this method
@@ -43,53 +44,55 @@ module Shipit
         # method returns false - has a non-zero exit status. We utilize
         # the POSIX 'test' command with no arguments which should
         # always have an exit status of 1.
-        Command.new('test', env: env, chdir: @stack.deploys_path)
+        Command.new('test', env:, chdir: @stack.deploys_path)
       end
     end
 
     def fetch_deployed_revision
       with_temporary_working_directory(commit: @stack.commits.reachable.last) do |dir|
-        spec = DeploySpec::FileSystem.new(dir, @stack.environment)
+        spec = DeploySpec::FileSystem.new(dir, @stack)
         outputs = spec.fetch_deployed_revision_steps!.map do |command_line|
-          Command.new(command_line, env: env, chdir: dir).run
+          Command.new(command_line, env:, chdir: dir).run
         end
         outputs.find(&:present?).try(:strip)
       end
     end
 
     def build_cacheable_deploy_spec
-      with_temporary_working_directory do |dir|
-        DeploySpec::FileSystem.new(dir, @stack.environment).cacheable
+      with_temporary_working_directory(recursive: false) do |dir|
+        DeploySpec::FileSystem.new(dir, @stack).cacheable
       end
     end
 
-    def with_temporary_working_directory(commit: nil)
+    def with_temporary_working_directory(commit: nil, recursive: true)
       commit ||= @stack.last_deployed_commit.presence || @stack.commits.reachable.last
 
       if !commit || !fetched?(commit).tap(&:run).success?
         @stack.acquire_git_cache_lock do
-          unless fetched?(commit).tap(&:run).success?
-            fetch.run!
-          end
+          fetch.run! unless fetched?(commit).tap(&:run).success?
         end
       end
 
+      git_args = []
+      git_args << '--recursive' if recursive
       Dir.mktmpdir do |dir|
         git(
           'clone', @stack.git_path, @stack.repo_name,
-          '--recursive', '--origin', 'cache',
+          *git_args, '--origin', 'cache',
           chdir: dir
         ).run!
 
         git_dir = File.join(dir, @stack.repo_name)
-        git(
-          '-c',
-          'advice.detachedHead=false',
-          'checkout',
-          '--quiet',
-          commit.sha,
-          chdir: git_dir
-        ).run! if commit
+        if commit
+          git(
+            '-c',
+            'advice.detachedHead=false',
+            'checkout',
+            *quiet_git_arg,
+            commit.sha,
+            chdir: git_dir
+          ).run!
+        end
         yield Pathname.new(git_dir)
       end
     end
@@ -106,19 +109,24 @@ module Shipit
         .success?
     end
 
-    def git_clone(url, path, branch: 'master', **kwargs)
-      git('clone', '--quiet', *modern_git_args, '--recursive', '--branch', branch, url, path, **kwargs)
+    def git_clone(url, path, branch: 'main', **kwargs)
+      git('clone', *quiet_git_arg, *modern_git_args, '--recursive', '--branch', branch, url, path, **kwargs)
     end
 
     def modern_git_args
       return [] unless git_version >= Gem::Version.new('1.7.10')
-      %w(--single-branch)
+
+      %w[--single-branch]
     end
 
     def create_directories
       FileUtils.mkdir_p(@stack.deploys_path)
     end
 
+    def quiet_git_arg
+      Shipit.git_progress_output ? [] : ['--quiet']
+    end
+
     private
 
     def github

data/lib/shipit/stat.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Shipit
   module Stat
     extend self

data/lib/shipit/task_commands.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
-# rubocop:disable Lint/MissingSuper
+
+# rubocop:disable Lint/MissingCopEnableDirective, Lint/MissingSuper
 module Shipit
   class TaskCommands < Commands
     delegate :fetch_commit, :fetch, :fetched?, to: :stack_commands
@@ -10,18 +11,18 @@ module Shipit
     end
 
     def deploy_spec
-      @deploy_spec ||= DeploySpec::FileSystem.new(@task.working_directory, @stack.environment)
+      @deploy_spec ||= DeploySpec::FileSystem.new(@task.working_directory, @stack)
     end
 
     def install_dependencies
       deploy_spec.dependencies_steps!.map do |command_line|
-        Command.new(command_line, env: env, chdir: steps_directory)
+        Command.new(command_line, env:, chdir: steps_directory)
       end
     end
 
     def perform
       steps.map do |command_line|
-        Command.new(command_line, env: env, chdir: steps_directory)
+        Command.new(command_line, env:, chdir: steps_directory)
       end
     end
 
@@ -40,7 +41,7 @@ module Shipit
           'TASK_ID' => @task.id.to_s,
           'IGNORED_SAFETIES' => @task.ignored_safeties? ? '1' : '0',
           'GIT_COMMITTER_NAME' => @task.user&.name || Shipit.committer_name,
-          'GIT_COMMITTER_EMAIL' => @task.user&.email || Shipit.committer_email,
+          'GIT_COMMITTER_EMAIL' => @task.user&.email || Shipit.committer_email
         )
         .merge(deploy_spec.machine_env)
         .merge(@task.env)
@@ -68,7 +69,7 @@ module Shipit
           @task.working_directory,
           chdir: @stack.deploys_path
         ),
-        git('remote', 'add', 'origin', @stack.repo_git_url, chdir: @task.working_directory),
+        git('remote', 'add', 'origin', @stack.repo_git_url, chdir: @task.working_directory)
       ]
     end
 

data/lib/shipit/version.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Shipit
-  VERSION = '0.38.0'
+  VERSION = '0.40.0'
 end

data/lib/shipit.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'active_support/all'
 require 'active_model_serializers'
 require 'state_machines-activerecord'
@@ -60,17 +61,17 @@ module Shipit
   extend self
 
   GithubOrganizationUnknown = Class.new(StandardError)
-  TOP_LEVEL_GH_KEYS = [:app_id, :installation_id, :webhook_secret, :private_key, :oauth, :domain]
+  TOP_LEVEL_GH_KEYS = [:app_id, :installation_id, :webhook_secret, :private_key, :oauth, :domain].freeze
 
   delegate :table_name_prefix, to: :secrets
 
   attr_accessor :disable_api_authentication, :timeout_exit_codes, :deployment_checks, :respect_bare_shipit_file,
-    :database_serializer
+                :database_serializer
   attr_writer(
     :internal_hook_receivers,
     :preferred_org_emails,
     :task_execution_strategy,
-    :task_logger,
+    :task_logger
   )
 
   def task_execution_strategy
@@ -80,7 +81,7 @@ module Shipit
   self.timeout_exit_codes = [].freeze
   self.respect_bare_shipit_file = true
 
-  alias_method :respect_bare_shipit_file?, :respect_bare_shipit_file
+  alias respect_bare_shipit_file? respect_bare_shipit_file
 
   def authentication_disabled?
     ENV['SHIPIT_DISABLE_AUTH'].present?
@@ -95,7 +96,7 @@ module Shipit
   end
 
   def redis_url
-    secrets.redis_url.present? ? URI(secrets.redis_url) : nil
+    secrets.redis_url.present? ? URI(secrets.redis_url) : ENV["REDIS_URL"]
   end
 
   def redis
@@ -104,14 +105,19 @@ module Shipit
       logger: Rails.logger,
       reconnect_attempts: 3,
       reconnect_delay: 0.5,
-      reconnect_delay_max: 1,
+      reconnect_delay_max: 1
     )
   end
 
+  def redis=(client)
+    @redis ||= client
+  end
+
   module SafeJSON
     class << self
       def load(serial)
         return nil if serial.nil?
+
         # JSON.load is unsafe, we should use parse instead
         JSON.parse(serial)
       end
@@ -136,6 +142,7 @@ module Shipit
 
       def dump(object)
         return if object.nil?
+
         JSON.dump(object)
       end
     end
@@ -144,7 +151,7 @@ module Shipit
   self.database_serializer = TransitionalSerializer
 
   def serialized_column(attribute_name, type: nil, coder: nil)
-    column = Paquito::SerializedColumn.new(database_serializer, type, attribute_name: attribute_name)
+    column = Paquito::SerializedColumn.new(database_serializer, type, attribute_name:)
     if coder
       Paquito.chain(coder, column)
     else
@@ -167,12 +174,14 @@ module Shipit
 
   def github_default_organization
     return nil unless secrets&.github
+
     org = secrets.github.keys.first
     TOP_LEVEL_GH_KEYS.include?(org) ? nil : org
   end
 
   def github_organizations
     return [nil] unless github_default_organization
+
     secrets.github.keys
   end
 
@@ -183,9 +192,9 @@ module Shipit
   end
 
   def legacy_github_api
-    if secrets&.github_api.present?
-      @legacy_github_api ||= github.new_client(access_token: secrets.github_api[:access_token])
-    end
+    return unless secrets&.github_api.present?
+
+    @legacy_github_api ||= github.new_client(access_token: secrets.github_api[:access_token])
   end
 
   def user
@@ -204,7 +213,7 @@ module Shipit
     if secrets.user_access_tokens_key.present?
       secrets.user_access_tokens_key
     elsif secrets.secret_key_base
-      Digest::SHA256.digest("user_access_tokens_key" + secrets.secret_key_base)
+      Digest::SHA256.digest("user_access_tokens_key#{secrets.secret_key_base}")
     end
   end
 
@@ -220,6 +229,10 @@ module Shipit
     secrets.update_latest_deployed_ref
   end
 
+  def git_progress_output
+    secrets.git_progress_output || false
+  end
+
   def enforce_publish_config
     secrets.enforce_publish_config.presence
   end
@@ -240,7 +253,7 @@ module Shipit
     @all_settings_present ||= [
       secrets.github, # TODO: handle GitHub settings
       redis_url,
-      host,
+      host
     ].all?(&:present?)
   end
 
@@ -254,10 +267,10 @@ module Shipit
 
   def revision
     @revision ||= if revision_file.exist?
-      revision_file.read
-    else
-      %x(git rev-parse HEAD)
-    end.strip
+                    revision_file.read
+                  else
+                    `git rev-parse HEAD`
+                  end.strip
   end
 
   def default_inactivity_timeout
@@ -291,7 +304,7 @@ module Shipit
   end
 
   def secrets
-    Rails.application.secrets
+    Rails.application.credentials
   end
 end
 

data/lib/tasks/cron.rake

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 namespace :cron do
   desc "Updates deployed revisions"
   task minutely: :environment do
@@ -10,7 +11,7 @@ namespace :cron do
     Shipit::ReviewStackProvisioningQueue.work
   end
 
-  task hourly: %i(rollup refresh_users clear_stale_caches delete_old_deployment_directories)
+  task hourly: %i[rollup refresh_users clear_stale_caches delete_old_deployment_directories]
 
   desc "Rolls-up output chunks for completed deploys older than an hour"
   task rollup: :environment do

data/lib/tasks/dev.rake

@@ -1,9 +1,10 @@
 # frozen_string_literal: true
+
 namespace :dev do
   desc "Appends chunks to the last deploy, or specify with DEPLOY=id"
   task stream: :environment do
     require 'faker'
-    logger = Logger.new(STDOUT)
+    logger = Logger.new($stdout)
 
     deploy = Shipit::Deploy.find(ENV['DEPLOY']) if ENV['DEPLOY']
     deploy ||= Deploy.last
@@ -25,7 +26,7 @@ namespace :dev do
 
       logger.error(sentence)
 
-      deploy.chunks.create(text: sentence + "\n")
+      deploy.chunks.create(text: "#{sentence}\n")
       sleep 1
     end
   end

data/lib/tasks/shipit.rake

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 namespace :shipit do
   desc "Deploy from a running instance. "
   task deploy: :environment do
@@ -12,12 +13,12 @@ namespace :shipit do
       class Task
         def write(text)
           p(text)
-          chunks.create!(text: text)
+          chunks.create!(text:)
         end
       end
     end
 
-    Shipit::Stack.run_deploy_in_foreground(stack: stack, revision: revision)
+    Shipit::Stack.run_deploy_in_foreground(stack:, revision:)
   rescue ArgumentError
     p("Use this command as follows:")
     p("bundle exec rake shipit:deploy stack='shopify/shipit/production' revision='$SHA'")

data/lib/tasks/teams.rake

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 namespace :teams do
   desc "Import the members of each team configured through the github.oauth.teams config"
   task fetch: :environment do
@@ -6,9 +7,9 @@ namespace :teams do
       puts "Fetching @#{team.handle} members"
       begin
         team.refresh_members!
-      rescue Octokit::Unauthorized, Octokit::NotFound => error
+      rescue Octokit::Unauthorized, Octokit::NotFound => e
         puts "Failed to fetch @#{team.handle} members. Do you have enough permissions?"
-        puts "#{error.class}: #{error.message}"
+        puts "#{e.class}: #{e.message}"
       end
     end
   end

data/test/controllers/api/base_controller_test.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'test_helper'
 
 module Shipit

data/test/controllers/api/ccmenu_controller_test.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'test_helper'
 
 module Shipit
@@ -32,7 +33,7 @@ module Shipit
       test "xml contains required attributes" do
         get :show, params: { stack_id: @stack.to_param }
         project = get_project_from_xml(response.body)
-        %w(name activity lastBuildStatus lastBuildLabel lastBuildTime webUrl).each do |attribute|
+        %w[name activity lastBuildStatus lastBuildLabel lastBuildTime webUrl].each do |attribute|
           assert_includes project, attribute, "Response missing required attribute: #{attribute}"
         end
       end
@@ -55,9 +56,9 @@ module Shipit
         Hash.from_xml(xml)['Projects']['Project']
       end
 
-      def assert_payload(k, v)
+      def assert_payload(key, value)
         @project ||= get_project_from_xml(response.body)
-        assert_equal(v, @project[k])
+        assert_equal(value, @project[key])
       end
     end
   end

data/test/controllers/api/commits_controller_test.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'test_helper'
 
 module Shipit

data/test/controllers/api/deploys_controller_test.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'test_helper'
 
 module Shipit
@@ -49,7 +50,7 @@ module Shipit
         request.headers['X-Shipit-User'] = @user.login
         post :create, params: { stack_id: @stack.to_param, sha: @commit.sha }
         deploy = Deploy.last
-        deploy.user == @user
+        assert_equal @user, deploy.user
       end
 
       test "#create normalises the claimed user" do
@@ -81,6 +82,7 @@ module Shipit
         end
 
         assert_response :conflict
+        assert_json 'error', 'A task is already running.'
       end
 
       test "#create refuses to deploy unsuccessful commits if the require_ci flag is passed" do
@@ -119,6 +121,29 @@ module Shipit
         assert_response :accepted
         assert_json 'status', 'pending'
       end
+
+      test "#create uses allow_concurrency param when provided" do
+        @stack.update!(lock_reason: 'Something broken')
+
+        assert_difference -> { @stack.deploys.count }, 1 do
+          post :create, params: { stack_id: @stack.to_param, sha: @commit.sha, force: 'true', allow_concurrency: 'false' }
+        end
+        assert_response :accepted
+        assert_json 'status', 'pending'
+        refute @stack.deploys.last.allow_concurrency
+      end
+
+      test "#create defaults allow_concurrency to force param when not provided" do
+        @stack.update!(lock_reason: 'Something broken')
+        expected_force = true
+
+        assert_difference -> { @stack.deploys.count }, 1 do
+          post :create, params: { stack_id: @stack.to_param, sha: @commit.sha, force: expected_force }
+        end
+        assert_response :accepted
+        assert_json 'status', 'pending'
+        assert_equal expected_force, @stack.deploys.last.allow_concurrency
+      end
     end
   end
 end

data/test/controllers/api/hooks_controller_test.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'test_helper'
 
 module Shipit
@@ -49,7 +50,7 @@ module Shipit
 
       test "#create adds a new hook" do
         assert_difference -> { Hook.count }, 1 do
-          post :create, params: { delivery_url: 'https://example.com/hook', events: %w(deploy rollback) }
+          post :create, params: { delivery_url: 'https://example.com/hook', events: %w[deploy rollback] }
         end
         hook = Hook.last
         assert_json 'delivery_url', 'https://example.com/hook'
@@ -60,14 +61,14 @@ module Shipit
       test "#create do not allow to set protected attributes" do
         post :create, params: {
           delivery_url: 'https://example.com/hook',
-          events: %w(deploy rollback),
-          created_at: 2.months.ago.to_formatted_s(:db),
+          events: %w[deploy rollback],
+          created_at: 2.months.ago.to_formatted_s(:db)
         }
-        Hook.last.created_at > 2.seconds.ago
+        assert_operator Hook.last.created_at, :>, 2.seconds.ago
       end
 
       test "#create returns validation errors" do
-        post :create, params: { delivery_url: '../etc/passwd', events: %w(deploy) }
+        post :create, params: { delivery_url: '../etc/passwd', events: %w[deploy] }
         assert_response :unprocessable_entity
         assert_json 'errors', 'delivery_url' => ['is not a valid URL']
       end

data/test/controllers/api/locks_controller_test.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'test_helper'
 
 module Shipit

data/test/controllers/api/merge_requests_controller_test.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'test_helper'
 
 module Shipit

data/test/controllers/api/outputs_controller_test.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'test_helper'
 
 module Shipit