shipit-engine 0.28.1 → 0.33.0

data/lib/shipit/first_parent_commits_iterator.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class FirstParentCommitsIterator < OctokitIterator
     def each

data/lib/shipit/flock.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'English'
 require 'timeout'
 require 'pathname'

data/lib/shipit/github_app.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class GitHubApp
     include Mutex_m
@@ -9,6 +10,8 @@ module Shipit
         end
       end
 
+      attr_reader :expires_at, :refresh_at
+
       def to_s
         @token
       end
@@ -16,16 +19,26 @@ module Shipit
       def initialize(token, expires_at)
         @token = token
         @expires_at = expires_at
+
+        # This needs to be lower than the token's lifetime, but higher than the cache expiry setting.
+        @refresh_at = expires_at - GITHUB_TOKEN_REFRESH_WINDOW
       end
 
       def blank?
-        @expires_at.past?
+        # Old tokens missing @refresh_at may be used upon deploy, so we should auto-correct for now.
+        # TODO: Remove this assignment at a later date.
+        @refresh_at ||= @expires_at - GITHUB_TOKEN_REFRESH_WINDOW
+        @refresh_at.past?
       end
     end
 
-    DOMAIN = 'github.com'.freeze
+    DOMAIN = 'github.com'
     AuthenticationFailed = Class.new(StandardError)
-    API_STATUS_ID = 'brv1bkgrwx7q'.freeze
+    API_STATUS_ID = 'brv1bkgrwx7q'
+
+    GITHUB_EXPECTED_TOKEN_LIFETIME = 60.minutes
+    GITHUB_TOKEN_RAILS_CACHE_LIFETIME = 50.minutes
+    GITHUB_TOKEN_REFRESH_WINDOW = GITHUB_EXPECTED_TOKEN_LIFETIME - GITHUB_TOKEN_RAILS_CACHE_LIFETIME - 2.minutes
 
     attr_reader :oauth_teams, :domain, :bot_login
 
@@ -39,7 +52,7 @@ module Shipit
       oauth = (@config[:oauth] || {}).with_indifferent_access
       @oauth_id = oauth[:id]
       @oauth_secret = oauth[:secret]
-      @oauth_teams = Array.wrap(oauth[:teams] || oauth[:teams])
+      @oauth_teams = Array.wrap(oauth[:teams])
     end
 
     def login
@@ -79,13 +92,21 @@ module Shipit
     end
 
     def fetch_new_token
-      Rails.cache.fetch('github:integration:access-token', expires_in: 50.minutes, race_condition_ttl: 10.minutes) do
+      # Rails can add 5 minutes to the cache entry expiration time when any TTL is provided,
+      # so our TTL setting can be lower, and TTL + expires_in should be lower than the GitHub token expiration.
+      Rails.cache.fetch(
+        'github:integration:access-token',
+        expires_in: GITHUB_TOKEN_RAILS_CACHE_LIFETIME,
+        race_condition_ttl: 4.minutes,
+      ) do
         response = new_client(bearer_token: authentication_payload).create_app_installation_access_token(
           installation_id,
           accept: 'application/vnd.github.machine-man-preview+json',
         )
         token = Token.from_github(response)
         raise AuthenticationFailed if token.blank?
+        Rails.logger.info("Created GitHub access token ending #{token.to_s[-5..-1]}, expires at #{token.expires_at}"\
+          " and will be refreshed at #{token&.refresh_at}")
         token
       end
     end
@@ -112,7 +133,7 @@ module Shipit
     end
 
     def url(*path)
-      @url ||= "https://#{domain}".freeze
+      @url ||= "https://#{domain}"
       path.empty? ? @url : File.join(@url, *path.map(&:to_s))
     end
 
@@ -120,12 +141,22 @@ module Shipit
       url('/api/v3/') if enterprise?
     end
 
+    def web_endpoint
+      url if enterprise?
+    end
+
     def enterprise?
       domain != DOMAIN
     end
 
     def new_client(options = {})
-      client = Octokit::Client.new(options.reverse_merge(api_endpoint: api_endpoint))
+      if enterprise?
+        options = options.reverse_merge(
+          api_endpoint: api_endpoint,
+          web_endpoint: web_endpoint,
+        )
+      end
+      client = Octokit::Client.new(options)
       client.middleware = faraday_stack
       client
     end
@@ -143,9 +174,9 @@ module Shipit
           logger: Rails.logger,
           serializer: NullSerializer,
         )
-        builder.use GitHubHTTPCacheMiddleware
-        builder.use Octokit::Response::RaiseError
-        builder.adapter Faraday.default_adapter
+        builder.use(GitHubHTTPCacheMiddleware)
+        builder.use(Octokit::Response::RaiseError)
+        builder.adapter(Faraday.default_adapter)
       end
     end
 

data/lib/shipit/github_http_cache_middleware.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class GitHubHTTPCacheMiddleware < Faraday::Middleware
     def call(request_env)

data/lib/shipit/null_serializer.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   module NullSerializer
     extend self

data/lib/shipit/octokit_check_runs.rb

@@ -1,8 +1,9 @@
+# frozen_string_literal: true
 module OctokitCheckRuns
   def check_runs(repo, sha, options = {})
-    paginate "#{Octokit::Repository.path repo}/commits/#{sha}/check-runs", options.reverse_merge(
+    paginate("#{Octokit::Repository.path(repo)}/commits/#{sha}/check-runs", options.reverse_merge(
       accept: 'application/vnd.github.antiope-preview+json',
-    )
+    ))
   end
 end
 

data/lib/shipit/octokit_iterator.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class OctokitIterator
     include Enumerable
@@ -6,8 +7,8 @@ module Shipit
       if relation
         @response = relation.get(per_page: 100)
       else
-        yield Shipit.github.api
-        @response = Shipit.github.api.last_response
+        data = yield Shipit.github.api
+        @response = Shipit.github.api.last_response if data.present?
       end
     end
 

data/lib/shipit/paginator.rb

@@ -1,6 +1,7 @@
+# frozen_string_literal: true
 module Shipit
   class Paginator
-    def initialize(resources, controller, order: {id: :desc}, max_page_size: 100, default_page_size: 30)
+    def initialize(resources, controller, order: { id: :desc }, max_page_size: 100, default_page_size: 30)
       @order = order
       @controller = controller
       @since = controller.params[:since].presence
@@ -15,7 +16,7 @@ module Shipit
     end
 
     def links
-      links = {first: link_to(since: nil, page_size: (page_size if page_size != @default_page_size))}
+      links = { first: link_to(since: nil, page_size: (page_size if page_size != @default_page_size)) }
       links[:next] = link_to(since: to_a.last.id) unless last_page?
       links
     end

data/lib/shipit/review_stack_commands.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require 'shipit/stack_commands'
+
+module Shipit
+  class ReviewStackCommands < Shipit::StackCommands
+  end
+end

data/lib/shipit/rollback_commands.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class RollbackCommands < DeployCommands
     def steps

data/lib/shipit/same_site_cookie_middleware.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+module Shipit
+  class SameSiteCookieMiddleware
+    COOKIE_SEPARATOR = "\n"
+
+    def initialize(app)
+      @app = app
+    end
+
+    def call(env)
+      status, headers, body = @app.call(env)
+
+      if headers && headers['Set-Cookie'] &&
+         Rack::Request.new(env).ssl?
+
+        set_cookies = headers['Set-Cookie'].split(COOKIE_SEPARATOR).compact
+        set_cookies.map! do |cookie|
+          cookie << '; Secure' if cookie !~ /;\s*secure/i
+          cookie << '; SameSite=None' unless cookie.match?(/;\s*samesite=/i)
+          cookie
+        end
+
+        headers['Set-Cookie'] = set_cookies.join(COOKIE_SEPARATOR)
+      end
+
+      [status, headers, body]
+    end
+  end
+end

data/lib/shipit/simple_message_verifier.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class SimpleMessageVerifier < ActiveSupport::MessageVerifier
     def initialize(secret, options = {})

data/lib/shipit/stack_commands.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+# rubocop:disable Lint/MissingSuper
 require 'pathname'
 require 'fileutils'
 
@@ -7,6 +9,10 @@ module Shipit
       @stack = stack
     end
 
+    def env
+      super.merge(@stack.env)
+    end
+
     def fetch
       create_directories
       if Dir.exist?(@stack.git_path)
@@ -26,7 +32,7 @@ module Shipit
     end
 
     def fetch_deployed_revision
-      with_temporary_working_directory(commit: @stack.commits.last) do |dir|
+      with_temporary_working_directory(commit: @stack.commits.reachable.last) do |dir|
         spec = DeploySpec::FileSystem.new(dir, @stack.environment)
         outputs = spec.fetch_deployed_revision_steps!.map do |command_line|
           Command.new(command_line, env: env, chdir: dir).run
@@ -42,18 +48,20 @@ module Shipit
     end
 
     def with_temporary_working_directory(commit: nil)
-      commit ||= @stack.last_deployed_commit.presence || @stack.commits.last
+      commit ||= @stack.last_deployed_commit.presence || @stack.commits.reachable.last
 
       if !commit || !fetched?(commit).tap(&:run).success?
         @stack.acquire_git_cache_lock do
-          fetch.run!
+          unless fetched?(commit).tap(&:run).success?
+            fetch.run!
+          end
         end
       end
 
       Dir.mktmpdir do |dir|
         git(
           'clone', @stack.git_path, @stack.repo_name,
-          '--origin', 'cache',
+          '--recursive', '--origin', 'cache',
           chdir: dir
         ).run!
 

data/lib/shipit/stat.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   module Stat
     extend self

data/lib/shipit/task_commands.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+# rubocop:disable Lint/MissingSuper
 module Shipit
   class TaskCommands < Commands
     delegate :fetch, :fetched?, to: :stack_commands
@@ -28,20 +30,20 @@ module Shipit
     end
 
     def env
-      normalized_name = ActiveSupport::Inflector.transliterate(@task.author.name)
-      super.merge(
-        'ENVIRONMENT' => @stack.environment,
-        'BRANCH' => @stack.branch,
-        'SHIPIT_USER' => "#{@task.author.login} (#{normalized_name}) via Shipit",
-        'EMAIL' => @task.author.email,
-        'BUNDLE_PATH' => Rails.root.join('data', 'bundler').to_s,
-        'SHIPIT_LINK' => @task.permalink,
-        'LAST_DEPLOYED_SHA' => @stack.last_deployed_commit.sha,
-        'TASK_ID' => @task.id.to_s,
-        'IGNORED_SAFETIES' => @task.ignored_safeties? ? '1' : '0',
-        'GIT_COMMITTER_NAME' => @task.user&.name || Shipit.committer_name,
-        'GIT_COMMITTER_EMAIL' => @task.user&.email || Shipit.committer_email,
-      ).merge(deploy_spec.machine_env).merge(@task.env)
+      super
+        .merge(@stack.env)
+        .merge(
+          'SHIPIT_USER' => "#{@task.author.login} (#{normalized_author_name}) via Shipit",
+          'EMAIL' => @task.author.email,
+          'BUNDLE_PATH' => Rails.root.join('data', 'bundler').to_s,
+          'SHIPIT_LINK' => @task.permalink,
+          'TASK_ID' => @task.id.to_s,
+          'IGNORED_SAFETIES' => @task.ignored_safeties? ? '1' : '0',
+          'GIT_COMMITTER_NAME' => @task.user&.name || Shipit.committer_name,
+          'GIT_COMMITTER_EMAIL' => @task.user&.email || Shipit.committer_email,
+        )
+        .merge(deploy_spec.machine_env)
+        .merge(@task.env)
     end
 
     def checkout(commit)
@@ -52,6 +54,7 @@ module Shipit
       [
         git(
           'clone',
+          '--quiet',
           '--local',
           '--origin', 'cache',
           @stack.git_path,
@@ -70,6 +73,12 @@ module Shipit
       FileUtils.rm_rf(@task.working_directory) if deploy_spec.clear_working_directory?
     end
 
+    private
+
+    def normalized_author_name
+      ActiveSupport::Inflector.transliterate(@task.author.name)
+    end
+
     protected
 
     def steps_directory

data/lib/shipit/version.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
-  VERSION = '0.28.1'.freeze
+  VERSION = '0.33.0'
 end

data/lib/snippets/release-gem

@@ -50,8 +50,12 @@ spec = Gem::Specification.load(spec_path)
 if RubygemsAPI.published?(spec.name, spec.version)
   puts "#{spec.name} version #{spec.version} is already published."
   exit 0
+elsif !spec.metadata['allowed_push_host']
+  puts "Can't release the gem: spec.metadata['allowed_push_host'] must be defined."
+  exit 1
 else
-  Git.tag_and_push(spec.version) do
+  is_successful = Git.tag_and_push(spec.version) do
     system(*release_command)
   end
+  is_successful ? exit(0) : exit(1)
 end

data/lib/tasks/cron.rake

@@ -1,13 +1,16 @@
+# frozen_string_literal: true
 namespace :cron do
   desc "Updates deployed revisions"
   task minutely: :environment do
     Shipit::Stack.refresh_deployed_revisions
     Shipit::Stack.schedule_continuous_delivery
     Shipit::GithubStatus.refresh_status
-    Shipit::PullRequest.schedule_merges
+    Shipit::MergeRequest.schedule_merges
+    Shipit::ReapDeadTasksJob.perform_later
+    Shipit::ReviewStackProvisioningQueue.work
   end
 
-  task hourly: %i(rollup refresh_users)
+  task hourly: %i(rollup refresh_users clear_stale_caches delete_old_deployment_directories)
 
   desc "Rolls-up output chunks for completed deploys older than an hour"
   task rollup: :environment do
@@ -17,4 +20,12 @@ namespace :cron do
   task refresh_users: :environment do
     Shipit::User.refresh_shard(Time.now.hour % 24, 24)
   end
+
+  task clear_stale_caches: :environment do
+    Shipit::ReviewStack.clear_stale_caches
+  end
+
+  task delete_old_deployment_directories: :environment do
+    Shipit::ReviewStack.delete_old_deployment_directories
+  end
 end

data/lib/tasks/dev.rake

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 namespace :dev do
   desc "Appends chunks to the last deploy, or specify with DEPLOY=id"
   task stream: :environment do
@@ -11,7 +12,7 @@ namespace :dev do
 
     deploy.update_attribute(:output, nil)
 
-    logger.info "Starting to stream data to deploy ##{deploy.id}"
+    logger.info("Starting to stream data to deploy ##{deploy.id}")
 
     loop do
       sentence = Faker::Lorem.sentence.split.map do |word|
@@ -22,7 +23,7 @@ namespace :dev do
         end
       end.join(' ')
 
-      logger.error sentence
+      logger.error(sentence)
 
       deploy.chunks.create(text: sentence + "\n")
       sleep 1

data/lib/tasks/shipit.rake

@@ -1,27 +1,26 @@
+# frozen_string_literal: true
 namespace :shipit do
   desc "Deploy from a running instance. "
   task deploy: :environment do
-    begin
-      stack = ENV['stack']
-      revision = ENV['revision']
+    stack = ENV['stack']
+    revision = ENV['revision']
 
-      raise ArgumentError.new('The first argument has to be a stack, e.g. shopify/shipit/production') if stack.nil?
-      raise ArgumentError.new('The second argument has to be a revision') if revision.nil?
+    raise ArgumentError, 'The first argument has to be a stack, e.g. shopify/shipit/production' if stack.nil?
+    raise ArgumentError, 'The second argument has to be a revision' if revision.nil?
 
-      module Shipit
-        class Task
-          def write(text)
-            p text
-            chunks.create!(text: text)
-          end
+    module Shipit
+      class Task
+        def write(text)
+          p(text)
+          chunks.create!(text: text)
         end
       end
-
-      Shipit::Stack.run_deploy_in_foreground(stack: stack, revision: revision)
-    rescue ArgumentError
-      p "Use this command as follows:"
-      p "bundle exec rake shipit:deploy stack='shopify/shipit/production' revision='$SHA'"
-      raise
     end
+
+    Shipit::Stack.run_deploy_in_foreground(stack: stack, revision: revision)
+  rescue ArgumentError
+    p("Use this command as follows:")
+    p("bundle exec rake shipit:deploy stack='shopify/shipit/production' revision='$SHA'")
+    raise
   end
 end