shipit-engine 0.28.0 → 0.32.0

data/lib/shipit/cast_value.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   module CastValue
     def to_boolean(value)

data/lib/shipit/command.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'pty'
 require 'shellwords'
 require 'fileutils'
@@ -27,7 +28,7 @@ module Shipit
 
     def initialize(*args, default_timeout: Shipit.default_inactivity_timeout, env: {}, chdir:)
       @args, options = parse_arguments(args)
-      @timeout = options['timeout'.freeze] || options[:timeout] || default_timeout
+      @timeout = options['timeout'] || options[:timeout] || default_timeout
       @env = env
       @chdir = chdir.to_s
       @timed_out = false
@@ -48,10 +49,7 @@ module Shipit
     def interpolate_environment_variables(argument)
       return argument.map { |a| interpolate_environment_variables(a) } if argument.is_a?(Array)
 
-      argument.gsub(/(\$\w+)/) do |variable|
-        variable.sub!('$', '')
-        Shellwords.escape(@env.fetch(variable) { ENV[variable] })
-      end
+      EnvironmentVariables.with(env).interpolate(argument)
     end
 
     def success?
@@ -149,18 +147,16 @@ module Shipit
     def read_stream(io)
       touch_last_output_at
       loop do
-        begin
-          yield_control
-          yield io.read_nonblock(MAX_READ)
-          touch_last_output_at
-        rescue IO::WaitReadable
-          if output_timed_out?
-            @timed_out = true
-            raise TimedOut
-          end
-          IO.select([io], nil, nil, 1)
-          retry
+        yield_control
+        yield io.read_nonblock(MAX_READ)
+        touch_last_output_at
+      rescue IO::WaitReadable
+        if output_timed_out?
+          @timed_out = true
+          raise TimedOut
         end
+        IO.select([io], nil, nil, 1)
+        retry
       end
     rescue EOFError
     end
@@ -196,7 +192,7 @@ module Shipit
         # If we let the child a little bit of time, it solves it.
         retry_count -= 1
         if retry_count > 0
-          sleep 0.05
+          sleep(0.05)
           retry
         end
       end
@@ -220,7 +216,7 @@ module Shipit
           argument
         end
       end
-      return args, options
+      [args, options]
     end
 
     def running?

data/lib/shipit/commands.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class Commands
     class << self
@@ -6,13 +7,13 @@ module Shipit
       end
 
       def git_version
-        @git_version ||= parse_git_version(`git --version`)
+        @git_version ||= parse_git_version(%x(git --version))
       end
 
       def parse_git_version(raw_git_version)
-        raw_git_version =~ /(\d+\.\d+\.\d+)/
-        raise 'git command not found' unless $1
-        Gem::Version.new($1)
+        match_info = raw_git_version.match(/(\d+\.\d+\.\d+)/)
+        raise 'git command not found' unless match_info
+        Gem::Version.new(match_info[1])
       end
     end
 

data/lib/shipit/csv_serializer.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   module CSVSerializer
     extend self

data/lib/shipit/deploy_commands.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class DeployCommands < TaskCommands
     def steps

data/lib/shipit/engine.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class Engine < ::Rails::Engine
     isolate_namespace Shipit
@@ -32,11 +33,19 @@ module Shipit
       ActiveModel::Serializer.include(Engine.routes.url_helpers)
 
       if Shipit.github.oauth?
-        OmniAuth::Strategies::GitHub.configure path_prefix: '/github/auth'
-        app.middleware.use OmniAuth::Builder do
+        OmniAuth::Strategies::GitHub.configure(path_prefix: '/github/auth')
+        app.middleware.use(OmniAuth::Builder) do
           provider(:github, *Shipit.github.oauth_config)
         end
       end
+
+      if Shipit.enable_samesite_middleware?
+        app.config.middleware.insert_after(::Rack::Runtime, Shipit::SameSiteCookieMiddleware)
+      end
+
+      app.config.after_initialize do
+        ActionController::Base.include(Shipit::ActiveModelSerializersPatch)
+      end
     end
   end
 end

data/lib/shipit/environment_variables.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class EnvironmentVariables
     NotPermitted = Class.new(StandardError)
@@ -14,6 +15,15 @@ module Shipit
       sanitize_env_vars(variable_definitions)
     end
 
+    def interpolate(argument)
+      return argument unless @env
+
+      argument.gsub(/(\$\w+)/) do |variable|
+        variable.sub!('$', '')
+        Shellwords.escape(@env.fetch(variable) { ENV[variable] })
+      end
+    end
+
     private
 
     def initialize(env)
@@ -26,7 +36,7 @@ module Shipit
       allowed, disallowed = @env.partition { |k, _| allowed_variables.include?(k) }.map(&:to_h)
 
       error_message = "Variables #{disallowed.keys.to_sentence} have not been whitelisted"
-      raise NotPermitted.new(error_message) unless disallowed.empty?
+      raise NotPermitted, error_message unless disallowed.empty?
 
       allowed
     end

data/lib/shipit/first_parent_commits_iterator.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class FirstParentCommitsIterator < OctokitIterator
     def each

data/lib/shipit/flock.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'English'
 require 'timeout'
 require 'pathname'

data/lib/shipit/github_app.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class GitHubApp
     include Mutex_m
@@ -9,6 +10,8 @@ module Shipit
         end
       end
 
+      attr_reader :expires_at, :refresh_at
+
       def to_s
         @token
       end
@@ -16,15 +19,26 @@ module Shipit
       def initialize(token, expires_at)
         @token = token
         @expires_at = expires_at
+
+        # This needs to be lower than the token's lifetime, but higher than the cache expiry setting.
+        @refresh_at = expires_at - GITHUB_TOKEN_REFRESH_WINDOW
       end
 
       def blank?
-        @expires_at.past?
+        # Old tokens missing @refresh_at may be used upon deploy, so we should auto-correct for now.
+        # TODO: Remove this assignment at a later date.
+        @refresh_at ||= @expires_at - GITHUB_TOKEN_REFRESH_WINDOW
+        @refresh_at.past?
       end
     end
 
-    DOMAIN = 'github.com'.freeze
+    DOMAIN = 'github.com'
     AuthenticationFailed = Class.new(StandardError)
+    API_STATUS_ID = 'brv1bkgrwx7q'
+
+    GITHUB_EXPECTED_TOKEN_LIFETIME = 60.minutes
+    GITHUB_TOKEN_RAILS_CACHE_LIFETIME = 50.minutes
+    GITHUB_TOKEN_REFRESH_WINDOW = GITHUB_EXPECTED_TOKEN_LIFETIME - GITHUB_TOKEN_RAILS_CACHE_LIFETIME - 2.minutes
 
     attr_reader :oauth_teams, :domain, :bot_login
 
@@ -53,6 +67,13 @@ module Shipit
       client
     end
 
+    def api_status
+      conn = Faraday.new(url: 'https://www.githubstatus.com')
+      response = conn.get('/api/v2/components.json')
+      parsed = JSON.parse(response.body, symbolize_names: true)
+      parsed[:components].find { |c| c[:id] == API_STATUS_ID }
+    end
+
     def verify_webhook_signature(signature, message)
       return true unless webhook_secret
 
@@ -71,13 +92,21 @@ module Shipit
     end
 
     def fetch_new_token
-      Rails.cache.fetch('github:integration:access-token', expires_in: 50.minutes, race_condition_ttl: 10.minutes) do
+      # Rails can add 5 minutes to the cache entry expiration time when any TTL is provided,
+      # so our TTL setting can be lower, and TTL + expires_in should be lower than the GitHub token expiration.
+      Rails.cache.fetch(
+        'github:integration:access-token',
+        expires_in: GITHUB_TOKEN_RAILS_CACHE_LIFETIME,
+        race_condition_ttl: 4.minutes,
+      ) do
         response = new_client(bearer_token: authentication_payload).create_app_installation_access_token(
           installation_id,
           accept: 'application/vnd.github.machine-man-preview+json',
         )
         token = Token.from_github(response)
         raise AuthenticationFailed if token.blank?
+        Rails.logger.info("Created GitHub access token ending #{token.to_s[-5..-1]}, expires at #{token.expires_at}"\
+          " and will be refreshed at #{token&.refresh_at}")
         token
       end
     end
@@ -104,7 +133,7 @@ module Shipit
     end
 
     def url(*path)
-      @url ||= "https://#{domain}".freeze
+      @url ||= "https://#{domain}"
       path.empty? ? @url : File.join(@url, *path.map(&:to_s))
     end
 
@@ -112,13 +141,23 @@ module Shipit
       url('/api/v3/') if enterprise?
     end
 
+    def web_endpoint
+      url if enterprise?
+    end
+
     def enterprise?
       domain != DOMAIN
     end
 
     def new_client(options = {})
-      client = Octokit::Client.new(options.reverse_merge(api_endpoint: api_endpoint))
-      client.middleware = Shipit.new_faraday_stack
+      if enterprise?
+        options = options.reverse_merge(
+          api_endpoint: api_endpoint,
+          web_endpoint: web_endpoint,
+        )
+      end
+      client = Octokit::Client.new(options)
+      client.middleware = faraday_stack
       client
     end
 
@@ -126,6 +165,21 @@ module Shipit
 
     attr_reader :webhook_secret, :oauth_id, :oauth_secret
 
+    def faraday_stack
+      @faraday_stack ||= Faraday::RackBuilder.new do |builder|
+        builder.use(
+          Faraday::HttpCache,
+          shared_cache: false,
+          store: Rails.cache,
+          logger: Rails.logger,
+          serializer: NullSerializer,
+        )
+        builder.use(GitHubHTTPCacheMiddleware)
+        builder.use(Octokit::Response::RaiseError)
+        builder.adapter(Faraday.default_adapter)
+      end
+    end
+
     def app_id
       @app_id ||= @config.fetch(:app_id)
     end

data/lib/shipit/github_http_cache_middleware.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+module Shipit
+  class GitHubHTTPCacheMiddleware < Faraday::Middleware
+    def call(request_env)
+      @app.call(request_env).on_complete do |response_env|
+        if headers = response_env[:response_headers]
+
+          # We're removing max-age and s-maxage because some endpoints,
+          # especially /<repo>/commits, returns a max-age=0. This means that
+          # if two commits are pushed within 1 minute of each others, we won't
+          # see the second one. So instead we still issue the request, but we
+          # revalidate it.
+          cache_control = parse(headers['cache-control'].to_s)
+          cache_control.delete('max-age')
+          cache_control.delete('s-maxage')
+          cache_control['no-cache'] = true
+          cache_control['must-revalidate'] = true
+          headers['cache-control'] = dump(cache_control)
+
+          # We're removing `Authorization` from the `Vary` header because
+          # Faraday::HttpCache has a very limited garbage collection support
+          # and this cause cache entries to grow indefinitely.
+          # See https://github.com/Shopify/shipit-engine/issues/935 for
+          # more details.
+          vary = parse(headers['vary'].to_s)
+          vary.delete('authorization')
+          headers['vary'] = dump(vary)
+        end
+      end
+    end
+
+    private
+
+    def dump(directives)
+      directives.map do |k, v|
+        if v == true
+          k
+        else
+          "#{k}=#{v}"
+        end
+      end.join(', ')
+    end
+
+    def parse(header)
+      directives = {}
+
+      header.delete(' ').split(',').each do |part|
+        next if part.empty?
+
+        name, value = part.split('=', 2)
+        directives[name.downcase] = (value || true) unless name.empty?
+      end
+
+      directives
+    end
+  end
+end

data/lib/shipit/null_serializer.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   module NullSerializer
     extend self

data/lib/shipit/octokit_check_runs.rb

@@ -1,8 +1,9 @@
+# frozen_string_literal: true
 module OctokitCheckRuns
   def check_runs(repo, sha, options = {})
-    paginate "#{Octokit::Repository.path repo}/commits/#{sha}/check-runs", options.reverse_merge(
+    paginate("#{Octokit::Repository.path(repo)}/commits/#{sha}/check-runs", options.reverse_merge(
       accept: 'application/vnd.github.antiope-preview+json',
-    )
+    ))
   end
 end
 

data/lib/shipit/octokit_iterator.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class OctokitIterator
     include Enumerable
@@ -6,8 +7,8 @@ module Shipit
       if relation
         @response = relation.get(per_page: 100)
       else
-        yield Shipit.github.api
-        @response = Shipit.github.api.last_response
+        data = yield Shipit.github.api
+        @response = Shipit.github.api.last_response if data.present?
       end
     end
 

data/lib/shipit/paginator.rb

@@ -1,6 +1,7 @@
+# frozen_string_literal: true
 module Shipit
   class Paginator
-    def initialize(resources, controller, order: {id: :desc}, max_page_size: 100, default_page_size: 30)
+    def initialize(resources, controller, order: { id: :desc }, max_page_size: 100, default_page_size: 30)
       @order = order
       @controller = controller
       @since = controller.params[:since].presence
@@ -15,7 +16,7 @@ module Shipit
     end
 
     def links
-      links = {first: link_to(since: nil, page_size: (page_size if page_size != @default_page_size))}
+      links = { first: link_to(since: nil, page_size: (page_size if page_size != @default_page_size)) }
       links[:next] = link_to(since: to_a.last.id) unless last_page?
       links
     end

data/lib/shipit/rollback_commands.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class RollbackCommands < DeployCommands
     def steps

data/lib/shipit/same_site_cookie_middleware.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+module Shipit
+  class SameSiteCookieMiddleware
+    COOKIE_SEPARATOR = "\n"
+
+    def initialize(app)
+      @app = app
+    end
+
+    def call(env)
+      status, headers, body = @app.call(env)
+
+      if headers && headers['Set-Cookie'] &&
+         Rack::Request.new(env).ssl?
+
+        set_cookies = headers['Set-Cookie'].split(COOKIE_SEPARATOR).compact
+        set_cookies.map! do |cookie|
+          cookie << '; Secure' if cookie !~ /;\s*secure/i
+          cookie << '; SameSite=None' unless cookie.match?(/;\s*samesite=/i)
+          cookie
+        end
+
+        headers['Set-Cookie'] = set_cookies.join(COOKIE_SEPARATOR)
+      end
+
+      [status, headers, body]
+    end
+  end
+end