shipit-engine 0.31.0 → 0.35.0

data/lib/shipit/csv_serializer.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   module CSVSerializer
     extend self

data/lib/shipit/deploy_commands.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class DeployCommands < TaskCommands
     def steps

data/lib/shipit/engine.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class Engine < ::Rails::Engine
     isolate_namespace Shipit
@@ -9,6 +10,8 @@ module Shipit
       Shipit::Engine.routes.default_url_options[:host] = Shipit.host
       Pubsubstub.redis_url = Shipit.redis_url.to_s
 
+      Rails.application.secrets.deep_symbolize_keys!
+
       app.config.assets.paths << Emoji.images_path
       app.config.assets.precompile += %w(
         favicon.ico
@@ -32,12 +35,16 @@ module Shipit
       ActiveModel::Serializer.include(Engine.routes.url_helpers)
 
       if Shipit.github.oauth?
-        OmniAuth::Strategies::GitHub.configure path_prefix: '/github/auth'
-        app.middleware.use OmniAuth::Builder do
+        OmniAuth::Strategies::GitHub.configure(path_prefix: '/github/auth')
+        app.middleware.use(OmniAuth::Builder) do
           provider(:github, *Shipit.github.oauth_config)
         end
       end
 
+      if Shipit.enable_samesite_middleware?
+        app.config.middleware.insert_after(::Rack::Runtime, Shipit::SameSiteCookieMiddleware)
+      end
+
       app.config.after_initialize do
         ActionController::Base.include(Shipit::ActiveModelSerializersPatch)
       end

data/lib/shipit/environment_variables.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class EnvironmentVariables
     NotPermitted = Class.new(StandardError)
@@ -35,7 +36,7 @@ module Shipit
       allowed, disallowed = @env.partition { |k, _| allowed_variables.include?(k) }.map(&:to_h)
 
       error_message = "Variables #{disallowed.keys.to_sentence} have not been whitelisted"
-      raise NotPermitted.new(error_message) unless disallowed.empty?
+      raise NotPermitted, error_message unless disallowed.empty?
 
       allowed
     end

data/lib/shipit/first_parent_commits_iterator.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class FirstParentCommitsIterator < OctokitIterator
     def each

data/lib/shipit/flock.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'English'
 require 'timeout'
 require 'pathname'
@@ -10,14 +11,21 @@ module Shipit
 
     def initialize(path)
       @path = Pathname.new(path)
+      @acquired = false
     end
 
     def lock(timeout:)
+      return yield if @acquired
       path.parent.mkpath
       path.open('w') do |file|
         if retrying(timeout: timeout) { file.flock(File::LOCK_EX | File::LOCK_NB) }
           file.write($PROCESS_ID.to_s)
-          return yield
+          @acquired = true
+          begin
+            yield
+          ensure
+            @acquired = false
+          end
         else
           raise TimeoutError, "Couldn't acquire lock for #{path} in #{timeout} seconds"
         end

data/lib/shipit/github_app.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class GitHubApp
     include Mutex_m
@@ -31,9 +32,9 @@ module Shipit
       end
     end
 
-    DOMAIN = 'github.com'.freeze
+    DOMAIN = 'github.com'
     AuthenticationFailed = Class.new(StandardError)
-    API_STATUS_ID = 'brv1bkgrwx7q'.freeze
+    API_STATUS_ID = 'brv1bkgrwx7q'
 
     GITHUB_EXPECTED_TOKEN_LIFETIME = 60.minutes
     GITHUB_TOKEN_RAILS_CACHE_LIFETIME = 50.minutes
@@ -41,8 +42,9 @@ module Shipit
 
     attr_reader :oauth_teams, :domain, :bot_login
 
-    def initialize(config)
+    def initialize(organization, config)
       super()
+      @organization = organization
       @config = (config || {}).with_indifferent_access
       @domain = @config[:domain] || DOMAIN
       @webhook_secret = @config[:webhook_secret].presence
@@ -51,7 +53,7 @@ module Shipit
       oauth = (@config[:oauth] || {}).with_indifferent_access
       @oauth_id = oauth[:id]
       @oauth_secret = oauth[:secret]
-      @oauth_teams = Array.wrap(oauth[:teams] || oauth[:teams])
+      @oauth_teams = Array.wrap(oauth[:teams])
     end
 
     def login
@@ -91,10 +93,11 @@ module Shipit
     end
 
     def fetch_new_token
+      cache_key = @organization.nil? ? '' : "#{@organization.downcase}:"
       # Rails can add 5 minutes to the cache entry expiration time when any TTL is provided,
       # so our TTL setting can be lower, and TTL + expires_in should be lower than the GitHub token expiration.
       Rails.cache.fetch(
-        'github:integration:access-token',
+        "github:integration:#{cache_key}access-token",
         expires_in: GITHUB_TOKEN_RAILS_CACHE_LIFETIME,
         race_condition_ttl: 4.minutes,
       ) do
@@ -132,7 +135,7 @@ module Shipit
     end
 
     def url(*path)
-      @url ||= "https://#{domain}".freeze
+      @url ||= "https://#{domain}"
       path.empty? ? @url : File.join(@url, *path.map(&:to_s))
     end
 
@@ -173,22 +176,22 @@ module Shipit
           logger: Rails.logger,
           serializer: NullSerializer,
         )
-        builder.use GitHubHTTPCacheMiddleware
-        builder.use Octokit::Response::RaiseError
-        builder.adapter Faraday.default_adapter
+        builder.use(GitHubHTTPCacheMiddleware)
+        builder.use(Octokit::Response::RaiseError)
+        builder.adapter(Faraday.default_adapter)
       end
     end
 
     def app_id
-      @app_id ||= @config.fetch(:app_id)
+      @config.fetch(:app_id)
     end
 
     def installation_id
-      @installation_id ||= @config.fetch(:installation_id)
+      @config.fetch(:installation_id)
     end
 
     def private_key
-      @private_key ||= @config.fetch(:private_key)
+      @config.fetch(:private_key)
     end
 
     def authentication_payload

data/lib/shipit/github_http_cache_middleware.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class GitHubHTTPCacheMiddleware < Faraday::Middleware
     def call(request_env)

data/lib/shipit/null_serializer.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   module NullSerializer
     extend self

data/lib/shipit/octokit_check_runs.rb

@@ -1,8 +1,9 @@
+# frozen_string_literal: true
 module OctokitCheckRuns
   def check_runs(repo, sha, options = {})
-    paginate "#{Octokit::Repository.path repo}/commits/#{sha}/check-runs", options.reverse_merge(
+    paginate("#{Octokit::Repository.path(repo)}/commits/#{sha}/check-runs", options.reverse_merge(
       accept: 'application/vnd.github.antiope-preview+json',
-    )
+    ))
   end
 end
 

data/lib/shipit/octokit_iterator.rb

@@ -1,13 +1,14 @@
+# frozen_string_literal: true
 module Shipit
   class OctokitIterator
     include Enumerable
 
-    def initialize(relation = nil)
+    def initialize(relation = nil, github_api: nil)
       if relation
         @response = relation.get(per_page: 100)
       else
-        yield Shipit.github.api
-        @response = Shipit.github.api.last_response
+        data = yield github_api
+        @response = github_api.last_response if data.present?
       end
     end
 

data/lib/shipit/paginator.rb

@@ -1,6 +1,7 @@
+# frozen_string_literal: true
 module Shipit
   class Paginator
-    def initialize(resources, controller, order: {id: :desc}, max_page_size: 100, default_page_size: 30)
+    def initialize(resources, controller, order: { id: :desc }, max_page_size: 100, default_page_size: 30)
       @order = order
       @controller = controller
       @since = controller.params[:since].presence
@@ -15,7 +16,7 @@ module Shipit
     end
 
     def links
-      links = {first: link_to(since: nil, page_size: (page_size if page_size != @default_page_size))}
+      links = { first: link_to(since: nil, page_size: (page_size if page_size != @default_page_size)) }
       links[:next] = link_to(since: to_a.last.id) unless last_page?
       links
     end

data/lib/shipit/review_stack_commands.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require 'shipit/stack_commands'
+
+module Shipit
+  class ReviewStackCommands < Shipit::StackCommands
+  end
+end

data/lib/shipit/rollback_commands.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class RollbackCommands < DeployCommands
     def steps

data/lib/shipit/same_site_cookie_middleware.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+module Shipit
+  class SameSiteCookieMiddleware
+    COOKIE_SEPARATOR = "\n"
+
+    def initialize(app)
+      @app = app
+    end
+
+    def call(env)
+      status, headers, body = @app.call(env)
+
+      if headers && headers['Set-Cookie'] &&
+         Rack::Request.new(env).ssl?
+
+        set_cookies = headers['Set-Cookie'].split(COOKIE_SEPARATOR).compact
+        set_cookies.map! do |cookie|
+          cookie << '; Secure' if cookie !~ /;\s*secure/i
+          cookie << '; SameSite=None' unless cookie.match?(/;\s*samesite=/i)
+          cookie
+        end
+
+        headers['Set-Cookie'] = set_cookies.join(COOKIE_SEPARATOR)
+      end
+
+      [status, headers, body]
+    end
+  end
+end

data/lib/shipit/simple_message_verifier.rb

@@ -1,8 +1,9 @@
+# frozen_string_literal: true
 module Shipit
   class SimpleMessageVerifier < ActiveSupport::MessageVerifier
-    def initialize(secret, options = {})
+    def initialize(secret, **options)
       options[:serializer] ||= ToS
-      super(secret, options)
+      super(secret, **options)
     end
 
     private

data/lib/shipit/stack_commands.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+# rubocop:disable Lint/MissingSuper
 require 'pathname'
 require 'fileutils'
 
@@ -7,26 +9,36 @@ module Shipit
       @stack = stack
     end
 
+    def env
+      super.merge(@stack.env)
+    end
+
     def fetch
       create_directories
-      if Dir.exist?(@stack.git_path)
+      if valid_git_repository?(@stack.git_path)
         git('fetch', 'origin', '--tags', @stack.branch, env: env, chdir: @stack.git_path)
       else
+        @stack.clear_git_cache!
         git_clone(@stack.repo_git_url, @stack.git_path, branch: @stack.branch, env: env, chdir: @stack.deploys_path)
       end
     end
 
     def fetched?(commit)
-      git_dir = File.join(@stack.git_path, '.git')
-      if Dir.exist?(git_dir)
+      if valid_git_repository?(@stack.git_path)
         git('rev-parse', '--quiet', '--verify', "#{commit.sha}^{commit}", env: env, chdir: @stack.git_path)
       else
-        Command.new('test', '-d', git_dir, env: env, chdir: @stack.deploys_path)
+        # When the stack's git cache is not valid, the commit is
+        # NOT fetched. To keep the interface of this method
+        # consistent, we must return a Shipit::Command whose #success?
+        # method returns false - has a non-zero exit status. We utilize
+        # the POSIX 'test' command with no arguments which should
+        # always have an exit status of 1.
+        Command.new('test', env: env, chdir: @stack.deploys_path)
       end
     end
 
     def fetch_deployed_revision
-      with_temporary_working_directory(commit: @stack.commits.last) do |dir|
+      with_temporary_working_directory(commit: @stack.commits.reachable.last) do |dir|
         spec = DeploySpec::FileSystem.new(dir, @stack.environment)
         outputs = spec.fetch_deployed_revision_steps!.map do |command_line|
           Command.new(command_line, env: env, chdir: dir).run
@@ -42,7 +54,7 @@ module Shipit
     end
 
     def with_temporary_working_directory(commit: nil)
-      commit ||= @stack.last_deployed_commit.presence || @stack.commits.last
+      commit ||= @stack.last_deployed_commit.presence || @stack.commits.reachable.last
 
       if !commit || !fetched?(commit).tap(&:run).success?
         @stack.acquire_git_cache_lock do
@@ -55,18 +67,30 @@ module Shipit
       Dir.mktmpdir do |dir|
         git(
           'clone', @stack.git_path, @stack.repo_name,
-          '--origin', 'cache',
+          '--recursive', '--origin', 'cache',
           chdir: dir
         ).run!
 
         git_dir = File.join(dir, @stack.repo_name)
-        git('checkout', commit.sha, chdir: git_dir).run! if commit
+        git('checkout', '--config', 'advice.detachedHead=false', commit.sha, chdir: git_dir).run! if commit
         yield Pathname.new(git_dir)
       end
     end
 
+    def valid_git_repository?(path)
+      path.exist? &&
+        !path.empty? &&
+        git_cmd_succeeds?(path)
+    end
+
+    def git_cmd_succeeds?(path)
+      git("rev-parse", "--git-dir", chdir: path)
+        .tap(&:run)
+        .success?
+    end
+
     def git_clone(url, path, branch: 'master', **kwargs)
-      git('clone', *modern_git_args, '--recursive', '--branch', branch, url, path, **kwargs)
+      git('clone', '--quiet', *modern_git_args, '--recursive', '--branch', branch, url, path, **kwargs)
     end
 
     def modern_git_args
@@ -77,5 +101,11 @@ module Shipit
     def create_directories
       FileUtils.mkdir_p(@stack.deploys_path)
     end
+
+    private
+
+    def github
+      Shipit.github(organization: @stack.repository.owner)
+    end
   end
 end

data/lib/shipit/stat.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   module Stat
     extend self

data/lib/shipit/task_commands.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+# rubocop:disable Lint/MissingSuper
 module Shipit
   class TaskCommands < Commands
     delegate :fetch, :fetched?, to: :stack_commands
@@ -52,6 +54,7 @@ module Shipit
       [
         git(
           'clone',
+          '--quiet',
           '--local',
           '--origin', 'cache',
           @stack.git_path,
@@ -85,5 +88,11 @@ module Shipit
         @task.working_directory
       end
     end
+
+    private
+
+    def github
+      Shipit.github(organization: @stack.repository.owner)
+    end
   end
 end

data/lib/shipit/version.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
-  VERSION = '0.31.0'.freeze
+  VERSION = '0.35.0'
 end

data/lib/shipit.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'active_support/all'
 require 'active_model_serializers'
 require 'state_machines-activerecord'
@@ -23,7 +24,6 @@ require 'safe_yaml/load'
 require 'securecompare'
 
 require 'redis-objects'
-require 'redis-namespace'
 
 require 'octokit'
 require 'faraday-http-cache'
@@ -42,12 +42,14 @@ require 'shipit/simple_message_verifier'
 require 'shipit/command'
 require 'shipit/commands'
 require 'shipit/stack_commands'
+require 'shipit/review_stack_commands'
 require 'shipit/task_commands'
 require 'shipit/deploy_commands'
 require 'shipit/rollback_commands'
 require 'shipit/environment_variables'
 require 'shipit/stat'
 require 'shipit/github_http_cache_middleware'
+require 'shipit/same_site_cookie_middleware'
 require 'shipit/cast_value'
 require 'shipit/line_buffer'
 
@@ -57,10 +59,22 @@ SafeYAML::OPTIONS[:deserialize_symbols] = false
 module Shipit
   extend self
 
+  GithubOrganizationUnknown = Class.new(StandardError)
+  TOP_LEVEL_GH_KEYS = [:app_id, :installation_id, :webhook_secret, :private_key, :oauth, :domain]
+
   delegate :table_name_prefix, to: :secrets
 
-  attr_accessor :disable_api_authentication, :timeout_exit_codes
-  attr_writer :internal_hook_receivers, :task_logger, :preferred_org_emails
+  attr_accessor :disable_api_authentication, :timeout_exit_codes, :deployment_checks
+  attr_writer(
+    :internal_hook_receivers,
+    :preferred_org_emails,
+    :task_execution_strategy,
+    :task_logger,
+  )
+
+  def task_execution_strategy
+    @task_execution_strategy ||= Shipit::TaskExecutionStrategy::Default
+  end
 
   self.timeout_exit_codes = [].freeze
 
@@ -68,6 +82,10 @@ module Shipit
     ENV['SHIPIT_DISABLE_AUTH'].present?
   end
 
+  def enable_samesite_middleware?
+    ENV['SHIPIT_ENABLE_SAMESITE_NONE'].present?
+  end
+
   def app_name
     @app_name ||= secrets.app_name || Rails.application.class.name.split(':').first || 'Shipit'
   end
@@ -76,7 +94,7 @@ module Shipit
     secrets.redis_url.present? ? URI(secrets.redis_url) : nil
   end
 
-  def redis(namespace = nil)
+  def redis
     @redis ||= Redis.new(
       url: redis_url.to_s.presence,
       logger: Rails.logger,
@@ -84,17 +102,41 @@ module Shipit
       reconnect_delay: 0.5,
       reconnect_delay_max: 1,
     )
-    return @redis unless namespace
-    Redis::Namespace.new(namespace, redis: @redis)
   end
 
-  def github
-    @github ||= GitHubApp.new(secrets.github)
+  def github(organization: github_default_organization)
+    # Backward compatibility
+    # nil signifies the single github app config schema is being used
+    if github_default_organization.nil?
+      config = secrets.github
+    else
+      config = github_app_config(organization)
+      raise GithubOrganizationUnknown, organization if config.nil?
+    end
+    @github ||= {}
+    @github[organization] ||= GitHubApp.new(organization, config)
+  end
+
+  def github_default_organization
+    return nil unless secrets&.github
+    org = secrets.github.keys.first
+    TOP_LEVEL_GH_KEYS.include?(org) ? nil : org
+  end
+
+  def github_organizations
+    return [nil] unless github_default_organization
+    secrets.github.keys
+  end
+
+  def github_app_config(organization)
+    github_config = secrets.github.deep_transform_keys(&:downcase)
+    github_organization = organization.downcase.to_sym
+    github_config[github_organization]
   end
 
   def legacy_github_api
     if secrets&.github_api.present?
-      @legacy_github_api ||= github.new_client(access_token: secrets.github_api['access_token'])
+      @legacy_github_api ||= github.new_client(access_token: secrets.github_api[:access_token])
     end
   end
 
@@ -122,6 +164,10 @@ module Shipit
     secrets.default_merge_method || 'merge'
   end
 
+  def update_latest_deployed_ref
+    secrets.update_latest_deployed_ref
+  end
+
   def enforce_publish_config
     secrets.enforce_publish_config.presence
   end
@@ -147,7 +193,7 @@ module Shipit
   end
 
   def env
-    {'SHIPIT' => '1'}.merge(secrets.env || {})
+    { 'SHIPIT' => '1' }.merge(secrets.env || {})
   end
 
   def shell_paths
@@ -155,13 +201,11 @@ module Shipit
   end
 
   def revision
-    @revision ||= begin
-      if revision_file.exist?
-        revision_file.read
-      else
-        `git rev-parse HEAD`
-      end.strip
-    end
+    @revision ||= if revision_file.exist?
+      revision_file.read
+    else
+      %x(git rev-parse HEAD)
+    end.strip
   end
 
   def default_inactivity_timeout