shipit-engine 0.32.0 → 0.35.1

data/config/secrets.development.example.yml

@@ -1,8 +1,10 @@
 host: 'localhost:3000'
 redis_url: 'redis://127.0.0.1:6379/0'
 
+# For creating an app see: https://github.com/Shopify/shipit-engine/blob/master/docs/setup.md#creating-the-github-app
 # Can be obtained there: https://github.com/settings/apps
 # Set the "Authorization callback URL" as `<host>/github/auth/github/callback`
+
 github:
   app_id:
   installation_id:
@@ -12,3 +14,25 @@ github:
     id:
     secret:
     teams: # Optional
+
+# Use this configuration schema if you are configuring multiple Github applications for different Github organizations
+
+# github:
+#   somegithuborg:
+#     app_id:
+#     installation_id:
+#     webhook_secret: # nil
+#     private_key:
+#     oauth:
+#       id:
+#       secret:
+#       teams: # Optional
+#   someothergithuborg:
+#     app_id:
+#     installation_id:
+#     webhook_secret: # nil
+#     private_key:
+#     oauth:
+#       id:
+#       secret:
+#       teams: # Optional

data/config/secrets.development.shopify.yml

@@ -1,13 +1,24 @@
 host: 'shipit-engine.myshopify.io'
 redis_url: 'redis://shipit-engine.railgun:6379'
 
-# TODO: document creating a dev app
+# For creating an app see: https://github.com/Shopify/shipit-engine/blob/master/docs/setup.md#creating-the-github-app
+
 github:
-  app_id:
-  installation_id:
-  webhook_secret: # nil
-  private_key:
-  oauth:
-    id:
-    secret:
-    teams:
+  somegithuborg:
+    app_id:
+    installation_id:
+    webhook_secret: # nil
+    private_key:
+    oauth:
+      id:
+      secret:
+      teams:
+  someothergithuborg:
+    app_id:
+    installation_id:
+    webhook_secret: # nil
+    private_key:
+    oauth:
+      id:
+      secret:
+      teams:

data/db/migrate/20200706145406_add_review_stacks.rb

@@ -0,0 +1,12 @@
+class AddReviewStacks < ActiveRecord::Migration[6.0]
+  def change
+    add_column :stacks, :provision_status, :string, null: false, default: :deprovisioned
+    add_index :stacks, :provision_status
+
+    add_column :stacks, :type, :string, default: "Shipit::Stack"
+    add_index :stacks, :type
+
+    add_column :stacks, :awaiting_provision, :boolean, null: false, default: false
+    add_index :stacks, :awaiting_provision
+  end
+end

data/db/migrate/20200804144639_rename_pull_request_to_merge_request.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class RenamePullRequestToMergeRequest < ActiveRecord::Migration[6.0]
+  def change
+    rename_table :pull_requests, :merge_requests
+  end
+end

data/db/migrate/20200804161512_rename_commits_pull_request_id_to_merge_request_id.rb

@@ -0,0 +1,5 @@
+class RenameCommitsPullRequestIdToMergeRequestId < ActiveRecord::Migration[6.0]
+  def change
+    rename_column :commits, :pull_request_id, :merge_request_id
+  end
+end

data/db/migrate/20200813134712_recreate_shipit_pull_requests.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+class RecreateShipitPullRequests < ActiveRecord::Migration[6.0]
+  def change
+    create_table :pull_requests do |t|
+      t.references :stack, null: false
+      t.integer :number, null: false
+      t.string :title, limit: 256
+      t.integer :github_id, limit: 8
+      t.string :api_url, limit: 1024
+      t.string :state
+      t.integer :additions, null: false, default: 0
+      t.integer :deletions, null: false, default: 0
+      t.integer :user_id
+      t.text :labels
+      t.references :head
+      t.timestamps
+
+      t.index [:stack_id, :number], unique: true
+      t.index [:stack_id, :github_id], unique: true
+    end
+  end
+end

data/db/migrate/20200813194056_create_pull_request_assignments.rb

@@ -0,0 +1,8 @@
+class CreatePullRequestAssignments < ActiveRecord::Migration[6.0]
+  def change
+    create_table :pull_request_assignments do |t|
+      t.references :pull_request
+      t.references :user
+    end
+  end
+end

data/db/migrate/20201001125502_add_provision_pr_stacks_flag_to_repositories.rb

@@ -0,0 +1,7 @@
+class AddProvisionPrStacksFlagToRepositories < ActiveRecord::Migration[6.0]
+  def change
+    add_column :repositories, :review_stacks_enabled, :boolean, default: false
+    add_column :repositories, :provisioning_behavior, :string, default: :allow_all
+    add_column :repositories, :provisioning_label_name, :string
+  end
+end

data/db/migrate/20201008145809_add_retry_attempt_to_tasks.rb

@@ -0,0 +1,5 @@
+class AddRetryAttemptToTasks < ActiveRecord::Migration[6.0]
+  def change
+    add_column :tasks, :retry_attempt, :integer, null: false, default: 0
+  end
+end

data/db/migrate/20201008152744_add_max_retries_to_tasks.rb

@@ -0,0 +1,5 @@
+class AddMaxRetriesToTasks < ActiveRecord::Migration[6.0]
+  def change
+    add_column :tasks, :max_retries, :integer, null: true
+  end
+end

data/db/migrate/20210325194053_remove_stacks_branch_default.rb

@@ -0,0 +1,5 @@
+class RemoveStacksBranchDefault < ActiveRecord::Migration[6.1]
+  def change
+    change_column_default(:stacks, :branch, from: 'master', to: nil)
+  end
+end

data/db/migrate/20210504200438_add_github_updated_at_to_check_runs.rb

@@ -0,0 +1,5 @@
+class AddGithubUpdatedAtToCheckRuns < ActiveRecord::Migration[6.1]
+  def change
+    add_column :check_runs, :github_updated_at, :datetime, default: nil
+  end
+end

data/db/migrate/20210823075617_change_check_runs_github_updated_at_default.rb

@@ -0,0 +1,5 @@
+class ChangeCheckRunsGithubUpdatedAtDefault < ActiveRecord::Migration[6.1]
+  def change
+    change_column_default :check_runs, :github_updated_at, nil
+  end
+end

data/lib/shipit/command.rb

@@ -13,7 +13,8 @@ module Shipit
     Denied = Class.new(Error)
     TimedOut = Class.new(Error)
 
-    BASE_ENV = Bundler.clean_env.merge((ENV.keys - Bundler.clean_env.keys).map { |k| [k, nil] }.to_h)
+    unbundled_env = Bundler.respond_to?(:unbundled_env) ? Bundler.unbundled_env : Bundler.clean_env
+    BASE_ENV = unbundled_env.merge((ENV.keys - unbundled_env.keys).map { |k| [k, nil] }.to_h)
 
     class Failed < Error
       attr_reader :exit_code
@@ -29,7 +30,7 @@ module Shipit
     def initialize(*args, default_timeout: Shipit.default_inactivity_timeout, env: {}, chdir:)
       @args, options = parse_arguments(args)
       @timeout = options['timeout'] || options[:timeout] || default_timeout
-      @env = env
+      @env = env.transform_values { |v| v&.to_s }
       @chdir = chdir.to_s
       @timed_out = false
     end
@@ -86,7 +87,7 @@ module Shipit
       @out = @pid = nil
       FileUtils.mkdir_p(@chdir)
       begin
-        @out, child_in, @pid = PTY.spawn(clean_env, *interpolated_arguments, chdir: @chdir)
+        @out, child_in, @pid = PTY.spawn(unbundled_env, *interpolated_arguments, chdir: @chdir)
         child_in.close
       rescue Errno::ENOENT
         raise NotFound, "#{Shellwords.split(interpolated_arguments.first).first}: command not found"
@@ -97,8 +98,8 @@ module Shipit
       self
     end
 
-    def clean_env
-      BASE_ENV.merge('PATH' => "#{ENV['PATH']}:#{Shipit.shell_paths.join(':')}").merge(@env.stringify_keys)
+    def unbundled_env
+      BASE_ENV.merge('PATH' => "#{Shipit.shell_paths.join(':')}:#{ENV['PATH']}").merge(@env.stringify_keys)
     end
 
     def stream(&block)
@@ -216,7 +217,7 @@ module Shipit
           argument
         end
       end
-      [args, options]
+      [args.map(&:to_s), options]
     end
 
     def running?

data/lib/shipit/commands.rb

@@ -20,15 +20,28 @@ module Shipit
     delegate :git_version, to: :class
 
     def env
-      @env ||= Shipit.env.merge(
-        'GITHUB_DOMAIN' => Shipit.github.domain,
-        'GITHUB_TOKEN' => Shipit.github.token,
+      base_env
+    end
+
+    def git(*args)
+      kwargs = args.extract_options!
+      kwargs[:env] ||= base_env
+      Command.new("git", *args, **kwargs)
+    end
+    ruby2_keywords :git if respond_to?(:ruby2_keywords, true)
+
+    private
+
+    def base_env
+      @base_env ||= Shipit.env.merge(
+        'GITHUB_DOMAIN' => github.domain,
+        'GITHUB_TOKEN' => github.token,
         'GIT_ASKPASS' => Shipit::Engine.root.join('lib', 'snippets', 'git-askpass').realpath.to_s,
       )
     end
 
-    def git(*args)
-      Command.new("git", *args)
+    def github
+      Shipit.github
     end
   end
 end

data/lib/shipit/engine.rb

@@ -10,6 +10,8 @@ module Shipit
       Shipit::Engine.routes.default_url_options[:host] = Shipit.host
       Pubsubstub.redis_url = Shipit.redis_url.to_s
 
+      Rails.application.secrets.deep_symbolize_keys!
+
       app.config.assets.paths << Emoji.images_path
       app.config.assets.precompile += %w(
         favicon.ico

data/lib/shipit/flock.rb

@@ -11,14 +11,21 @@ module Shipit
 
     def initialize(path)
       @path = Pathname.new(path)
+      @acquired = false
     end
 
     def lock(timeout:)
+      return yield if @acquired
       path.parent.mkpath
       path.open('w') do |file|
         if retrying(timeout: timeout) { file.flock(File::LOCK_EX | File::LOCK_NB) }
           file.write($PROCESS_ID.to_s)
-          return yield
+          @acquired = true
+          begin
+            yield
+          ensure
+            @acquired = false
+          end
         else
           raise TimeoutError, "Couldn't acquire lock for #{path} in #{timeout} seconds"
         end

data/lib/shipit/github_app.rb

@@ -42,8 +42,9 @@ module Shipit
 
     attr_reader :oauth_teams, :domain, :bot_login
 
-    def initialize(config)
+    def initialize(organization, config)
       super()
+      @organization = organization
       @config = (config || {}).with_indifferent_access
       @domain = @config[:domain] || DOMAIN
       @webhook_secret = @config[:webhook_secret].presence
@@ -52,7 +53,7 @@ module Shipit
       oauth = (@config[:oauth] || {}).with_indifferent_access
       @oauth_id = oauth[:id]
       @oauth_secret = oauth[:secret]
-      @oauth_teams = Array.wrap(oauth[:teams] || oauth[:teams])
+      @oauth_teams = Array.wrap(oauth[:teams])
     end
 
     def login
@@ -92,10 +93,11 @@ module Shipit
     end
 
     def fetch_new_token
+      cache_key = @organization.nil? ? '' : "#{@organization.downcase}:"
       # Rails can add 5 minutes to the cache entry expiration time when any TTL is provided,
       # so our TTL setting can be lower, and TTL + expires_in should be lower than the GitHub token expiration.
       Rails.cache.fetch(
-        'github:integration:access-token',
+        "github:integration:#{cache_key}access-token",
         expires_in: GITHUB_TOKEN_RAILS_CACHE_LIFETIME,
         race_condition_ttl: 4.minutes,
       ) do
@@ -181,15 +183,15 @@ module Shipit
     end
 
     def app_id
-      @app_id ||= @config.fetch(:app_id)
+      @config.fetch(:app_id)
     end
 
     def installation_id
-      @installation_id ||= @config.fetch(:installation_id)
+      @config.fetch(:installation_id)
     end
 
     def private_key
-      @private_key ||= @config.fetch(:private_key)
+      @config.fetch(:private_key)
     end
 
     def authentication_payload

data/lib/shipit/octokit_iterator.rb

@@ -3,12 +3,12 @@ module Shipit
   class OctokitIterator
     include Enumerable
 
-    def initialize(relation = nil)
+    def initialize(relation = nil, github_api: nil)
       if relation
         @response = relation.get(per_page: 100)
       else
-        data = yield Shipit.github.api
-        @response = Shipit.github.api.last_response if data.present?
+        data = yield github_api
+        @response = github_api.last_response if data.present?
       end
     end
 

data/lib/shipit/review_stack_commands.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require 'shipit/stack_commands'
+
+module Shipit
+  class ReviewStackCommands < Shipit::StackCommands
+  end
+end

data/lib/shipit/simple_message_verifier.rb

@@ -1,9 +1,9 @@
 # frozen_string_literal: true
 module Shipit
   class SimpleMessageVerifier < ActiveSupport::MessageVerifier
-    def initialize(secret, options = {})
+    def initialize(secret, **options)
       options[:serializer] ||= ToS
-      super(secret, options)
+      super(secret, **options)
     end
 
     private

data/lib/shipit/stack_commands.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+# rubocop:disable Lint/MissingSuper
 require 'pathname'
 require 'fileutils'
 
@@ -8,21 +9,31 @@ module Shipit
       @stack = stack
     end
 
+    def env
+      super.merge(@stack.env)
+    end
+
     def fetch
       create_directories
-      if Dir.exist?(@stack.git_path)
+      if valid_git_repository?(@stack.git_path)
         git('fetch', 'origin', '--tags', @stack.branch, env: env, chdir: @stack.git_path)
       else
+        @stack.clear_git_cache!
         git_clone(@stack.repo_git_url, @stack.git_path, branch: @stack.branch, env: env, chdir: @stack.deploys_path)
       end
     end
 
     def fetched?(commit)
-      git_dir = File.join(@stack.git_path, '.git')
-      if Dir.exist?(git_dir)
+      if valid_git_repository?(@stack.git_path)
         git('rev-parse', '--quiet', '--verify', "#{commit.sha}^{commit}", env: env, chdir: @stack.git_path)
       else
-        Command.new('test', '-d', git_dir, env: env, chdir: @stack.deploys_path)
+        # When the stack's git cache is not valid, the commit is
+        # NOT fetched. To keep the interface of this method
+        # consistent, we must return a Shipit::Command whose #success?
+        # method returns false - has a non-zero exit status. We utilize
+        # the POSIX 'test' command with no arguments which should
+        # always have an exit status of 1.
+        Command.new('test', env: env, chdir: @stack.deploys_path)
       end
     end
 
@@ -56,18 +67,30 @@ module Shipit
       Dir.mktmpdir do |dir|
         git(
           'clone', @stack.git_path, @stack.repo_name,
-          '--origin', 'cache',
+          '--recursive', '--origin', 'cache',
           chdir: dir
         ).run!
 
         git_dir = File.join(dir, @stack.repo_name)
-        git('checkout', commit.sha, chdir: git_dir).run! if commit
+        git('-c', 'advice.detachedHead=false', 'checkout', commit.sha, chdir: git_dir).run! if commit
         yield Pathname.new(git_dir)
       end
     end
 
+    def valid_git_repository?(path)
+      path.exist? &&
+        !path.empty? &&
+        git_cmd_succeeds?(path)
+    end
+
+    def git_cmd_succeeds?(path)
+      git("rev-parse", "--git-dir", chdir: path)
+        .tap(&:run)
+        .success?
+    end
+
     def git_clone(url, path, branch: 'master', **kwargs)
-      git('clone', *modern_git_args, '--recursive', '--branch', branch, url, path, **kwargs)
+      git('clone', '--quiet', *modern_git_args, '--recursive', '--branch', branch, url, path, **kwargs)
     end
 
     def modern_git_args
@@ -78,5 +101,11 @@ module Shipit
     def create_directories
       FileUtils.mkdir_p(@stack.deploys_path)
     end
+
+    private
+
+    def github
+      Shipit.github(organization: @stack.repository.owner)
+    end
   end
 end