shipit-engine 0.30.0 → 0.34.0

data/config/routes.rb

@@ -1,5 +1,7 @@
+# frozen_string_literal: true
 Shipit::Engine.routes.draw do
   stack_id_format = %r{[^/]+/[^/]+/[^/]+}
+  repository_id_format = %r{[^/]+/[^/]+}
   sha_format = /[\da-f]{6,40}/
   root to: 'stacks#index'
 
@@ -17,6 +19,7 @@ Shipit::Engine.routes.draw do
     scope '/stacks/*id', id: stack_id_format, as: :stack do
       get '/' => 'stacks#show'
       delete '/' => 'stacks#destroy'
+      patch '/' => 'stacks#update'
     end
 
     scope '/stacks/*stack_id', stack_id: stack_id_format, as: :stack do
@@ -28,8 +31,9 @@ Shipit::Engine.routes.draw do
       resources :deploys, only: %i(index create) do
         resources :release_statuses, only: %i(create)
       end
+      resources :rollbacks, only: %i(create)
       resources :commits, only: %i(index)
-      resources :pull_requests, only: %i(index show update destroy)
+      resources :merge_requests, only: %i(index show update destroy)
       post '/task/:task_name' => 'tasks#trigger', as: :trigger_task
       resources :hooks, only: %i(index create show update destroy)
     end
@@ -43,12 +47,20 @@ Shipit::Engine.routes.draw do
 
   # Browser extension
   get '/merge_status', action: :show, controller: :merge_status, as: :merge_status
-  put '/merge_status/*stack_id/pull/:number', action: :enqueue, controller: :merge_status, id: stack_id_format, as: :enqueue_pull_request
-  delete '/merge_status/*stack_id/pull/:number', action: :dequeue, controller: :merge_status, id: stack_id_format, as: :dequeue_pull_request
+  put '/merge_status/*stack_id/pull/:number', action: :enqueue, controller: :merge_status, id: stack_id_format, as: :enqueue_merge_request
+  delete '/merge_status/*stack_id/pull/:number', action: :dequeue, controller: :merge_status, id: stack_id_format, as: :dequeue_merge_request
 
   # Humans
   resources :api_clients
-  resources :stacks, only: %i(new create index)
+
+  resources :repositories, only: %i(new index create)
+  scope '/repositories/*id', id: repository_id_format, as: :repository do
+    get '/' => 'repositories#show'
+    patch '/' => 'repositories#update'
+    delete '/' => 'repositories#destroy'
+    get :settings, controller: :repositories
+    get 'stacks/new' => 'repositories#new_stack'
+  end
 
   scope '/github/auth/github', as: :github_authentication, controller: :github_authentication do
     get '/', action: :request
@@ -57,11 +69,13 @@ Shipit::Engine.routes.draw do
     get :logout
   end
 
+  resources :stacks, only: %i(new create index)
   scope '/*id', id: stack_id_format, as: :stack do
     get '/' => 'stacks#show'
     patch '/' => 'stacks#update'
     delete '/' => 'stacks#destroy'
     get :settings, controller: :stacks
+    get :all_tasks, controller: :stacks, as: :tasks_list
     get :statistics, controller: :stacks
     post :refresh, controller: :stacks
     get :refresh, controller: :stacks # For easier design, sorry :/
@@ -74,7 +88,7 @@ Shipit::Engine.routes.draw do
 
   scope '/*stack_id', stack_id: stack_id_format, as: :stack do
     get '/commit/:sha/checks' => 'commit_checks#show', as: :commit_checks
-    get '/commit/:sha/checks/tail' => 'commit_checks#tail', as: :tail_commit_checks, defaults: {format: :json}
+    get '/commit/:sha/checks/tail' => 'commit_checks#tail', as: :tail_commit_checks, defaults: { format: :json }
 
     get '/stats' => 'stats#show', as: :stats
 
@@ -89,7 +103,7 @@ Shipit::Engine.routes.draw do
 
       member do
         post :abort
-        get :tail, defaults: {format: :json}
+        get :tail, defaults: { format: :json }
       end
     end
 
@@ -103,7 +117,11 @@ Shipit::Engine.routes.draw do
       resources :release_statuses, only: %i(create)
     end
 
-    resources :pull_requests, only: %i(index destroy create)
+    resources :merge_requests, only: %i(index destroy create)
   end
   get '/stacks/:id' => 'stacks#lookup'
+
+  scope '/*repo', repo: %r{[^/]+/[^/]+}, as: :stack_search do
+    get '/' => 'stacks#index'
+  end
 end

data/config/secrets.development.example.yml

@@ -1,8 +1,10 @@
 host: 'localhost:3000'
 redis_url: 'redis://127.0.0.1:6379/0'
 
+# For creating an app see: https://github.com/Shopify/shipit-engine/blob/master/docs/setup.md#creating-the-github-app
 # Can be obtained there: https://github.com/settings/apps
 # Set the "Authorization callback URL" as `<host>/github/auth/github/callback`
+
 github:
   app_id:
   installation_id:
@@ -12,3 +14,25 @@ github:
     id:
     secret:
     teams: # Optional
+
+# Use this configuration schema if you are configuring multiple Github applications for different Github organizations
+
+# github:
+#   somegithuborg:
+#     app_id:
+#     installation_id:
+#     webhook_secret: # nil
+#     private_key:
+#     oauth:
+#       id:
+#       secret:
+#       teams: # Optional
+#   someothergithuborg:
+#     app_id:
+#     installation_id:
+#     webhook_secret: # nil
+#     private_key:
+#     oauth:
+#       id:
+#       secret:
+#       teams: # Optional

data/config/secrets.development.shopify.yml

@@ -1,13 +1,24 @@
 host: 'shipit-engine.myshopify.io'
 redis_url: 'redis://shipit-engine.railgun:6379'
 
-# TODO: document creating a dev app
+# For creating an app see: https://github.com/Shopify/shipit-engine/blob/master/docs/setup.md#creating-the-github-app
+
 github:
-  app_id:
-  installation_id:
-  webhook_secret: # nil
-  private_key:
-  oauth:
-    id:
-    secret:
-    teams:
+  somegithuborg:
+    app_id:
+    installation_id:
+    webhook_secret: # nil
+    private_key:
+    oauth:
+      id:
+      secret:
+      teams:
+  someothergithuborg:
+    app_id:
+    installation_id:
+    webhook_secret: # nil
+    private_key:
+    oauth:
+      id:
+      secret:
+      teams:

data/db/migrate/20200226211925_add_index_to_tasks_status.rb

@@ -0,0 +1,5 @@
+class AddIndexToTasksStatus < ActiveRecord::Migration[6.0]
+  def change
+    add_index :tasks, :status
+  end
+end

data/db/migrate/20200427135152_add_pull_request_head_sha_to_commit.rb

@@ -0,0 +1,5 @@
+class AddPullRequestHeadShaToCommit < ActiveRecord::Migration[6.0]
+  def change
+    add_column :commits, :pull_request_head_sha, :string, limit: 40
+  end
+end

data/db/migrate/20200615181558_add_rollback_once_aborted_to.rb

@@ -0,0 +1,5 @@
+class AddRollbackOnceAbortedTo < ActiveRecord::Migration[6.0]
+  def change
+    add_column :tasks, :rollback_once_aborted_to_id, :integer
+  end
+end

data/db/migrate/20200706145406_add_review_stacks.rb

@@ -0,0 +1,12 @@
+class AddReviewStacks < ActiveRecord::Migration[6.0]
+  def change
+    add_column :stacks, :provision_status, :string, null: false, default: :deprovisioned
+    add_index :stacks, :provision_status
+
+    add_column :stacks, :type, :string, default: "Shipit::Stack"
+    add_index :stacks, :type
+
+    add_column :stacks, :awaiting_provision, :boolean, null: false, default: false
+    add_index :stacks, :awaiting_provision
+  end
+end

data/db/migrate/20200804144639_rename_pull_request_to_merge_request.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class RenamePullRequestToMergeRequest < ActiveRecord::Migration[6.0]
+  def change
+    rename_table :pull_requests, :merge_requests
+  end
+end

data/db/migrate/20200804161512_rename_commits_pull_request_id_to_merge_request_id.rb

@@ -0,0 +1,5 @@
+class RenameCommitsPullRequestIdToMergeRequestId < ActiveRecord::Migration[6.0]
+  def change
+    rename_column :commits, :pull_request_id, :merge_request_id
+  end
+end

data/db/migrate/20200813134712_recreate_shipit_pull_requests.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+class RecreateShipitPullRequests < ActiveRecord::Migration[6.0]
+  def change
+    create_table :pull_requests do |t|
+      t.references :stack, null: false
+      t.integer :number, null: false
+      t.string :title, limit: 256
+      t.integer :github_id, limit: 8
+      t.string :api_url, limit: 1024
+      t.string :state
+      t.integer :additions, null: false, default: 0
+      t.integer :deletions, null: false, default: 0
+      t.integer :user_id
+      t.text :labels
+      t.references :head
+      t.timestamps
+
+      t.index [:stack_id, :number], unique: true
+      t.index [:stack_id, :github_id], unique: true
+    end
+  end
+end

data/db/migrate/20200813194056_create_pull_request_assignments.rb

@@ -0,0 +1,8 @@
+class CreatePullRequestAssignments < ActiveRecord::Migration[6.0]
+  def change
+    create_table :pull_request_assignments do |t|
+      t.references :pull_request
+      t.references :user
+    end
+  end
+end

data/db/migrate/20201001125502_add_provision_pr_stacks_flag_to_repositories.rb

@@ -0,0 +1,7 @@
+class AddProvisionPrStacksFlagToRepositories < ActiveRecord::Migration[6.0]
+  def change
+    add_column :repositories, :review_stacks_enabled, :boolean, default: false
+    add_column :repositories, :provisioning_behavior, :string, default: :allow_all
+    add_column :repositories, :provisioning_label_name, :string
+  end
+end

data/db/migrate/20201008145809_add_retry_attempt_to_tasks.rb

@@ -0,0 +1,5 @@
+class AddRetryAttemptToTasks < ActiveRecord::Migration[6.0]
+  def change
+    add_column :tasks, :retry_attempt, :integer, null: false, default: 0
+  end
+end

data/db/migrate/20201008152744_add_max_retries_to_tasks.rb

@@ -0,0 +1,5 @@
+class AddMaxRetriesToTasks < ActiveRecord::Migration[6.0]
+  def change
+    add_column :tasks, :max_retries, :integer, null: true
+  end
+end

data/db/migrate/20210325194053_remove_stacks_branch_default.rb

@@ -0,0 +1,5 @@
+class RemoveStacksBranchDefault < ActiveRecord::Migration[6.1]
+  def change
+    change_column_default(:stacks, :branch, from: 'master', to: nil)
+  end
+end

data/db/migrate/20210504200438_add_github_updated_at_to_check_runs.rb

@@ -0,0 +1,5 @@
+class AddGithubUpdatedAtToCheckRuns < ActiveRecord::Migration[6.1]
+  def change
+    add_column :check_runs, :github_updated_at, :datetime, default: 0
+  end
+end

data/lib/shipit.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'active_support/all'
 require 'active_model_serializers'
 require 'state_machines-activerecord'
@@ -23,7 +24,6 @@ require 'safe_yaml/load'
 require 'securecompare'
 
 require 'redis-objects'
-require 'redis-namespace'
 
 require 'octokit'
 require 'faraday-http-cache'
@@ -42,12 +42,14 @@ require 'shipit/simple_message_verifier'
 require 'shipit/command'
 require 'shipit/commands'
 require 'shipit/stack_commands'
+require 'shipit/review_stack_commands'
 require 'shipit/task_commands'
 require 'shipit/deploy_commands'
 require 'shipit/rollback_commands'
 require 'shipit/environment_variables'
 require 'shipit/stat'
 require 'shipit/github_http_cache_middleware'
+require 'shipit/same_site_cookie_middleware'
 require 'shipit/cast_value'
 require 'shipit/line_buffer'
 
@@ -57,10 +59,22 @@ SafeYAML::OPTIONS[:deserialize_symbols] = false
 module Shipit
   extend self
 
+  GithubOrganizationUnknown = Class.new(StandardError)
+  TOP_LEVEL_GH_KEYS = [:app_id, :installation_id, :webhook_secret, :private_key, :oauth, :domain]
+
   delegate :table_name_prefix, to: :secrets
 
-  attr_accessor :disable_api_authentication, :timeout_exit_codes
-  attr_writer :internal_hook_receivers, :task_logger, :preferred_org_emails
+  attr_accessor :disable_api_authentication, :timeout_exit_codes, :deployment_checks
+  attr_writer(
+    :internal_hook_receivers,
+    :preferred_org_emails,
+    :task_execution_strategy,
+    :task_logger,
+  )
+
+  def task_execution_strategy
+    @task_execution_strategy ||= Shipit::TaskExecutionStrategy::Default
+  end
 
   self.timeout_exit_codes = [].freeze
 
@@ -68,6 +82,10 @@ module Shipit
     ENV['SHIPIT_DISABLE_AUTH'].present?
   end
 
+  def enable_samesite_middleware?
+    ENV['SHIPIT_ENABLE_SAMESITE_NONE'].present?
+  end
+
   def app_name
     @app_name ||= secrets.app_name || Rails.application.class.name.split(':').first || 'Shipit'
   end
@@ -76,7 +94,7 @@ module Shipit
     secrets.redis_url.present? ? URI(secrets.redis_url) : nil
   end
 
-  def redis(namespace = nil)
+  def redis
     @redis ||= Redis.new(
       url: redis_url.to_s.presence,
       logger: Rails.logger,
@@ -84,17 +102,41 @@ module Shipit
       reconnect_delay: 0.5,
       reconnect_delay_max: 1,
     )
-    return @redis unless namespace
-    Redis::Namespace.new(namespace, redis: @redis)
   end
 
-  def github
-    @github ||= GitHubApp.new(secrets.github)
+  def github(organization: github_default_organization)
+    # Backward compatibility
+    # nil signifies the single github app config schema is being used
+    if github_default_organization.nil?
+      config = secrets.github
+    else
+      config = github_app_config(organization)
+      raise GithubOrganizationUnknown, organization if config.nil?
+    end
+    @github ||= {}
+    @github[organization] ||= GitHubApp.new(organization, config)
+  end
+
+  def github_default_organization
+    return nil unless secrets&.github
+    org = secrets.github.keys.first
+    TOP_LEVEL_GH_KEYS.include?(org) ? nil : org
+  end
+
+  def github_organizations
+    return [nil] unless github_default_organization
+    secrets.github.keys
+  end
+
+  def github_app_config(organization)
+    github_config = secrets.github.deep_transform_keys(&:downcase)
+    github_organization = organization.downcase.to_sym
+    github_config[github_organization]
   end
 
   def legacy_github_api
     if secrets&.github_api.present?
-      @legacy_github_api ||= github.new_client(access_token: secrets.github_api['access_token'])
+      @legacy_github_api ||= github.new_client(access_token: secrets.github_api[:access_token])
     end
   end
 
@@ -122,6 +164,10 @@ module Shipit
     secrets.default_merge_method || 'merge'
   end
 
+  def update_latest_deployed_ref
+    secrets.update_latest_deployed_ref
+  end
+
   def enforce_publish_config
     secrets.enforce_publish_config.presence
   end
@@ -147,7 +193,7 @@ module Shipit
   end
 
   def env
-    {'SHIPIT' => '1'}.merge(secrets.env || {})
+    { 'SHIPIT' => '1' }.merge(secrets.env || {})
   end
 
   def shell_paths
@@ -155,13 +201,11 @@ module Shipit
   end
 
   def revision
-    @revision ||= begin
-      if revision_file.exist?
-        revision_file.read
-      else
-        `git rev-parse HEAD`
-      end.strip
-    end
+    @revision ||= if revision_file.exist?
+      revision_file.read
+    else
+      %x(git rev-parse HEAD)
+    end.strip
   end
 
   def default_inactivity_timeout

data/lib/shipit/cast_value.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   module CastValue
     def to_boolean(value)