shipit-engine 0.21.0 → 0.22.0

data/app/models/shipit/rollback.rb

@@ -1,6 +1,6 @@
 module Shipit
   class Rollback < Deploy
-    belongs_to :deploy, foreign_key: :parent_id
+    belongs_to :deploy, foreign_key: :parent_id, inverse_of: false
 
     state_machine :status do
       after_transition to: :success, do: :lock_reverted_commits

data/app/models/shipit/stack.rb

@@ -28,7 +28,10 @@ module Shipit
     has_many :tasks, dependent: :destroy
     has_many :deploys
     has_many :rollbacks
-    has_many :deploys_and_rollbacks, -> { where(type: %w(Shipit::Deploy Shipit::Rollback)) }, class_name: 'Task'
+    has_many :deploys_and_rollbacks,
+             -> { where(type: %w(Shipit::Deploy Shipit::Rollback)) },
+             class_name: 'Task',
+             inverse_of: :stack
     has_many :github_hooks, dependent: :destroy, class_name: 'Shipit::GithubHook::Repo'
     has_many :hooks, dependent: :destroy
     has_many :api_clients, dependent: :destroy
@@ -39,7 +42,12 @@ module Shipit
     end
 
     def lock_author=(user)
-      super(user.try!(:logged_in?) ? user : nil)
+      super(user&.logged_in? ? user : nil)
+    end
+
+    def self.repo(full_name)
+      repo_owner, repo_name = full_name.downcase.split('/')
+      where(repo_owner: repo_owner, repo_name: repo_name)
     end
 
     before_validation :update_defaults
@@ -51,7 +59,7 @@ module Shipit
     after_commit :emit_removed_hooks, on: :destroy
     after_commit :broadcast_update, on: :update
     after_commit :emit_merge_status_hooks, on: :update
-    after_commit :setup_hooks, :sync_github, on: :create
+    after_commit :sync_github, on: :create
     after_commit :schedule_merges_if_necessary, on: :update
 
     validates :repo_name, uniqueness: {scope: %i(repo_owner environment),
@@ -84,7 +92,7 @@ module Shipit
 
     def trigger_task(definition_id, user, env: nil, force: false)
       definition = find_task_definition(definition_id)
-      env = env.try!(:to_h) || {}
+      env = env&.to_h || {}
 
       definition.variables_with_defaults.each do |variable|
         env[variable.name] ||= variable.default
@@ -110,7 +118,7 @@ module Shipit
         user_id: user.id,
         until_commit: until_commit,
         since_commit: since_commit,
-        env: filter_deploy_envs(env.try!(:to_h) || {}),
+        env: filter_deploy_envs(env&.to_h || {}),
         allow_concurrency: force,
         ignored_safeties: force || !until_commit.deployable?,
       )
@@ -177,7 +185,7 @@ module Shipit
 
     def update_deployed_revision(sha)
       last_deploy = deploys_and_rollbacks.last
-      return if last_deploy.try!(:active?)
+      return if last_deploy&.active?
 
       actual_deployed_commit = commits.reachable.by_sha(sha)
       return unless actual_deployed_commit
@@ -196,7 +204,7 @@ module Shipit
     end
 
     def head
-      commits.reachable.first.try!(:sha)
+      commits.reachable.first&.sha
     end
 
     def merge_status(backlog_leniency_factor: 2.0)
@@ -256,7 +264,7 @@ module Shipit
     end
 
     def last_deployed_commit
-      last_successful_deploy.try!(:until_commit) || NoDeployedCommit
+      last_successful_deploy&.until_commit || NoDeployedCommit
     end
 
     def deployable?
@@ -268,19 +276,19 @@ module Shipit
     end
 
     def repo_name=(name)
-      super(name.try!(:downcase))
+      super(name&.downcase)
     end
 
     def repo_owner=(name)
-      super(name.try!(:downcase))
+      super(name&.downcase)
     end
 
     def repo_http_url
-      Shipit.github_url("#{repo_owner}/#{repo_name}")
+      Shipit.github.url("#{repo_owner}/#{repo_name}")
     end
 
     def repo_git_url
-      "git@#{Shipit.github_domain}:#{repo_owner}/#{repo_name}.git"
+      "git@#{Shipit.github.domain}:#{repo_owner}/#{repo_name}.git"
     end
 
     def base_path
@@ -319,7 +327,7 @@ module Shipit
 
     def github_commits
       handle_github_redirections do
-        Shipit.github_api.commits(github_repo_name, sha: branch)
+        Shipit.github.api.commits(github_repo_name, sha: branch)
       end
     rescue Octokit::Conflict
       [] # Repository is empty...
@@ -337,9 +345,9 @@ module Shipit
     end
 
     def refresh_repository!
-      resource = Shipit.github_api.repo(github_repo_name)
+      resource = Shipit.github.api.repo(github_repo_name)
       if resource.try(:message) == 'Moved Permanently'
-        resource = Shipit.github_api.get(resource.url)
+        resource = Shipit.github.api.get(resource.url)
       end
       update!(repo_owner: resource.owner.login, repo_name: resource.name)
     end
@@ -413,13 +421,6 @@ module Shipit
       )
     end
 
-    def setup_hooks
-      REQUIRED_HOOKS.each do |event|
-        hook = github_hooks.find_or_create_by!(event: event)
-        hook.schedule_setup!
-      end
-    end
-
     def schedule_for_destroy!
       DestroyStackJob.perform_later(self)
     end
@@ -463,16 +464,16 @@ module Shipit
       tasks.where(type: 'Shipit::Deploy').success.order(id: :desc).limit(100).durations
     end
 
+    def sync_github
+      GithubSyncJob.perform_later(stack_id: id)
+    end
+
     private
 
     def clear_cache
       remove_instance_variable(:@active_task) if defined?(@active_task)
     end
 
-    def sync_github
-      GithubSyncJob.perform_later(stack_id: id)
-    end
-
     def clear_local_files
       FileUtils.rm_rf(base_path.to_s)
     end

data/app/models/shipit/task.rb

@@ -11,7 +11,7 @@ module Shipit
 
     attr_accessor :pid
 
-    belongs_to :deploy, foreign_key: :parent_id, required: false # required for fixtures
+    belongs_to :deploy, foreign_key: :parent_id, required: false, inverse_of: false # required for fixtures
 
     belongs_to :user, optional: true
     belongs_to :aborted_by, class_name: 'User', optional: true
@@ -21,7 +21,7 @@ module Shipit
 
     deferred_touch stack: :updated_at
 
-    has_many :chunks, -> { order(:id) }, class_name: 'OutputChunk', dependent: :delete_all
+    has_many :chunks, -> { order(:id) }, class_name: 'OutputChunk', dependent: :delete_all, inverse_of: :task
 
     serialize :definition, TaskDefinition
     serialize :env, Hash

data/app/models/shipit/team.rb

@@ -9,9 +9,8 @@ module Shipit
       -> { where(event: REQUIRED_HOOKS) },
       foreign_key: :organization,
       primary_key: :organization,
-      class_name: 'GithubHook::Organization'
-
-    after_commit :setup_hooks, if: :automatically_setup_hooks?
+      class_name: 'GithubHook::Organization',
+      inverse_of: false
 
     class << self
       def find_or_create_by_handle(handle)
@@ -26,7 +25,7 @@ module Shipit
       end
 
       def find_team_on_github(organization, slug)
-        teams = Shipit::OctokitIterator.new { Shipit.github_api.org_teams(organization, per_page: 100) }
+        teams = Shipit::OctokitIterator.new { Shipit.github.api.org_teams(organization, per_page: 100) }
         teams.find { |t| t.slug == slug }
       rescue Octokit::NotFound
       end
@@ -40,20 +39,8 @@ module Shipit
       members.append(member) unless members.include?(member)
     end
 
-    attr_writer :automatically_setup_hooks
-    def automatically_setup_hooks?
-      @automatically_setup_hooks
-    end
-
-    def setup_hooks(async: true)
-      REQUIRED_HOOKS.each do |event|
-        hook = github_hooks.find_or_create_by!(event: event)
-        async ? hook.schedule_setup! : hook.setup!
-      end
-    end
-
     def refresh_members!
-      github_members = Shipit::OctokitIterator.new(Shipit.github_api.get(api_url).rels[:members])
+      github_members = Shipit::OctokitIterator.new(Shipit.github.api.get(api_url).rels[:members])
       members = github_members.map { |u| User.find_or_create_from_github(u) }
       self.members = members
       save!

data/app/models/shipit/user.rb

@@ -12,7 +12,7 @@ module Shipit
 
     def self.find_or_create_by_login!(login)
       find_or_create_by!(login: login) do |user|
-        user.github_user = Shipit.github_api.user(login)
+        user.github_user = Shipit.github.api.user(login)
       end
     end
 
@@ -44,7 +44,7 @@ module Shipit
     end
 
     def github_api
-      return Shipit.github_api unless github_access_token
+      return Shipit.github.api unless github_access_token
 
       @github_api ||= begin
         client = Octokit::Client.new(access_token: github_access_token)
@@ -71,7 +71,7 @@ module Shipit
     end
 
     def refresh_from_github!
-      update!(github_user: Shipit.github_api.user(github_id))
+      update!(github_user: Shipit.github.api.user(github_id))
     rescue Octokit::NotFound
       identify_renamed_user!
     end

data/app/serializers/shipit/task_serializer.rb

@@ -43,7 +43,7 @@ module Shipit
     end
 
     def action
-      object.definition.try!(:action)
+      object.definition&.action
     end
 
     def include_action?
@@ -51,7 +51,7 @@ module Shipit
     end
 
     def description
-      object.definition.try!(:action)
+      object.definition&.action
     end
 
     def include_description?

data/app/serializers/shipit/user_serializer.rb

@@ -1,5 +1,5 @@
 module Shipit
   class UserSerializer < ActiveModel::Serializer
-    attributes :id, :name, :email, :login, :avatar_url, :created_at, :updated_at
+    attributes :id, :name, :email, :login, :avatar_url, :created_at, :updated_at, :github_id
   end
 end

data/app/views/shipit/missing_settings.html.erb

@@ -17,52 +17,21 @@
 <div class="wrapper">
   <section>
     <header class="section-header">
-      <h2>GitHub application</h2>
+      <h2>GitHub App</h2>
     </header>
 
-    <% if Shipit.github_oauth_id.blank? || Shipit.github_oauth_secret.blank? %>
-      <p><%= missing_github_oauth_message %></p>
-    <% end %>
-    <p id="github_oauth_id">
-      ID:
-    <% if Shipit.github_oauth_id.present? %>
+    <p id="github_app">
+      Config:
+    <% if Rails.application.secrets.github.present? %>
        Success!
     <% else %>
       <span class="missing">
-        <%= missing_github_oauth_id_message %>
-      </span>
-    <% end %>
-    </p>
-
-    <p id="github_oauth_secret">
-    Secret:
-    <% if Shipit.github_oauth_secret.present? %>
-      Success!
-    <% else %>
-      <span class="missing">
-        <%= missing_github_oauth_secret_message %>
+        <%= missing_github_app_message %>
       </span>
     <% end %>
     </p>
   </section>
 
-  <section>
-    <header class="section-header">
-      <h2>GitHub API</h2>
-    </header>
-
-    <p id="github_api">
-      Token:
-      <% if Shipit.github_api_credentials.present? %>
-        Success!
-      <% else %>
-        <span class="missing">
-          <%= missing_github_api_credentials_message %>
-        </span>
-      <% end %>
-    </p>
-  </section>
-
   <section>
     <header class="section-header">
       <h2>Redis</h2>

data/app/views/shipit/stacks/new.html.erb

@@ -9,7 +9,7 @@
         <div class="field-wrapper">
           <%= label_tag "Repo" %>
           <br>
-          <%= Shipit.github_url %>
+          <%= Shipit.github.url %>
           /
           <%= f.text_field :repo_owner, placeholder: 'e.g. Shopify', required: true, class: "repo" %>
           /

data/app/views/shipit/stacks/settings.html.erb

@@ -64,10 +64,6 @@
     <div class="setting-section">
       <h5>Resynchronize this stack</h5>
       <table>
-        <tr>
-          <td><%= button_to "Webhooks", stack_sync_webhooks_path(@stack), class: "btn", method: "post" %></td>
-          <td>Ensure that all required webhooks have been created.</td>
-        </tr>
         <tr>
           <td><%= button_to "Clear Git Cache", stack_clear_git_cache_path(@stack), class: "btn", method: "post" %></td>
           <td>Delete the local git mirror in case it's in a bad state.</td>

data/config/routes.rb

@@ -8,18 +8,7 @@ Shipit::Engine.routes.draw do
   # Robots
   get '/status/version' => 'status#version', as: :version
 
-  resources :stacks, only: %i(new create index) do
-    resource :webhooks, only: [] do
-      post :push, :state
-    end
-  end
-
-  resources :webhooks, only: [] do
-    collection do
-      post :membership
-      get :membership
-    end
-  end
+  resources :webhooks, only: :create
 
   # API
   namespace :api do
@@ -50,6 +39,8 @@ Shipit::Engine.routes.draw do
   end
 
   # Humans
+  resources :stacks, only: %i(new create index)
+
   scope '/github/auth/github', as: :github_authentication, controller: :github_authentication do
     get '/', action: :request
     post :callback
@@ -64,7 +55,6 @@ Shipit::Engine.routes.draw do
     get :settings, controller: :stacks
     post :refresh, controller: :stacks
     get :refresh, controller: :stacks # For easier design, sorry :/
-    post :sync_webhooks, controller: :stacks
     post :clear_git_cache, controller: :stacks
   end
 

data/config/secrets.development.shopify.yml

@@ -1,19 +1,14 @@
 host: 'https://shipit-engine.myshopify.io'
 redis_url: 'redis://shipit-engine.railgun:6379'
 
-github_api:
-  # Can be obtained there: https://github.com/settings/tokens/new
-  # The required permissions are: `admin:org_hook`, `admin:repo_hook`, `read:org` and `repo`
+# TODO: document creating a dev app
+github:
+  app_id:
+  installation_id:
+  webhook_secret: # nil
+  private_key:
+  oauth:
+    id:
+    secret:
+    teams:
   access_token:
-
-# Can be obtained there: https://github.com/settings/applications/new
-# Set the "Authorization callback URL" as `<host>/github/auth/github/callback`
-github_oauth: 
-  id:
-  secret:
-  # teams: # Optional
-
-# To work on the kubernetes deploy script
-# env:
-#   KUBECONFIG: # Path of the kubeconfig you want to use.
-#   GOOGLE_APPLICATION_CREDENTIALS: # Used if your kube auth provider is gcp, in which case it should be path of the service account credentials you want to use. Alternatively, you can run `gcloud auth application-default login` (and remove this variable) to use your own google account locally.

data/config/secrets.development.yml

@@ -1,7 +1,7 @@
 host: 'http://shipit-engine.localhost'
 redis_url: 'redis://shipit-engine.railgun:6379'
 
-github_api:
+github:
   access_token: 4cc410d7a8fb04f6095076f519f7e0981d915136
 
 # If you want to test GitHub Authentication

data/db/migrate/20180417130436_remove_all_github_hooks.rb

@@ -0,0 +1,11 @@
+class RemoveAllGithubHooks < ActiveRecord::Migration[5.1]
+  def change
+    if !Shipit.legacy_github_api && Shipit::GithubHook.any?
+      Rails.logger.error("Can't destroy github hooks because no legacy token is configred")
+    else
+      Shipit::GithubHook.find_each do |hook|
+        Shipit::DestroyJob.perform_later(hook)
+      end
+    end
+  end
+end

data/lib/shipit.rb

@@ -26,10 +26,13 @@ require 'redis-objects'
 require 'redis-namespace'
 
 require 'octokit'
+require 'shipit/octokit_bot_users_patch'
+
 require 'faraday-http-cache'
 
 require 'shipit/version'
 
+require 'shipit/github_app'
 require 'shipit/paginator'
 require 'shipit/null_serializer'
 require 'shipit/csv_serializer'
@@ -72,39 +75,24 @@ module Shipit
     Redis::Namespace.new(namespace, redis: @redis)
   end
 
-  def github_domain
-    @github_domain ||= secrets.github_domain.presence || 'github.com'.freeze
-  end
-
-  def github_enterprise?
-    github_domain != 'github.com'
+  def github
+    @github ||= GitHubApp.new(secrets.github)
   end
 
-  def github_url(path = nil)
-    @github_url ||= "https://#{github_domain}".freeze
-    path ? File.join(@github_url, path) : @github_url
-  end
-
-  def github_api_endpoint
-    github_url('/api/v3/') if github_enterprise?
+  def legacy_github_api
+    if secrets&.github_api.present?
+      @legacy_github_api ||= Octokit::Client.new(access_token: secrets.github_api['access_token'])
+    end
   end
 
   def user
-    if github_api.login
-      User.find_or_create_by_login!(github_api.login)
+    if github.bot_login
+      User.find_or_create_by_login!(github.bot_login)
     else
       AnonymousUser.new
     end
   end
 
-  def github_api
-    @github_api ||= begin
-      client = Octokit::Client.new(github_api_credentials)
-      client.middleware = new_faraday_stack
-      client
-    end
-  end
-
   def new_faraday_stack
     Faraday::RackBuilder.new do |builder|
       builder.use(
@@ -121,10 +109,6 @@ module Shipit
     end
   end
 
-  def github_api_credentials
-    {api_endpoint: github_api_endpoint}.merge((Rails.application.secrets.github_api || {}).symbolize_keys)
-  end
-
   def api_clients_secret
     secrets.api_clients_secret.presence || secrets.secret_key_base
   end
@@ -150,39 +134,12 @@ module Shipit
   end
 
   def github_teams
-    @github_teams ||= github_teams_handles.map { |t| Team.find_or_create_by_handle(t) }
-  end
-
-  def github_teams_handles
-    (Array(github_oauth_credentials['team']) + Array(github_oauth_credentials['teams'])).sort.uniq
-  end
-
-  def github_oauth_id
-    github_oauth_credentials['id']
-  end
-
-  def github_oauth_secret
-    github_oauth_credentials['secret']
-  end
-
-  def github_oauth_credentials
-    (secrets.github_oauth || {}).to_h.stringify_keys
-  end
-
-  def github_oauth_options
-    return {} unless github_enterprise?
-    {
-      site: github_api_endpoint,
-      authorize_url: github_url('/login/oauth/authorize'),
-      token_url: github_url('/login/oauth/access_token'),
-    }
+    @github_teams ||= github.oauth_teams.map { |t| Team.find_or_create_by_handle(t) }
   end
 
   def all_settings_present?
     @all_settings_present ||= [
-      github_oauth_id,
-      github_oauth_secret,
-      github_api_credentials,
+      secrets.github, # TODO: handle GitHub settings
       redis_url,
       host,
     ].all?(&:present?)