shipit-engine 0.33.0 → 0.34.0

data/app/models/shipit/status/group.rb

@@ -31,7 +31,7 @@ module Shipit
       end
 
       delegate :pending?, :success?, :error?, :failure?, :unknown?, :missing?, :state, :simple_state,
-               to: :significant_status
+        to: :significant_status
       delegate :each, :size, :map, to: :statuses
       delegate :required_statuses, to: :commit
 

data/app/models/shipit/task.rb

@@ -39,7 +39,7 @@ module Shipit
     scope :last_seven_days, -> { where("created_at > ?", 7.days.ago) }
     scope :previous_seven_days, -> { where(created_at: 14.days.ago..7.days.ago) }
 
-    scope :due_for_rollup, -> { completed.where(rolled_up: false).where('created_at <= ?', 1.hour.ago) }
+    scope :due_for_rollup, -> { not_active.where(rolled_up: false).where('created_at <= ?', 1.hour.ago) }
 
     after_save :record_status_change
     after_create :prevent_concurrency, unless: :allow_concurrency?
@@ -175,7 +175,7 @@ module Shipit
     end
 
     delegate :acquire_git_cache_lock, :async_refresh_deployed_revision, :async_update_estimated_deploy_duration,
-             to: :stack
+      to: :stack
 
     delegate :checklist, to: :definition
 
@@ -283,6 +283,10 @@ module Shipit
       user || AnonymousUser.new
     end
 
+    def author=(user)
+      super(user.presence)
+    end
+
     def finished?
       !pending? && !running? && !aborting?
     end

data/app/models/shipit/task_execution_strategy/default.rb

@@ -83,19 +83,18 @@ module Shipit
       end
 
       def capture!(command)
-        started_at = Time.now
+        started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
         command.start do
           @task.ping
           check_for_abort
         end
-        @task.write("$ #{command}\npid: #{command.pid}\nstarted at: #{started_at}\n")
+        @task.write("\n$ #{command}\npid: #{command.pid}\n")
         @task.pid = command.pid
         command.stream! do |line|
           @task.write(line)
         end
-        @task.write("\n")
-        finished_at = Time.now
-        @task.write("pid: #{command.pid}\nfinished at: #{finished_at}\nran in: #{finished_at - started_at} seconds\n")
+        finished_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        @task.write("pid: #{command.pid} finished in: #{finished_at - started_at} seconds\n")
         command.success?
       end
 

data/app/models/shipit/team.rb

@@ -26,7 +26,8 @@ module Shipit
       end
 
       def find_team_on_github(organization, slug)
-        teams = Shipit::OctokitIterator.new { Shipit.github.api.org_teams(organization, per_page: 100) }
+        gh_api = Shipit.github(organization: organization).api
+        teams = Shipit::OctokitIterator.new(github_api: gh_api) { gh_api.org_teams(organization, per_page: 100) }
         teams.find { |t| t.slug == slug }
       rescue Octokit::NotFound
       end
@@ -41,7 +42,8 @@ module Shipit
     end
 
     def refresh_members!
-      github_members = Shipit::OctokitIterator.new(Shipit.github.api.get(api_url).rels[:members])
+      github_api = Shipit.github(organization: organization).api
+      github_members = Shipit::OctokitIterator.new(github_api.get(api_url).rels[:members])
       members = github_members.map { |u| User.find_or_create_from_github(u) }
       self.members = members
       save!

data/app/models/shipit/user.rb

@@ -15,6 +15,8 @@ module Shipit
 
     def self.find_or_create_by_login!(login)
       find_or_create_by!(login: login) do |user|
+        # Users are global, any app can be used
+        # This will not work for users that only exist in an Enterprise install
         user.github_user = Shipit.github.api.user(login)
       end
     end
@@ -91,6 +93,8 @@ module Shipit
     end
 
     def refresh_from_github!
+      # Users are global, any app can be used
+      # This will not work for users that only exist in an Enterprise install
       update!(github_user: Shipit.github.api.user(github_id))
     rescue Octokit::NotFound
       identify_renamed_user!

data/app/models/shipit/webhooks/handlers/pull_request/review_stack_adapter.rb

@@ -44,8 +44,8 @@ module Shipit
             return unless stack.archived?
 
             stack.transaction do
-              stack.unarchive!(*args, &block)
               Shipit::ReviewStackProvisioningQueue.add(stack)
+              stack.unarchive!(*args, &block)
             end
           end
 

data/app/models/shipit/webhooks/handlers/push_handler.rb

@@ -7,7 +7,10 @@ module Shipit
           requires :ref
         end
         def process
-          stacks.where(branch: branch).each(&:sync_github)
+          stacks
+            .not_archived
+            .where(branch: branch)
+            .find_each(&:sync_github)
         end
 
         private

data/app/serializers/shipit/merge_request_serializer.rb

@@ -8,7 +8,7 @@ module Shipit
     has_one :head, serializer: ShortCommitSerializer
 
     attributes :id, :number, :title, :github_id, :additions, :deletions, :state, :merge_status, :mergeable,
-               :merge_requested_at, :rejection_reason, :html_url, :branch, :base_ref
+      :merge_requested_at, :rejection_reason, :html_url, :branch, :base_ref
 
     def html_url
       github_pull_request_url(object)

data/app/validators/subset_validator.rb

@@ -3,6 +3,6 @@ class SubsetValidator < ActiveModel::EachValidator
   def validate_each(record, attribute, value)
     superset = options[:of]
     rest = value - superset
-    record.errors.add(attribute, :subset, options) unless rest.empty?
+    record.errors.add(attribute, :subset, **options) unless rest.empty?
   end
 end

data/app/views/layouts/merge_status.html.erb

@@ -4,7 +4,7 @@
     <%= stylesheet_link_tag *(params[:stylesheets] || []).select { |url| url.start_with?('https://assets-cdn.github.com/', 'https://github.githubassets.com/') } %>
     <%= stylesheet_link_tag 'merge_status' %>
   </head>
-  <body>
+  <body data-color-mode="<%= params[:mode] %>">
     <div class="merge-status-container" data-layout-content><%= yield %></div>
     <%= javascript_include_tag 'merge_status' %>
   </body>

data/app/views/shipit/stacks/_banners.html.erb

@@ -69,7 +69,8 @@
         <h2 class="banner__title">Continuous Delivery Delayed!</h2>
         <p class="banner__text">
           Continuous Delivery for this stack is currently paused because
-          <%= link_to 'the pre-deploy checks failed', stack_commit_checks_path(stack, sha: stack.next_commit_to_deploy.sha) %>.
+
+          <%= link_to_if stack.deployment_checks_passed?, 'the pre-deploy checks failed', stack_commit_checks_path(stack, sha: stack.next_commit_to_deploy.sha) %>.
           You can either wait for them to pass, or trigger a deploy manually.
         </p>
       </div>

data/app/views/shipit/stacks/new.html.erb

@@ -17,7 +17,7 @@
         </div>
         <div class="field-wrapper">
           <%= f.label :branch %>
-          <%= f.text_field :branch, placeholder: 'master' %>
+          <%= f.text_field :branch, placeholder: '<default branch>' %>
         </div>
         <div class="field-wrapper">
           <%= f.label :environment %>

data/config/secrets.development.example.yml

@@ -1,8 +1,10 @@
 host: 'localhost:3000'
 redis_url: 'redis://127.0.0.1:6379/0'
 
+# For creating an app see: https://github.com/Shopify/shipit-engine/blob/master/docs/setup.md#creating-the-github-app
 # Can be obtained there: https://github.com/settings/apps
 # Set the "Authorization callback URL" as `<host>/github/auth/github/callback`
+
 github:
   app_id:
   installation_id:
@@ -12,3 +14,25 @@ github:
     id:
     secret:
     teams: # Optional
+
+# Use this configuration schema if you are configuring multiple Github applications for different Github organizations
+
+# github:
+#   somegithuborg:
+#     app_id:
+#     installation_id:
+#     webhook_secret: # nil
+#     private_key:
+#     oauth:
+#       id:
+#       secret:
+#       teams: # Optional
+#   someothergithuborg:
+#     app_id:
+#     installation_id:
+#     webhook_secret: # nil
+#     private_key:
+#     oauth:
+#       id:
+#       secret:
+#       teams: # Optional

data/config/secrets.development.shopify.yml

@@ -1,13 +1,24 @@
 host: 'shipit-engine.myshopify.io'
 redis_url: 'redis://shipit-engine.railgun:6379'
 
-# TODO: document creating a dev app
+# For creating an app see: https://github.com/Shopify/shipit-engine/blob/master/docs/setup.md#creating-the-github-app
+
 github:
-  app_id:
-  installation_id:
-  webhook_secret: # nil
-  private_key:
-  oauth:
-    id:
-    secret:
-    teams:
+  somegithuborg:
+    app_id:
+    installation_id:
+    webhook_secret: # nil
+    private_key:
+    oauth:
+      id:
+      secret:
+      teams:
+  someothergithuborg:
+    app_id:
+    installation_id:
+    webhook_secret: # nil
+    private_key:
+    oauth:
+      id:
+      secret:
+      teams:

data/db/migrate/20210325194053_remove_stacks_branch_default.rb

@@ -0,0 +1,5 @@
+class RemoveStacksBranchDefault < ActiveRecord::Migration[6.1]
+  def change
+    change_column_default(:stacks, :branch, from: 'master', to: nil)
+  end
+end

data/db/migrate/20210504200438_add_github_updated_at_to_check_runs.rb

@@ -0,0 +1,5 @@
+class AddGithubUpdatedAtToCheckRuns < ActiveRecord::Migration[6.1]
+  def change
+    add_column :check_runs, :github_updated_at, :datetime, default: 0
+  end
+end

data/lib/shipit.rb

@@ -24,7 +24,6 @@ require 'safe_yaml/load'
 require 'securecompare'
 
 require 'redis-objects'
-require 'redis-namespace'
 
 require 'octokit'
 require 'faraday-http-cache'
@@ -60,9 +59,12 @@ SafeYAML::OPTIONS[:deserialize_symbols] = false
 module Shipit
   extend self
 
+  GithubOrganizationUnknown = Class.new(StandardError)
+  TOP_LEVEL_GH_KEYS = [:app_id, :installation_id, :webhook_secret, :private_key, :oauth, :domain]
+
   delegate :table_name_prefix, to: :secrets
 
-  attr_accessor :disable_api_authentication, :timeout_exit_codes
+  attr_accessor :disable_api_authentication, :timeout_exit_codes, :deployment_checks
   attr_writer(
     :internal_hook_receivers,
     :preferred_org_emails,
@@ -92,7 +94,7 @@ module Shipit
     secrets.redis_url.present? ? URI(secrets.redis_url) : nil
   end
 
-  def redis(namespace = nil)
+  def redis
     @redis ||= Redis.new(
       url: redis_url.to_s.presence,
       logger: Rails.logger,
@@ -100,17 +102,41 @@ module Shipit
       reconnect_delay: 0.5,
       reconnect_delay_max: 1,
     )
-    return @redis unless namespace
-    Redis::Namespace.new(namespace, redis: @redis)
   end
 
-  def github
-    @github ||= GitHubApp.new(secrets.github)
+  def github(organization: github_default_organization)
+    # Backward compatibility
+    # nil signifies the single github app config schema is being used
+    if github_default_organization.nil?
+      config = secrets.github
+    else
+      config = github_app_config(organization)
+      raise GithubOrganizationUnknown, organization if config.nil?
+    end
+    @github ||= {}
+    @github[organization] ||= GitHubApp.new(organization, config)
+  end
+
+  def github_default_organization
+    return nil unless secrets&.github
+    org = secrets.github.keys.first
+    TOP_LEVEL_GH_KEYS.include?(org) ? nil : org
+  end
+
+  def github_organizations
+    return [nil] unless github_default_organization
+    secrets.github.keys
+  end
+
+  def github_app_config(organization)
+    github_config = secrets.github.deep_transform_keys(&:downcase)
+    github_organization = organization.downcase.to_sym
+    github_config[github_organization]
   end
 
   def legacy_github_api
     if secrets&.github_api.present?
-      @legacy_github_api ||= github.new_client(access_token: secrets.github_api['access_token'])
+      @legacy_github_api ||= github.new_client(access_token: secrets.github_api[:access_token])
     end
   end
 
@@ -175,13 +201,11 @@ module Shipit
   end
 
   def revision
-    @revision ||= begin
-      if revision_file.exist?
-        revision_file.read
-      else
-        %x(git rev-parse HEAD)
-      end.strip
-    end
+    @revision ||= if revision_file.exist?
+      revision_file.read
+    else
+      %x(git rev-parse HEAD)
+    end.strip
   end
 
   def default_inactivity_timeout

data/lib/shipit/command.rb

@@ -13,7 +13,8 @@ module Shipit
     Denied = Class.new(Error)
     TimedOut = Class.new(Error)
 
-    BASE_ENV = Bundler.clean_env.merge((ENV.keys - Bundler.clean_env.keys).map { |k| [k, nil] }.to_h)
+    unbundled_env = Bundler.respond_to?(:unbundled_env) ? Bundler.unbundled_env : Bundler.clean_env
+    BASE_ENV = unbundled_env.merge((ENV.keys - unbundled_env.keys).map { |k| [k, nil] }.to_h)
 
     class Failed < Error
       attr_reader :exit_code
@@ -29,7 +30,7 @@ module Shipit
     def initialize(*args, default_timeout: Shipit.default_inactivity_timeout, env: {}, chdir:)
       @args, options = parse_arguments(args)
       @timeout = options['timeout'] || options[:timeout] || default_timeout
-      @env = env
+      @env = env.transform_values { |v| v&.to_s }
       @chdir = chdir.to_s
       @timed_out = false
     end
@@ -86,7 +87,7 @@ module Shipit
       @out = @pid = nil
       FileUtils.mkdir_p(@chdir)
       begin
-        @out, child_in, @pid = PTY.spawn(clean_env, *interpolated_arguments, chdir: @chdir)
+        @out, child_in, @pid = PTY.spawn(unbundled_env, *interpolated_arguments, chdir: @chdir)
         child_in.close
       rescue Errno::ENOENT
         raise NotFound, "#{Shellwords.split(interpolated_arguments.first).first}: command not found"
@@ -97,8 +98,8 @@ module Shipit
       self
     end
 
-    def clean_env
-      BASE_ENV.merge('PATH' => "#{ENV['PATH']}:#{Shipit.shell_paths.join(':')}").merge(@env.stringify_keys)
+    def unbundled_env
+      BASE_ENV.merge('PATH' => "#{Shipit.shell_paths.join(':')}:#{ENV['PATH']}").merge(@env.stringify_keys)
     end
 
     def stream(&block)
@@ -216,7 +217,7 @@ module Shipit
           argument
         end
       end
-      [args, options]
+      [args.map(&:to_s), options]
     end
 
     def running?

data/lib/shipit/commands.rb

@@ -21,8 +21,8 @@ module Shipit
 
     def env
       @env ||= Shipit.env.merge(
-        'GITHUB_DOMAIN' => Shipit.github.domain,
-        'GITHUB_TOKEN' => Shipit.github.token,
+        'GITHUB_DOMAIN' => github.domain,
+        'GITHUB_TOKEN' => github.token,
         'GIT_ASKPASS' => Shipit::Engine.root.join('lib', 'snippets', 'git-askpass').realpath.to_s,
       )
     end
@@ -30,5 +30,12 @@ module Shipit
     def git(*args)
       Command.new("git", *args)
     end
+    ruby2_keywords :git if respond_to?(:ruby2_keywords, true)
+
+    private
+
+    def github
+      Shipit.github
+    end
   end
 end

data/lib/shipit/engine.rb

@@ -10,6 +10,8 @@ module Shipit
       Shipit::Engine.routes.default_url_options[:host] = Shipit.host
       Pubsubstub.redis_url = Shipit.redis_url.to_s
 
+      Rails.application.secrets.deep_symbolize_keys!
+
       app.config.assets.paths << Emoji.images_path
       app.config.assets.precompile += %w(
         favicon.ico

data/lib/shipit/flock.rb

@@ -11,14 +11,21 @@ module Shipit
 
     def initialize(path)
       @path = Pathname.new(path)
+      @acquired = false
     end
 
     def lock(timeout:)
+      return yield if @acquired
       path.parent.mkpath
       path.open('w') do |file|
         if retrying(timeout: timeout) { file.flock(File::LOCK_EX | File::LOCK_NB) }
           file.write($PROCESS_ID.to_s)
-          return yield
+          @acquired = true
+          begin
+            yield
+          ensure
+            @acquired = false
+          end
         else
           raise TimeoutError, "Couldn't acquire lock for #{path} in #{timeout} seconds"
         end