shipit-engine 0.15.0 → 0.16.0

data/app/models/shipit/deploy_spec/file_system.rb

@@ -2,11 +2,11 @@ module Shipit
   class DeploySpec
     class FileSystem < DeploySpec
       include NpmDiscovery
-      include KubernetesDiscovery
       include PypiDiscovery
       include RubygemsDiscovery
       include CapistranoDiscovery
       include BundlerDiscovery
+      include KubernetesDiscovery
 
       def initialize(app_dir, env)
         @app_dir = Pathname(app_dir)
@@ -29,6 +29,11 @@ module Shipit
 
       def cacheable_config
         (config || {}).deep_merge(
+          'merge' => {
+            'require' => pull_request_required_statuses,
+            'ignore' => pull_request_ignored_statuses,
+            'revalidate_after' => revalidate_pull_requests_after.try!(:to_i),
+          },
           'ci' => {
             'hide' => hidden_statuses,
             'allow_failures' => soft_failing_statuses,

data/app/models/shipit/deploy_spec/kubernetes_discovery.rb

@@ -9,24 +9,21 @@ module Shipit
         discover_kubernetes || super
       end
 
-      def discover_machine_env
-        env = super
-        env = env.merge('K8S_TEMPLATE_FOLDER' => kube_config['template_dir']) if kube_config['template_dir']
-        env
-      end
-
       private
 
       def discover_kubernetes
         return unless kube_config.present?
 
-        [
-          Shellwords.join([
-            "kubernetes-deploy",
-            kube_config['namespace'],
-            kube_config['context'],
-          ]),
-        ]
+        cmd = ["kubernetes-deploy"]
+        if kube_config['template_dir']
+          cmd << '--template-dir'
+          cmd << kube_config['template_dir']
+        end
+
+        cmd << kube_config['namespace']
+        cmd << kube_config['context']
+
+        [Shellwords.join(cmd)]
       end
 
       def kube_config

data/app/models/shipit/deploy_spec/npm_discovery.rb

@@ -62,7 +62,8 @@ module Shipit
 
       def publish_npm_package
         check_tags = 'assert-npm-version-tag'
-        publish = js_command('publish')
+        # `yarn publish` requires user input, so always use npm.
+        publish = 'npm publish'
 
         [check_tags, publish]
       end

data/app/models/shipit/duration.rb

@@ -1,5 +1,7 @@
 module Shipit
   class Duration < ActiveSupport::Duration
+    ParseError = Class.new(ArgumentError)
+
     FORMAT = /
       \A
       (?<days>\d+d)?
@@ -18,7 +20,7 @@ module Shipit
     class << self
       def parse(value)
         unless match = FORMAT.match(value.to_s)
-          raise ArgumentError, "not a duration: #{value.inspect}"
+          raise ParseError, "not a duration: #{value.inspect}"
         end
         parts = []
         UNITS.values.each do |unit|

data/app/models/shipit/hook.rb

@@ -16,6 +16,7 @@ module Shipit
       commit_status
       deployable_status
       merge_status
+      merge
     ).freeze
 
     belongs_to :stack, required: false

data/app/models/shipit/pull_request.rb

@@ -0,0 +1,252 @@
+module Shipit
+  class PullRequest < ApplicationRecord
+    include DeferredTouch
+
+    WAITING_STATUSES = %w(fetching pending).freeze
+    QUEUED_STATUSES = %w(pending revalidating).freeze
+    REJECTION_REASONS = %w(ci_failing merge_conflict).freeze
+    InvalidTransition = Class.new(StandardError)
+    NotReady = Class.new(StandardError)
+
+    class StatusChecker < Status::Group
+      def initialize(commit, statuses, deploy_spec)
+        @deploy_spec = deploy_spec
+        super(commit, statuses)
+      end
+
+      private
+
+      attr_reader :deploy_spec
+
+      def reject_hidden(statuses)
+        statuses.reject { |s| ignored_statuses.include?(s.context) }
+      end
+
+      def reject_allowed_to_fail(statuses)
+        statuses.reject { |s| ignored_statuses.include?(s.context) }
+      end
+
+      def ignored_statuses
+        deploy_spec.try!(:pull_request_ignored_statuses) || []
+      end
+
+      def required_statuses
+        deploy_spec.try!(:pull_request_required_statuses) || []
+      end
+    end
+
+    belongs_to :stack
+    belongs_to :head, class_name: 'Shipit::Commit'
+    belongs_to :merge_requested_by, class_name: 'Shipit::User'
+    has_one :merge_commit, class_name: 'Shipit::Commit'
+
+    deferred_touch stack: :updated_at
+
+    validates :number, presence: true, uniqueness: {scope: :stack_id}
+
+    scope :waiting, -> { where(merge_status: WAITING_STATUSES) }
+    scope :pending, -> { where(merge_status: 'pending') }
+    scope :to_be_merged, -> { pending.order(merge_requested_at: :asc) }
+    scope :queued, -> { where(merge_status: QUEUED_STATUSES).order(merge_requested_at: :asc) }
+
+    after_save :record_merge_status_change
+    after_commit :emit_hooks
+
+    state_machine :merge_status, initial: :fetching do
+      state :fetching
+      state :pending
+      state :rejected
+      state :canceled
+      state :merged
+      state :revalidating
+
+      event :fetched do
+        transition fetching: :pending
+      end
+
+      event :reject do
+        transition pending: :rejected
+      end
+
+      event :revalidate do
+        transition pending: :revalidating
+      end
+
+      event :cancel do
+        transition any => :canceled
+      end
+
+      event :complete do
+        transition pending: :merged
+      end
+
+      event :retry do
+        transition %i(rejected canceled revalidating) => :pending
+      end
+
+      before_transition rejected: any do |pr|
+        pr.rejection_reason = nil
+      end
+
+      before_transition %i(fetching rejected canceled) => :pending do |pr|
+        pr.merge_requested_at = Time.now.utc
+      end
+
+      before_transition any => :pending do |pr|
+        pr.revalidated_at = Time.now.utc
+      end
+    end
+
+    def self.schedule_merges
+      Shipit::Stack.where(id: pending.uniq.pluck(:stack_id)).find_each(&:schedule_merges)
+    end
+
+    def self.extract_number(stack, number_or_url)
+      case number_or_url
+      when /\A#?(\d+)\z/
+        $1.to_i
+      when %r{\Ahttps://#{Regexp.escape(Shipit.github_domain)}/([^/]+)/([^/]+)/pull/(\d+)}
+        return unless $1.downcase == stack.repo_owner.downcase
+        return unless $2.downcase == stack.repo_name.downcase
+        return $3.to_i
+      end
+    end
+
+    def self.request_merge!(stack, number, user)
+      now = Time.now.utc
+      pull_request = begin
+        create_with(
+          merge_requested_at: now,
+          merge_requested_by: user.presence,
+        ).find_or_create_by!(
+          stack: stack,
+          number: number,
+        )
+      rescue ActiveRecord::RecordNotUnique
+        retry
+      end
+      pull_request.update!(merge_requested_by: user.presence)
+      pull_request.retry! if pull_request.rejected? || pull_request.canceled? || pull_request.revalidating?
+      pull_request.schedule_refresh!
+      pull_request
+    end
+
+    def reject!(reason)
+      unless REJECTION_REASONS.include?(reason)
+        raise ArgumentError, "invalid reason: #{reason.inspect}, must be one of: #{REJECTION_REASONS.inspect}"
+      end
+      self.rejection_reason = reason.presence
+      super()
+      true
+    end
+
+    def reject_unless_mergeable!
+      return reject!('merge_conflict') if merge_conflict?
+      return reject!('ci_failing') unless all_status_checks_passed?
+      false
+    end
+
+    def merge!
+      raise InvalidTransition unless pending?
+
+      raise NotReady if not_mergeable_yet?
+      if need_revalidation?
+        revalidate!
+        return false
+      end
+
+      Shipit.github_api.merge_pull_request(
+        stack.github_repo_name,
+        number,
+        merge_message,
+        sha: head.sha,
+        commit_message: 'Merged by Shipit',
+        merge_method: 'merge',
+      )
+      begin
+        Shipit.github_api.delete_branch(stack.github_repo_name, branch)
+      rescue Octokit::UnprocessableEntity
+        # branch was already deleted somehow
+      end
+      complete!
+      return true
+    rescue Octokit::MethodNotAllowed # merge conflict
+      reject!('merge_conflict')
+      return false
+    rescue Octokit::Conflict # shas didn't match, PR was updated.
+      raise NotReady
+    end
+
+    def all_status_checks_passed?
+      StatusChecker.new(head, head.statuses, stack.cached_deploy_spec).success?
+    end
+
+    def waiting?
+      WAITING_STATUSES.include?(merge_status)
+    end
+
+    def need_revalidation?
+      timeout = stack.cached_deploy_spec.try!(:revalidate_pull_requests_after)
+      return false unless timeout
+      (revalidated_at + timeout).past?
+    end
+
+    def merge_conflict?
+      mergeable == false
+    end
+
+    def not_mergeable_yet?
+      mergeable.nil?
+    end
+
+    def schedule_refresh!
+      RefreshPullRequestJob.perform_later(self)
+    end
+
+    def refresh!
+      update!(github_pull_request: Shipit.github_api.pull_request(stack.github_repo_name, number))
+      head.refresh_statuses!
+      fetched! if fetching?
+    end
+
+    def github_pull_request=(github_pull_request)
+      self.github_id = github_pull_request.id
+      self.api_url = github_pull_request.url
+      self.title = github_pull_request.title
+      self.state = github_pull_request.state
+      self.mergeable = github_pull_request.mergeable
+      self.additions = github_pull_request.additions
+      self.deletions = github_pull_request.deletions
+      self.branch = github_pull_request.head.ref
+      self.head = find_or_create_commit_from_github_by_sha!(github_pull_request.head.sha, detached: true)
+    end
+
+    def merge_message
+      return title unless merge_requested_by
+      "#{title}\n\nMerge-Requested-By: #{merge_requested_by.login}\n"
+    end
+
+    private
+
+    def record_merge_status_change
+      @merge_status_changed ||= merge_status_changed?
+    end
+
+    def emit_hooks
+      return unless @merge_status_changed
+      @merge_status_changed = nil
+      Hook.emit('merge', stack, pull_request: self, status: merge_status, stack: stack)
+    end
+
+    def find_or_create_commit_from_github_by_sha!(sha, attributes)
+      if commit = stack.commits.by_sha(sha)
+        return commit
+      else
+        github_commit = Shipit.github_api.commit(stack.github_repo_name, sha)
+        stack.commits.create_from_github!(github_commit, attributes)
+      end
+    rescue ActiveRecord::RecordNotUnique
+      retry
+    end
+  end
+end

data/app/models/shipit/stack.rb

@@ -24,6 +24,7 @@ module Shipit
     REQUIRED_HOOKS = %i(push status).freeze
 
     has_many :commits, dependent: :destroy
+    has_many :pull_requests, dependent: :destroy
     has_many :tasks, dependent: :destroy
     has_many :deploys
     has_many :rollbacks
@@ -50,6 +51,7 @@ module Shipit
     after_commit :broadcast_update, on: :update
     after_commit :emit_merge_status_hooks, on: :update
     after_commit :setup_hooks, :sync_github, on: :create
+    after_commit :schedule_merges_if_necessary, on: :update
 
     validates :repo_name, uniqueness: {scope: %i(repo_owner environment),
                                        message: 'cannot be used more than once with this environment'}
@@ -138,6 +140,10 @@ module Shipit
       trigger_deploy(commit, Shipit.user, env: cached_deploy_spec.default_deploy_env)
     end
 
+    def schedule_merges
+      MergePullRequestsJob.perform_later(self)
+    end
+
     def next_commit_to_deploy
       commits_to_deploy = commits.order(id: :asc).newer_than(last_deployed_commit).reachable.preload(:statuses)
       commits_to_deploy = commits_to_deploy.limit(maximum_commits_per_deploy) if maximum_commits_per_deploy
@@ -179,15 +185,21 @@ module Shipit
     end
 
     def merge_status
-      if locked?
-        'locked'
+      return 'locked' if locked?
+      return 'failure' if %w(failure error).freeze.include?(branch_status)
+      if maximum_commits_per_deploy && (undeployed_commits_count > maximum_commits_per_deploy * 1.5)
+        'backlogged'
       else
-        significant_statuses = undeployed_commits.map(&:significant_status)
-        significant_statuses << last_deployed_commit.significant_status unless last_deployed_commit.blank?
+        'success'
+      end
+    end
 
-        last_finalized_status = significant_statuses.reject { |s| %w(pending unknown).include?(s.state) }.first
-        last_finalized_status.try!(:simple_state) || 'pending'
+    def branch_status
+      undeployed_commits.each do |commit|
+        state = commit.status.simple_state
+        return state unless %w(pending unknown missing).freeze.include?(state)
       end
+      'pending'
     end
 
     def status
@@ -217,18 +229,14 @@ module Shipit
       end
     end
 
-    def filter_visible_statuses(statuses)
-      statuses.reject { |s| hidden_statuses.include?(s.context) }
-    end
-
-    def filter_meaningful_statuses(statuses)
-      filter_visible_statuses(statuses).reject { |s| soft_failing_statuses.include?(s.context) }
-    end
-
     def deployable?
       !locked? && !active_task?
     end
 
+    def allows_merges?
+      merge_queue_enabled? && !locked? && merge_status == 'success'
+    end
+
     def repo_name=(name)
       super(name.try!(:downcase))
     end
@@ -369,9 +377,11 @@ module Shipit
     end
 
     def broadcast_update
-      payload = {url: Shipit::Engine.routes.url_helpers.stack_path(self)}.to_json
-      event = Pubsubstub::Event.new(payload, name: "stack.update")
-      Pubsubstub::RedisPubSub.publish("stack.#{id}", event)
+      Pubsubstub.publish(
+        "stack.#{id}",
+        {id: id, updated_at: updated_at}.to_json,
+        name: 'update',
+      )
     end
 
     def setup_hooks
@@ -443,6 +453,12 @@ module Shipit
       self.branch = 'master' if branch.blank?
     end
 
+    def schedule_merges_if_necessary
+      if previous_changes.include?('lock_reason') && previous_changes['lock_reason'].last.blank?
+        schedule_merges
+      end
+    end
+
     def emit_lock_hooks
       return unless previous_changes.include?('lock_reason')
       Hook.emit(:lock, self, locked: locked?, stack: self)