shipit-engine 0.31.0 → 0.35.0

data/app/models/shipit/deploy_spec/file_system.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class DeploySpec
     class FileSystem < DeploySpec
@@ -12,6 +13,7 @@ module Shipit
       def initialize(app_dir, env)
         @app_dir = Pathname(app_dir)
         @env = env
+        super(nil)
       end
 
       def cacheable
@@ -31,10 +33,10 @@ module Shipit
       def cacheable_config
         (config || {}).deep_merge(
           'merge' => {
-            'require' => pull_request_required_statuses,
-            'ignore' => pull_request_ignored_statuses,
-            'revalidate_after' => revalidate_pull_requests_after&.to_i,
-            'method' => pull_request_merge_method,
+            'require' => merge_request_required_statuses,
+            'ignore' => merge_request_ignored_statuses,
+            'revalidate_after' => revalidate_merge_requests_after&.to_i,
+            'method' => merge_request_merge_method,
             'max_divergence' => {
               'commits' => max_divergence_commits&.to_i,
               'age' => max_divergence_age&.to_i,
@@ -61,14 +63,19 @@ module Shipit
             'context' => release_status_context,
             'delay' => release_status_delay,
           },
-          'dependencies' => {'override' => dependencies_steps},
+          'dependencies' => { 'override' => dependencies_steps },
+          'provision' => { 'handler_name' => provisioning_handler_name },
           'deploy' => {
             'override' => deploy_steps,
             'variables' => deploy_variables.map(&:to_h),
             'max_commits' => maximum_commits_per_deploy,
             'interval' => pause_between_deploys,
+            'retries' => retries_on_deploy,
+          },
+          'rollback' => {
+            'override' => rollback_steps,
+            'retries' => retries_on_rollback,
           },
-          'rollback' => {'override' => rollback_steps},
           'fetch' => fetch_deployed_revision_steps,
           'tasks' => cacheable_tasks,
         )
@@ -84,10 +91,16 @@ module Shipit
       end
 
       def load_config
-        read_config(file("shipit.#{@env}.yml", root: true)) ||
+        read_config(file("#{app_name}.#{@env}.yml", root: true)) ||
+          read_config(file("#{app_name}.yml", root: true)) ||
+          read_config(file("shipit.#{@env}.yml", root: true)) ||
           read_config(file('shipit.yml', root: true))
       end
 
+      def app_name
+        @app_name ||= Shipit.app_name.downcase
+      end
+
       def read_config(path)
         SafeYAML.load(path.read) if path.exist?
       end

data/app/models/shipit/deploy_spec/kubernetes_discovery.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class DeploySpec
     module KubernetesDiscovery

data/app/models/shipit/deploy_spec/lerna_discovery.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'json'
 
 module Shipit
@@ -75,10 +76,18 @@ module Shipit
       end
 
       def publish_independent_packages
-        [
-          'assert-lerna-independent-version-tags',
-          'publish-lerna-independent-packages',
-        ]
+        command = if lerna_lerna >= LATEST_MAJOR_VERSION
+          [
+            'assert-lerna-independent-version-tags',
+            'publish-lerna-independent-packages',
+          ]
+        else
+          [
+            'assert-lerna-independent-version-tags',
+            'publish-lerna-independent-packages-legacy',
+          ]
+        end
+        command
       end
 
       def publish_fixed_version_packages

data/app/models/shipit/deploy_spec/npm_discovery.rb

@@ -1,13 +1,14 @@
+# frozen_string_literal: true
 require 'json'
 
 module Shipit
   class DeploySpec
     module NpmDiscovery
       # https://docs.npmjs.com/cli/publish
-      PUBLIC = 'public'.freeze
-      PRIVATE = 'restricted'.freeze
+      PUBLIC = 'public'
+      PRIVATE = 'restricted'
       VALID_ACCESS = [PUBLIC, PRIVATE].freeze
-      NPM_REGISTRY = "https://registry.npmjs.org/".freeze
+      NPM_REGISTRY = "https://registry.npmjs.org/"
 
       def discover_dependencies_steps
         discover_package_json || super
@@ -57,7 +58,7 @@ module Shipit
         # are treated as 'next' npm dist-tags.
         # An 1.0.0-beta.1 would be installable using both:
         # `yarn add package@1.0.0-beta.1` and `yarn add package@next`
-        return 'next' if ['-beta', '-alpha', '-rc', '-next'].any? { |tag| version.include? tag }
+        return 'next' if ['-beta', '-alpha', '-rc', '-next'].any? { |tag| version.include?(tag) }
         'latest'
       end
 

data/app/models/shipit/deploy_spec/pypi_discovery.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class DeploySpec
     module PypiDiscovery

data/app/models/shipit/deploy_spec/rubygems_discovery.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class DeploySpec
     module RubygemsDiscovery

data/app/models/shipit/deploy_spec.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'pathname'
 
 module Shipit
@@ -5,20 +6,34 @@ module Shipit
     Error = Class.new(StandardError)
 
     class << self
+      attr_accessor :pretty_generate
+
       def load(json)
         config = json.blank? ? {} : JSON.parse(json)
         new(config)
       end
 
       def dump(spec)
-        JSON.dump(spec.cacheable.config) if spec
+        return unless spec
+
+        if pretty_generate?
+          JSON.pretty_generate(spec.cacheable.config)
+        else
+          JSON.dump(spec.cacheable.config)
+        end
       end
 
       def bundle_path
         Rails.root.join('data', 'bundler')
       end
+
+      def pretty_generate?
+        @pretty_generate
+      end
     end
 
+    self.pretty_generate = false
+
     def initialize(config)
       @config = config
     end
@@ -40,6 +55,10 @@ module Shipit
       end
     end
 
+    def blank?
+      config.empty?
+    end
+
     def supports_fetch_deployed_revision?
       fetch_deployed_revision_steps.present?
     end
@@ -85,6 +104,10 @@ module Shipit
       Duration.parse(config('deploy', 'interval') { 0 })
     end
 
+    def provisioning_handler_name
+      config('provision', 'handler_name')
+    end
+
     def deploy_steps
       around_steps('deploy') do
         config('deploy', 'override') { discover_deploy_steps }
@@ -103,6 +126,10 @@ module Shipit
       deploy_variables.map { |v| [v.name, v.default] }.to_h
     end
 
+    def retries_on_deploy
+      config('deploy', 'retries') { nil }
+    end
+
     def rollback_steps
       around_steps('rollback') do
         config('rollback', 'override') { discover_rollback_steps }
@@ -113,6 +140,10 @@ module Shipit
       rollback_steps || cant_detect!(:rollback)
     end
 
+    def retries_on_rollback
+      config('rollback', 'retries') { nil }
+    end
+
     def fetch_deployed_revision_steps
       config('fetch') || discover_fetch_deployed_revision_steps
     end
@@ -160,12 +191,12 @@ module Shipit
       Array.wrap(config('ci', 'blocking'))
     end
 
-    def pull_request_merge_method
+    def merge_request_merge_method
       method = config('merge', 'method')
       method if %w(merge rebase squash).include?(method)
     end
 
-    def pull_request_required_statuses
+    def merge_request_required_statuses
       if config('merge', 'require') || config('merge', 'ignore')
         Array.wrap(config('merge', 'require'))
       else
@@ -173,7 +204,7 @@ module Shipit
       end
     end
 
-    def pull_request_ignored_statuses
+    def merge_request_ignored_statuses
       if config('merge', 'require') || config('merge', 'ignore')
         Array.wrap(config('merge', 'ignore')) + [release_status_context].compact
       else
@@ -181,7 +212,7 @@ module Shipit
       end
     end
 
-    def revalidate_pull_requests_after
+    def revalidate_merge_requests_after
       if timeout = config('merge', 'revalidate_after')
         begin
           Duration.parse(timeout)
@@ -255,11 +286,11 @@ module Shipit
     end
 
     def task_not_found!(id)
-      raise TaskDefinition::NotFound.new("No definition for task #{id.inspect}")
+      raise TaskDefinition::NotFound, "No definition for task #{id.inspect}"
     end
 
     def cant_detect!(type)
-      raise DeploySpec::Error.new(I18n.t("deploy_spec.hint.#{type}"))
+      raise DeploySpec::Error, I18n.t("deploy_spec.hint.#{type}")
     end
   end
 end

data/app/models/shipit/deploy_stats.rb

@@ -1,10 +1,11 @@
+# frozen_string_literal: true
 module Shipit
   class DeployStats
     delegate :empty?, to: :@deploys
 
     def initialize(deploys)
       @deploys = deploys
-      @durations = @deploys.map { |d| d.duration.value }.compact
+      @durations = @deploys.map { |d| d.duration&.value }.compact
     end
 
     def count

data/app/models/shipit/duration.rb

@@ -1,15 +1,16 @@
+# frozen_string_literal: true
 module Shipit
   class Duration < ActiveSupport::Duration
     ParseError = Class.new(ArgumentError)
 
-    FORMAT = /
+    FORMAT = %r{
       \A
       (?<days>\d+d)?
       (?<hours>\d+h)?
       (?<minutes>\d+m)?
       (?<seconds>\d+s?)?
       \z
-    /x
+    }x
     UNITS = {
       's' => :seconds,
       'm' => :minutes,
@@ -19,6 +20,8 @@ module Shipit
 
     class << self
       def parse(value)
+        return new(-1) if value.to_s == "-1"
+
         unless match = FORMAT.match(value.to_s)
           raise ParseError, "not a duration: #{value.inspect}"
         end

data/app/models/shipit/ephemeral_commit_checks.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class EphemeralCommitChecks
     FINAL_STATUSES = %w(failed error success).freeze

data/app/models/shipit/github_hook.rb

@@ -1,5 +1,6 @@
+# frozen_string_literal: true
 module Shipit
-  class GithubHook < ActiveRecord::Base
+  class GithubHook < Record
     # TODO: app-migration, delete class
     belongs_to :stack, required: false # Required for fixtures
 

data/app/models/shipit/github_status.rb

@@ -1,6 +1,7 @@
+# frozen_string_literal: true
 module Shipit
   module GithubStatus
-    CACHE_KEY = 'github::status'.freeze
+    CACHE_KEY = 'github::status'
 
     class << self
       def status

data/app/models/shipit/hook.rb

@@ -1,11 +1,28 @@
+# frozen_string_literal: true
 module Shipit
-  class Hook < ActiveRecord::Base
+  class Hook < Record
+    class DeliverySigner
+      attr_reader :secret
+
+      ALGORITHM = 'sha256'
+
+      def initialize(secret)
+        @secret = secret
+      end
+
+      def sign(payload)
+        hmac = OpenSSL::HMAC.hexdigest(ALGORITHM, secret, payload)
+        "#{ALGORITHM}=#{hmac}"
+      end
+    end
+
     class DeliverySpec
-      def initialize(event:, url:, content_type:, payload:)
+      def initialize(event:, url:, content_type:, payload:, secret:)
         @event = event
         @url = url
         @content_type = content_type
         @payload = payload
+        @secret = secret
       end
 
       def send!
@@ -14,12 +31,12 @@ module Shipit
 
       private
 
-      attr_reader :event, :url, :content_type, :payload
+      attr_reader :event, :url, :content_type, :payload, :secret
 
       def http
         Faraday::Connection.new do |connection|
           connection.headers = headers
-          connection.adapter Faraday.default_adapter
+          connection.adapter(Faraday.default_adapter)
         end
       end
 
@@ -28,9 +45,16 @@ module Shipit
           'User-Agent' => 'Shipit Webhook',
           'Content-Type' => content_type,
           'X-Shipit-Event' => event,
+          'X-Shipit-Signature' => signature,
           'Accept' => '*/*',
         }
       end
+
+      def signature
+        return nil if secret.blank?
+
+        DeliverySigner.new(secret).sign(payload)
+      end
     end
 
     default_scope { order :id }
@@ -44,6 +68,7 @@ module Shipit
 
     EVENTS = %w(
       stack
+      review_stack
       task
       deploy
       rollback
@@ -52,14 +77,15 @@ module Shipit
       deployable_status
       merge_status
       merge
+      pull_request
     ).freeze
 
     belongs_to :stack, required: false
     has_many :deliveries
 
-    validates :delivery_url, presence: true, url: {no_local: true, allow_blank: true}
-    validates :content_type, presence: true, inclusion: {in: CONTENT_TYPES.keys}
-    validates :events, presence: true, subset: {of: EVENTS}
+    validates :delivery_url, presence: true, url: { no_local: true, allow_blank: true }
+    validates :content_type, presence: true, inclusion: { in: CONTENT_TYPES.keys }
+    validates :events, presence: true, subset: { of: EVENTS }
 
     serialize :events, Shipit::CSVSerializer
 
@@ -116,6 +142,7 @@ module Shipit
         url: delivery_url,
         content_type: CONTENT_TYPES[content_type],
         payload: serialize_payload(payload),
+        secret: secret,
       )
     end
 

data/app/models/shipit/membership.rb

@@ -1,8 +1,9 @@
+# frozen_string_literal: true
 module Shipit
-  class Membership < ActiveRecord::Base
+  class Membership < Record
     belongs_to :team, required: true
     belongs_to :user, required: true
 
-    validates :user_id, uniqueness: {scope: :team_id}
+    validates :user_id, uniqueness: { scope: :team_id }
   end
 end

data/app/models/shipit/merge_request.rb

@@ -0,0 +1,304 @@
+# frozen_string_literal: true
+module Shipit
+  class MergeRequest < ApplicationRecord
+    include DeferredTouch
+
+    MERGE_REQUEST_FIELD = 'Merge-Requested-By'
+
+    WAITING_STATUSES = %w(fetching pending).freeze
+    QUEUED_STATUSES = %w(pending revalidating).freeze
+    REJECTION_REASONS = %w(ci_missing ci_failing merge_conflict requires_rebase).freeze
+    InvalidTransition = Class.new(StandardError)
+    NotReady = Class.new(StandardError)
+
+    class StatusChecker < Status::Group
+      def initialize(commit, statuses, deploy_spec)
+        @deploy_spec = deploy_spec
+        super(commit, statuses)
+      end
+
+      private
+
+      attr_reader :deploy_spec
+
+      def reject_hidden(statuses)
+        statuses.reject { |s| ignored_statuses.include?(s.context) }
+      end
+
+      def reject_allowed_to_fail(statuses)
+        statuses.reject { |s| ignored_statuses.include?(s.context) }
+      end
+
+      def ignored_statuses
+        deploy_spec&.merge_request_ignored_statuses || []
+      end
+
+      def required_statuses
+        deploy_spec&.merge_request_required_statuses || []
+      end
+    end
+
+    belongs_to :stack
+    belongs_to :head, class_name: 'Shipit::Commit', optional: true
+    belongs_to :base_commit, class_name: 'Shipit::Commit', optional: true
+    belongs_to :merge_requested_by, class_name: 'Shipit::User', optional: true
+    has_one :merge_commit, class_name: 'Shipit::Commit'
+
+    deferred_touch stack: :updated_at
+
+    validates :number, presence: true, uniqueness: { scope: :stack_id }
+
+    scope :waiting, -> { where(merge_status: WAITING_STATUSES) }
+    scope :pending, -> { where(merge_status: 'pending') }
+    scope :to_be_merged, -> { pending.order(merge_requested_at: :asc) }
+    scope :queued, -> { where(merge_status: QUEUED_STATUSES).order(merge_requested_at: :asc) }
+
+    after_save :record_merge_status_change
+    after_commit :emit_hooks
+
+    state_machine :merge_status, initial: :fetching do
+      state :fetching
+      state :pending
+      state :rejected
+      state :canceled
+      state :merged
+      state :revalidating
+
+      event :fetched do
+        transition fetching: :pending
+      end
+
+      event :reject do
+        transition pending: :rejected
+      end
+
+      event :revalidate do
+        transition pending: :revalidating
+      end
+
+      event :cancel do
+        transition any => :canceled
+      end
+
+      event :complete do
+        transition pending: :merged
+      end
+
+      event :retry do
+        transition %i(rejected canceled revalidating) => :pending
+      end
+
+      before_transition rejected: any do |pr|
+        pr.rejection_reason = nil
+      end
+
+      before_transition %i(fetching rejected canceled) => :pending do |pr|
+        pr.merge_requested_at = Time.now.utc
+      end
+
+      before_transition any => :pending do |pr|
+        pr.revalidated_at = Time.now.utc
+      end
+
+      before_transition %i(pending) => :merged do |pr|
+        Stack.increment_counter(:undeployed_commits_count, pr.stack_id)
+      end
+    end
+
+    def self.schedule_merges
+      Shipit::Stack.where(merge_queue_enabled: true).find_each(&:schedule_merges)
+    end
+
+    def self.extract_number(stack, number_or_url)
+      org = stack.repository.owner
+      case number_or_url
+      when /\A#?(\d+)\z/
+        $1.to_i
+      when %r{\Ahttps://#{Regexp.escape(Shipit.github(organization: org).domain)}/([^/]+)/([^/]+)/pull/(\d+)}
+        return unless $1.downcase == stack.repo_owner.downcase
+        return unless $2.downcase == stack.repo_name.downcase
+        $3.to_i
+      end
+    end
+
+    def self.request_merge!(stack, number, user)
+      now = Time.now.utc
+      merge_request = begin
+        create_with(
+          merge_requested_at: now,
+          merge_requested_by: user.presence,
+        ).find_or_create_by!(
+          stack: stack,
+          number: number,
+        )
+      rescue ActiveRecord::RecordNotUnique
+        retry
+      end
+      merge_request.update!(merge_requested_by: user.presence)
+      merge_request.retry! if merge_request.rejected? || merge_request.canceled? || merge_request.revalidating?
+      merge_request.schedule_refresh!
+      merge_request
+    end
+
+    def reject!(reason)
+      unless REJECTION_REASONS.include?(reason)
+        raise ArgumentError, "invalid reason: #{reason.inspect}, must be one of: #{REJECTION_REASONS.inspect}"
+      end
+      self.rejection_reason = reason.presence
+      super()
+      true
+    end
+
+    def reject_unless_mergeable!
+      return reject!('merge_conflict') if merge_conflict?
+      return reject!('ci_missing') if any_status_checks_missing?
+      return reject!('ci_failing') if any_status_checks_failed?
+      return reject!('requires_rebase') if stale?
+      false
+    end
+
+    def merge!
+      raise InvalidTransition unless pending?
+
+      raise NotReady if not_mergeable_yet?
+
+      stack.github_api.merge_pull_request(
+        stack.github_repo_name,
+        number,
+        merge_message,
+        sha: head.sha,
+        commit_message: 'Merged by Shipit',
+        merge_method: stack.merge_method,
+      )
+      begin
+        if stack.github_api.pull_requests(stack.github_repo_name, base: branch).empty?
+          stack.github_api.delete_branch(stack.github_repo_name, branch)
+        end
+      rescue Octokit::UnprocessableEntity
+        # branch was already deleted somehow
+      end
+      complete!
+      true
+    rescue Octokit::MethodNotAllowed # merge conflict
+      reject!('merge_conflict')
+      false
+    rescue Octokit::Conflict # shas didn't match, PR was updated.
+      raise NotReady
+    end
+
+    def all_status_checks_passed?
+      return false unless head
+      StatusChecker.new(head, head.statuses_and_check_runs, stack.cached_deploy_spec).success?
+    end
+
+    def any_status_checks_failed?
+      status = StatusChecker.new(head, head.statuses_and_check_runs, stack.cached_deploy_spec)
+      status.failure? || status.error?
+    end
+
+    def any_status_checks_missing?
+      StatusChecker.new(head, head.statuses_and_check_runs, stack.cached_deploy_spec).missing?
+    end
+
+    def waiting?
+      WAITING_STATUSES.include?(merge_status)
+    end
+
+    def need_revalidation?
+      timeout = stack.cached_deploy_spec&.revalidate_merge_requests_after
+      return false unless timeout
+      (revalidated_at + timeout).past?
+    end
+
+    def merge_conflict?
+      mergeable == false
+    end
+
+    def not_mergeable_yet?
+      mergeable.nil?
+    end
+
+    def schedule_refresh!
+      RefreshMergeRequestJob.perform_later(self)
+    end
+
+    def closed?
+      state == "closed"
+    end
+
+    def merged_upstream?
+      closed? && merged_at
+    end
+
+    def refresh!
+      update!(github_pull_request: stack.github_api.pull_request(stack.github_repo_name, number))
+      head.refresh_statuses!
+      head.refresh_check_runs!
+      fetched! if fetching?
+      @comparison = nil
+    end
+
+    def github_pull_request=(github_pull_request)
+      self.github_id = github_pull_request.id
+      self.api_url = github_pull_request.url
+      self.title = github_pull_request.title
+      self.state = github_pull_request.state
+      self.mergeable = github_pull_request.mergeable
+      self.additions = github_pull_request.additions
+      self.deletions = github_pull_request.deletions
+      self.branch = github_pull_request.head.ref
+      self.head = find_or_create_commit_from_github_by_sha!(github_pull_request.head.sha, detached: true)
+      self.merged_at = github_pull_request.merged_at
+      self.base_ref = github_pull_request.base.ref
+      self.base_commit = find_or_create_commit_from_github_by_sha!(github_pull_request.base.sha, detached: true)
+    end
+
+    def merge_message
+      return title unless merge_requested_by
+      "#{title}\n\n#{MERGE_REQUEST_FIELD}: #{merge_requested_by.login}\n"
+    end
+
+    def stale?
+      return false unless base_commit
+      spec = stack.cached_deploy_spec
+      if max_branch_age = spec.max_divergence_age
+        return true if Time.now.utc - head.committed_at > max_branch_age
+      end
+      if commit_count_limit = spec.max_divergence_commits
+        return true if comparison.behind_by > commit_count_limit
+      end
+      false
+    end
+
+    def comparison
+      @comparison ||= stack.github_api.compare(
+        stack.github_repo_name,
+        base_ref,
+        head.sha,
+      )
+    end
+
+    private
+
+    def record_merge_status_change
+      @merge_status_changed ||= saved_change_to_attribute?(:merge_status)
+    end
+
+    def emit_hooks
+      return unless @merge_status_changed
+      @merge_status_changed = nil
+      Hook.emit('merge', stack, merge_request: self, status: merge_status, stack: stack)
+    end
+
+    def find_or_create_commit_from_github_by_sha!(sha, attributes)
+      if commit = stack.commits.by_sha(sha)
+        commit
+      else
+        github_commit = stack.github_api.commit(stack.github_repo_name, sha)
+        stack.commits.create_from_github!(github_commit, attributes)
+      end
+    rescue ActiveRecord::RecordNotUnique
+      retry
+    end
+  end
+end