shipit-engine 0.20.0 → 0.21.0

data/app/models/shipit/deploy_spec/lerna_discovery.rb

@@ -0,0 +1,85 @@
+require 'json'
+
+module Shipit
+  class DeploySpec
+    module LernaDiscovery
+      def discover_dependencies_steps
+        discover_lerna_json || super
+      end
+
+      def discover_lerna_json
+        lerna_install if lerna?
+      end
+
+      def lerna_install
+        [js_command('install --no-progress'), 'node_modules/.bin/lerna bootstrap']
+      end
+
+      def discover_review_checklist
+        discover_lerna_checklist || super
+      end
+
+      def discover_lerna_checklist
+        if lerna?
+          [%(
+            <strong>Don't forget version and tag before publishing!</strong>
+            You can do this with:<br/>
+            <pre>
+            lerna publish --skip-npm
+            && git add -A
+            && git push --follow-tags
+            </pre>
+          )]
+        end
+      end
+
+      def lerna?
+        lerna_json.exist?
+      end
+
+      def lerna_json
+        file('lerna.json')
+      end
+
+      def lerna_version
+        lerna_config = lerna_json.read
+        JSON.parse(lerna_config)['version']
+      end
+
+      def discover_lerna_packages
+        publish_lerna_packages if lerna?
+      end
+
+      def discover_deploy_steps
+        discover_lerna_packages || super
+      end
+
+      def publish_lerna_packages
+        return publish_independent_packages if lerna_version == 'independent'
+        publish_fixed_version_packages
+      end
+
+      def publish_independent_packages
+        [
+          'assert-lerna-independent-version-tags',
+          'publish-lerna-independent-packages',
+        ]
+      end
+
+      def publish_fixed_version_packages
+        check_tags = 'assert-lerna-fixed-version-tag'
+        # `yarn publish` requires user input, so always use npm.
+        version = lerna_version
+        publish =
+          "node_modules/.bin/lerna publish " \
+          "--yes " \
+          "--skip-git " \
+          "--repo-version #{version} " \
+          "--force-publish=* " \
+          "--npm-tag #{dist_tag(version)}"
+
+        [check_tags, publish]
+      end
+    end
+  end
+end

data/app/models/shipit/deploy_spec/npm_discovery.rb

@@ -3,6 +3,12 @@ require 'json'
 module Shipit
   class DeploySpec
     module NpmDiscovery
+      # https://docs.npmjs.com/cli/publish
+      PUBLIC = 'public'.freeze
+      PRIVATE = 'restricted'.freeze
+      VALID_ACCESS = [PUBLIC, PRIVATE].freeze
+      NPM_REGISTRY = "https://registry.npmjs.org/".freeze
+
       def discover_dependencies_steps
         discover_package_json || super
       end
@@ -20,13 +26,19 @@ module Shipit
       end
 
       def discover_yarn_checklist
-        [%(<strong>Don't forget version and tag before publishing!</strong> You can do this with:<br/>
-          yarn version --new-version <strong>&lt;major|minor|patch&gt;</strong> && git push --tags</pre>)] if yarn?
+        if yarn?
+          [%(<strong>Don't forget version and tag before publishing!</strong> You can do this with:<br/>
+            yarn version --new-version <strong>&lt;major|minor|patch&gt;</strong>
+            && git push --follow-tags</pre>)]
+        end
       end
 
       def discover_npm_checklist
-        [%(<strong>Don't forget version and tag before publishing!</strong> You can do this with:<br/>
-          npm version <strong>&lt;major|minor|patch&gt;</strong> && git push --tags</pre>)] if npm?
+        if npm?
+          [%(<strong>Don't forget version and tag before publishing!</strong> You can do this with:<br/>
+            npm version <strong>&lt;major|minor|patch&gt;</strong>
+            && git push --follow-tags</pre>)]
+        end
       end
 
       def npm?
@@ -40,10 +52,23 @@ module Shipit
         JSON.parse(file.read)['private'].blank?
       end
 
+      def dist_tag(version)
+        # Pre-release SemVer tags such as 'beta', 'alpha', 'rc' and 'next'
+        # are treated as 'next' npm dist-tags.
+        # An 1.0.0-beta.1 would be installable using both:
+        # `yarn add package@1.0.0-beta.1` and `yarn add package@next`
+        return 'next' if ['-beta', '-alpha', '-rc', '-next'].any? { |tag| version.include? tag }
+        'latest'
+      end
+
       def package_json
         file('package.json')
       end
 
+      def package_json_contents
+        @package_json_contents ||= JSON.parse(package_json.read)
+      end
+
       def yarn?
         yarn_lock.exist? && public?
       end
@@ -60,11 +85,84 @@ module Shipit
         discover_npm_package || super
       end
 
+      def package_name
+        package_json_contents['name']
+      end
+
+      def package_version
+        package_json_contents['version']
+      end
+
+      def publish_config
+        package_json_contents['publishConfig']
+      end
+
+      def publish_config_access
+        config = publish_config
+
+        # default to private deploy when we enforce a publishConfig
+        if enforce_publish_config?
+          return PRIVATE if config.blank?
+          config['access'] || PRIVATE
+        end
+
+        return PUBLIC if config.blank?
+        config['access'] || PUBLIC
+      end
+
+      def scoped_package?
+        return false if Shipit.npm_org_scope.nil?
+        package_name.start_with?(Shipit.npm_org_scope)
+      end
+
+      def enforce_publish_config?
+        enforce = Shipit.enforce_publish_config
+        return false if enforce.nil? || enforce.to_s == "0"
+        true
+      end
+
+      def valid_publish_config?
+        return true unless enforce_publish_config?
+        return false if Shipit.private_npm_registry.nil?
+        return false if publish_config.blank?
+        return true if publish_config_access == PUBLIC
+
+        valid_publish_config_access? && private_scoped_package?
+      end
+
+      def valid_publish_config_access?
+        VALID_ACCESS.include?(publish_config_access)
+      end
+
+      # ensure private packages are scoped
+      def private_scoped_package?
+        publish_config_access == PRIVATE && scoped_package?
+      end
+
+      def local_npmrc
+        file(".npmrc")
+      end
+
+      def registry
+        scope = Shipit.npm_org_scope
+        prefix = scoped_package? ? "#{scope}:registry" : "registry"
+
+        if publish_config_access == PUBLIC
+          return "#{prefix}=#{NPM_REGISTRY}"
+        end
+
+        "#{prefix}=#{Shipit.private_npm_registry}"
+      end
+
       def publish_npm_package
+        return ['misconfigured-npm-publish-config'] unless valid_publish_config?
+
+        generate_npmrc = "generate-local-npmrc \"#{registry}\""
         check_tags = 'assert-npm-version-tag'
         # `yarn publish` requires user input, so always use npm.
-        publish = 'npm publish'
+        publish = "npm publish --tag #{dist_tag(package_version)} --access #{publish_config_access}"
 
+        return [check_tags, generate_npmrc, publish] if enforce_publish_config?
         [check_tags, publish]
       end
 

data/app/models/shipit/deploy_spec/pypi_discovery.rb

@@ -14,8 +14,10 @@ module Shipit
       end
 
       def discover_pypi_checklist
-        [%(<strong>Don't forget to add a tag before deploying!</strong> You can do this with:
-          git tag -a -m "Version <strong>x.y.z</strong>" v<strong>x.y.z</strong> && git push --tags)] if egg?
+        if egg?
+          [%(<strong>Don't forget to add a tag before deploying!</strong> You can do this with:
+            git tag -a -m "Version <strong>x.y.z</strong>" v<strong>x.y.z</strong> && git push --tags)]
+        end
       end
 
       def egg?

data/app/models/shipit/deploy_spec/rubygems_discovery.rb

@@ -14,8 +14,10 @@ module Shipit
       end
 
       def discover_gem_checklist
-        [%(<strong>Don't forget to add a tag before deploying!</strong> You can do this with:
-          git tag v<strong>x.y.z</strong> && git push --tags)] if gem?
+        if gem?
+          [%(<strong>Don't forget to add a tag before deploying!</strong> You can do this with:
+            git tag v<strong>x.y.z</strong> && git push --tags)]
+        end
       end
 
       def gem?

data/app/models/shipit/deploy_spec.rb

@@ -15,7 +15,7 @@ module Shipit
       end
 
       def bundle_path
-        Rails.root.join('data/bundler')
+        Rails.root.join('data', 'bundler')
       end
     end
 
@@ -135,13 +135,17 @@ module Shipit
     end
 
     def required_statuses
-      Array.wrap(config('ci', 'require'))
+      (Array.wrap(config('ci', 'require')) + blocking_statuses).uniq
     end
 
     def soft_failing_statuses
       Array.wrap(config('ci', 'allow_failures'))
     end
 
+    def blocking_statuses
+      Array.wrap(config('ci', 'blocking'))
+    end
+
     def pull_request_required_statuses
       if config('merge', 'require') || config('merge', 'ignore')
         Array.wrap(config('merge', 'require'))
@@ -167,6 +171,19 @@ module Shipit
       end
     end
 
+    def max_divergence_commits
+      config('merge', 'max_divergence', 'commits')
+    end
+
+    def max_divergence_age
+      if timeout = config('merge', 'max_divergence', 'age')
+        begin
+          Duration.parse(timeout)
+        rescue Duration::ParseError
+        end
+      end
+    end
+
     def review_checks
       config('review', 'checks') || []
     end

data/app/models/shipit/duration.rb

@@ -23,7 +23,7 @@ module Shipit
           raise ParseError, "not a duration: #{value.inspect}"
         end
         parts = []
-        UNITS.values.each do |unit|
+        UNITS.each_value do |unit|
           if value = match[unit]
             parts << [unit, value.to_i]
           end

data/app/models/shipit/github_status.rb

@@ -9,7 +9,7 @@ module Shipit
 
       def refresh_status
         Rails.cache.write(CACHE_KEY, Shipit.github_api.github_status)
-      rescue Net::OpenTimeout, Octokit::ServerError
+      rescue Faraday::Error, Octokit::ServerError
       end
     end
   end

data/app/models/shipit/hook.rb

@@ -31,8 +31,8 @@ module Shipit
     serialize :events, Shipit::CSVSerializer
 
     scope :global, -> { where(stack_id: nil) }
-    scope :scoped_to, -> (stack) { where(stack_id: stack.id) }
-    scope :for_stack, -> (stack_id) { where(stack_id: [nil, stack_id]) }
+    scope :scoped_to, ->(stack) { where(stack_id: stack.id) }
+    scope :for_stack, ->(stack_id) { where(stack_id: [nil, stack_id]) }
 
     class << self
       def emit(event, stack, payload)
@@ -80,9 +80,8 @@ module Shipit
     end
 
     def purge_old_deliveries!(keep: DELIVERIES_LOG_SIZE)
-      if cut_off_time = deliveries.order(created_at: :desc).limit(1).offset(keep).pluck(:created_at).first
-        deliveries.where('created_at > ?', cut_off_time).delete_all
-      end
+      delivery_ids = deliveries.sent.order(id: :desc).offset(keep).pluck(:id)
+      deliveries.where(id: delivery_ids).delete_all
     end
 
     private

data/app/models/shipit/output_chunk.rb

@@ -2,7 +2,7 @@ module Shipit
   class OutputChunk < ActiveRecord::Base
     belongs_to :task
 
-    scope :tail, -> (start) { order(id: :asc).where('id > ?', start || 0) }
+    scope :tail, ->(start) { order(id: :asc).where('id > ?', start || 0) }
 
     def text=(string)
       super(string.force_encoding(Encoding::UTF_8).scrub)

data/app/models/shipit/pull_request.rb

@@ -4,7 +4,7 @@ module Shipit
 
     WAITING_STATUSES = %w(fetching pending).freeze
     QUEUED_STATUSES = %w(pending revalidating).freeze
-    REJECTION_REASONS = %w(ci_failing merge_conflict).freeze
+    REJECTION_REASONS = %w(ci_failing merge_conflict requires_rebase).freeze
     InvalidTransition = Class.new(StandardError)
     NotReady = Class.new(StandardError)
 
@@ -37,6 +37,7 @@ module Shipit
 
     belongs_to :stack
     belongs_to :head, class_name: 'Shipit::Commit', optional: true
+    belongs_to :base_commit, class_name: 'Shipit::Commit', optional: true
     belongs_to :merge_requested_by, class_name: 'Shipit::User', optional: true
     has_one :merge_commit, class_name: 'Shipit::Commit'
 
@@ -102,7 +103,7 @@ module Shipit
     end
 
     def self.schedule_merges
-      Shipit::Stack.where(id: pending.uniq.pluck(:stack_id)).find_each(&:schedule_merges)
+      Shipit::Stack.where(merge_queue_enabled: true).find_each(&:schedule_merges)
     end
 
     def self.extract_number(stack, number_or_url)
@@ -112,7 +113,7 @@ module Shipit
       when %r{\Ahttps://#{Regexp.escape(Shipit.github_domain)}/([^/]+)/([^/]+)/pull/(\d+)}
         return unless $1.downcase == stack.repo_owner.downcase
         return unless $2.downcase == stack.repo_name.downcase
-        return $3.to_i
+        $3.to_i
       end
     end
 
@@ -146,7 +147,8 @@ module Shipit
 
     def reject_unless_mergeable!
       return reject!('merge_conflict') if merge_conflict?
-      return reject!('ci_failing') unless all_status_checks_passed?
+      return reject!('ci_failing') if any_status_checks_failed?
+      return reject!('requires_rebase') if stale?
       false
     end
 
@@ -154,10 +156,6 @@ module Shipit
       raise InvalidTransition unless pending?
 
       raise NotReady if not_mergeable_yet?
-      if need_revalidation?
-        revalidate!
-        return false
-      end
 
       Shipit.github_api.merge_pull_request(
         stack.github_repo_name,
@@ -184,9 +182,15 @@ module Shipit
     end
 
     def all_status_checks_passed?
+      return false unless head
       StatusChecker.new(head, head.statuses, stack.cached_deploy_spec).success?
     end
 
+    def any_status_checks_failed?
+      status = StatusChecker.new(head, head.statuses, stack.cached_deploy_spec)
+      status.failure? || status.error?
+    end
+
     def waiting?
       WAITING_STATUSES.include?(merge_status)
     end
@@ -221,6 +225,7 @@ module Shipit
       update!(github_pull_request: Shipit.github_api.pull_request(stack.github_repo_name, number))
       head.refresh_statuses!
       fetched! if fetching?
+      @comparison = nil
     end
 
     def github_pull_request=(github_pull_request)
@@ -234,6 +239,8 @@ module Shipit
       self.branch = github_pull_request.head.ref
       self.head = find_or_create_commit_from_github_by_sha!(github_pull_request.head.sha, detached: true)
       self.merged_at = github_pull_request.merged_at
+      self.base_ref = github_pull_request.base.ref
+      self.base_commit = find_or_create_commit_from_github_by_sha!(github_pull_request.base.sha, detached: true)
     end
 
     def merge_message
@@ -241,16 +248,30 @@ module Shipit
       "#{title}\n\nMerge-Requested-By: #{merge_requested_by.login}\n"
     end
 
-    private
-
-    if Rails.gem_version >= Gem::Version.new('5.1.0.beta1')
-      def record_merge_status_change
-        @merge_status_changed ||= saved_change_to_attribute?(:merge_status)
+    def stale?
+      return false unless base_commit
+      spec = stack.cached_deploy_spec
+      if max_branch_age = spec.max_divergence_age
+        return true if Time.now.utc - head.committed_at > max_branch_age
       end
-    else
-      def record_merge_status_change
-        @merge_status_changed ||= merge_status_changed?
+      if commit_count_limit = spec.max_divergence_commits
+        return true if comparison.behind_by > commit_count_limit
       end
+      false
+    end
+
+    def comparison
+      @comparison ||= Shipit.github_api.compare(
+        stack.github_repo_name,
+        base_ref,
+        head.sha,
+      )
+    end
+
+    private
+
+    def record_merge_status_change
+      @merge_status_changed ||= saved_change_to_attribute?(:merge_status)
     end
 
     def emit_hooks

data/app/models/shipit/stack.rb

@@ -44,6 +44,7 @@ module Shipit
 
     before_validation :update_defaults
     before_destroy :clear_local_files
+    before_save :set_locked_since
     after_commit :emit_lock_hooks
     after_commit :emit_added_hooks, on: :create
     after_commit :emit_updated_hooks, on: :update
@@ -198,14 +199,14 @@ module Shipit
       commits.reachable.first.try!(:sha)
     end
 
-    def merge_status(backlog_leniency_factor: 1.5)
+    def merge_status(backlog_leniency_factor: 2.0)
       return 'locked' if locked?
       return 'failure' if %w(failure error).freeze.include?(branch_status)
       return 'backlogged' if backlogged?(backlog_leniency_factor: backlog_leniency_factor)
       'success'
     end
 
-    def backlogged?(backlog_leniency_factor: 1.5)
+    def backlogged?(backlog_leniency_factor: 2.0)
       maximum_commits_per_deploy && (undeployed_commits_count > maximum_commits_per_deploy * backlog_leniency_factor)
     end
 
@@ -255,11 +256,7 @@ module Shipit
     end
 
     def last_deployed_commit
-      if deploy = last_successful_deploy
-        deploy.until_commit
-      else
-        NoDeployedCommit
-      end
+      last_successful_deploy.try!(:until_commit) || NoDeployedCommit
     end
 
     def deployable?
@@ -287,7 +284,7 @@ module Shipit
     end
 
     def base_path
-      Rails.root.join('data/stacks', repo_owner, repo_name, environment)
+      Rails.root.join('data', 'stacks', repo_owner, repo_name, environment)
     end
 
     def deploys_path
@@ -362,7 +359,6 @@ module Shipit
 
     def lock(reason, user)
       params = {lock_reason: reason, lock_author: user}
-      params[:locked_since] = Time.current if locked_since.nil?
       update!(params)
     end
 
@@ -384,8 +380,8 @@ module Shipit
     end
 
     delegate :plugins, :task_definitions, :hidden_statuses, :required_statuses, :soft_failing_statuses,
-             :deploy_variables, :filter_task_envs, :filter_deploy_envs, :maximum_commits_per_deploy,
-             :pause_between_deploys, to: :cached_deploy_spec
+             :blocking_statuses, :deploy_variables, :filter_task_envs, :filter_deploy_envs,
+             :maximum_commits_per_deploy, :pause_between_deploys, to: :cached_deploy_spec
 
     def monitoring?
       monitoring.present?
@@ -486,6 +482,16 @@ module Shipit
       self.branch = 'master' if branch.blank?
     end
 
+    def set_locked_since
+      return unless lock_reason_changed?
+
+      if lock_reason.blank?
+        self.locked_since = nil
+      else
+        self.locked_since ||= Time.now
+      end
+    end
+
     def schedule_merges_if_necessary
       if previous_changes.include?('lock_reason') && previous_changes['lock_reason'].last.blank?
         schedule_merges

data/app/models/shipit/status/common.rb

@@ -37,6 +37,10 @@ module Shipit
         commit.hidden_statuses.include?(context)
       end
 
+      def blocking?
+        !success? && commit.blocking_statuses.include?(context)
+      end
+
       def required?
         commit.required_statuses.include?(context)
       end

data/app/models/shipit/status/group.rb

@@ -52,6 +52,10 @@ module Shipit
         true
       end
 
+      def blocking?
+        statuses.any?(&:blocking?)
+      end
+
       private
 
       def reject_hidden(statuses)

data/app/models/shipit/task.rb

@@ -7,13 +7,14 @@ module Shipit
     PRESENCE_CHECK_TIMEOUT = 15
     ACTIVE_STATUSES = %w(pending running aborting).freeze
     COMPLETED_STATUSES = %w(success error failed flapping aborted).freeze
-    UNSUCCESSFUL_STATUSES = %w(error failed aborted flapping).freeze
+    UNSUCCESSFUL_STATUSES = %w(error failed aborted flapping timedout).freeze
 
     attr_accessor :pid
 
     belongs_to :deploy, foreign_key: :parent_id, required: false # required for fixtures
 
     belongs_to :user, optional: true
+    belongs_to :aborted_by, class_name: 'User', optional: true
     belongs_to :stack, counter_cache: true
     belongs_to :until_commit, class_name: 'Commit'
     belongs_to :since_commit, class_name: 'Commit'
@@ -56,11 +57,11 @@ module Shipit
         task.started_at ||= Time.now.utc
       end
 
-      before_transition any => %i(success failed error) do |task|
+      before_transition any => %i(success failed error timedout) do |task|
         task.ended_at ||= Time.now.utc
       end
 
-      after_transition any => %i(success failed error) do |task|
+      after_transition any => %i(success failed error timedout) do |task|
         task.async_refresh_deployed_revision
       end
 
@@ -88,6 +89,10 @@ module Shipit
         transition all => :error
       end
 
+      event :giveup do # :timeout would cause a name clash
+        transition all => :timedout
+      end
+
       event :aborting do
         transition all - %i(aborted) => :aborting
       end
@@ -97,7 +102,7 @@ module Shipit
       end
 
       event :flap do
-        transition %i(failed error success) => :flapping
+        transition %i(failed error timedout success) => :flapping
       end
 
       state :pending
@@ -105,6 +110,7 @@ module Shipit
       state :failed
       state :success
       state :error
+      state :timedout
       state :aborting
       state :aborted
       state :flapping
@@ -114,15 +120,21 @@ module Shipit
       status.in?(ACTIVE_STATUSES)
     end
 
-    def report_failure!(_error)
+    def report_failure!(error)
       reload
       if aborting?
         aborted!
+      elsif error.respond_to?(:exit_code) && Shipit.timeout_exit_codes.include?(error.exit_code)
+        giveup!
       else
         failure!
       end
     end
 
+    def report_timeout!(_error)
+      giveup!
+    end
+
     def report_error!(error)
       write("#{error.class}: #{error.message}\n\t#{error.backtrace.join("\n\t")}\n")
       error!
@@ -177,7 +189,7 @@ module Shipit
     def output
       gzip = self[:gzip_output]
 
-      if gzip.nil? || gzip.empty?
+      if gzip.blank?
         ''
       else
         ActiveSupport::Gzip.decompress(gzip)
@@ -236,8 +248,8 @@ module Shipit
       end
     end
 
-    def abort!(rollback_once_aborted: false)
-      update!(rollback_once_aborted: rollback_once_aborted)
+    def abort!(rollback_once_aborted: false, aborted_by:)
+      update!(rollback_once_aborted: rollback_once_aborted, aborted_by_id: aborted_by.id)
 
       if alive?
         aborting