shipit-engine 0.20.1 → 0.21.0

data/app/models/shipit/commit_deployment.rb

@@ -22,7 +22,7 @@ module Shipit
 
       response = begin
         create_deployment_on_github(author.github_api)
-      rescue Octokit::NotFound, Octokit::Forbidden
+      rescue Octokit::ClientError
         raise if Shipit.github_api == author.github_api
         # If the deploy author didn't gave us the permission to create the deployment we falback the the main shipit
         # user.

data/app/models/shipit/commit_deployment_status.rb

@@ -10,7 +10,7 @@ module Shipit
       return if github_id?
       response = begin
         create_status_on_github(author.github_api)
-      rescue Octokit::NotFound, Octokit::Forbidden
+      rescue Octokit::ClientError
         raise if Shipit.github_api == author.github_api
         # If the deploy author didn't gave us the permission to create the deployment we falback the the main shipit
         # user.

data/app/models/shipit/deploy_spec.rb

@@ -15,7 +15,7 @@ module Shipit
       end
 
       def bundle_path
-        Rails.root.join('data/bundler')
+        Rails.root.join('data', 'bundler')
       end
     end
 
@@ -135,13 +135,17 @@ module Shipit
     end
 
     def required_statuses
-      Array.wrap(config('ci', 'require'))
+      (Array.wrap(config('ci', 'require')) + blocking_statuses).uniq
     end
 
     def soft_failing_statuses
       Array.wrap(config('ci', 'allow_failures'))
     end
 
+    def blocking_statuses
+      Array.wrap(config('ci', 'blocking'))
+    end
+
     def pull_request_required_statuses
       if config('merge', 'require') || config('merge', 'ignore')
         Array.wrap(config('merge', 'require'))
@@ -167,6 +171,19 @@ module Shipit
       end
     end
 
+    def max_divergence_commits
+      config('merge', 'max_divergence', 'commits')
+    end
+
+    def max_divergence_age
+      if timeout = config('merge', 'max_divergence', 'age')
+        begin
+          Duration.parse(timeout)
+        rescue Duration::ParseError
+        end
+      end
+    end
+
     def review_checks
       config('review', 'checks') || []
     end

data/app/models/shipit/deploy_spec/bundler_discovery.rb

@@ -54,16 +54,7 @@ module Shipit
       end
 
       def coerce_task_definition(config)
-        coerced_steps = Array(config['steps']).map do |command|
-          should_prepend_bundle_exec?(command) ? bundle_exec(command) : command
-        end
-        config.merge('steps' => coerced_steps)
-      end
-
-      private
-
-      def should_prepend_bundle_exec?(command)
-        Shipit.automatically_prepend_bundle_exec && !command.start_with?('bundle exec')
+        config.merge('steps' => Array(config['steps']))
       end
     end
   end

data/app/models/shipit/deploy_spec/file_system.rb

@@ -2,6 +2,7 @@ module Shipit
   class DeploySpec
     class FileSystem < DeploySpec
       include NpmDiscovery
+      include LernaDiscovery
       include PypiDiscovery
       include RubygemsDiscovery
       include CapistranoDiscovery
@@ -33,11 +34,16 @@ module Shipit
             'require' => pull_request_required_statuses,
             'ignore' => pull_request_ignored_statuses,
             'revalidate_after' => revalidate_pull_requests_after.try!(:to_i),
+            'max_divergence' => {
+              'commits' => max_divergence_commits.try!(:to_i),
+              'age' => max_divergence_age.try!(:to_i),
+            },
           },
           'ci' => {
             'hide' => hidden_statuses,
             'allow_failures' => soft_failing_statuses,
             'require' => required_statuses,
+            'blocking' => blocking_statuses,
           },
           'machine' => {
             'environment' => discover_machine_env.merge(machine_env),

data/app/models/shipit/deploy_spec/kubernetes_discovery.rb

@@ -26,7 +26,7 @@ module Shipit
       private
 
       def discover_kubernetes
-        return unless kube_config.present?
+        return if kube_config.blank?
 
         cmd = ["kubernetes-deploy"]
         if kube_config['template_dir']

data/app/models/shipit/deploy_spec/lerna_discovery.rb

@@ -0,0 +1,85 @@
+require 'json'
+
+module Shipit
+  class DeploySpec
+    module LernaDiscovery
+      def discover_dependencies_steps
+        discover_lerna_json || super
+      end
+
+      def discover_lerna_json
+        lerna_install if lerna?
+      end
+
+      def lerna_install
+        [js_command('install --no-progress'), 'node_modules/.bin/lerna bootstrap']
+      end
+
+      def discover_review_checklist
+        discover_lerna_checklist || super
+      end
+
+      def discover_lerna_checklist
+        if lerna?
+          [%(
+            <strong>Don't forget version and tag before publishing!</strong>
+            You can do this with:<br/>
+            <pre>
+            lerna publish --skip-npm
+            && git add -A
+            && git push --follow-tags
+            </pre>
+          )]
+        end
+      end
+
+      def lerna?
+        lerna_json.exist?
+      end
+
+      def lerna_json
+        file('lerna.json')
+      end
+
+      def lerna_version
+        lerna_config = lerna_json.read
+        JSON.parse(lerna_config)['version']
+      end
+
+      def discover_lerna_packages
+        publish_lerna_packages if lerna?
+      end
+
+      def discover_deploy_steps
+        discover_lerna_packages || super
+      end
+
+      def publish_lerna_packages
+        return publish_independent_packages if lerna_version == 'independent'
+        publish_fixed_version_packages
+      end
+
+      def publish_independent_packages
+        [
+          'assert-lerna-independent-version-tags',
+          'publish-lerna-independent-packages',
+        ]
+      end
+
+      def publish_fixed_version_packages
+        check_tags = 'assert-lerna-fixed-version-tag'
+        # `yarn publish` requires user input, so always use npm.
+        version = lerna_version
+        publish =
+          "node_modules/.bin/lerna publish " \
+          "--yes " \
+          "--skip-git " \
+          "--repo-version #{version} " \
+          "--force-publish=* " \
+          "--npm-tag #{dist_tag(version)}"
+
+        [check_tags, publish]
+      end
+    end
+  end
+end

data/app/models/shipit/deploy_spec/npm_discovery.rb

@@ -3,6 +3,12 @@ require 'json'
 module Shipit
   class DeploySpec
     module NpmDiscovery
+      # https://docs.npmjs.com/cli/publish
+      PUBLIC = 'public'.freeze
+      PRIVATE = 'restricted'.freeze
+      VALID_ACCESS = [PUBLIC, PRIVATE].freeze
+      NPM_REGISTRY = "https://registry.npmjs.org/".freeze
+
       def discover_dependencies_steps
         discover_package_json || super
       end
@@ -20,13 +26,19 @@ module Shipit
       end
 
       def discover_yarn_checklist
-        [%(<strong>Don't forget version and tag before publishing!</strong> You can do this with:<br/>
-          yarn version --new-version <strong>&lt;major|minor|patch&gt;</strong> && git push --tags</pre>)] if yarn?
+        if yarn?
+          [%(<strong>Don't forget version and tag before publishing!</strong> You can do this with:<br/>
+            yarn version --new-version <strong>&lt;major|minor|patch&gt;</strong>
+            && git push --follow-tags</pre>)]
+        end
       end
 
       def discover_npm_checklist
-        [%(<strong>Don't forget version and tag before publishing!</strong> You can do this with:<br/>
-          npm version <strong>&lt;major|minor|patch&gt;</strong> && git push --tags</pre>)] if npm?
+        if npm?
+          [%(<strong>Don't forget version and tag before publishing!</strong> You can do this with:<br/>
+            npm version <strong>&lt;major|minor|patch&gt;</strong>
+            && git push --follow-tags</pre>)]
+        end
       end
 
       def npm?
@@ -40,10 +52,23 @@ module Shipit
         JSON.parse(file.read)['private'].blank?
       end
 
+      def dist_tag(version)
+        # Pre-release SemVer tags such as 'beta', 'alpha', 'rc' and 'next'
+        # are treated as 'next' npm dist-tags.
+        # An 1.0.0-beta.1 would be installable using both:
+        # `yarn add package@1.0.0-beta.1` and `yarn add package@next`
+        return 'next' if ['-beta', '-alpha', '-rc', '-next'].any? { |tag| version.include? tag }
+        'latest'
+      end
+
       def package_json
         file('package.json')
       end
 
+      def package_json_contents
+        @package_json_contents ||= JSON.parse(package_json.read)
+      end
+
       def yarn?
         yarn_lock.exist? && public?
       end
@@ -60,11 +85,84 @@ module Shipit
         discover_npm_package || super
       end
 
+      def package_name
+        package_json_contents['name']
+      end
+
+      def package_version
+        package_json_contents['version']
+      end
+
+      def publish_config
+        package_json_contents['publishConfig']
+      end
+
+      def publish_config_access
+        config = publish_config
+
+        # default to private deploy when we enforce a publishConfig
+        if enforce_publish_config?
+          return PRIVATE if config.blank?
+          config['access'] || PRIVATE
+        end
+
+        return PUBLIC if config.blank?
+        config['access'] || PUBLIC
+      end
+
+      def scoped_package?
+        return false if Shipit.npm_org_scope.nil?
+        package_name.start_with?(Shipit.npm_org_scope)
+      end
+
+      def enforce_publish_config?
+        enforce = Shipit.enforce_publish_config
+        return false if enforce.nil? || enforce.to_s == "0"
+        true
+      end
+
+      def valid_publish_config?
+        return true unless enforce_publish_config?
+        return false if Shipit.private_npm_registry.nil?
+        return false if publish_config.blank?
+        return true if publish_config_access == PUBLIC
+
+        valid_publish_config_access? && private_scoped_package?
+      end
+
+      def valid_publish_config_access?
+        VALID_ACCESS.include?(publish_config_access)
+      end
+
+      # ensure private packages are scoped
+      def private_scoped_package?
+        publish_config_access == PRIVATE && scoped_package?
+      end
+
+      def local_npmrc
+        file(".npmrc")
+      end
+
+      def registry
+        scope = Shipit.npm_org_scope
+        prefix = scoped_package? ? "#{scope}:registry" : "registry"
+
+        if publish_config_access == PUBLIC
+          return "#{prefix}=#{NPM_REGISTRY}"
+        end
+
+        "#{prefix}=#{Shipit.private_npm_registry}"
+      end
+
       def publish_npm_package
+        return ['misconfigured-npm-publish-config'] unless valid_publish_config?
+
+        generate_npmrc = "generate-local-npmrc \"#{registry}\""
         check_tags = 'assert-npm-version-tag'
         # `yarn publish` requires user input, so always use npm.
-        publish = 'npm publish'
+        publish = "npm publish --tag #{dist_tag(package_version)} --access #{publish_config_access}"
 
+        return [check_tags, generate_npmrc, publish] if enforce_publish_config?
         [check_tags, publish]
       end
 

data/app/models/shipit/deploy_spec/pypi_discovery.rb

@@ -14,8 +14,10 @@ module Shipit
       end
 
       def discover_pypi_checklist
-        [%(<strong>Don't forget to add a tag before deploying!</strong> You can do this with:
-          git tag -a -m "Version <strong>x.y.z</strong>" v<strong>x.y.z</strong> && git push --tags)] if egg?
+        if egg?
+          [%(<strong>Don't forget to add a tag before deploying!</strong> You can do this with:
+            git tag -a -m "Version <strong>x.y.z</strong>" v<strong>x.y.z</strong> && git push --tags)]
+        end
       end
 
       def egg?

data/app/models/shipit/deploy_spec/rubygems_discovery.rb

@@ -14,8 +14,10 @@ module Shipit
       end
 
       def discover_gem_checklist
-        [%(<strong>Don't forget to add a tag before deploying!</strong> You can do this with:
-          git tag v<strong>x.y.z</strong> && git push --tags)] if gem?
+        if gem?
+          [%(<strong>Don't forget to add a tag before deploying!</strong> You can do this with:
+            git tag v<strong>x.y.z</strong> && git push --tags)]
+        end
       end
 
       def gem?

data/app/models/shipit/duration.rb

@@ -23,7 +23,7 @@ module Shipit
           raise ParseError, "not a duration: #{value.inspect}"
         end
         parts = []
-        UNITS.values.each do |unit|
+        UNITS.each_value do |unit|
           if value = match[unit]
             parts << [unit, value.to_i]
           end

data/app/models/shipit/github_status.rb

@@ -9,7 +9,7 @@ module Shipit
 
       def refresh_status
         Rails.cache.write(CACHE_KEY, Shipit.github_api.github_status)
-      rescue Net::OpenTimeout, Octokit::ServerError
+      rescue Faraday::Error, Octokit::ServerError
       end
     end
   end

data/app/models/shipit/hook.rb

@@ -31,8 +31,8 @@ module Shipit
     serialize :events, Shipit::CSVSerializer
 
     scope :global, -> { where(stack_id: nil) }
-    scope :scoped_to, -> (stack) { where(stack_id: stack.id) }
-    scope :for_stack, -> (stack_id) { where(stack_id: [nil, stack_id]) }
+    scope :scoped_to, ->(stack) { where(stack_id: stack.id) }
+    scope :for_stack, ->(stack_id) { where(stack_id: [nil, stack_id]) }
 
     class << self
       def emit(event, stack, payload)
@@ -80,9 +80,8 @@ module Shipit
     end
 
     def purge_old_deliveries!(keep: DELIVERIES_LOG_SIZE)
-      if cut_off_time = deliveries.order(created_at: :desc).limit(1).offset(keep).pluck(:created_at).first
-        deliveries.where('created_at <= ?', cut_off_time).delete_all
-      end
+      delivery_ids = deliveries.sent.order(id: :desc).offset(keep).pluck(:id)
+      deliveries.where(id: delivery_ids).delete_all
     end
 
     private

data/app/models/shipit/output_chunk.rb

@@ -2,7 +2,7 @@ module Shipit
   class OutputChunk < ActiveRecord::Base
     belongs_to :task
 
-    scope :tail, -> (start) { order(id: :asc).where('id > ?', start || 0) }
+    scope :tail, ->(start) { order(id: :asc).where('id > ?', start || 0) }
 
     def text=(string)
       super(string.force_encoding(Encoding::UTF_8).scrub)

data/app/models/shipit/pull_request.rb

@@ -4,7 +4,7 @@ module Shipit
 
     WAITING_STATUSES = %w(fetching pending).freeze
     QUEUED_STATUSES = %w(pending revalidating).freeze
-    REJECTION_REASONS = %w(ci_failing merge_conflict).freeze
+    REJECTION_REASONS = %w(ci_failing merge_conflict requires_rebase).freeze
     InvalidTransition = Class.new(StandardError)
     NotReady = Class.new(StandardError)
 
@@ -37,6 +37,7 @@ module Shipit
 
     belongs_to :stack
     belongs_to :head, class_name: 'Shipit::Commit', optional: true
+    belongs_to :base_commit, class_name: 'Shipit::Commit', optional: true
     belongs_to :merge_requested_by, class_name: 'Shipit::User', optional: true
     has_one :merge_commit, class_name: 'Shipit::Commit'
 
@@ -112,7 +113,7 @@ module Shipit
       when %r{\Ahttps://#{Regexp.escape(Shipit.github_domain)}/([^/]+)/([^/]+)/pull/(\d+)}
         return unless $1.downcase == stack.repo_owner.downcase
         return unless $2.downcase == stack.repo_name.downcase
-        return $3.to_i
+        $3.to_i
       end
     end
 
@@ -146,7 +147,8 @@ module Shipit
 
     def reject_unless_mergeable!
       return reject!('merge_conflict') if merge_conflict?
-      return reject!('ci_failing') unless all_status_checks_passed?
+      return reject!('ci_failing') if any_status_checks_failed?
+      return reject!('requires_rebase') if stale?
       false
     end
 
@@ -154,10 +156,6 @@ module Shipit
       raise InvalidTransition unless pending?
 
       raise NotReady if not_mergeable_yet?
-      if need_revalidation?
-        revalidate!
-        return false
-      end
 
       Shipit.github_api.merge_pull_request(
         stack.github_repo_name,
@@ -184,9 +182,15 @@ module Shipit
     end
 
     def all_status_checks_passed?
+      return false unless head
       StatusChecker.new(head, head.statuses, stack.cached_deploy_spec).success?
     end
 
+    def any_status_checks_failed?
+      status = StatusChecker.new(head, head.statuses, stack.cached_deploy_spec)
+      status.failure? || status.error?
+    end
+
     def waiting?
       WAITING_STATUSES.include?(merge_status)
     end
@@ -221,6 +225,7 @@ module Shipit
       update!(github_pull_request: Shipit.github_api.pull_request(stack.github_repo_name, number))
       head.refresh_statuses!
       fetched! if fetching?
+      @comparison = nil
     end
 
     def github_pull_request=(github_pull_request)
@@ -234,6 +239,8 @@ module Shipit
       self.branch = github_pull_request.head.ref
       self.head = find_or_create_commit_from_github_by_sha!(github_pull_request.head.sha, detached: true)
       self.merged_at = github_pull_request.merged_at
+      self.base_ref = github_pull_request.base.ref
+      self.base_commit = find_or_create_commit_from_github_by_sha!(github_pull_request.base.sha, detached: true)
     end
 
     def merge_message
@@ -241,16 +248,30 @@ module Shipit
       "#{title}\n\nMerge-Requested-By: #{merge_requested_by.login}\n"
     end
 
-    private
-
-    if Rails.gem_version >= Gem::Version.new('5.1.0.beta1')
-      def record_merge_status_change
-        @merge_status_changed ||= saved_change_to_attribute?(:merge_status)
+    def stale?
+      return false unless base_commit
+      spec = stack.cached_deploy_spec
+      if max_branch_age = spec.max_divergence_age
+        return true if Time.now.utc - head.committed_at > max_branch_age
       end
-    else
-      def record_merge_status_change
-        @merge_status_changed ||= merge_status_changed?
+      if commit_count_limit = spec.max_divergence_commits
+        return true if comparison.behind_by > commit_count_limit
       end
+      false
+    end
+
+    def comparison
+      @comparison ||= Shipit.github_api.compare(
+        stack.github_repo_name,
+        base_ref,
+        head.sha,
+      )
+    end
+
+    private
+
+    def record_merge_status_change
+      @merge_status_changed ||= saved_change_to_attribute?(:merge_status)
     end
 
     def emit_hooks