shipit-engine 0.33.0 → 0.34.0

data/app/models/shipit/commit_checks.rb

@@ -10,14 +10,14 @@ module Shipit
     end
 
     def synchronize(&block)
-      @lock ||= Redis::Lock.new('lock', redis, expiration: 1, timeout: 2)
+      @lock ||= Redis::Lock.new(key('lock'), Shipit.redis, expiration: 1, timeout: 2)
       @lock.lock(&block)
     end
 
     def schedule
-      return false if redis.get('status').present?
+      return false if Shipit.redis.get(key('status')).present?
       synchronize do
-        return false if redis.get('status').present?
+        return false if Shipit.redis.get(key('status')).present?
 
         initialize_redis_state
       end
@@ -26,34 +26,34 @@ module Shipit
     end
 
     def initialize_redis_state
-      redis.pipelined do
-        redis.set('output', '', ex: OUTPUT_TTL)
-        redis.set('status', 'scheduled', ex: OUTPUT_TTL)
-      end
+      Shipit.redis.set(key('status'), 'scheduled', ex: OUTPUT_TTL)
       @status = 'scheduled'
     end
 
     def status
-      @status ||= redis.get('status')
+      @status ||= Shipit.redis.get(key('status'))
     end
 
     def status=(status)
-      redis.set('status', status)
+      Shipit.redis.set(key('status'), status)
       @status = status
     end
 
     def output(since: 0)
-      redis.getrange('output', since, -1)
+      Shipit.redis.getrange(key('output'), since, -1)
     end
 
     def write(output)
-      redis.append('output', output)
+      Shipit.redis.pipelined do
+        Shipit.redis.append(key('output'), output)
+        Shipit.redis.expire(key('output'), OUTPUT_TTL)
+      end
     end
 
     private
 
-    def redis
-      @redis ||= Shipit.redis("commit:#{commit.id}:checks")
+    def key(key)
+      "commit:#{commit.id}:checks:#{key}"
     end
   end
 end

data/app/models/shipit/commit_deployment.rb

@@ -20,15 +20,15 @@ module Shipit
       return if github_id?
 
       response = begin
-        create_deployment_on_github(author.github_api)
+        create_deployment_on_github(stack.github_api)
       rescue Octokit::ClientError
-        raise if Shipit.github.api == author.github_api
+        raise if Shipit.github(organization: stack.repository.owner).api == stack.github_api
         # If the deploy author didn't gave us the permission to create the deployment we falback the the main shipit
         # user.
         #
         # Octokit currently raise NotFound, but I'm convinced it should be Forbidden if the user can see the repository.
         # So to be future proof I catch boths.
-        create_deployment_on_github(Shipit.github.api)
+        create_deployment_on_github(stack.github_api)
       end
       update!(github_id: response.id, api_url: response.url)
     end

data/app/models/shipit/commit_deployment_status.rb

@@ -12,15 +12,15 @@ module Shipit
     def create_on_github!
       return if github_id?
       response = begin
-        create_status_on_github(author.github_api)
+        create_status_on_github(stack.github_api)
       rescue Octokit::ClientError
-        raise if Shipit.github.api == author.github_api
+        raise if Shipit.github(organization: stack.repository.owner).api == stack.github_api
         # If the deploy author didn't gave us the permission to create the deployment we falback the the main shipit
         # user.
         #
         # Octokit currently raise NotFound, but I'm convinced it should be Forbidden if the user can see the repository.
         # So to be future proof I catch boths.
-        create_status_on_github(Shipit.github.api)
+        create_status_on_github(stack.github_api)
       end
       update!(github_id: response.id, api_url: response.url)
     end

data/app/models/shipit/deploy.rb

@@ -15,8 +15,8 @@ module Shipit
       after_transition any => any, do: :update_last_deploy_time
     end
 
-    belongs_to :until_commit, class_name: 'Commit', required: true, inverse_of: :deploys
-    belongs_to :since_commit, class_name: 'Commit', required: true, inverse_of: :deploys
+    belongs_to :until_commit, class_name: 'Commit', required: true
+    belongs_to :since_commit, class_name: 'Commit', required: true
     has_many :commit_deployments, dependent: :destroy, inverse_of: :task, foreign_key: :task_id do
       GITHUB_STATUSES = {
         'pending' => 'pending',
@@ -97,14 +97,16 @@ module Shipit
     end
 
     # Rolls the stack back to this deploy
-    def trigger_rollback(user = AnonymousUser.new, env: nil, force: false)
+    def trigger_rollback(user = AnonymousUser.new, env: nil, force: false, lock: true)
       rollback = build_rollback(user, env: env, force: force)
       rollback.save!
       rollback.enqueue
 
-      lock_reason = "A rollback for #{rollback.since_commit.sha} has been triggered. " \
-        "Please make sure the reason for the rollback has been addressed before deploying again."
-      stack.update!(lock_reason: lock_reason, lock_author_id: user.id)
+      if lock
+        lock_reason = "A rollback for #{rollback.since_commit.sha} has been triggered. " \
+          "Please make sure the reason for the rollback has been addressed before deploying again."
+        stack.update!(lock_reason: lock_reason, lock_author_id: user.id)
+      end
 
       rollback
     end
@@ -209,7 +211,7 @@ module Shipit
     end
 
     def report_complete!
-      if stack.release_status? && stack.release_status_delay.positive?
+      if stack.release_status? && !stack.release_status_delay.zero?
         enter_validation!
       else
         super
@@ -269,10 +271,13 @@ module Shipit
       when 'aborted', 'aborting'
         append_release_status('failure', "The deploy on #{stack.environment} was canceled")
       when 'validating'
-        if stack.release_status_delay.positive?
-          append_release_status('pending', "The deploy on #{stack.environment} succeeded")
-          MarkDeployHealthyJob.set(wait: stack.release_status_delay).perform_later(self)
-        end
+        append_release_status(
+          'pending',
+          "The deploy on #{stack.environment} succeeded"
+        ) unless stack.release_status_delay.zero?
+
+        MarkDeployHealthyJob.set(wait: stack.release_status_delay)
+          .perform_later(self) if stack.release_status_delay.positive?
       when 'success'
         if stack.release_status_delay.zero?
           append_release_status('success', "The deploy on #{stack.environment} succeeded")

data/app/models/shipit/deploy_spec/lerna_discovery.rb

@@ -76,10 +76,18 @@ module Shipit
       end
 
       def publish_independent_packages
-        [
-          'assert-lerna-independent-version-tags',
-          'publish-lerna-independent-packages',
-        ]
+        command = if lerna_lerna >= LATEST_MAJOR_VERSION
+          [
+            'assert-lerna-independent-version-tags',
+            'publish-lerna-independent-packages',
+          ]
+        else
+          [
+            'assert-lerna-independent-version-tags',
+            'publish-lerna-independent-packages-legacy',
+          ]
+        end
+        command
       end
 
       def publish_fixed_version_packages

data/app/models/shipit/duration.rb

@@ -20,6 +20,8 @@ module Shipit
 
     class << self
       def parse(value)
+        return new(-1) if value.to_s == "-1"
+
         unless match = FORMAT.match(value.to_s)
           raise ParseError, "not a duration: #{value.inspect}"
         end

data/app/models/shipit/hook.rb

@@ -1,12 +1,28 @@
 # frozen_string_literal: true
 module Shipit
   class Hook < Record
+    class DeliverySigner
+      attr_reader :secret
+
+      ALGORITHM = 'sha256'
+
+      def initialize(secret)
+        @secret = secret
+      end
+
+      def sign(payload)
+        hmac = OpenSSL::HMAC.hexdigest(ALGORITHM, secret, payload)
+        "#{ALGORITHM}=#{hmac}"
+      end
+    end
+
     class DeliverySpec
-      def initialize(event:, url:, content_type:, payload:)
+      def initialize(event:, url:, content_type:, payload:, secret:)
         @event = event
         @url = url
         @content_type = content_type
         @payload = payload
+        @secret = secret
       end
 
       def send!
@@ -15,7 +31,7 @@ module Shipit
 
       private
 
-      attr_reader :event, :url, :content_type, :payload
+      attr_reader :event, :url, :content_type, :payload, :secret
 
       def http
         Faraday::Connection.new do |connection|
@@ -29,9 +45,16 @@ module Shipit
           'User-Agent' => 'Shipit Webhook',
           'Content-Type' => content_type,
           'X-Shipit-Event' => event,
+          'X-Shipit-Signature' => signature,
           'Accept' => '*/*',
         }
       end
+
+      def signature
+        return nil if secret.blank?
+
+        DeliverySigner.new(secret).sign(payload)
+      end
     end
 
     default_scope { order :id }
@@ -119,6 +142,7 @@ module Shipit
         url: delivery_url,
         content_type: CONTENT_TYPES[content_type],
         payload: serialize_payload(payload),
+        secret: secret,
       )
     end
 

data/app/models/shipit/merge_request.rb

@@ -110,10 +110,11 @@ module Shipit
     end
 
     def self.extract_number(stack, number_or_url)
+      org = stack.repository.owner
       case number_or_url
       when /\A#?(\d+)\z/
         $1.to_i
-      when %r{\Ahttps://#{Regexp.escape(Shipit.github.domain)}/([^/]+)/([^/]+)/pull/(\d+)}
+      when %r{\Ahttps://#{Regexp.escape(Shipit.github(organization: org).domain)}/([^/]+)/([^/]+)/pull/(\d+)}
         return unless $1.downcase == stack.repo_owner.downcase
         return unless $2.downcase == stack.repo_name.downcase
         $3.to_i
@@ -161,7 +162,7 @@ module Shipit
 
       raise NotReady if not_mergeable_yet?
 
-      Shipit.github.api.merge_pull_request(
+      stack.github_api.merge_pull_request(
         stack.github_repo_name,
         number,
         merge_message,
@@ -170,8 +171,8 @@ module Shipit
         merge_method: stack.merge_method,
       )
       begin
-        if Shipit.github.api.pull_requests(stack.github_repo_name, base: branch).empty?
-          Shipit.github.api.delete_branch(stack.github_repo_name, branch)
+        if stack.github_api.pull_requests(stack.github_repo_name, base: branch).empty?
+          stack.github_api.delete_branch(stack.github_repo_name, branch)
         end
       rescue Octokit::UnprocessableEntity
         # branch was already deleted somehow
@@ -230,8 +231,9 @@ module Shipit
     end
 
     def refresh!
-      update!(github_pull_request: Shipit.github.api.pull_request(stack.github_repo_name, number))
+      update!(github_pull_request: stack.github_api.pull_request(stack.github_repo_name, number))
       head.refresh_statuses!
+      head.refresh_check_runs!
       fetched! if fetching?
       @comparison = nil
     end
@@ -269,7 +271,7 @@ module Shipit
     end
 
     def comparison
-      @comparison ||= Shipit.github.api.compare(
+      @comparison ||= stack.github_api.compare(
         stack.github_repo_name,
         base_ref,
         head.sha,
@@ -292,7 +294,7 @@ module Shipit
       if commit = stack.commits.by_sha(sha)
         commit
       else
-        github_commit = Shipit.github.api.commit(stack.github_repo_name, sha)
+        github_commit = stack.github_api.commit(stack.github_repo_name, sha)
         stack.commits.create_from_github!(github_commit, attributes)
       end
     rescue ActiveRecord::RecordNotUnique

data/app/models/shipit/pull_request.rb

@@ -53,7 +53,7 @@ module Shipit
       if commit = stack.commits.by_sha(sha)
         commit
       else
-        github_commit = Shipit.github.api.commit(stack.github_repo_name, sha)
+        github_commit = stack.github_api.commit(stack.github_repo_name, sha)
         stack.commits.create_from_github!(github_commit)
       end
     rescue ActiveRecord::RecordNotUnique

data/app/models/shipit/release_status.rb

@@ -25,7 +25,7 @@ module Shipit
     private
 
     def create_status_on_github
-      Shipit.github.api.create_status(
+      stack.github_api.create_status(
         stack.github_repo_name,
         commit.sha,
         state,

data/app/models/shipit/repository.rb

@@ -38,7 +38,7 @@ module Shipit
     private_constant :NAME_MAX_SIZE
 
     validates :name, uniqueness: { scope: %i(owner), case_sensitive: false,
-                                   message: 'cannot be used more than once' }
+                                   message: 'cannot be used more than once', }
     validates :owner, :name, presence: true, ascii_only: true
     validates :owner, format: { with: /\A[a-z0-9_\-\.]+\z/ }, length: { maximum: OWNER_MAX_SIZE }
     validates :name, format: { with: /\A[a-z0-9_\-\.]+\z/ }, length: { maximum: NAME_MAX_SIZE }
@@ -67,7 +67,7 @@ module Shipit
     end
 
     def http_url
-      Shipit.github.url(full_name)
+      github_app.url(full_name)
     end
 
     def full_name
@@ -75,7 +75,7 @@ module Shipit
     end
 
     def git_url
-      "https://#{Shipit.github.domain}/#{owner}/#{name}.git"
+      "https://#{github_app.domain}/#{owner}/#{name}.git"
     end
 
     def schedule_for_destroy!
@@ -93,5 +93,11 @@ module Shipit
         name: repo_name.downcase,
       ).first!
     end
+
+    protected
+
+    def github_app
+      Shipit.github(organization: owner)
+    end
   end
 end

data/app/models/shipit/review_stack.rb

@@ -7,8 +7,8 @@ module Shipit
         "archived_since > :earliest AND archived_since < :latest",
         earliest: 1.day.ago,
         latest: 1.hour.ago
-      ).each do |review_stack|
-        Shipit::ClearGitCacheJob.perform_later(review_stack)
+      ).find_each do |review_stack|
+        review_stack.clear_local_files
       end
     end
 
@@ -22,6 +22,20 @@ module Shipit
       end
     end
 
+    def update_latest_deployed_ref
+      # noop: last deployed ref is useless for review stacks
+    end
+
+    model_name.class_eval do
+      def route_key
+        "stacks"
+      end
+
+      def singular_route_key
+        "stack"
+      end
+    end
+
     has_one :pull_request, foreign_key: :stack_id
 
     after_commit :emit_added_hooks, on: :create

data/app/models/shipit/stack.rb

@@ -32,9 +32,9 @@ module Shipit
     has_many :deploys
     has_many :rollbacks
     has_many :deploys_and_rollbacks,
-             -> { where(type: %w(Shipit::Deploy Shipit::Rollback)) },
-             class_name: 'Task',
-             inverse_of: :stack
+      -> { where(type: %w(Shipit::Deploy Shipit::Rollback)) },
+      class_name: 'Task',
+      inverse_of: :stack
     has_many :github_hooks, dependent: :destroy, class_name: 'Shipit::GithubHook::Repo'
     has_many :hooks, dependent: :destroy
     has_many :api_clients, dependent: :destroy
@@ -83,13 +83,21 @@ module Shipit
     after_commit :emit_merge_status_hooks, on: :update
     after_commit :sync_github, on: :create
     after_commit :schedule_merges_if_necessary, on: :update
+    after_commit :sync_github_if_necessary, on: :update
+
+    def sync_github_if_necessary
+      if (archived_since_previously_changed? && archived_since.nil?) || branch_previously_changed?
+        sync_github
+      end
+    end
 
     validates :repository, uniqueness: {
       scope: %i(environment), case_sensitive: false,
-      message: 'cannot be used more than once with this environment. Check archived stacks.'
+      message: 'cannot be used more than once with this environment. Check archived stacks.',
     }
     validates :environment, format: { with: /\A[a-z0-9\-_\:]+\z/ }, length: { maximum: ENVIRONMENT_MAX_SIZE }
     validates :deploy_url, format: { with: URI.regexp(%w(http https ssh)) }, allow_blank: true
+    validates :branch, presence: true
 
     validates :lock_reason, length: { maximum: 4096 }
 
@@ -184,7 +192,7 @@ module Shipit
     end
 
     def continuous_delivery_delayed?
-      continuous_delivery_delayed_since? && continuous_deployment? && checks?
+      continuous_delivery_delayed_since? && continuous_deployment? && (checks? || deployment_checks?)
     end
 
     def continuous_delivery_delayed!
@@ -353,7 +361,7 @@ module Shipit
     end
 
     def deployable?
-      !locked? && !active_task? && !awaiting_provision?
+      !locked? && !active_task? && !awaiting_provision? && deployment_checks_passed?
     end
 
     def allows_merges?
@@ -372,26 +380,27 @@ module Shipit
     delegate :git_url, to: :repository, prefix: :repo
 
     def base_path
-      Rails.root.join('data', 'stacks', repo_owner, repo_name, environment)
+      @base_path ||= Rails.root.join('data', 'stacks', repo_owner, repo_name, environment)
     end
 
     def deploys_path
-      File.join(base_path, "deploys")
+      @deploys_path ||= base_path.join("deploys")
     end
 
     def git_path
-      File.join(base_path, "git")
+      @git_path ||= base_path.join("git")
     end
 
     def acquire_git_cache_lock(timeout: 15, &block)
-      Flock.new(git_path.to_s + '.lock').lock(timeout: timeout, &block)
+      @git_cache_lock ||= Flock.new(git_path.to_s + '.lock')
+      @git_cache_lock.lock(timeout: timeout, &block)
     end
 
     def clear_git_cache!
       tmp_path = "#{git_path}-#{SecureRandom.hex}"
-      return unless File.exist?(git_path)
+      return unless git_path.exist?
       acquire_git_cache_lock do
-        File.rename(git_path, tmp_path)
+        git_path.rename(tmp_path)
       end
       FileUtils.rm_rf(tmp_path)
     end
@@ -402,12 +411,20 @@ module Shipit
 
     def github_commits
       handle_github_redirections do
-        Shipit.github.api.commits(github_repo_name, sha: branch)
+        github_api.commits(github_repo_name, sha: branch)
       end
     rescue Octokit::Conflict
       [] # Repository is empty...
     end
 
+    def github_api
+      github_app.api
+    end
+
+    def github_app
+      Shipit.github(organization: repository.owner)
+    end
+
     def handle_github_redirections
       # https://developer.github.com/v3/#http-redirects
       resource = yield
@@ -420,9 +437,9 @@ module Shipit
     end
 
     def refresh_repository!
-      resource = Shipit.github.api.repo(github_repo_name)
+      resource = github_api.repo(github_repo_name)
       if resource.try(:message) == 'Moved Permanently'
-        resource = Shipit.github.api.get(resource.url)
+        resource = github_api.get(resource.url)
       end
       repository.update!(owner: resource.owner.login, name: resource.name)
     end
@@ -487,9 +504,9 @@ module Shipit
     end
 
     delegate :plugins, :task_definitions, :hidden_statuses, :required_statuses, :soft_failing_statuses,
-             :blocking_statuses, :deploy_variables, :filter_task_envs, :filter_deploy_envs,
-             :maximum_commits_per_deploy, :pause_between_deploys, :retries_on_deploy, :retries_on_rollback,
-             to: :cached_deploy_spec
+      :blocking_statuses, :deploy_variables, :filter_task_envs, :filter_deploy_envs,
+      :maximum_commits_per_deploy, :pause_between_deploys, :retries_on_deploy, :retries_on_rollback,
+      to: :cached_deploy_spec
 
     def monitoring?
       monitoring.present?
@@ -581,19 +598,31 @@ module Shipit
       links_spec.transform_values { |url| context.interpolate(url) }
     end
 
+    def clear_local_files
+      FileUtils.rm_rf(base_path.to_s)
+    end
+
+    def deployment_checks_passed?
+      return true unless deployment_checks?
+
+      Shipit.deployment_checks.call(self)
+    end
+
     private
 
     def clear_cache
       remove_instance_variable(:@active_task) if defined?(@active_task)
     end
 
-    def clear_local_files
-      FileUtils.rm_rf(base_path.to_s)
-    end
-
     def update_defaults
       self.environment = 'production' if environment.blank?
-      self.branch = 'master' if branch.blank?
+      self.branch = default_branch_name if branch.blank?
+    end
+
+    def default_branch_name
+      Shipit.github.api.repo(github_repo_name).default_branch
+    rescue Octokit::NotFound, Octokit::InvalidRepository
+      nil
     end
 
     def set_locked_since
@@ -607,7 +636,7 @@ module Shipit
     end
 
     def schedule_merges_if_necessary
-      if previous_changes.include?('lock_reason') && previous_changes['lock_reason'].last.blank?
+      if lock_reason_previously_changed? && lock_reason.blank?
         schedule_merges
       end
     end
@@ -644,7 +673,7 @@ module Shipit
     end
 
     def should_resume_continuous_delivery?(commit)
-      !deployable? ||
+      (deployment_checks_passed? && !deployable?) ||
         deployed_too_recently? ||
         commit.nil? ||
         commit.deployed?
@@ -653,7 +682,12 @@ module Shipit
     def should_delay_continuous_delivery?(commit)
       commit.deploy_failed? ||
         (checks? && !EphemeralCommitChecks.new(commit).run.success?) ||
+        !deployment_checks_passed? ||
         commit.recently_pushed?
     end
+
+    def deployment_checks?
+      Shipit.deployment_checks.present?
+    end
   end
 end