shipit-engine 0.28.0 → 0.32.0

data/app/models/shipit/deploy_spec/kubernetes_discovery.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class DeploySpec
     module KubernetesDiscovery

data/app/models/shipit/deploy_spec/lerna_discovery.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'json'
 
 module Shipit

data/app/models/shipit/deploy_spec/npm_discovery.rb

@@ -1,13 +1,14 @@
+# frozen_string_literal: true
 require 'json'
 
 module Shipit
   class DeploySpec
     module NpmDiscovery
       # https://docs.npmjs.com/cli/publish
-      PUBLIC = 'public'.freeze
-      PRIVATE = 'restricted'.freeze
+      PUBLIC = 'public'
+      PRIVATE = 'restricted'
       VALID_ACCESS = [PUBLIC, PRIVATE].freeze
-      NPM_REGISTRY = "https://registry.npmjs.org/".freeze
+      NPM_REGISTRY = "https://registry.npmjs.org/"
 
       def discover_dependencies_steps
         discover_package_json || super
@@ -57,7 +58,7 @@ module Shipit
         # are treated as 'next' npm dist-tags.
         # An 1.0.0-beta.1 would be installable using both:
         # `yarn add package@1.0.0-beta.1` and `yarn add package@next`
-        return 'next' if ['-beta', '-alpha', '-rc', '-next'].any? { |tag| version.include? tag }
+        return 'next' if ['-beta', '-alpha', '-rc', '-next'].any? { |tag| version.include?(tag) }
         'latest'
       end
 

data/app/models/shipit/deploy_spec/pypi_discovery.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class DeploySpec
     module PypiDiscovery

data/app/models/shipit/deploy_spec/rubygems_discovery.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class DeploySpec
     module RubygemsDiscovery

data/app/models/shipit/deploy_stats.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+module Shipit
+  class DeployStats
+    delegate :empty?, to: :@deploys
+
+    def initialize(deploys)
+      @deploys = deploys
+      @durations = @deploys.map { |d| d.duration&.value }.compact
+    end
+
+    def count
+      @deploys.length
+    end
+
+    def average_duration
+      return if empty?
+      @durations.sum / @durations.length.to_f
+    end
+
+    def max_duration
+      @durations.max
+    end
+
+    def min_duration
+      @durations.min
+    end
+
+    def median_duration
+      return if @durations.empty?
+      (sorted_durations[(@durations.length - 1) / 2] + sorted_durations[@durations.length / 2]) / 2.0
+    end
+
+    def success_rate
+      return if empty?
+      (@deploys.count(&:success?) / @deploys.length.to_f) * 100
+    end
+
+    def compare(compare_stats)
+      {
+        count: percent_change(compare_stats.count, count),
+        average_duration: percent_change(compare_stats.average_duration, average_duration),
+        median_duration: percent_change(compare_stats.median_duration, median_duration),
+      }
+    end
+
+    protected
+
+    def sorted_durations
+      @sorted ||= @durations.sort
+    end
+
+    def percent_change(from, to)
+      return if to.nil? || from.nil?
+      return to * 100 if from.zero?
+      ((to - from) / from.to_f) * 100
+    end
+  end
+end

data/app/models/shipit/duration.rb

@@ -1,15 +1,16 @@
+# frozen_string_literal: true
 module Shipit
   class Duration < ActiveSupport::Duration
     ParseError = Class.new(ArgumentError)
 
-    FORMAT = /
+    FORMAT = %r{
       \A
       (?<days>\d+d)?
       (?<hours>\d+h)?
       (?<minutes>\d+m)?
       (?<seconds>\d+s?)?
       \z
-    /x
+    }x
     UNITS = {
       's' => :seconds,
       'm' => :minutes,

data/app/models/shipit/ephemeral_commit_checks.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class EphemeralCommitChecks
     FINAL_STATUSES = %w(failed error success).freeze

data/app/models/shipit/github_hook.rb

@@ -1,5 +1,6 @@
+# frozen_string_literal: true
 module Shipit
-  class GithubHook < ActiveRecord::Base
+  class GithubHook < Record
     # TODO: app-migration, delete class
     belongs_to :stack, required: false # Required for fixtures
 

data/app/models/shipit/github_status.rb

@@ -1,6 +1,7 @@
+# frozen_string_literal: true
 module Shipit
   module GithubStatus
-    CACHE_KEY = 'github::status'.freeze
+    CACHE_KEY = 'github::status'
 
     class << self
       def status
@@ -8,7 +9,7 @@ module Shipit
       end
 
       def refresh_status
-        Rails.cache.write(CACHE_KEY, Shipit.github.api.github_status)
+        Rails.cache.write(CACHE_KEY, Shipit.github.api_status)
       rescue Faraday::Error, Octokit::ServerError
       end
     end

data/app/models/shipit/hook.rb

@@ -1,5 +1,6 @@
+# frozen_string_literal: true
 module Shipit
-  class Hook < ActiveRecord::Base
+  class Hook < Record
     class DeliverySpec
       def initialize(event:, url:, content_type:, payload:)
         @event = event
@@ -19,7 +20,7 @@ module Shipit
       def http
         Faraday::Connection.new do |connection|
           connection.headers = headers
-          connection.adapter Faraday.default_adapter
+          connection.adapter(Faraday.default_adapter)
         end
       end
 
@@ -57,9 +58,9 @@ module Shipit
     belongs_to :stack, required: false
     has_many :deliveries
 
-    validates :delivery_url, presence: true, url: {no_local: true, allow_blank: true}
-    validates :content_type, presence: true, inclusion: {in: CONTENT_TYPES.keys}
-    validates :events, presence: true, subset: {of: EVENTS}
+    validates :delivery_url, presence: true, url: { no_local: true, allow_blank: true }
+    validates :content_type, presence: true, inclusion: { in: CONTENT_TYPES.keys }
+    validates :events, presence: true, subset: { of: EVENTS }
 
     serialize :events, Shipit::CSVSerializer
 

data/app/models/shipit/membership.rb

@@ -1,8 +1,9 @@
+# frozen_string_literal: true
 module Shipit
-  class Membership < ActiveRecord::Base
+  class Membership < Record
     belongs_to :team, required: true
     belongs_to :user, required: true
 
-    validates :user_id, uniqueness: {scope: :team_id}
+    validates :user_id, uniqueness: { scope: :team_id }
   end
 end

data/app/models/shipit/output_chunk.rb

@@ -1,11 +1,16 @@
+# frozen_string_literal: true
 module Shipit
-  class OutputChunk < ActiveRecord::Base
+  class OutputChunk < Record
     belongs_to :task
 
     scope :tail, ->(start) { order(id: :asc).where('id > ?', start || 0) }
 
     def text=(string)
-      super(string.force_encoding(Encoding::UTF_8).scrub)
+      if string.frozen?
+        super(string)
+      else
+        super(string.force_encoding(Encoding::UTF_8).scrub)
+      end
     end
   end
 end

data/app/models/shipit/pull_request.rb

@@ -1,12 +1,13 @@
+# frozen_string_literal: true
 module Shipit
   class PullRequest < ApplicationRecord
     include DeferredTouch
 
-    MERGE_REQUEST_FIELD = 'Merge-Requested-By'.freeze
+    MERGE_REQUEST_FIELD = 'Merge-Requested-By'
 
     WAITING_STATUSES = %w(fetching pending).freeze
     QUEUED_STATUSES = %w(pending revalidating).freeze
-    REJECTION_REASONS = %w(ci_failing merge_conflict requires_rebase).freeze
+    REJECTION_REASONS = %w(ci_missing ci_failing merge_conflict requires_rebase).freeze
     InvalidTransition = Class.new(StandardError)
     NotReady = Class.new(StandardError)
 
@@ -45,7 +46,7 @@ module Shipit
 
     deferred_touch stack: :updated_at
 
-    validates :number, presence: true, uniqueness: {scope: :stack_id}
+    validates :number, presence: true, uniqueness: { scope: :stack_id }
 
     scope :waiting, -> { where(merge_status: WAITING_STATUSES) }
     scope :pending, -> { where(merge_status: 'pending') }
@@ -149,6 +150,7 @@ module Shipit
 
     def reject_unless_mergeable!
       return reject!('merge_conflict') if merge_conflict?
+      return reject!('ci_missing') if any_status_checks_missing?
       return reject!('ci_failing') if any_status_checks_failed?
       return reject!('requires_rebase') if stale?
       false
@@ -175,10 +177,10 @@ module Shipit
         # branch was already deleted somehow
       end
       complete!
-      return true
+      true
     rescue Octokit::MethodNotAllowed # merge conflict
       reject!('merge_conflict')
-      return false
+      false
     rescue Octokit::Conflict # shas didn't match, PR was updated.
       raise NotReady
     end
@@ -190,7 +192,11 @@ module Shipit
 
     def any_status_checks_failed?
       status = StatusChecker.new(head, head.statuses_and_check_runs, stack.cached_deploy_spec)
-      status.failure? || status.error? || status.missing?
+      status.failure? || status.error?
+    end
+
+    def any_status_checks_missing?
+      StatusChecker.new(head, head.statuses_and_check_runs, stack.cached_deploy_spec).missing?
     end
 
     def waiting?
@@ -284,7 +290,7 @@ module Shipit
 
     def find_or_create_commit_from_github_by_sha!(sha, attributes)
       if commit = stack.commits.by_sha(sha)
-        return commit
+        commit
       else
         github_commit = Shipit.github.api.commit(stack.github_repo_name, sha)
         stack.commits.create_from_github!(github_commit, attributes)

data/app/models/shipit/record.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+module Shipit
+  class Record < ActiveRecord::Base
+    self.abstract_class = true
+
+    class << self
+      def serializer_class
+        if defined? @serializer_class
+          @serializer_class
+        else
+          @serializer_class = "#{name}Serializer".safe_constantize
+        end
+      end
+    end
+
+    delegate :serializer_class, to: :class
+  end
+end

data/app/models/shipit/release_status.rb

@@ -1,5 +1,6 @@
+# frozen_string_literal: true
 module Shipit
-  class ReleaseStatus < ActiveRecord::Base
+  class ReleaseStatus < Record
     MAX_DESCRIPTION_LENGTH = 140
     include DeferredTouch
 
@@ -13,7 +14,7 @@ module Shipit
     scope :to_be_created, -> { where(github_id: nil).order(id: :asc) }
 
     STATES = %w(pending success failure error).freeze
-    validates :state, presence: true, inclusion: {in: STATES}
+    validates :state, presence: true, inclusion: { in: STATES }
 
     def create_status_on_github!
       return true if github_id?

data/app/models/shipit/repository.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+module Shipit
+  class Repository < ApplicationRecord
+    OWNER_MAX_SIZE = 39
+    private_constant :OWNER_MAX_SIZE
+
+    NAME_MAX_SIZE = 100
+    private_constant :NAME_MAX_SIZE
+
+    validates :name, uniqueness: { scope: %i(owner), case_sensitive: false,
+                                   message: 'cannot be used more than once' }
+    validates :owner, :name, presence: true, ascii_only: true
+    validates :owner, format: { with: /\A[a-z0-9_\-\.]+\z/ }, length: { maximum: OWNER_MAX_SIZE }
+    validates :name, format: { with: /\A[a-z0-9_\-\.]+\z/ }, length: { maximum: NAME_MAX_SIZE }
+
+    has_many :stacks, dependent: :destroy
+
+    def self.from_github_repo_name(github_repo_name)
+      repo_owner, repo_name = github_repo_name.downcase.split('/')
+      find_by(owner: repo_owner, name: repo_name)
+    end
+
+    def name=(n)
+      super(n&.downcase)
+    end
+
+    def owner=(o)
+      super(o&.downcase)
+    end
+
+    def http_url
+      Shipit.github.url(full_name)
+    end
+
+    def full_name
+      "#{owner}/#{name}"
+    end
+
+    def git_url
+      "https://#{Shipit.github.domain}/#{owner}/#{name}.git"
+    end
+  end
+end

data/app/models/shipit/rollback.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class Rollback < Deploy
     belongs_to :deploy, foreign_key: :parent_id, inverse_of: false

data/app/models/shipit/stack.rb

@@ -1,7 +1,8 @@
+# frozen_string_literal: true
 require 'fileutils'
 
 module Shipit
-  class Stack < ActiveRecord::Base
+  class Stack < Record
     module NoDeployedCommit
       extend self
 
@@ -22,8 +23,6 @@ module Shipit
       end
     end
 
-    REPO_OWNER_MAX_SIZE = 39
-    REPO_NAME_MAX_SIZE = 100
     ENVIRONMENT_MAX_SIZE = 50
     REQUIRED_HOOKS = %i(push status).freeze
 
@@ -40,6 +39,27 @@ module Shipit
     has_many :hooks, dependent: :destroy
     has_many :api_clients, dependent: :destroy
     belongs_to :lock_author, class_name: :User, optional: true
+    belongs_to :repository
+    validates_associated :repository
+
+    scope :not_archived, -> { where(archived_since: nil) }
+
+    default_scope { preload(:repository) }
+
+    def env
+      {
+        'ENVIRONMENT' => environment,
+        'LAST_DEPLOYED_SHA' => last_deployed_commit.sha,
+        'GITHUB_REPO_OWNER' => repository.owner,
+        'GITHUB_REPO_NAME' => repository.name,
+        'DEPLOY_URL' => deploy_url,
+        'BRANCH' => branch,
+      }
+    end
+
+    def repository
+      super || build_repository
+    end
 
     def lock_author(*)
       super || AnonymousUser.new
@@ -49,11 +69,6 @@ module Shipit
       super(user&.logged_in? ? user : nil)
     end
 
-    def self.repo(full_name)
-      repo_owner, repo_name = full_name.downcase.split('/')
-      where(repo_owner: repo_owner, repo_name: repo_name)
-    end
-
     before_validation :update_defaults
     before_destroy :clear_local_files
     before_save :set_locked_since
@@ -66,18 +81,17 @@ module Shipit
     after_commit :sync_github, on: :create
     after_commit :schedule_merges_if_necessary, on: :update
 
-    validates :repo_name, uniqueness: {scope: %i(repo_owner environment),
-                                       message: 'cannot be used more than once with this environment'}
-    validates :repo_owner, :repo_name, :environment, presence: true, ascii_only: true
-    validates :repo_owner, format: {with: /\A[a-z0-9_\-\.]+\z/}, length: {maximum: REPO_OWNER_MAX_SIZE}
-    validates :repo_name, format: {with: /\A[a-z0-9_\-\.]+\z/}, length: {maximum: REPO_NAME_MAX_SIZE}
-    validates :environment, format: {with: /\A[a-z0-9\-_\:]+\z/}, length: {maximum: ENVIRONMENT_MAX_SIZE}
-    validates :deploy_url, format: {with: URI.regexp(%w(http https ssh))}, allow_blank: true
+    validates :repository, uniqueness: {
+      scope: %i(environment), case_sensitive: false,
+      message: 'cannot be used more than once with this environment. Check archived stacks.'
+    }
+    validates :environment, format: { with: /\A[a-z0-9\-_\:]+\z/ }, length: { maximum: ENVIRONMENT_MAX_SIZE }
+    validates :deploy_url, format: { with: URI.regexp(%w(http https ssh)) }, allow_blank: true
 
-    validates :lock_reason, length: {maximum: 4096}
+    validates :lock_reason, length: { maximum: 4096 }
 
     serialize :cached_deploy_spec, DeploySpec
-    delegate :find_task_definition, :supports_rollback?, :links, :release_status?, :release_status_delay,
+    delegate :find_task_definition, :supports_rollback?, :release_status?, :release_status_delay,
              :release_status_context, :supports_fetch_deployed_revision?, to: :cached_deploy_spec, allow_nil: true
 
     def self.refresh_deployed_revisions
@@ -129,9 +143,24 @@ module Shipit
     end
 
     def trigger_deploy(*args, **kwargs)
+      if changed?
+        # If this is the first deploy since the spec changed it's possible the record will be dirty here, meaning we
+        # cant lock. In this one case persist the changes, otherwise log a warning and let the lock raise, so we
+        # can debug what's going on here. We don't expect anything other than the deploy spec to dirty the model
+        # instance, because of how that field is serialised.
+        if changes.keys == ['cached_deploy_spec']
+          save!
+        else
+          Rails.logger.warning("#{changes.keys} field(s) were unexpectedly modified on stack #{id} while deploying")
+        end
+      end
+
       run_now = kwargs.delete(:run_now)
-      deploy = build_deploy(*args, **kwargs)
-      deploy.save!
+      deploy = with_lock do
+        deploy = build_deploy(*args, **kwargs)
+        deploy.save!
+        deploy
+      end
       run_now ? deploy.run_now! : deploy.enqueue
       continuous_delivery_resumed!
       deploy
@@ -150,14 +179,16 @@ module Shipit
     end
 
     def trigger_continuous_delivery
+      return if cached_deploy_spec.blank?
+
       commit = next_commit_to_deploy
 
-      if !deployable? || deployed_too_recently? || commit.nil? || commit.deployed?
+      if should_resume_continuous_delivery?(commit)
         continuous_delivery_resumed!
         return
       end
 
-      if commit.deploy_failed? || (checks? && !EphemeralCommitChecks.new(commit).run.success?)
+      if should_delay_continuous_delivery?(commit)
         continuous_delivery_delayed!
         return
       end
@@ -189,7 +220,7 @@ module Shipit
     def async_refresh_deployed_revision
       async_refresh_deployed_revision!
     rescue => error
-      logger.warn "Failed to dispatch FetchDeployedRevisionJob: [#{error.class.name}] #{error.message}"
+      logger.warn("Failed to dispatch FetchDeployedRevisionJob: [#{error.class.name}] #{error.message}")
     end
 
     def async_refresh_deployed_revision!
@@ -257,8 +288,8 @@ module Shipit
         next if commits_to_lock.empty?
 
         affected_rows += commits
-                         .where(id: commits_to_lock.map(&:id).uniq)
-                         .lock_all(revert.author)
+          .where(id: commits_to_lock.map(&:id).uniq)
+          .lock_all(revert.author)
       end
 
       touch if affected_rows > 1
@@ -320,21 +351,12 @@ module Shipit
       cached_deploy_spec&.pull_request_merge_method || Shipit.default_merge_method
     end
 
-    def repo_name=(name)
-      super(name&.downcase)
-    end
-
-    def repo_owner=(name)
-      super(name&.downcase)
-    end
-
-    def repo_http_url
-      Shipit.github.url("#{repo_owner}/#{repo_name}")
-    end
-
-    def repo_git_url
-      "https://#{Shipit.github.domain}/#{repo_owner}/#{repo_name}.git"
-    end
+    delegate :name=, to: :repository, prefix: :repo
+    delegate :name, to: :repository, prefix: :repo
+    delegate :owner=, to: :repository, prefix: :repo
+    delegate :owner, to: :repository, prefix: :repo
+    delegate :http_url, to: :repository, prefix: :repo
+    delegate :git_url, to: :repository, prefix: :repo
 
     def base_path
       Rails.root.join('data', 'stacks', repo_owner, repo_name, environment)
@@ -354,8 +376,8 @@ module Shipit
 
     def clear_git_cache!
       tmp_path = "#{git_path}-#{SecureRandom.hex}"
+      return unless File.exist?(git_path)
       acquire_git_cache_lock do
-        return unless File.exist?(git_path)
         File.rename(git_path, tmp_path)
       end
       FileUtils.rm_rf(tmp_path)
@@ -389,7 +411,7 @@ module Shipit
       if resource.try(:message) == 'Moved Permanently'
         resource = Shipit.github.api.get(resource.url)
       end
-      update!(repo_owner: resource.owner.login, repo_name: resource.name)
+      repository.update!(owner: resource.owner.login, name: resource.name)
     end
 
     def active_task?
@@ -406,7 +428,7 @@ module Shipit
     end
 
     def lock(reason, user)
-      params = {lock_reason: reason, lock_author: user}
+      params = { lock_reason: reason, lock_author: user }
       update!(params)
     end
 
@@ -414,6 +436,18 @@ module Shipit
       update!(lock_reason: nil, lock_author: nil, locked_since: nil)
     end
 
+    def archived?
+      archived_since.present?
+    end
+
+    def archive!(user)
+      update!(archived_since: Time.now, lock_reason: "Archived", lock_author: user)
+    end
+
+    def unarchive!
+      update!(archived_since: nil, lock_reason: nil, lock_author: nil, locked_since: nil)
+    end
+
     def to_param
       [repo_owner, repo_name, environment].join('/')
     end
@@ -429,11 +463,14 @@ module Shipit
 
     def self.from_param!(param)
       repo_owner, repo_name, environment = param.split('/')
-      where(
-        repo_owner: repo_owner.downcase,
-        repo_name: repo_name.downcase,
-        environment: environment,
-      ).first!
+      includes(:repository)
+        .where(
+          repositories: {
+            owner: repo_owner.downcase,
+            name: repo_name.downcase,
+          },
+          environment: environment,
+        ).first!
     end
 
     delegate :plugins, :task_definitions, :hidden_statuses, :required_statuses, :soft_failing_statuses,
@@ -463,13 +500,15 @@ module Shipit
     end
 
     def update_latest_deployed_ref
-      UpdateGithubLastDeployedRefJob.perform_later(self)
+      if Shipit.update_latest_deployed_ref
+        UpdateGithubLastDeployedRefJob.perform_later(self)
+      end
     end
 
     def broadcast_update
       Pubsubstub.publish(
         "stack.#{id}",
-        {id: id, updated_at: updated_at}.to_json,
+        { id: id, updated_at: updated_at }.to_json,
         name: 'update',
       )
     end
@@ -521,6 +560,13 @@ module Shipit
       GithubSyncJob.perform_later(stack_id: id)
     end
 
+    def links
+      links_spec = cached_deploy_spec&.links || {}
+      context = EnvironmentVariables.with(env)
+
+      links_spec.transform_values { |url| context.interpolate(url) }
+    end
+
     private
 
     def clear_cache
@@ -556,7 +602,7 @@ module Shipit
       return unless previous_changes.include?('lock_reason')
 
       lock_details = if previous_changes['lock_reason'].last.blank?
-        {from: previous_changes['locked_since'].first, until: Time.zone.now}
+        { from: previous_changes['locked_since'].first, until: Time.zone.now }
       end
 
       Hook.emit(:lock, self, locked: locked?, lock_details: lock_details, stack: self)
@@ -582,5 +628,18 @@ module Shipit
     def ci_enabled_cache_key
       "stacks:#{id}:ci_enabled"
     end
+
+    def should_resume_continuous_delivery?(commit)
+      !deployable? ||
+        deployed_too_recently? ||
+        commit.nil? ||
+        commit.deployed?
+    end
+
+    def should_delay_continuous_delivery?(commit)
+      commit.deploy_failed? ||
+        (checks? && !EphemeralCommitChecks.new(commit).run.success?) ||
+        commit.recently_pushed?
+    end
   end
 end