shipit-engine 0.32.0 → 0.33.0

data/app/models/shipit/review_stack.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+module Shipit
+  class ReviewStack < Shipit::Stack
+    def self.clear_stale_caches
+      Shipit::ReviewStack.where(
+        "archived_since > :earliest AND archived_since < :latest",
+        earliest: 1.day.ago,
+        latest: 1.hour.ago
+      ).each do |review_stack|
+        Shipit::ClearGitCacheJob.perform_later(review_stack)
+      end
+    end
+
+    def self.delete_old_deployment_directories
+      Shipit::Deploy.not_active.where(
+        "created_at > :earliest AND updated_at < :latest",
+        earliest: 1.day.ago,
+        latest: 1.hour.ago
+      ).find_each do |deploy|
+        Shipit::Commands.for(deploy).clear_working_directory
+      end
+    end
+
+    has_one :pull_request, foreign_key: :stack_id
+
+    after_commit :emit_added_hooks, on: :create
+    after_commit :emit_updated_hooks, on: :update
+    after_commit :emit_removed_hooks, on: :destroy
+
+    state_machine :provision_status, initial: :deprovisioned do
+      state :provisioned
+      state :provisioning
+      state :deprovisioning
+      state :deprovisioned
+
+      event :provision do
+        transition deprovisioned: :provisioning
+      end
+
+      event :provision_success do
+        transition provisioning: :provisioned
+      end
+
+      event :provision_failure do
+        transition provisioning: :deprovisioned
+      end
+
+      event :deprovision do
+        transition provisioned: :deprovisioning
+      end
+
+      event :deprovision_success do
+        transition deprovisioning: :deprovisioned
+      end
+
+      event :deprovision_failure do
+        transition deprovisioning: :provisioned
+      end
+
+      after_transition deprovisioned: :provisioning do |stack, _|
+        stack.provisioner.up
+      end
+
+      after_transition provisioned: :deprovisioning do |stack, _|
+        stack.provisioner.down
+      end
+    end
+
+    def env
+      return super unless pull_request.present?
+
+      super
+        .merge(
+          pull_request
+            .labels
+            .each_with_object({}) { |label_name, labels| labels[label_name.upcase] = "true" }
+        )
+    end
+
+    def provisioner
+      provisioner_class.new(self)
+    end
+
+    def provisioner_class
+      ProvisioningHandler.fetch(provisioning_handler_name)
+    end
+
+    def enqueue_for_provisioning
+      return if awaiting_provision
+      update!(awaiting_provision: true)
+    end
+
+    def remove_from_provisioning_queue
+      return unless awaiting_provision
+      update!(awaiting_provision: false)
+    end
+
+    def to_partial_path
+      "shipit/stacks/stack"
+    end
+
+    def emit_added_hooks
+      Hook.emit(:review_stack, self, action: :added, review_stack: self)
+    end
+
+    def emit_updated_hooks
+      changed = !(previous_changes.keys - %w(updated_at)).empty?
+      Hook.emit(:review_stack, self, action: :updated, review_stack: self) if changed
+    end
+
+    def emit_removed_hooks
+      Hook.emit(:review_stack, self, action: :removed, review_stack: self)
+    end
+  end
+end

data/app/models/shipit/review_stack_provisioning_queue.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Shipit
+  class ReviewStackProvisioningQueue
+    def self.work
+      new.work
+    end
+
+    def self.add(stack)
+      stack.enqueue_for_provisioning
+    end
+
+    def self.queued_stacks
+      new.queued_stacks
+    end
+
+    def work
+      queued_stacks.find_each(&method(:provision))
+    end
+
+    def queued_stacks
+      @queued_stacks ||= Shipit::ReviewStack
+        .with_provision_status(:deprovisioned)
+        .where(awaiting_provision: true)
+    end
+
+    private
+
+    def provision(stack)
+      if stack.provisioner.provision?
+        stack.provision
+      else
+        Rails.logger.info(
+          "Putting review ReviewStack<#{stack.id}> back into the provisioning queue - #provision? was falsey."
+        )
+      end
+    end
+  end
+end

data/app/models/shipit/stack.rb

@@ -27,7 +27,7 @@ module Shipit
     REQUIRED_HOOKS = %i(push status).freeze
 
     has_many :commits, dependent: :destroy
-    has_many :pull_requests, dependent: :destroy
+    has_many :merge_requests, dependent: :destroy
     has_many :tasks, dependent: :destroy
     has_many :deploys
     has_many :rollbacks
@@ -44,6 +44,9 @@ module Shipit
 
     scope :not_archived, -> { where(archived_since: nil) }
 
+    include DeferredTouch
+    deferred_touch repository: :updated_at
+
     default_scope { preload(:repository) }
 
     def env
@@ -91,8 +94,17 @@ module Shipit
     validates :lock_reason, length: { maximum: 4096 }
 
     serialize :cached_deploy_spec, DeploySpec
-    delegate :find_task_definition, :supports_rollback?, :release_status?, :release_status_delay,
-             :release_status_context, :supports_fetch_deployed_revision?, to: :cached_deploy_spec, allow_nil: true
+    delegate(
+      :provisioning_handler_name,
+      :find_task_definition,
+      :release_status?,
+      :release_status_context,
+      :release_status_delay,
+      :supports_fetch_deployed_revision?,
+      :supports_rollback?,
+      to: :cached_deploy_spec,
+      allow_nil: true
+    )
 
     def self.refresh_deployed_revisions
       find_each.select(&:supports_fetch_deployed_revision?).each(&:async_refresh_deployed_revision)
@@ -139,6 +151,7 @@ module Shipit
         env: filter_deploy_envs(env&.to_h || {}),
         allow_concurrency: force,
         ignored_safeties: force || !until_commit.deployable?,
+        max_retries: retries_on_deploy,
       )
     end
 
@@ -200,7 +213,7 @@ module Shipit
     end
 
     def schedule_merges
-      MergePullRequestsJob.perform_later(self)
+      ProcessMergeRequestsJob.perform_later(self)
     end
 
     def next_commit_to_deploy
@@ -340,7 +353,7 @@ module Shipit
     end
 
     def deployable?
-      !locked? && !active_task?
+      !locked? && !active_task? && !awaiting_provision?
     end
 
     def allows_merges?
@@ -348,7 +361,7 @@ module Shipit
     end
 
     def merge_method
-      cached_deploy_spec&.pull_request_merge_method || Shipit.default_merge_method
+      cached_deploy_spec&.merge_request_merge_method || Shipit.default_merge_method
     end
 
     delegate :name=, to: :repository, prefix: :repo
@@ -384,7 +397,7 @@ module Shipit
     end
 
     def github_repo_name
-      [repo_owner, repo_name].join('/')
+      repository.github_repo_name
     end
 
     def github_commits
@@ -475,7 +488,8 @@ module Shipit
 
     delegate :plugins, :task_definitions, :hidden_statuses, :required_statuses, :soft_failing_statuses,
              :blocking_statuses, :deploy_variables, :filter_task_envs, :filter_deploy_envs,
-             :maximum_commits_per_deploy, :pause_between_deploys, to: :cached_deploy_spec
+             :maximum_commits_per_deploy, :pause_between_deploys, :retries_on_deploy, :retries_on_rollback,
+             to: :cached_deploy_spec
 
     def monitoring?
       monitoring.present?

data/app/models/shipit/task.rb

@@ -27,8 +27,6 @@ module Shipit
 
     deferred_touch stack: :updated_at
 
-    has_many :chunks, -> { order(:id) }, class_name: 'OutputChunk', dependent: :delete_all, inverse_of: :task
-
     serialize :definition, TaskDefinition
     serialize :env, Hash
 
@@ -79,6 +77,10 @@ module Shipit
         task.async_refresh_deployed_revision
       end
 
+      after_transition any => %i(aborted success failed error timedout) do |task|
+        task.schedule_rollup_chunks
+      end
+
       after_transition any => :flapping do |task|
         task.update!(confirmations: 0)
       end
@@ -87,6 +89,10 @@ module Shipit
         task.async_update_estimated_deploy_duration
       end
 
+      after_transition any => %i(failed error timedout) do |task|
+        task.retry_if_necessary
+      end
+
       event :run do
         transition pending: :running
       end
@@ -197,16 +203,16 @@ module Shipit
 
     def write(text)
       log_output(text)
-      chunks.create!(text: text)
+      Shipit.redis.append(output_key, text)
     end
 
     def chunk_output
       if rolled_up?
         output
       else
-        blob = chunks.pluck(:text).join
+        blob = Shipit.redis.get(output_key)
 
-        if blob.size > OUTPUT_SIZE_LIMIT
+        if blob && blob.size > OUTPUT_SIZE_LIMIT
           Rails.logger.warn("Task #{id} output exceeds limit of #{HUMAN_READABLE_OUTPUT_LIMIT}, and will be truncated.")
           blob = blob.last(OUTPUT_SIZE_LIMIT - OUTPUT_TRUNCATED_MESSAGE.size)
           blob = OUTPUT_TRUNCATED_MESSAGE + blob
@@ -216,15 +222,30 @@ module Shipit
       end
     end
 
+    def chunk_output_size
+      return 0 if rolled_up?
+
+      Shipit.redis.strlen(output_key)
+    end
+
+    def tail_output(range_start)
+      Shipit.redis.getrange(output_key, range_start || 0, -1)
+    end
+
     def schedule_rollup_chunks
       ChunkRollupJob.perform_later(self)
     end
 
     def rollup_chunks
       ActiveRecord::Base.transaction do
-        self.output = chunk_output
-        chunks.delete_all
+        chunks = Shipit::OutputChunk.where(task: self).pluck(:text)
+        chunks << chunk_output
+        self.output = chunks.join("\n")
+
         update_attribute(:rolled_up, true)
+
+        Shipit.redis.del(output_key)
+        Shipit::OutputChunk.where(task: self).delete_all
       end
     end
 
@@ -382,12 +403,32 @@ module Shipit
         .reject(&:alive?)
     end
 
+    def retry_if_necessary
+      return unless retries_configured? && !stack.reload.locked?
+
+      if retry_attempt < max_retries
+        retry_task = duplicate_task
+        retry_task.retry_attempt = duplicate_task.retry_attempt + 1
+        retry_task.save!
+
+        retry_task.enqueue
+      end
+    end
+
+    def retries_configured?
+      !max_retries.nil? && max_retries > 0
+    end
+
     private
 
     def prevent_concurrency
       raise ConcurrentTaskRunning if stack.tasks.active.exclusive.count > 1
     end
 
+    def output_key
+      "#{status_key}:output"
+    end
+
     def status_key
       "shipit:task:#{id}"
     end
@@ -405,5 +446,13 @@ module Shipit
     def output_line_buffer
       @output_line_buffer ||= LineBuffer.new
     end
+
+    def duplicate_task
+      copy_task = dup
+      copy_task.status = 'pending'
+      copy_task.started_at = nil
+      copy_task.ended_at = nil
+      copy_task
+    end
   end
 end

data/app/models/shipit/task_execution_strategy/base.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Shipit
+  module TaskExecutionStrategy
+    class Base
+      def initialize(task)
+        self.task = task
+      end
+
+      def execute
+        raise(
+          NotImplmentedError,
+          "subclasses of TaskExectuionStrategy::Base must implement the #execute method"
+        )
+      end
+
+      attr_accessor :task
+    end
+  end
+end

data/app/models/shipit/task_execution_strategy/default.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+module Shipit
+  module TaskExecutionStrategy
+    class Default < Base
+      def execute
+        @commands = Commands.for(@task)
+        unless @task.pending?
+          Rails.logger.error("Task ##{@task.id} already in `#{@task.status}` state. Aborting.")
+          return
+        end
+        run
+      ensure
+        @commands.clear_working_directory
+      end
+
+      def run
+        @task.ping
+        @task.run!
+        checkout_repository
+        perform_task
+        @task.write("\nCompleted successfully\n")
+        @task.report_complete!
+      rescue Command::TimedOut => error
+        @task.write("\n#{error.message}\n")
+        @task.report_timeout!(error)
+      rescue Command::Error => error
+        @task.write("\n#{error.message}\n")
+        @task.report_failure!(error)
+      rescue StandardError => error
+        @task.report_error!(error)
+      rescue Exception => error
+        @task.report_error!(error)
+        raise
+      end
+
+      def abort!(signal: 'TERM')
+        pid = @task.pid
+        if pid
+          @task.write("$ kill #{pid}\n")
+          Process.kill(signal, pid)
+        else
+          @task.write("Can't abort, no recorded pid, WTF?\n")
+        end
+      rescue SystemCallError => error
+        @task.write("kill: (#{pid}) - #{error.message}\n")
+      end
+
+      def check_for_abort
+        @task.should_abort? do |times_killed|
+          if times_killed > 3
+            abort!(signal: 'KILL')
+          else
+            abort!
+          end
+        end
+      end
+
+      def perform_task
+        capture_all!(@commands.install_dependencies)
+        capture_all!(@commands.perform)
+      end
+
+      def checkout_repository
+        unless @commands.fetched?(@task.until_commit).tap(&:run).success?
+          # acquire_git_cache_lock can take upto 15 seconds
+          # to process. Try to make sure that the job isn't
+          # marked dead while we attempt to acquire the lock.
+          @task.ping
+          @task.acquire_git_cache_lock do
+            @task.ping
+            unless @commands.fetched?(@task.until_commit).tap(&:run).success?
+              capture!(@commands.fetch)
+            end
+          end
+        end
+        capture_all!(@commands.clone)
+        capture!(@commands.checkout(@task.until_commit))
+      end
+
+      def capture_all!(commands)
+        commands.map { |c| capture!(c) }
+      end
+
+      def capture!(command)
+        started_at = Time.now
+        command.start do
+          @task.ping
+          check_for_abort
+        end
+        @task.write("$ #{command}\npid: #{command.pid}\nstarted at: #{started_at}\n")
+        @task.pid = command.pid
+        command.stream! do |line|
+          @task.write(line)
+        end
+        @task.write("\n")
+        finished_at = Time.now
+        @task.write("pid: #{command.pid}\nfinished at: #{finished_at}\nran in: #{finished_at - started_at} seconds\n")
+        command.success?
+      end
+
+      def capture(command)
+        capture!(command)
+        command.success?
+      rescue Command::Error
+        false
+      end
+    end
+  end
+end