shipit-engine 0.30.0 → 0.34.0

data/app/models/shipit/status.rb

@@ -1,5 +1,6 @@
+# frozen_string_literal: true
 module Shipit
-  class Status < ActiveRecord::Base
+  class Status < Record
     include Common
     include DeferredTouch
 
@@ -11,7 +12,7 @@ module Shipit
 
     deferred_touch commit: :updated_at
 
-    validates :state, inclusion: {in: STATES, allow_blank: true}, presence: true
+    validates :state, inclusion: { in: STATES, allow_blank: true }, presence: true
 
     after_create :enable_ci_on_stack
     after_commit :schedule_continuous_delivery, :broadcast_update, on: :create

data/app/models/shipit/status/common.rb

@@ -1,24 +1,25 @@
+# frozen_string_literal: true
 module Shipit
   class Status
     module Common
       def unknown?
-        state == 'unknown'.freeze
+        state == 'unknown'
       end
 
       def pending?
-        state == 'pending'.freeze
+        state == 'pending'
       end
 
       def success?
-        state == 'success'.freeze
+        state == 'success'
       end
 
       def error?
-        state == 'error'.freeze
+        state == 'error'
       end
 
       def failure?
-        state == 'failure'.freeze
+        state == 'failure'
       end
 
       def missing?
@@ -30,7 +31,7 @@ module Shipit
       end
 
       def simple_state
-        state == 'error'.freeze ? 'failure'.freeze : state
+        state == 'error' ? 'failure' : state
       end
 
       def allowed_to_fail?

data/app/models/shipit/status/group.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class Status
     class Group
@@ -30,7 +31,7 @@ module Shipit
       end
 
       delegate :pending?, :success?, :error?, :failure?, :unknown?, :missing?, :state, :simple_state,
-               to: :significant_status
+        to: :significant_status
       delegate :each, :size, :map, to: :statuses
       delegate :required_statuses, to: :commit
 

data/app/models/shipit/status/missing.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class Status
     class Missing
@@ -15,7 +16,7 @@ module Shipit
       end
 
       def state
-        'pending'.freeze
+        'pending'
       end
 
       def missing?

data/app/models/shipit/status/unknown.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class Status
     class Unknown
@@ -21,7 +22,7 @@ module Shipit
       end
 
       def state
-        'unknown'.freeze
+        'unknown'
       end
 
       def missing?

data/app/models/shipit/task.rb

@@ -1,13 +1,17 @@
+# frozen_string_literal: true
 module Shipit
-  class Task < ActiveRecord::Base
+  class Task < Record
     include DeferredTouch
 
     ConcurrentTaskRunning = Class.new(StandardError)
 
-    PRESENCE_CHECK_TIMEOUT = 15
+    PRESENCE_CHECK_TIMEOUT = 30
     ACTIVE_STATUSES = %w(pending running aborting).freeze
     COMPLETED_STATUSES = %w(success flapping faulty validating).freeze
     UNSUCCESSFUL_STATUSES = %w(error failed aborted flapping timedout faulty).freeze
+    OUTPUT_SIZE_LIMIT = 16.megabytes # A MySQL mediumblob
+    HUMAN_READABLE_OUTPUT_LIMIT = ActionController::Base.helpers.number_to_human_size(OUTPUT_SIZE_LIMIT)
+    OUTPUT_TRUNCATED_MESSAGE = "Output exceeded the limit of #{HUMAN_READABLE_OUTPUT_LIMIT} and was truncated\n"
 
     attr_accessor :pid
 
@@ -19,9 +23,9 @@ module Shipit
     belongs_to :until_commit, class_name: 'Commit', required: false
     belongs_to :since_commit, class_name: 'Commit', required: false
 
-    deferred_touch stack: :updated_at
+    belongs_to :rollback_once_aborted_to, class_name: 'Task', optional: true
 
-    has_many :chunks, -> { order(:id) }, class_name: 'OutputChunk', dependent: :delete_all, inverse_of: :task
+    deferred_touch stack: :updated_at
 
     serialize :definition, TaskDefinition
     serialize :env, Hash
@@ -35,7 +39,7 @@ module Shipit
     scope :last_seven_days, -> { where("created_at > ?", 7.days.ago) }
     scope :previous_seven_days, -> { where(created_at: 14.days.ago..7.days.ago) }
 
-    scope :due_for_rollup, -> { completed.where(rolled_up: false).where('created_at <= ?', 1.hour.ago) }
+    scope :due_for_rollup, -> { not_active.where(rolled_up: false).where('created_at <= ?', 1.hour.ago) }
 
     after_save :record_status_change
     after_create :prevent_concurrency, unless: :allow_concurrency?
@@ -73,6 +77,10 @@ module Shipit
         task.async_refresh_deployed_revision
       end
 
+      after_transition any => %i(aborted success failed error timedout) do |task|
+        task.schedule_rollup_chunks
+      end
+
       after_transition any => :flapping do |task|
         task.update!(confirmations: 0)
       end
@@ -81,6 +89,10 @@ module Shipit
         task.async_update_estimated_deploy_duration
       end
 
+      after_transition any => %i(failed error timedout) do |task|
+        task.retry_if_necessary
+      end
+
       event :run do
         transition pending: :running
       end
@@ -163,7 +175,7 @@ module Shipit
     end
 
     delegate :acquire_git_cache_lock, :async_refresh_deployed_revision, :async_update_estimated_deploy_duration,
-             to: :stack
+      to: :stack
 
     delegate :checklist, to: :definition
 
@@ -191,26 +203,49 @@ module Shipit
 
     def write(text)
       log_output(text)
-      chunks.create!(text: text)
+      Shipit.redis.append(output_key, text)
     end
 
     def chunk_output
       if rolled_up?
         output
       else
-        chunks.pluck(:text).join
+        blob = Shipit.redis.get(output_key)
+
+        if blob && blob.size > OUTPUT_SIZE_LIMIT
+          Rails.logger.warn("Task #{id} output exceeds limit of #{HUMAN_READABLE_OUTPUT_LIMIT}, and will be truncated.")
+          blob = blob.last(OUTPUT_SIZE_LIMIT - OUTPUT_TRUNCATED_MESSAGE.size)
+          blob = OUTPUT_TRUNCATED_MESSAGE + blob
+        end
+
+        blob
       end
     end
 
+    def chunk_output_size
+      return 0 if rolled_up?
+
+      Shipit.redis.strlen(output_key)
+    end
+
+    def tail_output(range_start)
+      Shipit.redis.getrange(output_key, range_start || 0, -1)
+    end
+
     def schedule_rollup_chunks
       ChunkRollupJob.perform_later(self)
     end
 
     def rollup_chunks
       ActiveRecord::Base.transaction do
-        self.output = chunk_output
-        chunks.delete_all
+        chunks = Shipit::OutputChunk.where(task: self).pluck(:text)
+        chunks << chunk_output
+        self.output = chunks.join("\n")
+
         update_attribute(:rolled_up, true)
+
+        Shipit.redis.del(output_key)
+        Shipit::OutputChunk.where(task: self).delete_all
       end
     end
 
@@ -248,6 +283,10 @@ module Shipit
       user || AnonymousUser.new
     end
 
+    def author=(user)
+      super(user.presence)
+    end
+
     def finished?
       !pending? && !running? && !aborting?
     end
@@ -280,8 +319,12 @@ module Shipit
       end
     end
 
-    def abort!(rollback_once_aborted: false, aborted_by:)
-      update!(rollback_once_aborted: rollback_once_aborted, aborted_by_id: aborted_by.id)
+    def abort!(rollback_once_aborted: false, rollback_once_aborted_to: nil, aborted_by:)
+      update!(
+        rollback_once_aborted: rollback_once_aborted,
+        rollback_once_aborted_to: rollback_once_aborted_to,
+        aborted_by_id: aborted_by.id
+      )
 
       if alive?
         aborting
@@ -349,12 +392,47 @@ module Shipit
       end
     end
 
+    def self.recently_created_at
+      5.minutes.ago
+    end
+
+    ZOMBIE_STATES = %w(running aborting).freeze
+    private_constant :ZOMBIE_STATES
+    def self.zombies
+      where(status: ZOMBIE_STATES)
+        .where(
+          "created_at <= :recently",
+          recently: recently_created_at,
+        )
+        .reject(&:alive?)
+    end
+
+    def retry_if_necessary
+      return unless retries_configured? && !stack.reload.locked?
+
+      if retry_attempt < max_retries
+        retry_task = duplicate_task
+        retry_task.retry_attempt = duplicate_task.retry_attempt + 1
+        retry_task.save!
+
+        retry_task.enqueue
+      end
+    end
+
+    def retries_configured?
+      !max_retries.nil? && max_retries > 0
+    end
+
     private
 
     def prevent_concurrency
       raise ConcurrentTaskRunning if stack.tasks.active.exclusive.count > 1
     end
 
+    def output_key
+      "#{status_key}:output"
+    end
+
     def status_key
       "shipit:task:#{id}"
     end
@@ -372,5 +450,13 @@ module Shipit
     def output_line_buffer
       @output_line_buffer ||= LineBuffer.new
     end
+
+    def duplicate_task
+      copy_task = dup
+      copy_task.status = 'pending'
+      copy_task.started_at = nil
+      copy_task.ended_at = nil
+      copy_task
+    end
   end
 end

data/app/models/shipit/task_definition.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   class TaskDefinition
     NotFound = Class.new(StandardError)

data/app/models/shipit/task_execution_strategy/base.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Shipit
+  module TaskExecutionStrategy
+    class Base
+      def initialize(task)
+        self.task = task
+      end
+
+      def execute
+        raise(
+          NotImplmentedError,
+          "subclasses of TaskExectuionStrategy::Base must implement the #execute method"
+        )
+      end
+
+      attr_accessor :task
+    end
+  end
+end

data/app/models/shipit/task_execution_strategy/default.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+module Shipit
+  module TaskExecutionStrategy
+    class Default < Base
+      def execute
+        @commands = Commands.for(@task)
+        unless @task.pending?
+          Rails.logger.error("Task ##{@task.id} already in `#{@task.status}` state. Aborting.")
+          return
+        end
+        run
+      ensure
+        @commands.clear_working_directory
+      end
+
+      def run
+        @task.ping
+        @task.run!
+        checkout_repository
+        perform_task
+        @task.write("\nCompleted successfully\n")
+        @task.report_complete!
+      rescue Command::TimedOut => error
+        @task.write("\n#{error.message}\n")
+        @task.report_timeout!(error)
+      rescue Command::Error => error
+        @task.write("\n#{error.message}\n")
+        @task.report_failure!(error)
+      rescue StandardError => error
+        @task.report_error!(error)
+      rescue Exception => error
+        @task.report_error!(error)
+        raise
+      end
+
+      def abort!(signal: 'TERM')
+        pid = @task.pid
+        if pid
+          @task.write("$ kill #{pid}\n")
+          Process.kill(signal, pid)
+        else
+          @task.write("Can't abort, no recorded pid, WTF?\n")
+        end
+      rescue SystemCallError => error
+        @task.write("kill: (#{pid}) - #{error.message}\n")
+      end
+
+      def check_for_abort
+        @task.should_abort? do |times_killed|
+          if times_killed > 3
+            abort!(signal: 'KILL')
+          else
+            abort!
+          end
+        end
+      end
+
+      def perform_task
+        capture_all!(@commands.install_dependencies)
+        capture_all!(@commands.perform)
+      end
+
+      def checkout_repository
+        unless @commands.fetched?(@task.until_commit).tap(&:run).success?
+          # acquire_git_cache_lock can take upto 15 seconds
+          # to process. Try to make sure that the job isn't
+          # marked dead while we attempt to acquire the lock.
+          @task.ping
+          @task.acquire_git_cache_lock do
+            @task.ping
+            unless @commands.fetched?(@task.until_commit).tap(&:run).success?
+              capture!(@commands.fetch)
+            end
+          end
+        end
+        capture_all!(@commands.clone)
+        capture!(@commands.checkout(@task.until_commit))
+      end
+
+      def capture_all!(commands)
+        commands.map { |c| capture!(c) }
+      end
+
+      def capture!(command)
+        started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        command.start do
+          @task.ping
+          check_for_abort
+        end
+        @task.write("\n$ #{command}\npid: #{command.pid}\n")
+        @task.pid = command.pid
+        command.stream! do |line|
+          @task.write(line)
+        end
+        finished_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        @task.write("pid: #{command.pid} finished in: #{finished_at - started_at} seconds\n")
+        command.success?
+      end
+
+      def capture(command)
+        capture!(command)
+        command.success?
+      rescue Command::Error
+        false
+      end
+    end
+  end
+end