shipit-engine 0.32.0 → 0.35.1

data/app/models/shipit/status/group.rb

@@ -31,7 +31,7 @@ module Shipit
       end
 
       delegate :pending?, :success?, :error?, :failure?, :unknown?, :missing?, :state, :simple_state,
-               to: :significant_status
+        to: :significant_status
       delegate :each, :size, :map, to: :statuses
       delegate :required_statuses, to: :commit
 

data/app/models/shipit/task.rb

@@ -27,8 +27,6 @@ module Shipit
 
     deferred_touch stack: :updated_at
 
-    has_many :chunks, -> { order(:id) }, class_name: 'OutputChunk', dependent: :delete_all, inverse_of: :task
-
     serialize :definition, TaskDefinition
     serialize :env, Hash
 
@@ -41,7 +39,7 @@ module Shipit
     scope :last_seven_days, -> { where("created_at > ?", 7.days.ago) }
     scope :previous_seven_days, -> { where(created_at: 14.days.ago..7.days.ago) }
 
-    scope :due_for_rollup, -> { completed.where(rolled_up: false).where('created_at <= ?', 1.hour.ago) }
+    scope :due_for_rollup, -> { not_active.where(rolled_up: false).where('created_at <= ?', 1.hour.ago) }
 
     after_save :record_status_change
     after_create :prevent_concurrency, unless: :allow_concurrency?
@@ -79,6 +77,10 @@ module Shipit
         task.async_refresh_deployed_revision
       end
 
+      after_transition any => %i(aborted success failed error timedout) do |task|
+        task.schedule_rollup_chunks
+      end
+
       after_transition any => :flapping do |task|
         task.update!(confirmations: 0)
       end
@@ -87,6 +89,10 @@ module Shipit
         task.async_update_estimated_deploy_duration
       end
 
+      after_transition any => %i(failed error timedout) do |task|
+        task.retry_if_necessary
+      end
+
       event :run do
         transition pending: :running
       end
@@ -169,7 +175,7 @@ module Shipit
     end
 
     delegate :acquire_git_cache_lock, :async_refresh_deployed_revision, :async_update_estimated_deploy_duration,
-             to: :stack
+      to: :stack
 
     delegate :checklist, to: :definition
 
@@ -197,16 +203,16 @@ module Shipit
 
     def write(text)
       log_output(text)
-      chunks.create!(text: text)
+      Shipit.redis.append(output_key, text)
     end
 
     def chunk_output
       if rolled_up?
         output
       else
-        blob = chunks.pluck(:text).join
+        blob = Shipit.redis.get(output_key)
 
-        if blob.size > OUTPUT_SIZE_LIMIT
+        if blob && blob.size > OUTPUT_SIZE_LIMIT
           Rails.logger.warn("Task #{id} output exceeds limit of #{HUMAN_READABLE_OUTPUT_LIMIT}, and will be truncated.")
           blob = blob.last(OUTPUT_SIZE_LIMIT - OUTPUT_TRUNCATED_MESSAGE.size)
           blob = OUTPUT_TRUNCATED_MESSAGE + blob
@@ -216,15 +222,30 @@ module Shipit
       end
     end
 
+    def chunk_output_size
+      return 0 if rolled_up?
+
+      Shipit.redis.strlen(output_key)
+    end
+
+    def tail_output(range_start)
+      Shipit.redis.getrange(output_key, range_start || 0, -1)
+    end
+
     def schedule_rollup_chunks
       ChunkRollupJob.perform_later(self)
     end
 
     def rollup_chunks
       ActiveRecord::Base.transaction do
-        self.output = chunk_output
-        chunks.delete_all
+        chunks = Shipit::OutputChunk.where(task: self).pluck(:text)
+        chunks << chunk_output
+        self.output = chunks.join("\n")
+
         update_attribute(:rolled_up, true)
+
+        Shipit.redis.del(output_key)
+        Shipit::OutputChunk.where(task: self).delete_all
       end
     end
 
@@ -262,6 +283,10 @@ module Shipit
       user || AnonymousUser.new
     end
 
+    def author=(user)
+      super(user.presence)
+    end
+
     def finished?
       !pending? && !running? && !aborting?
     end
@@ -382,12 +407,32 @@ module Shipit
         .reject(&:alive?)
     end
 
+    def retry_if_necessary
+      return unless retries_configured? && !stack.reload.locked?
+
+      if retry_attempt < max_retries
+        retry_task = duplicate_task
+        retry_task.retry_attempt = duplicate_task.retry_attempt + 1
+        retry_task.save!
+
+        retry_task.enqueue
+      end
+    end
+
+    def retries_configured?
+      !max_retries.nil? && max_retries > 0
+    end
+
     private
 
     def prevent_concurrency
       raise ConcurrentTaskRunning if stack.tasks.active.exclusive.count > 1
     end
 
+    def output_key
+      "#{status_key}:output"
+    end
+
     def status_key
       "shipit:task:#{id}"
     end
@@ -405,5 +450,13 @@ module Shipit
     def output_line_buffer
       @output_line_buffer ||= LineBuffer.new
     end
+
+    def duplicate_task
+      copy_task = dup
+      copy_task.status = 'pending'
+      copy_task.started_at = nil
+      copy_task.ended_at = nil
+      copy_task
+    end
   end
 end

data/app/models/shipit/task_execution_strategy/base.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Shipit
+  module TaskExecutionStrategy
+    class Base
+      def initialize(task)
+        self.task = task
+      end
+
+      def execute
+        raise(
+          NotImplmentedError,
+          "subclasses of TaskExectuionStrategy::Base must implement the #execute method"
+        )
+      end
+
+      attr_accessor :task
+    end
+  end
+end

data/app/models/shipit/task_execution_strategy/default.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+module Shipit
+  module TaskExecutionStrategy
+    class Default < Base
+      def execute
+        @commands = Commands.for(@task)
+        unless @task.pending?
+          Rails.logger.error("Task ##{@task.id} already in `#{@task.status}` state. Aborting.")
+          return
+        end
+        run
+      ensure
+        @commands.clear_working_directory
+      end
+
+      def run
+        @task.ping
+        @task.run!
+        checkout_repository
+        perform_task
+        @task.write("\nCompleted successfully\n")
+        @task.report_complete!
+      rescue Command::TimedOut => error
+        @task.write("\n#{error.message}\n")
+        @task.report_timeout!(error)
+      rescue Command::Error => error
+        @task.write("\n#{error.message}\n")
+        @task.report_failure!(error)
+      rescue StandardError => error
+        @task.report_error!(error)
+      rescue Exception => error
+        @task.report_error!(error)
+        raise
+      end
+
+      def abort!(signal: 'TERM')
+        pid = @task.pid
+        if pid
+          @task.write("$ kill #{pid}\n")
+          Process.kill(signal, pid)
+        else
+          @task.write("Can't abort, no recorded pid, WTF?\n")
+        end
+      rescue SystemCallError => error
+        @task.write("kill: (#{pid}) - #{error.message}\n")
+      end
+
+      def check_for_abort
+        @task.should_abort? do |times_killed|
+          if times_killed > 3
+            abort!(signal: 'KILL')
+          else
+            abort!
+          end
+        end
+      end
+
+      def perform_task
+        capture_all!(@commands.install_dependencies)
+        capture_all!(@commands.perform)
+      end
+
+      def checkout_repository
+        unless @commands.fetched?(@task.until_commit).tap(&:run).success?
+          # acquire_git_cache_lock can take upto 15 seconds
+          # to process. Try to make sure that the job isn't
+          # marked dead while we attempt to acquire the lock.
+          @task.ping
+          @task.acquire_git_cache_lock do
+            @task.ping
+            unless @commands.fetched?(@task.until_commit).tap(&:run).success?
+              capture!(@commands.fetch)
+            end
+          end
+        end
+        capture_all!(@commands.clone)
+        capture!(@commands.checkout(@task.until_commit))
+      end
+
+      def capture_all!(commands)
+        commands.map { |c| capture!(c) }
+      end
+
+      def capture!(command)
+        started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        command.start do
+          @task.ping
+          check_for_abort
+        end
+        @task.write("\n$ #{command}\npid: #{command.pid}\n")
+        @task.pid = command.pid
+        command.stream! do |line|
+          @task.write(line)
+        end
+        finished_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        @task.write("pid: #{command.pid} finished in: #{finished_at - started_at} seconds\n")
+        command.success?
+      end
+
+      def capture(command)
+        capture!(command)
+        command.success?
+      rescue Command::Error
+        false
+      end
+    end
+  end
+end

data/app/models/shipit/team.rb

@@ -26,7 +26,8 @@ module Shipit
       end
 
       def find_team_on_github(organization, slug)
-        teams = Shipit::OctokitIterator.new { Shipit.github.api.org_teams(organization, per_page: 100) }
+        gh_api = Shipit.github(organization: organization).api
+        teams = Shipit::OctokitIterator.new(github_api: gh_api) { gh_api.org_teams(organization, per_page: 100) }
         teams.find { |t| t.slug == slug }
       rescue Octokit::NotFound
       end
@@ -41,7 +42,8 @@ module Shipit
     end
 
     def refresh_members!
-      github_members = Shipit::OctokitIterator.new(Shipit.github.api.get(api_url).rels[:members])
+      github_api = Shipit.github(organization: organization).api
+      github_members = Shipit::OctokitIterator.new(github_api.get(api_url).rels[:members])
       members = github_members.map { |u| User.find_or_create_from_github(u) }
       self.members = members
       save!

data/app/models/shipit/user.rb

@@ -15,6 +15,8 @@ module Shipit
 
     def self.find_or_create_by_login!(login)
       find_or_create_by!(login: login) do |user|
+        # Users are global, any app can be used
+        # This will not work for users that only exist in an Enterprise install
         user.github_user = Shipit.github.api.user(login)
       end
     end
@@ -24,7 +26,7 @@ module Shipit
     end
 
     def self.find_or_create_author_from_github_commit(github_commit)
-      if (match_info = github_commit.commit.message.match(/^#{PullRequest::MERGE_REQUEST_FIELD}: ([\w\-\.]+)$/))
+      if (match_info = github_commit.commit.message.match(/^#{MergeRequest::MERGE_REQUEST_FIELD}: ([\w\-\.]+)$/))
         begin
           return find_or_create_by_login!(match_info[1])
         rescue Octokit::NotFound
@@ -80,12 +82,19 @@ module Shipit
       @authorized ||= Shipit.github_teams.empty? || teams.where(id: Shipit.github_teams.map(&:id)).exists?
     end
 
+    def repositories_contributed_to
+      return [] unless id
+      Stack.where(id: stacks_contributed_to).distinct.pluck(:repository_id)
+    end
+
     def stacks_contributed_to
       return [] unless id
       Commit.where('author_id = :id or committer_id = :id', id: id).distinct.pluck(:stack_id)
     end
 
     def refresh_from_github!
+      # Users are global, any app can be used
+      # This will not work for users that only exist in an Enterprise install
       update!(github_user: Shipit.github.api.user(github_id))
     rescue Octokit::NotFound
       identify_renamed_user!

data/app/models/shipit/webhooks/handlers/pull_request/assigned_handler.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module Shipit
+  module Webhooks
+    module Handlers
+      module PullRequest
+        class AssignedHandler < Shipit::Webhooks::Handlers::Handler
+          params do
+            requires :action, String
+            requires :number, Integer
+            requires :pull_request do
+              requires :id, Integer
+              requires :number, Integer
+              requires :url, String
+              requires :title, String
+              requires :state, String
+              requires :additions, Integer
+              requires :deletions, Integer
+              requires :head do
+                requires :sha, String
+                requires :ref, String
+              end
+              requires :user do
+                requires :login, String
+              end
+              requires :assignees, Array do
+                requires :login, String
+              end
+              requires :labels, Array do
+                requires :name, String
+              end
+            end
+            requires :repository do
+              requires :full_name, String
+            end
+            requires :sender do
+              requires :login, String
+            end
+          end
+
+          def process
+            return unless respond_to_assignee_change?
+
+            pull_request.update(github_pull_request: params.pull_request) if pull_request.present?
+          end
+
+          private
+
+          def respond_to_assignee_change?
+            %w[assigned unassigned].include?(params.action)
+          end
+
+          def pull_request
+            @pull_request ||= Shipit::PullRequest
+              .joins(:stack, stack: :repository)
+              .find_by(
+                number: params.number,
+                stacks: {
+                  repositories:
+                    {
+                      id: repository.id,
+                    },
+                }
+              )
+          end
+
+          def repository
+            Shipit::Repository.from_github_repo_name(params.repository.full_name) || Shipit::NullRepository.new
+          end
+        end
+      end
+    end
+  end
+end

data/app/models/shipit/webhooks/handlers/pull_request/closed_handler.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Shipit
+  module Webhooks
+    module Handlers
+      module PullRequest
+        class ClosedHandler < Shipit::Webhooks::Handlers::Handler
+          params do
+            requires :action, String
+            requires :number, Integer
+            requires :pull_request do
+              requires :id, Integer
+              requires :number, Integer
+              requires :url, String
+              requires :title, String
+              requires :state, String
+              requires :additions, Integer
+              requires :deletions, Integer
+              requires :head do
+                requires :sha, String
+                requires :ref, String
+              end
+              requires :user do
+                requires :login, String
+              end
+              requires :assignees, Array do
+                requires :login, String
+              end
+              requires :labels, Array do
+                requires :name, String
+              end
+            end
+            requires :repository do
+              requires :full_name, String
+            end
+            requires :sender do
+              requires :login, String
+            end
+          end
+
+          def process
+            return unless respond_to_pull_request_closed?
+
+            review_stack.archive!
+          end
+
+          private
+
+          def repository
+            @repository ||=
+              Shipit::Repository.from_github_repo_name(params.repository.full_name) ||
+              Shipit::NullRepository.new
+          end
+
+          def review_stack
+            @review_stack ||=
+              Shipit::Webhooks::Handlers::PullRequest::ReviewStackAdapter
+                .new(params, scope: repository.review_stacks)
+          end
+
+          def respond_to_pull_request_closed?
+            params.action == "closed"
+          end
+        end
+      end
+    end
+  end
+end