container_broker 1.0.1

data/app/jobs/application_job.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class ApplicationJob < ActiveJob::Base
+  # Automatically retry jobs that encountered a deadlock
+  # retry_on ActiveRecord::Deadlocked
+
+  # Most jobs are safe to ignore if the underlying records are no longer available
+  # discard_on ActiveJob::DeserializationError
+end

data/app/jobs/collect_load_metrics_job.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class CollectLoadMetricsJob < ContainerBrokerBaseJob
+  queue_as :default
+
+  def perform
+    CollectLoadMetrics.new.perform
+  end
+end

data/app/jobs/container_broker_base_job.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+class ContainerBrokerBaseJob < ApplicationJob
+  JOB_METRIC = "jobs"
+
+  around_perform do |job, block|
+    time = Benchmark.realtime { block.call }
+
+    Metrics.new(JOB_METRIC).count(
+      job_id: job.job_id,
+      job_class: job.class.to_s,
+      executions: job.executions,
+      queue_name: job.queue_name,
+      hostname: Socket.gethostname,
+      time: time
+    )
+  end
+
+  around_perform do |job, block|
+    request_id = job.class.request_id_from_args(job.arguments.first)
+
+    if request_id
+      Rails.logger.tagged(" request_id=#{request_id} ") do
+        CurrentThreadRequestId.set(request_id) { block.call }
+      end
+    else
+      block.call
+    end
+  end
+
+  def self.request_id_from_args(_args); end
+end

data/app/jobs/migrate_tasks_from_dead_node_job.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+class MigrateTasksFromDeadNodeJob < ContainerBrokerBaseJob
+  queue_as :default
+
+  def perform(node:)
+    if node.available?
+      Rails.logger.debug("Not migrating tasks because #{node} returned to available status")
+      return
+    end
+
+    node.run_with_lock_no_wait do
+      Rails.logger.debug("Migrating tasks from #{node}")
+      node.slots.reject(&:available?).each do |slot|
+        Rails.logger.debug("Migrating task for #{slot}")
+        current_task = slot.current_task
+        if current_task
+          Rails.logger.debug("Retrying slot current task #{current_task}")
+          current_task.mark_as_retry if current_task.starting? || current_task.started?
+        else
+          Rails.logger.debug("Slot does not have current task")
+        end
+
+        MigrateRunner.new(runner_id: slot.runner_id).migrate
+
+        Rails.logger.debug("Releasing #{slot}")
+        slot.release
+        Rails.logger.debug("#{slot} released")
+      end
+    end
+  end
+end

data/app/jobs/monitor_unresponsive_node_job.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+class MonitorUnresponsiveNodeJob < ContainerBrokerBaseJob
+  queue_as :default
+
+  def perform(node:)
+    node.run_with_lock_no_wait do
+      node.runner_service(:node_availability).perform(node: node)
+
+      Rails.logger.debug("Marking #{node} as available again")
+      node.register_success
+      node.available!
+
+      RunTasksForAllExecutionTypesJob.perform_later
+    end
+  rescue StandardError => e
+    node.register_error("#{e.class}: #{e.message}")
+
+    Rails.logger.info("#{node} still unresponsive")
+  end
+end

data/app/jobs/monitor_unresponsive_nodes_job.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class MonitorUnresponsiveNodesJob < ContainerBrokerBaseJob
+  def perform
+    Node.where(:status.in => %w[unstable unavailable]).each do |node|
+      MonitorUnresponsiveNodeJob.perform_later(node: node)
+    end
+  end
+end

data/app/jobs/release_slot_job.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+class ReleaseSlotJob < ContainerBrokerBaseJob
+  class InvalidSlotContainerId < StandardError; end
+  queue_as :default
+
+  def perform(slot:, runner_id:)
+    Rails.logger.debug("ReleaseSlotJob for #{slot} and container #{runner_id}")
+
+    if MigrateRunner.new(runner_id: runner_id).migrated?
+      Rails.logger.debug("Ignores release slot for #{slot} because it's migrated")
+      return
+    end
+
+    check_same_runner_id(slot: slot, runner_id: runner_id)
+
+    UpdateTaskStatusJob.perform_now(slot.current_task.reload)
+
+    Rails.logger.debug("Enqueueing container removal")
+    RemoveRunnerJob.perform_later(node: MongoidSerializableModel.new(slot.node), runner_id: slot.runner_id) if Settings.delete_container_after_run
+
+    check_for_slot_removal = CheckForSlotRemoval.new(slot: slot)
+    check_for_slot_removal.perform
+    if check_for_slot_removal.removed?
+      Rails.logger.debug("Slot removed and wont be released")
+    else
+      slot.release
+      Rails.logger.debug("Slot released (#{slot.status})")
+    end
+  rescue Runners::RunnerIdNotFoundError => e
+    Rails.logger.debug("Runner #{runner_id} not found (#{e.message}). Task will be rescheduled in UpdateNodeStatus.")
+  rescue StandardError => e
+    Rails.logger.debug("Error in ReleaseSlotJob for #{slot}: #{e}")
+    slot.node.register_error(e.message)
+    raise
+  end
+
+  def check_same_runner_id(slot:, runner_id:)
+    return if runner_id == slot.runner_id
+
+    error_message = "Current container id (#{slot.runner_id}) in #{slot} is different than the provided (#{runner_id})"
+
+    Rails.logger.error(error_message)
+
+    raise InvalidSlotContainerId, error_message
+  end
+end

data/app/jobs/remove_runner_job.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class RemoveRunnerJob < ContainerBrokerBaseJob
+  queue_as :default
+
+  def perform(node:, runner_id:)
+    node
+      .runner_service(:remove_runner)
+      .perform(node: node, runner_id: runner_id)
+  end
+end

data/app/jobs/remove_unused_tags_job.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+class RemoveUnusedTagsJob < ContainerBrokerBaseJob
+  queue_as :default
+
+  def perform(*_args)
+    remove_unreferenced_tags
+  end
+
+  def remove_unreferenced_tags
+    TaskTag
+      .all
+      .to_a
+      .reject { |task_tag| any_task_referencing_tag?(task_tag) }
+      .each(&:destroy!)
+  end
+
+  def tag_expression(task_tag)
+    :"tags.#{task_tag.name}"
+  end
+
+  def any_task_referencing_tag?(task_tag)
+    Task.where(tag_expression(task_tag).exists => true).exists?
+  end
+end

data/app/jobs/request_id_from_task.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module RequestIdFromTask
+  def request_id_from_args(args)
+    args[:task]&.request_id
+  end
+end

data/app/jobs/run_task_job.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+class RunTaskJob < ContainerBrokerBaseJob
+  extend RequestIdFromTask
+
+  queue_as :default
+
+  def perform(task:, slot:)
+    # TODO: remove after successful deploy
+    task.update!(storage_mounts: { "ingest-nfs" => task.attributes["ingest_storage_mount"] }) if task.attributes["ingest_storage_mount"] && task.storage_mounts.blank?
+
+    Rails.logger.debug("Performing RunTaskJob for #{task} #{slot}")
+
+    raise "Invalid task status - #{task}" unless task.starting?
+    raise "Invalid slot status - #{slot}" unless slot.attaching?
+
+    runner_id = task.generate_runner_id
+
+    task.update!(runner_id: runner_id)
+    slot.update!(runner_id: runner_id)
+
+    slot
+      .node
+      .runner_service(:run_task)
+      .perform(task: task, slot: slot, runner_id: runner_id)
+
+    task.mark_as_started!(runner_id: runner_id, slot: slot)
+    Rails.logger.debug("#{task} marked as started")
+
+    slot.mark_as_running(current_task: task, runner_id: runner_id)
+    Rails.logger.debug("#{slot} marked as running")
+
+    add_metric(task)
+    task
+  rescue StandardError => e
+    Rails.logger.debug("Error in RunTaskJob: #{e}")
+
+    slot.node.register_error(e.message) if e.is_a?(Node::NodeConnectionError)
+
+    slot.release
+    Rails.logger.debug("#{slot} released")
+
+    task.mark_as_retry(error: e.message)
+    Rails.logger.debug("#{task} marked as retry")
+
+    add_metric(task)
+
+    Rails.logger.debug("Performed RunTaskJob for #{task} #{slot}")
+  end
+
+  def add_metric(task)
+    Metrics.new("tasks").count(
+      task_id: task.id,
+      name: task&.name,
+      type: task&.execution_type,
+      slot: task&.slot&.name,
+      node: task&.slot&.node&.name,
+      started_at: task.started_at,
+      duration: task.milliseconds_waiting,
+      error: task.error,
+      status: task.status
+    )
+  end
+end

data/app/jobs/run_tasks_for_all_execution_types_job.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class RunTasksForAllExecutionTypesJob < ContainerBrokerBaseJob
+  queue_as :default
+
+  def perform
+    Slot.pluck(:execution_type).uniq.each do |execution_type|
+      RunTasksJob.perform_later(execution_type: execution_type)
+    end
+  end
+end

data/app/jobs/run_tasks_job.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+class RunTasksJob < ContainerBrokerBaseJob
+  attr_reader :execution_type
+
+  def perform(execution_type:)
+    @execution_type = execution_type
+
+    enqueue_tasks
+  end
+
+  private
+
+  def enqueue_tasks
+    while have_pending_tasks? && (slot = lock_slot)
+      task = lock_task
+      if task
+        Rails.logger.debug "Perform_later RunTaskJob for #{slot} #{task}"
+        RunTaskJob.perform_later(slot: slot, task: task)
+      else
+        slot.available!
+        break
+      end
+    end
+  end
+
+  def lock_slot
+    LockSlot.new(execution_type: execution_type).perform
+  end
+
+  def lock_task
+    lock_task_service.perform
+  end
+
+  def have_pending_tasks?
+    lock_task_service.any_pending?
+  end
+
+  def lock_task_service
+    @lock_task_service ||= LockTask.new(execution_type: execution_type)
+  end
+end

data/app/jobs/timeout_failed_tasks_job.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+class TimeoutFailedTasksJob < ContainerBrokerBaseJob
+  queue_as :default
+
+  def perform(*_args)
+    timeout_failed_tasks
+  end
+
+  def timeout_failed_tasks
+    failed_tasks_to_timeout.map do |task|
+      Rails.logger.debug("Marking task as error due to timeout: #{task.uuid}")
+
+      task.error!
+
+      persist_logs(task)
+    end
+  end
+
+  def failed_tasks_to_timeout
+    Task.failed.where(:finished_at.lt => Time.current - Settings.timeout_tasks_after_hours)
+  end
+
+  private
+
+  def persist_logs(task)
+    task.set_logs("#{task.get_logs}\nThis task was automatically marked as error due to timeout.\n".dup)
+
+    task.save!
+  end
+end

data/app/jobs/update_all_nodes_status_job.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class UpdateAllNodesStatusJob < ContainerBrokerBaseJob
+  def perform
+    Node.available.each do |node|
+      UpdateNodeStatusJob.perform_later(node: node)
+    end
+  end
+end

data/app/jobs/update_node_status_job.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+class UpdateNodeStatusJob < ContainerBrokerBaseJob
+  queue_as :default
+
+  def perform(node:)
+    Rails.logger.debug("Waiting for lock to update status of #{node}")
+
+    updated = LockManager.new(type: self.class.to_s, id: node.id, expire: 1.minute, wait: false).lock do
+      Rails.logger.debug("Lock acquired for update status of #{node}")
+
+      node.runner_service(:update_node_status).perform(node: node)
+
+      Rails.logger.debug("Releasing lock for update status of #{node}")
+      true
+    end
+
+    if updated
+      Rails.logger.debug("Lock released for update status of #{node}")
+    else
+      Rails.logger.debug("Node updating is locked by another job and will be ignored now")
+    end
+  end
+end

data/app/jobs/update_task_status_job.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+class UpdateTaskStatusJob < ContainerBrokerBaseJob
+  extend RequestIdFromTask
+
+  class InvalidContainerStatusError < StandardError; end
+
+  queue_as :default
+
+  def perform(task)
+    Rails.logger.debug("Updating status for task #{task}")
+    Rails.logger.debug("Task #{task} is running in slot #{task.slot}")
+
+    execution_info = task.slot.node.runner_service(:fetch_execution_info).perform(task: task)
+
+    Rails.logger.debug("Got runner #{execution_info.id} with state #{execution_info.status}")
+
+    unless execution_info.terminated?
+      Rails.logger.debug("Runner should be terminated but it is #{execution_info.status}. Execution info is #{execution_info.to_h}")
+      raise InvalidContainerStatusError,
+            "Runner should be terminated (current status: #{execution_info.status})"
+    end
+
+    Rails.logger.debug("Container is in status #{execution_info.status} and exit code #{execution_info.exit_code}")
+
+    task.exit_code = execution_info.exit_code
+    task.started_at = execution_info.started_at
+    task.finished_at = execution_info.finished_at
+
+    persist_logs(task)
+
+    if execution_info.success?
+      Rails.logger.debug("Marking task as completed and no errors")
+      task.error = nil
+      task.completed!
+    else
+      Rails.logger.debug("Marked task for retry and set error as #{execution_info.error}")
+      task.mark_as_retry(error: execution_info.error)
+    end
+
+    task.save!
+
+    add_metric(task)
+  end
+
+  def persist_logs(task)
+    return unless task.persist_logs
+
+    Rails.logger.debug("Persisting logs for #{task}")
+    container_logs = task.slot.node.runner_service(:fetch_logs).perform(task: task)
+    task.set_logs(container_logs)
+  end
+
+  def add_metric(task)
+    Metrics.new("tasks").count(
+      task_id: task.id,
+      event_id: task&.tags&.dig("event_id"),
+      api_id: task&.tags&.dig("api_id").to_i,
+      name: task&.name,
+      type: task&.execution_type,
+      slot: task&.slot&.name,
+      node: task&.slot&.node&.name,
+      started_at: task.started_at,
+      finished_at: task.finished_at,
+      duration: task.milliseconds_running,
+      processing_time: task.seconds_running.to_i,
+      error: task.error,
+      status: task.status
+    )
+  end
+end