container_broker 1.0.1

data/app/services/lock_task.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+class LockTask
+  attr_reader :execution_type
+
+  def initialize(execution_type:)
+    @execution_type = execution_type
+  end
+
+  def perform
+    task = all_pending
+           .find_one_and_update(
+             {
+               "$set" => { status: "starting" }
+             }, return_document: :after
+           )
+    return unless task
+
+    task.reload
+
+    persist_metrics(task)
+
+    task
+  end
+
+  def any_pending?
+    all_pending.any?
+  end
+
+  private
+
+  def all_pending
+    Task
+      .where(execution_type: execution_type)
+      .where(:status.in => %w[waiting retry])
+  end
+
+  def persist_metrics(task)
+    Metrics.new("tasks").count(
+      task_id: task.id,
+      name: task&.name,
+      status: task.status
+    )
+  end
+end

data/app/services/metrics.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require "measures"
+
+class Metrics
+  attr_reader :metric
+
+  def initialize(metric)
+    @metric = metric
+  end
+
+  def count(data = {})
+    return unless enabled?
+
+    client.count(metric, data.merge(
+                           origin: "container-broker"
+                         ))
+  rescue StandardError => e
+    Rails.logger.warn("Error sending metrics to measures: #{e}")
+  end
+
+  def duration(data = {})
+    if enabled?
+      client.time(metric, data) { yield data if block_given? }
+    else
+      yield data if block_given?
+    end
+  end
+
+  private
+
+  def enabled?
+    Settings.measures.enabled
+  end
+
+  def transport
+    Measures::Transports::UDP.new(Settings.measures.host, Settings.measures.port)
+  end
+
+  def client
+    Measures::Client.new(transport, Settings.measures.index, Settings.measures.owner)
+  end
+end

data/app/services/migrate_runner.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+# It's important to persist migrated ids because there may be some tasks in the execution queue for the same runner id
+class MigrateRunner
+  TTL = Rails.env.development? ? 10.hours : 1.hour
+  KEY_PREFIX = "migrated_ids"
+
+  attr_reader :runner_id
+
+  def initialize(runner_id:)
+    @runner_id = runner_id
+  end
+
+  def migrate
+    Rails.logger.info("Migrate runner id #{runner_id}")
+    self.class.redis_client.set("#{KEY_PREFIX}_#{runner_id}", 1, ex: TTL)
+  end
+
+  def migrated?
+    self.class.redis_client.exists?("#{KEY_PREFIX}_#{runner_id}")
+  end
+
+  def self.redis_client
+    Redis.new(RedisUrlParser.call(Settings.redis_url))
+  end
+end

data/app/services/node_task_acceptance.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class NodeTaskAcceptance
+  attr_reader :node
+
+  def initialize(node:)
+    @node = node
+  end
+
+  def accept!
+    @node.update!(accept_new_tasks: true)
+    RunTasksForAllExecutionTypesJob.perform_later
+  end
+
+  def reject!
+    @node.update!(accept_new_tasks: false)
+  end
+end

data/app/services/node_usage_percentage_per_execution_type.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+class NodeUsagePercentagePerExecutionType
+  def initialize(node)
+    @node = node
+  end
+
+  def perform
+    execution_type_groups.map do |execution_type_group|
+      {
+        execution_type: execution_type_group[0],
+        usage_percent: SlotsUsagePercentage.new(execution_type_group[1]).perform
+      }
+    end
+  end
+
+  private
+
+  def execution_type_groups
+    @node.slots.group_by(&:execution_type)
+  end
+end

data/app/services/reschedule_tasks_for_missing_runners.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+class RescheduleTasksForMissingRunners
+  attr_reader :started_tasks, :runner_ids
+
+  def initialize(runner_ids:, started_tasks:)
+    @started_tasks = started_tasks
+    @runner_ids = runner_ids
+  end
+
+  def perform
+    tasks_without_runner.each do |runner_id|
+      task = started_tasks_group_by_runner_id[runner_id]
+      message = "Task retryied because runner #{runner_id} is missing (#{task} #{task&.slot})"
+      Rails.logger.debug(message)
+
+      report_event(message: message, task: task, runner_id: runner_id)
+
+      slot = task.slot
+      task.mark_as_retry(error: message)
+      slot&.release
+    end
+  end
+
+  private
+
+  def report_event(message:, task:, runner_id:)
+    return unless Settings.sentry.enabled
+
+    slot = task&.slot
+    node = slot&.node
+    Raven.capture_exception(
+      message,
+      level: :info,
+      extra: {
+        runner: slot&.node&.runner_provider,
+        runner_id: runner_id,
+        slot: {
+          id: slot&.id,
+          name: slot&.name,
+          status: slot&.status,
+          runner_id: slot&.runner_id
+        },
+        node: {
+          id: node&.id,
+          name: node&.name,
+          status: node&.status
+        },
+        task: {
+          id: task.id,
+          name: task.name,
+          status: task.status
+        }
+      }
+    )
+  end
+
+  def tasks_without_runner
+    started_tasks_group_by_runner_id.keys - runner_ids
+  end
+
+  def started_tasks_group_by_runner_id
+    @started_tasks_group_by_runner_id ||= started_tasks
+                                          .map(&:reload)
+                                          .select(&:started?)
+                                          .select(&:runner_id)
+                                          .group_by(&:runner_id)
+                                          .transform_values(&:first)
+  end
+end

data/app/services/runners.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+module Runners
+end

data/app/services/runners/docker/create_connection.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Runners
+  module Docker
+    class CreateConnection
+      def perform(node:)
+        raise(Runners::InvalidRunner, "Node must be a docker runner") unless node.docker?
+
+        ::Docker::Connection.new(
+          node.hostname,
+          connect_timeout: 5,
+          read_timeout: 15,
+          write_timeout: 5
+        )
+      end
+    end
+  end
+end

data/app/services/runners/docker/create_execution_info.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module Runners
+  module Docker
+    class CreateExecutionInfo
+      attr_reader :container
+
+      def perform(container:)
+        @container = container
+
+        execution_info_data = {
+          id: container_name(container: container),
+          status: status
+        }
+
+        if full_state_present?
+          execution_info_data.merge!(
+            exit_code: state["ExitCode"],
+            started_at: state["StartedAt"],
+            finished_at: state["FinishedAt"],
+            error: state["Error"]
+          )
+        end
+
+        Runners::ExecutionInfo.new(execution_info_data)
+      end
+
+      private
+
+      def status
+        if waiting?
+          "pending"
+        elsif running?
+          "running"
+        elsif terminated_with_success?
+          "success"
+        elsif terminated_with_error?
+          "error"
+        elsif terminated?
+          "exited"
+        end
+      end
+
+      def container_name(container:)
+        name = container.info["Name"] || container.info["Names"].first
+
+        name.remove(%r{^/})
+      end
+
+      def waiting?
+        state_status == "created"
+      end
+
+      def running?
+        state_status == "running"
+      end
+
+      def terminated?
+        state_status == "exited"
+      end
+
+      def terminated_with_success?
+        terminated? && state["ExitCode"]&.zero?
+      end
+
+      def terminated_with_error?
+        terminated? && state["ExitCode"]&.positive?
+      end
+
+      def full_state_present?
+        state.is_a?(Hash)
+      end
+
+      def state
+        container.info["State"]
+      end
+
+      def state_status
+        if full_state_present?
+          state["Status"]
+        else
+          state
+        end
+      end
+    end
+  end
+end

data/app/services/runners/docker/fetch_execution_info.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Runners
+  module Docker
+    class FetchExecutionInfo
+      def perform(task:)
+        container = Runners::Docker::FetchTaskContainer
+                    .new
+                    .perform(task: task)
+
+        CreateExecutionInfo.new.perform(container: container)
+      rescue ::Docker::Error::NotFoundError => e
+        raise Runners::RunnerIdNotFoundError, e.message
+      end
+    end
+  end
+end

data/app/services/runners/docker/fetch_logs.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Runners
+  module Docker
+    class FetchLogs
+      def perform(task:)
+        # streaming_logs avoids some encoding issues and should be safe since container status = exited
+        # (see https://github.com/swipely/docker-api/issues/290 for reference)
+        Runners::Docker::FetchTaskContainer
+          .new
+          .perform(task: task)
+          .streaming_logs(stdout: true, stderr: true, tail: 1_000)
+      rescue ::Docker::Error::NotFoundError => e
+        raise Runners::RunnerIdNotFoundError, e.message
+      end
+    end
+  end
+end

data/app/services/runners/docker/fetch_task_container.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Runners
+  module Docker
+    class FetchTaskContainer
+      def perform(task:)
+        ::Docker::Container.get(
+          task.runner_id,
+          { all: true },
+          CreateConnection.new.perform(node: task.slot.node)
+        )
+      end
+    end
+  end
+end

data/app/services/runners/docker/filer.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Runners
+  module Docker
+    class Filer
+      class InvalidMountName < StandardError; end
+
+      def perform(task_storage_mounts:)
+        task_storage_mounts.map do |task_mount_name, task_mount_path|
+          node_mount_path = Settings.storage_mounts.docker[task_mount_name]
+
+          raise InvalidMountName unless node_mount_path
+
+          [node_mount_path, task_mount_path].join(":")
+        end
+      end
+    end
+  end
+end

data/app/services/runners/docker/kill_slot_runner.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Runners
+  module Docker
+    class KillSlotRunner
+      def perform(slot:)
+        return unless slot.runner_id.present?
+
+        ::Docker::Container
+          .get(slot.runner_id, {}, CreateConnection.new.perform(node: slot.node))
+          .kill!
+      rescue ::Docker::Error::NotFoundError => e
+        Rails.logger.info("Container #{slot.runner_id} already removed - #{e.message} (e.class)")
+      rescue Excon::Error => e
+        Rails.logger.info("Error removing container: #{e}")
+      end
+    end
+  end
+end