container_broker 1.0.1

data/app/services/runners/docker/node_availability.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Runners
+  module Docker
+    class NodeAvailability
+      def perform(node:)
+        ::Docker.info(CreateConnection.new.perform(node: node))
+      end
+    end
+  end
+end

data/app/services/runners/docker/remove_runner.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Runners
+  module Docker
+    class RemoveRunner
+      def perform(node:, runner_id:)
+        Rails.logger.info("Removing container #{runner_id} from node #{node}")
+        container = ::Docker::Container.get(runner_id, { all: true }, CreateConnection.new.perform(node: node))
+        container.kill if container.info["State"]["Status"] == "running"
+        container.delete
+      rescue ::Docker::Error::NotFoundError, ::Docker::Error::ConflictError => e
+        Rails.logger.info("Container #{runner_id} already removed - #{e.message} (e.class)")
+      rescue Excon::Error, ::Docker::Error::TimeoutError => e
+        node.register_error(e.message)
+      end
+    end
+  end
+end

data/app/services/runners/docker/run_task.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Runners
+  module Docker
+    class RunTask
+      def perform(task:, slot:, runner_id:)
+        Rails.logger.debug("Performing Docker::RunTask for #{task} #{slot}")
+
+        pull_image(task: task, slot: slot)
+        Rails.logger.debug("Image pulled for #{task} #{slot}")
+
+        container = create_container(task: task, slot: slot, name: runner_id)
+        Rails.logger.debug("Container #{container.id} created for #{task} #{slot} with name #{runner_id}")
+
+        container.start
+        Rails.logger.debug("Container #{container.id} started")
+
+        runner_id
+      rescue Excon::Error, ::Docker::Error::TimeoutError => e then
+        message = "Docker connection error: #{e.message}"
+        message += "\n#{e.response.body}" if e.respond_to?(:response)
+        raise Node::NodeConnectionError, message
+      rescue ::Docker::Error::NotFoundError => e
+        raise "Docker image not found: #{e.message}"
+      end
+
+      private
+
+      def pull_image(task:, slot:)
+        return if ::Docker::Image.exist?(task.image, {}, CreateConnection.new.perform(node: slot.node))
+
+        image_name, image_tag = task.image.split(":")
+        image_tag ||= "latest"
+
+        ::Docker::Image.create({ "fromImage" => image_name, "tag" => image_tag }, nil, CreateConnection.new.perform(node: slot.node))
+      end
+
+      def create_container(task:, slot:, name:)
+        binds = Filer.new.perform(task_storage_mounts: task.storage_mounts)
+
+        user = [
+          Settings.run_container_as.user_id,
+          Settings.run_container_as.group_id
+        ].join(":")
+
+        ::Docker::Container.create(
+          {
+            "name" => name,
+            "Image" => task.image,
+            "User" => user,
+            "HostConfig" => {
+              "Binds" => binds,
+              "NetworkMode" => ENV["DOCKER_CONTAINERS_NETWORK"].to_s
+            },
+            "Entrypoint" => [],
+            "Cmd" => ["sh", "-c", task.cmd]
+          },
+          CreateConnection.new.perform(node: slot.node)
+        )
+      end
+    end
+  end
+end

data/app/services/runners/docker/update_node_status.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Runners
+  module Docker
+    class UpdateNodeStatus
+      include UpdateNodeStatusHelper
+
+      attr_reader :node
+
+      def perform(node:)
+        @node = node
+        Rails.logger.debug("Start updating node status for #{node}")
+
+        # Other tasks can be started at this time. Because of this it's necessary to load the tasks first and then the containers
+        started_tasks = Task.started.where(:slot.in => node.slots.pluck(:id)).to_a
+
+        Rails.logger.debug("Got #{containers.count} containers")
+
+        execution_infos = containers.map do |container|
+          execution_info = CreateExecutionInfo.new.perform(container: container)
+
+          slot = node.slots.find_by(runner_id: execution_info.id)
+          if slot
+            Rails.logger.debug("Slot found for container #{execution_info.id}: #{slot}")
+
+            if execution_info.terminated?
+              Rails.logger.debug("Container #{execution_info.id} exited")
+
+              check_slot_release(slot: slot, runner_id: execution_info.id)
+            elsif started_with_error?(container: container, docker_connection: CreateConnection.new.perform(node: node))
+              container.start
+            end
+          else
+            remove_unknown_runner(node: node, runner_id: execution_info.id)
+          end
+
+          execution_info
+        end
+
+        RescheduleTasksForMissingRunners
+          .new(runner_ids: execution_infos.map(&:id), started_tasks: started_tasks)
+          .perform
+
+        node.register_success
+
+        send_metrics(node: node, execution_infos: execution_infos)
+      rescue Excon::Error, ::Docker::Error::DockerError => e
+        node.register_error(e.message)
+      end
+
+      private
+
+      def containers
+        @containers ||= ::Docker::Container.all({ all: true }, CreateConnection.new.perform(node: node))
+      end
+
+      def started_with_error?(container:, docker_connection:)
+        container.info["State"] == "created" && ::Docker::Container.get(container.id, { all: true }, docker_connection).info["State"]["ExitCode"].positive?
+      end
+    end
+  end
+end

data/app/services/runners/execution_info.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Runners
+  class UnknownCompletionInformation < StandardError; end
+
+  ExecutionInfo = Struct.new(:id, :status, :exit_code, :started_at, :finished_at, :error, :schedule_pending, keyword_init: true) do
+    def success?
+      check_completion_information_available
+
+      status == "success"
+    end
+
+    def error?
+      check_completion_information_available
+
+      status == "error"
+    end
+
+    def running?
+      status == "running"
+    end
+
+    def pending?
+      status == "pending"
+    end
+
+    def terminated?
+      exited_without_completion_information? || success? || error?
+    end
+
+    def exited_without_completion_information?
+      status == "exited"
+    end
+
+    def schedule_pending?
+      schedule_pending
+    end
+
+    private
+
+    def check_completion_information_available
+      # Some execution infos return just the "exited" status and not the complete state
+      # So in this point, if the user is asking for success or error, then we need to force it
+      # to fetch the complete status (which has the exit code)
+
+      raise(UnknownCompletionInformation, "Complete status not available") if exited_without_completion_information?
+    end
+  end
+end

data/app/services/runners/invalid_config.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Runners
+  class InvalidConfig < StandardError; end
+end

data/app/services/runners/invalid_runner.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Runners
+  class InvalidRunner < StandardError; end
+end

data/app/services/runners/kubernetes/create_client.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Runners
+  module Kubernetes
+    class CreateClient
+      attr_reader :node
+
+      def perform(node:)
+        @node = node
+
+        raise(Runners::InvalidRunner, "Node must be a kubernetes runner") unless node.kubernetes?
+
+        raise(Runners::InvalidConfig, "Invalid configuration (#{node.runner_config}) for kubernetes") unless valid?
+
+        KubernetesClient.new(
+          uri: node.hostname,
+          bearer_token: node.runner_config["bearer_token"],
+          namespace: node.runner_config["namespace"]
+        )
+      end
+
+      private
+
+      def valid?
+        %w[bearer_token namespace nfs_server nfs_path node_selector].none? { |field| node.runner_config[field].blank? }
+      end
+    end
+  end
+end

data/app/services/runners/kubernetes/create_execution_info.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module Runners
+  module Kubernetes
+    class CreateExecutionInfo
+      attr_reader :pod
+
+      ERROR_REASONS = %w[
+        ImagePullBackOff
+        ErrImagePull
+      ].freeze
+
+      def perform(pod:)
+        @pod = pod
+
+        Runners::ExecutionInfo.new(
+          id: pod&.metadata&.name,
+          status: status,
+          exit_code: container_status&.state&.terminated&.exitCode,
+          started_at: started_at,
+          finished_at: container_status&.state&.terminated&.finishedAt,
+          error: error_message,
+          schedule_pending: schedule_pending?
+        )
+      end
+
+      private
+
+      def status
+        if running?
+          "running"
+        elsif terminated_with_success?
+          "success"
+        elsif error?
+          "error"
+        elsif waiting? || schedule_pending?
+          "pending"
+        end
+      end
+
+      def waiting?
+        container_status&.state&.waiting.present?
+      end
+
+      def running?
+        container_status&.state&.running&.present?
+      end
+
+      def terminated_with_error?
+        container_status&.state&.terminated&.exitCode&.positive?
+      end
+
+      def terminated_with_success?
+        container_status&.state&.terminated&.exitCode&.zero?
+      end
+
+      def reason_is_error?
+        waiting? && ERROR_REASONS.include?(reason[:reason])
+      end
+
+      def error?
+        terminated_with_error? || reason_is_error?
+      end
+
+      def schedule_pending?
+        unschedulable_error_messsage.present?
+      end
+
+      def unschedulable_error_messsage
+        return if pod.status&.phase != "Pending"
+
+        found = pod&.status&.conditions&.find { |condition| condition.reason == "Unschedulable" }
+        "#{found.reason}: #{found.message}" if found
+      end
+
+      def error_message
+        if error?
+          reason.values.compact.join(": ")
+        elsif unschedulable_error_messsage.present?
+          unschedulable_error_messsage
+        end
+      end
+
+      def started_at
+        (container_status&.state&.terminated || container_status&.state&.running)&.startedAt
+      end
+
+      def reason
+        reason_value = container_status&.state&.to_hash&.values&.first
+        return {} unless reason_value
+
+        {
+          reason: reason_value[:reason],
+          message: reason_value[:message]
+        }
+      end
+
+      def container_status
+        pod.status&.containerStatuses&.first
+      end
+    end
+  end
+end

data/app/services/runners/kubernetes/fetch_execution_info.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Runners
+  module Kubernetes
+    class FetchExecutionInfo
+      def perform(task:)
+        pod = CreateClient.new.perform(node: task.slot.node).fetch_pod(pod_name: task.runner_id)
+
+        CreateExecutionInfo.new.perform(pod: pod)
+      rescue KubernetesClient::PodNotFoundError => e
+        raise Runners::RunnerIdNotFoundError, e.message
+      end
+    end
+  end
+end

data/app/services/runners/kubernetes/fetch_logs.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Runners
+  module Kubernetes
+    class FetchLogs
+      def perform(task:)
+        CreateClient.new.perform(node: task.slot.node).fetch_pod_logs(pod_name: task.runner_id)
+      rescue KubernetesClient::PodNotFoundError => e
+        raise Runners::RunnerIdNotFoundError, e.message
+      rescue KubernetesClient::LogsNotFoundError
+        Rails.logger.error("Error on fetching kubernetes pod logs")
+
+        "Logs not found"
+      end
+    end
+  end
+end

data/app/services/runners/kubernetes/filer.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Runners
+  module Kubernetes
+    class Filer
+      class InvalidMountName < StandardError; end
+
+      attr_reader :task_storage_mounts
+
+      def perform(task_storage_mounts:)
+        @task_storage_mounts = task_storage_mounts
+
+        {
+          internal: internal,
+          external: external
+        }
+      end
+
+      private
+
+      def internal
+        task_storage_mounts.map do |task_mount_name, task_mount_path|
+          {
+            name: task_mount_name,
+            mountPath: task_mount_path
+          }
+        end
+      end
+
+      def external
+        task_storage_mounts.map do |task_mount_name, _task_mount_path|
+          node_mount_path = Settings.to_hash[:storage_mounts][:kubernetes][task_mount_name.to_sym]
+
+          raise InvalidMountName unless node_mount_path
+
+          node_mount_path.merge(name: task_mount_name)
+        end
+      end
+    end
+  end
+end

data/app/services/runners/kubernetes/kill_slot_runner.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Runners
+  module Kubernetes
+    class KillSlotRunner
+      def perform(slot:)
+        RemoveRunner.new.perform(node: slot.node, runner_id: slot.runner_id)
+      end
+    end
+  end
+end