container_broker 1.0.1

data/app/services/runners/kubernetes/node_availability.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Runners
+  module Kubernetes
+    class NodeAvailability
+      def perform(node:)
+        CreateClient.new.perform(node: node).api_info
+      end
+    end
+  end
+end

data/app/services/runners/kubernetes/remove_runner.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Runners
+  module Kubernetes
+    class RemoveRunner
+      def perform(node:, runner_id:)
+        Rails.logger.debug("Deleting pod")
+        begin
+          CreateClient.new.perform(node: node).force_delete_pod(pod_name: runner_id)
+          Rails.logger.debug("Pod #{runner_id} removed")
+        rescue KubernetesClient::PodNotFoundError
+          Rails.logger.debug("Pod #{runner_id} already removed")
+        rescue KubernetesClient::NetworkError => e
+          node.register_error(e.message)
+        end
+      end
+    end
+  end
+end

data/app/services/runners/kubernetes/run_task.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Runners
+  module Kubernetes
+    class RunTask
+      NFS_NAME = "nfs"
+
+      def perform(task:, slot:, runner_id:)
+        create_pod(task: task, node: slot.node, runner_id: runner_id)
+      rescue KubernetesClient::NetworkError => e then
+        raise Node::NodeConnectionError, "#{e.class}: #{e.message}"
+      end
+
+      private
+
+      def create_pod(task:, node:, runner_id:)
+        CreateClient.new.perform(node: node).create_pod(
+          pod_name: runner_id,
+          image: task.image,
+          cmd: task.cmd,
+          internal_mounts: internal_mounts(task: task),
+          external_mounts: external_mounts(task: task),
+          node_selector: node.runner_config["node_selector"]
+        )
+      end
+
+      def filer(task:)
+        Filer.new.perform(task_storage_mounts: task.storage_mounts)
+      end
+
+      def internal_mounts(task:)
+        filer(task: task)[:internal]
+      end
+
+      def external_mounts(task:)
+        filer(task: task)[:external]
+      end
+
+      def add_metric(task)
+        Metrics.new("tasks").count(
+          task_id: task.id,
+          name: task&.name,
+          type: task&.execution_type,
+          slot: task&.slot&.name,
+          node: task&.slot&.node&.name,
+          started_at: task.started_at,
+          duration: task.milliseconds_waiting,
+          error: task.error,
+          status: task.status
+        )
+      end
+    end
+  end
+end

data/app/services/runners/kubernetes/update_node_status.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Runners
+  module Kubernetes
+    class UpdateNodeStatus
+      include UpdateNodeStatusHelper
+
+      attr_reader :node
+
+      def perform(node:)
+        @node = node
+
+        # Other tasks can be started at this time. Because of this it's necessary to load the tasks first and then the containers
+        started_tasks = Task.started.where(:slot.in => node.slots.pluck(:id)).to_a
+
+        node.update!(runner_capacity_reached: pending_schedule_pods?)
+
+        execution_infos.each do |execution_info|
+          runner_id = execution_info.id
+          slot = node.slots.find_by(runner_id: runner_id)
+
+          if slot
+            if execution_info.terminated?
+              Rails.logger.debug("Pod #{runner_id} Complete")
+              check_slot_release(slot: slot, runner_id: runner_id)
+            else
+              slot.current_task&.update!(error: execution_info.error) if execution_info.error
+              Rails.logger.debug("Pod is not terminated (it is #{execution_info.status}). Ignoring.")
+            end
+          else
+            remove_unknown_runner(node: node, runner_id: runner_id)
+          end
+        end
+
+        RescheduleTasksForMissingRunners
+          .new(runner_ids: pods.keys, started_tasks: started_tasks)
+          .perform
+
+        node.register_success
+
+        send_metrics(node: node, execution_infos: execution_infos)
+      rescue KubernetesClient::NetworkError => e
+        Rails.logger.debug("Error #{e.class}: #{e}")
+        node.register_error(e.message)
+      end
+
+      private
+
+      def pods
+        @pods ||= CreateClient.new.perform(node: node)
+                              .fetch_pods
+                              .tap { |pods| Rails.logger.debug("Fetched #{pods.count} pods") }
+      end
+
+      def execution_infos
+        @execution_infos ||= pods.values.map { |pod| CreateExecutionInfo.new.perform(pod: pod) }
+      end
+
+      def pending_schedule_pods?
+        execution_infos.any?(&:schedule_pending?)
+      end
+    end
+  end
+end

data/app/services/runners/runner_id_not_found_error.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Runners
+  class RunnerIdNotFoundError < StandardError; end
+end

data/app/services/runners/services_factory.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Runners
+  class ServicesFactory
+    class ServiceNotFoundForRunner < StandardError; end
+
+    SERVICES = {
+      kubernetes: {
+        update_node_status: Runners::Kubernetes::UpdateNodeStatus,
+        node_availability: Runners::Kubernetes::NodeAvailability,
+        run_task: Runners::Kubernetes::RunTask,
+        kill_slot_runner: Runners::Kubernetes::KillSlotRunner,
+        remove_runner: Runners::Kubernetes::RemoveRunner,
+        fetch_logs: Runners::Kubernetes::FetchLogs,
+        fetch_execution_info: Runners::Kubernetes::FetchExecutionInfo,
+        filer: Runners::Kubernetes::Filer
+      },
+      docker: {
+        update_node_status: Runners::Docker::UpdateNodeStatus,
+        node_availability: Runners::Docker::NodeAvailability,
+        run_task: Runners::Docker::RunTask,
+        kill_slot_runner: Runners::Docker::KillSlotRunner,
+        remove_runner: Runners::Docker::RemoveRunner,
+        fetch_logs: Runners::Docker::FetchLogs,
+        fetch_execution_info: Runners::Docker::FetchExecutionInfo,
+        filer: Runners::Docker::Filer
+      }
+    }.freeze
+
+    def self.fabricate(runner:, service:)
+      service_class = SERVICES.dig(runner.to_sym, service)
+
+      raise ServiceNotFoundForRunner, "No service #{service} found for #{runner}" unless service_class
+
+      service_class.new
+    end
+  end
+end

data/app/services/runners/update_node_status_helper.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Runners
+  module UpdateNodeStatusHelper
+    def check_slot_release(slot:, runner_id:)
+      if slot.running?
+        slot.releasing!
+        Rails.logger.debug("Slot was running. Marked as releasing. Slot: #{slot}. Current task: #{slot.current_task}")
+        ReleaseSlotJob.perform_later(slot: MongoidSerializableModel.new(slot), runner_id: runner_id)
+      else
+        Rails.logger.debug("Slot was not running (it was #{slot.status}). Ignoring.")
+      end
+    end
+
+    def remove_unknown_runner(node:, runner_id:)
+      Rails.logger.debug("Slot not found for container #{runner_id}")
+
+      if Settings.ignore_containers.none? { |ignored_name| runner_id.start_with?(ignored_name) }
+        # It is needed to select the container using just any of its names
+        RemoveRunnerJob.perform_later(node: node, runner_id: runner_id)
+      else
+        Rails.logger.debug("Container #{runner_id} is ignored for removal")
+      end
+    end
+
+    def send_metrics(node:, execution_infos:)
+      runners_count = execution_infos
+                      .group_by(&:status)
+                      .transform_keys { |k| "#{k}_runners".to_sym }
+                      .transform_values(&:count)
+
+      data = {
+        hostname: node.hostname,
+        runner_type: node.runner_provider,
+        capacity_reached: node.runner_capacity_reached,
+        schedule_pending: execution_infos.count(&:schedule_pending?),
+        total_runners: execution_infos.count
+      }
+
+      Metrics.new("runners").count(data.merge(runners_count))
+    end
+  end
+end

data/app/services/slots_usage_percentage.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class SlotsUsagePercentage
+  def initialize(slots)
+    @slots = slots
+  end
+
+  def perform
+    (((@slots.size - available_slots.size).to_f / @slots.size) * 100)
+      .round(2)
+  end
+
+  private
+
+  def available_slots
+    @slots.select(&:available?)
+  end
+end

data/config/application.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require_relative "boot"
+
+require "action_controller/railtie"
+# require "action_view/railtie"
+# require "action_mailer/railtie"
+require "active_job/railtie"
+# require "action_cable/engine"
+# require "rails/test_unit/railtie"
+# require "sprockets/railtie"
+
+# Require the gems listed in Gemfile, including any gems
+# you've limited to :test, :development, or :production.
+Bundler.require(*Rails.groups)
+
+module ContainerBroker
+  class Application < Rails::Application
+    # Initialize configuration defaults for originally generated Rails version.
+    config.load_defaults 5.1
+
+    config.api_only = true
+
+    # Settings in config/environments/* take precedence over those specified here.
+    # Application configuration should go into files in config/initializers
+    # -- all .rb files in that directory are automatically loaded.
+
+    config.eager_load_paths << Rails.root.join("lib")
+
+    config.log_tags = [
+      ->(request) { " request_id=#{request.request_id} " }
+    ]
+  end
+end

data/config/boot.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../Gemfile", __dir__)
+
+require "bundler/setup" # Set up gems listed in the Gemfile.

data/config/environment.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+# Load the Rails application.
+require_relative "application"
+
+# Initialize the Rails application.
+Rails.application.initialize!

data/config/environments/test.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+Rails.application.configure do
+  # Settings specified here will take precedence over those in config/application.rb.
+
+  # The test environment is used exclusively to run your application's
+  # test suite. You never need to work with it otherwise. Remember that
+  # your test database is "scratch space" for the test suite and is wiped
+  # and recreated between test runs. Don't rely on the data there!
+  config.cache_classes = true
+
+  # Do not eager load code on boot. This avoids loading your whole application
+  # just for the purpose of running a single test. If you are using a tool that
+  # preloads Rails for running tests, you may have to set it to true.
+  config.eager_load = false
+
+  # Configure public file server for tests with Cache-Control for performance.
+  config.public_file_server.enabled = true
+  config.public_file_server.headers = {
+    "Cache-Control" => "public, max-age=#{1.hour.seconds.to_i}"
+  }
+
+  # Show full error reports and disable caching.
+  config.consider_all_requests_local       = true
+  config.action_controller.perform_caching = false
+
+  # Raise exceptions instead of rendering exception templates.
+  config.action_dispatch.show_exceptions = false
+
+  # Disable request forgery protection in test environment.
+  config.action_controller.allow_forgery_protection = false
+  # config.action_mailer.perform_caching = false
+
+  # Tell Action Mailer not to deliver emails to the real world.
+  # The :test delivery method accumulates sent emails in the
+  # ActionMailer::Base.deliveries array.
+  # config.action_mailer.delivery_method = :test
+
+  # Print deprecation notices to the stderr.
+  config.active_support.deprecation = :stderr
+
+  # Raises error for missing translations
+  # config.action_view.raise_on_missing_translations = true
+end

data/config/initializers/application_controller_renderer.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+# Be sure to restart your server when you modify this file.
+
+# ActiveSupport::Reloader.to_prepare do
+#   ApplicationController.renderer.defaults.merge!(
+#     http_host: 'example.org',
+#     https: false
+#   )
+# end

data/config/initializers/backtrace_silencers.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+# Be sure to restart your server when you modify this file.
+
+# You can add backtrace silencers for libraries that you're using but don't wish to see in your backtraces.
+# Rails.backtrace_cleaner.add_silencer { |line| line =~ /my_noisy_library/ }
+
+# You can also remove all the silencers if you're trying to debug a problem that might stem from framework code.
+# Rails.backtrace_cleaner.remove_silencers!

data/config/initializers/config.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+Config.setup do |config|
+  # Name of the constant exposing loaded settings
+  config.const_name = "Settings"
+
+  # Ability to remove elements of the array set in earlier loaded settings file. For example value: '--'.
+  #
+  # config.knockout_prefix = nil
+
+  # Overwrite an existing value when merging a `nil` value.
+  # When set to `false`, the existing value is retained after merge.
+  #
+  # config.merge_nil_values = true
+
+  # Overwrite arrays found in previously loaded settings file. When set to `false`, arrays will be merged.
+  #
+  # config.overwrite_arrays = true
+
+  # Load environment variables from the `ENV` object and override any settings defined in files.
+  #
+  # config.use_env = false
+
+  # Define ENV variable prefix deciding which variables to load into config.
+  #
+  # config.env_prefix = 'Settings'
+
+  # What string to use as level separator for settings loaded from ENV variables. Default value of '.' works well
+  # with Heroku, but you might want to change it for example for '__' to easy override settings from command line, where
+  # using dots in variable names might not be allowed (eg. Bash).
+  #
+  # config.env_separator = '.'
+
+  # Ability to process variables names:
+  #   * nil  - no change
+  #   * :downcase - convert to lower case
+  #
+  # config.env_converter = :downcase
+
+  # Parse numeric values as integers instead of strings.
+  #
+  # config.env_parse_values = true
+
+  # Validate presence and type of specific config values. Check https://github.com/dry-rb/dry-validation for details.
+  #
+  # config.schema do
+  #   required(:name).filled
+  #   required(:age).maybe(:int?)
+  #   required(:email).filled(format?: EMAIL_REGEX)
+  # end
+end

data/config/initializers/cookies_serializer.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+# Be sure to restart your server when you modify this file.
+
+# Specify a serializer for the signed and encrypted cookie jars.
+# Valid options are :json, :marshal, and :hybrid.
+Rails.application.config.action_dispatch.cookies_serializer = :json