container_broker 1.0.1

data/app/models/mongoid_serializable_model.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class MongoidSerializableModel
+  attr_reader :model
+  include GlobalID::Identification
+
+  def initialize(model)
+    @model = model
+  end
+
+  def to_global_id
+    model.to_global_id
+  end
+end

data/app/models/node.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+class Node
+  class NodeConnectionError < StandardError; end
+
+  include Mongoid::Document
+  include Mongoid::Uuid
+  include Mongoid::Timestamps
+  include GlobalID::Identification
+  include MongoidEnumerable
+
+  field :name, type: String
+  field :hostname, type: String
+  field :last_error, type: String
+  field :last_success_at, type: DateTime
+  field :accept_new_tasks, type: Boolean, default: true
+  field :runner_capacity_reached, type: Boolean, default: false
+  field :slots_execution_types, type: Hash, default: {}
+  field :runner_config, type: Hash, default: {}
+
+  enumerable :status, %w[available unstable unavailable], default: "unavailable"
+  enumerable :runner_provider, %w[docker kubernetes], default: :docker
+
+  has_many :slots
+
+  scope :accepting_new_tasks, -> { where(accept_new_tasks: true, :runner_capacity_reached.in => [nil, false]) }
+
+  validates :hostname, presence: true
+  validates :slots_execution_types, presence: true
+  validate :execution_types_format
+
+  def usage_per_execution_type
+    NodeUsagePercentagePerExecutionType.new(self).perform
+  end
+
+  def available_slot_with_execution_type(execution_type)
+    available_slots.find_by(execution_type: execution_type)
+  end
+
+  def available_slots
+    slots.available
+  end
+
+  def destroy_slots
+    slots.destroy_all
+  end
+
+  def runner_service(service)
+    Runners::ServicesFactory.fabricate(runner: runner_provider, service: service)
+  end
+
+  def register_error(error)
+    Rails.logger.info("Registering error in #{self}: #{error}")
+
+    update!(last_error: "#{error} at #{Time.zone.now}")
+
+    if available?
+      unstable!
+      Rails.logger.debug("#{self} marked as unstable")
+    elsif unstable?
+      if unstable_period_expired?
+        unavailable!
+        Rails.logger.debug("#{self} marked as unavailable because the unstable period has expired (last success was at #{last_success_at}). Migrating all tasks.")
+        MigrateTasksFromDeadNodeJob.perform_later(node: self)
+      else
+        Rails.logger.debug("#{self} still unstable until the limit period be expired (last success was at #{last_success_at})")
+      end
+    end
+  end
+
+  def unstable_period_expired?
+    last_success_at && last_success_at < Settings.node_unavailable_after_seconds.seconds.ago
+  end
+
+  def register_success
+    Rails.logger.debug("Registering success in #{self}")
+    update!(last_success_at: Time.zone.now)
+  end
+
+  def to_s
+    last_success = ", last success at #{last_success_at}" unless available?
+
+    "Node #{name} #{uuid} #{runner_provider} (#{status}#{last_success})"
+  end
+
+  def run_with_lock_no_wait
+    LockManager.new(type: self.class.to_s, id: id, wait: false, expire: 5.minutes).lock do
+      yield
+    end
+  end
+
+  private
+
+  def execution_types_format
+    valid = slots_execution_types
+            .keys
+            .all? { |execution_type| execution_type.match?(Constants::ExecutionType::FORMAT) }
+
+    errors.add(:slots_execution_types, Constants::ExecutionType::INVALID_FORMAT_MESSAGE) unless valid
+  end
+end

data/app/models/slot.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+class Slot
+  include Mongoid::Document
+  include Mongoid::Uuid
+  include GlobalID::Identification
+  include MongoidEnumerable
+
+  enumerable :status, %w[available attaching running releasing]
+
+  field :name, type: String
+  field :execution_type, type: String
+  field :runner_id, type: String
+  belongs_to :current_task, class_name: "Task", optional: true
+
+  belongs_to :node, optional: true
+
+  index(runner_id: 1)
+  index(node_id: 1)
+  index(execution_type: 1, status: 1)
+
+  validates :execution_type, presence: true
+  validates :execution_type, format: {
+    with: Constants::ExecutionType::FORMAT,
+    message: Constants::ExecutionType::INVALID_FORMAT_MESSAGE
+  }
+
+  scope :working, -> { where(:status.in => %w[attaching running releasing]) }
+
+  def mark_as_running(current_task:, runner_id:)
+    update!(status: :running, current_task: current_task, runner_id: runner_id)
+  end
+
+  def release
+    update!(status: :available, runner_id: nil, current_task: nil)
+    RunTasksJob.perform_later(execution_type: execution_type)
+  end
+
+  def to_s
+    "Slot #{name} #{uuid} (#{status} runner_id: #{runner_id})"
+  end
+end

data/app/models/task.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+class Task
+  include GlobalID::Identification
+  include Mongoid::Document
+  include Mongoid::Uuid
+  include MongoidEnumerable
+  extend Observable
+
+  field :name, type: String
+  field :runner_id, type: String
+  field :image, type: String
+  field :execution_type, type: String
+  field :cmd, type: String
+  field :storage_mounts, type: Hash, default: {}
+  field :exit_code, type: Integer
+  field :error, type: String
+  field :logs, type: BSON::Binary
+  field :created_at, type: DateTime
+  field :started_at, type: DateTime
+  field :finished_at, type: DateTime
+  field :progress, type: String
+  field :try_count, type: Integer, default: 0
+  field :persist_logs, type: Boolean, default: false
+  field :tags, type: Hash, default: {}
+
+  enumerable :status, %w[waiting starting started retry failed completed error], after_change: :status_changed
+
+  belongs_to :slot, optional: true
+
+  index({ created_at: 1 }, expire_after_seconds: 1.month)
+  index(tags: 1)
+  index(status: 1)
+  index(request_id: 1)
+  TaskTag.distinct(:name).each { |key| index("tags.#{key}" => 1) }
+
+  before_validation :normalize_tags
+  before_create { |task| task.created_at = Time.zone.now }
+  after_create do
+    RunTasksJob.perform_later(execution_type: execution_type)
+    AddTaskTagsJob.perform_later(task: self)
+  end
+
+  validates :name, :image, :cmd, :execution_type, presence: true
+  validates :execution_type, format: {
+    with: Constants::ExecutionType::FORMAT,
+    message: Constants::ExecutionType::INVALID_FORMAT_MESSAGE
+  }
+  validate :storage_mount_identifiers_exist
+
+  def set_logs(logs)
+    self.logs = BSON::Binary.new(logs.dup, :generic)
+  end
+
+  def get_logs
+    if started?
+      slot.node.runner_service(:fetch_logs).perform(task: self)
+    else
+      logs.try(:data)
+    end
+  end
+
+  def mark_as_started!(runner_id:, slot:)
+    update!(started_at: Time.zone.now, runner_id: runner_id, slot: slot)
+
+    started!
+  end
+
+  def mark_as_retry(error: nil)
+    update!(error: error)
+
+    if try_count < Settings.task_retry_count
+      update(try_count: try_count + 1, slot: nil, runner_id: nil)
+      retry!
+      RunTasksJob.perform_later(execution_type: execution_type)
+    else
+      failed!
+    end
+  end
+
+  def milliseconds_waiting
+    if started? || completed? || failed?
+      calculate_millisecond_span(created_at, started_at)
+    else
+      calculate_millisecond_span(created_at, Time.zone.now.to_datetime)
+    end
+  end
+
+  def milliseconds_running
+    if completed? || failed?
+      calculate_millisecond_span(started_at, finished_at)
+    elsif started?
+      calculate_millisecond_span(started_at, Time.zone.now.to_datetime)
+    end
+  end
+
+  def seconds_running
+    milliseconds_running&.div(1000)
+  end
+
+  def calculate_millisecond_span(start, finish)
+    ((finish - start) * 1.day.in_milliseconds).to_i if finish.present? && start.present?
+  end
+
+  def force_retry!
+    update(try_count: 0)
+    retry!
+    RunTasksJob.perform_later(execution_type: execution_type)
+  end
+
+  def normalize_tags
+    tags.transform_values!(&:to_s)
+  end
+
+  def to_s
+    "Task #{name} #{uuid} (#{status} runner_id: #{runner_id}) request_id=#{request_id}"
+  end
+
+  def generate_runner_id
+    prefix = name.gsub("_", "-").parameterize
+    random_suffix = SecureRandom.alphanumeric(8).downcase
+    max_prefix_size = Constants::Runner::MAX_NAME_SIZE - random_suffix.length - 1
+
+    "#{prefix.truncate(max_prefix_size, omission: "")}-#{random_suffix}"
+  end
+
+  def request_id
+    tags&.dig("request_id")
+  end
+
+  private
+
+  def status_changed(old_value, new_value)
+    self.class.observer_instances_for(self).each do |observer|
+      observer.status_change(old_value, new_value)
+    end
+  end
+
+  def storage_mount_identifiers_exist
+    valid = Node.pluck(:runner_provider).uniq.all? do |runner|
+      (storage_mounts.keys.map(&:to_s) - Settings.storage_mounts[runner].keys.map(&:to_s)).empty?
+    end
+
+    return if valid
+
+    errors.add(:storage_mounts, "Storage mounts are invalid")
+  end
+end

data/app/models/task_tag.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class TaskTag
+  include Mongoid::Document
+  include Mongoid::Uuid
+
+  field :name, type: String
+  field :values, type: Array, default: []
+
+  index({ name: 1 }, unique: true)
+end

data/app/observers/observable.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Observable
+  mattr_accessor :observers
+
+  def self.extended(model)
+    model.observers = Set.new
+  end
+
+  def add_observer(observer)
+    observers << observer
+  end
+
+  def remove_observer(observer)
+    observers.delete(observer)
+  end
+
+  def observer_instances_for(model)
+    observers.map do |observer|
+      observer.new(model)
+    end
+  end
+end

data/app/observers/task_observer.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class TaskObserver
+  attr_reader :task
+
+  def initialize(task)
+    @task = task
+  end
+
+  def status_change(_old_value, _new_value); end
+end

data/app/serializers/node_healthcheck_serializer.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class NodeHealthcheckSerializer < ActiveModel::Serializer
+  attributes :uuid, :name, :hostname, :status, :last_error, :created_at, :updated_at
+end

data/app/serializers/node_serializer.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class NodeSerializer < ActiveModel::Serializer
+  attributes :uuid, :hostname, :status, :accept_new_tasks, :slots_execution_types, :runner_provider, :runner_capacity_reached
+end

data/app/serializers/status_panel_node_serializer.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class StatusPanelNodeSerializer < ActiveModel::Serializer
+  attributes :uuid, :name, :hostname, :status, :last_error, :last_success_at,
+             :usage_per_execution_type, :slots_execution_types, :accept_new_tasks,
+             :runner_provider, :runner_capacity_reached
+
+  has_many :slots, serializer: StatusPanelSlotSerializer
+end

data/app/serializers/status_panel_slot_serializer.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class StatusPanelSlotSerializer < ActiveModel::Serializer
+  attributes :uuid, :name, :runner_id, :status, :execution_type
+end

data/app/serializers/status_panel_task_serializer.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+class StatusPanelTaskSerializer < ActiveModel::Serializer
+  attributes :uuid, :name, :image, :cmd, :status, :exit_code, :error, :try_count, :created_at,
+             :started_at, :finished_at, :progress, :seconds_running, :tags, :runner_id,
+             :storage_mounts, :slot, :execution_type
+
+  def slot
+    if object.slot
+      {
+        uuid: object.slot.uuid,
+        name: object.slot.name
+      }
+    end
+  end
+end

data/app/serializers/task_healthcheck_serializer.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class TaskHealthcheckSerializer < ActiveModel::Serializer
+  attributes :uuid, :status, :error, :created_at, :started_at, :finished_at, :execution_type
+end

data/app/serializers/task_serializer.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class TaskSerializer < ActiveModel::Serializer
+  attributes :uuid, :status, :exit_code, :error, :try_count,
+             :created_at, :started_at, :finished_at, :progress, :seconds_running,
+             :execution_type
+end

data/app/services/adjust_execution_type_slots.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+class AdjustExecutionTypeSlots
+  attr_reader :node, :execution_type
+
+  def initialize(node:, execution_type:)
+    @node = node
+    @execution_type = execution_type
+  end
+
+  def perform
+    increment_slots
+
+    decrement_slots
+
+    FriendlyNameSlots.new(node: node).perform
+  end
+
+  def increment?
+    amount > count_by_execution_type
+  end
+
+  def decrement?
+    amount < count_by_execution_type
+  end
+
+  private
+
+  def count_by_execution_type
+    node.slots.where(execution_type: execution_type).count
+  end
+
+  def amount
+    node.slots_execution_types[execution_type].to_i
+  end
+
+  def increment_slots
+    node.slots.create!(execution_type: execution_type) while increment?
+
+    RunTasksJob.perform_later(execution_type: execution_type)
+  end
+
+  def decrement_slots
+    while decrement?
+      slot = LockSlot.new(execution_type: execution_type, node: node).perform
+      break unless slot
+
+      slot.destroy!
+    end
+  end
+end