container_broker 1.0.1

data/config/initializers/docker_config.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+::Docker.logger = Logger.new(STDOUT)

data/config/initializers/filter_parameter_logging.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+# Be sure to restart your server when you modify this file.
+
+# Configure sensitive parameters which will be filtered from the log file.
+Rails.application.config.filter_parameters += [:password]

data/config/initializers/idempotent_request.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+Rails.application.config.middleware.use(
+  IdempotentRequest::Middleware,
+  storage: IdempotentRequest::RedisStorage.new(
+    Redis.new(RedisUrlParser.call(Settings.redis_url)),
+    expire_time: 1.day
+  ),
+  header_key: "X-Idempotency-Key",
+  policy: IdempotentRequest::Policy,
+  callback: IdempotentRequest::Callback
+)

data/config/initializers/inflections.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+# Be sure to restart your server when you modify this file.
+
+# Add new inflection rules using the following format. Inflections
+# are locale specific, and you may define rules for as many different
+# locales as you wish. All of these examples are active by default:
+# ActiveSupport::Inflector.inflections(:en) do |inflect|
+#   inflect.plural /^(ox)$/i, '\1en'
+#   inflect.singular /^(ox)en/i, '\1'
+#   inflect.irregular 'person', 'people'
+#   inflect.uncountable %w( fish sheep )
+# end
+
+# These inflection rules are supported but not enabled by default:
+# ActiveSupport::Inflector.inflections(:en) do |inflect|
+#   inflect.acronym 'RESTful'
+# end

data/config/initializers/mime_types.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+# Be sure to restart your server when you modify this file.
+
+# Add new mime types for use in respond_to blocks:
+# Mime::Type.register "text/richtext", :rtf

data/config/initializers/mongoid.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+Mongoid.raise_not_found_error = false

data/config/initializers/new_framework_defaults_6_0.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+# Be sure to restart your server when you modify this file.
+#
+# This file contains migration options to ease your Rails 6.0 upgrade.
+#
+# Once upgraded flip defaults one by one to migrate to the new default.
+#
+# Read the Guide for Upgrading Ruby on Rails for more info on each option.
+
+# Don't force requests from old versions of IE to be UTF-8 encoded.
+Rails.application.config.action_view.default_enforce_utf8 = false
+
+# Embed purpose and expiry metadata inside signed and encrypted
+# cookies for increased security.
+#
+# This option is not backwards compatible with earlier Rails versions.
+# It's best enabled when your entire app is migrated and stable on 6.0.
+Rails.application.config.action_dispatch.use_cookies_with_metadata = true
+
+# Change the return value of `ActionDispatch::Response#content_type` to Content-Type header without modification.
+Rails.application.config.action_dispatch.return_only_media_type_on_content_type = false
+
+# Return false instead of self when enqueuing is aborted from a callback.
+Rails.application.config.active_job.return_false_on_aborted_enqueue = true
+
+# Send Active Storage analysis and purge jobs to dedicated queues.
+# Rails.application.config.active_storage.queues.analysis = :active_storage_analysis
+# Rails.application.config.active_storage.queues.purge    = :active_storage_purge
+
+# When assigning to a collection of attachments declared via `has_many_attached`, replace existing
+# attachments instead of appending. Use #attach to add new attachments without replacing existing ones.
+# Rails.application.config.active_storage.replace_on_assign_to_many = true
+
+# Use ActionMailer::MailDeliveryJob for sending parameterized and normal mail.
+#
+# The default delivery jobs (ActionMailer::Parameterized::DeliveryJob, ActionMailer::DeliveryJob),
+# will be removed in Rails 6.1. This setting is not backwards compatible with earlier Rails versions.
+# If you send mail in the background, job workers need to have a copy of
+# MailDeliveryJob to ensure all delivery jobs are processed properly.
+# Make sure your entire app is migrated and stable on 6.0 before using this setting.
+# Rails.application.config.action_mailer.delivery_job = "ActionMailer::MailDeliveryJob"
+
+# Enable the same cache key to be reused when the object being cached of type
+# `ActiveRecord::Relation` changes by moving the volatile information (max updated at and count)
+# of the relation's cache key into the cache version to support recycling cache key.
+# Rails.application.config.active_record.collection_cache_versioning = true

data/config/initializers/raven.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+if Settings.sentry.enabled
+  require "raven"
+
+  Raven.configure do |config|
+    config.dsn = "https://#{Settings.sentry.app.public_key}:#{Settings.sentry.app.private_key}@#{Settings.sentry.host}/#{Settings.sentry.app.project_id}"
+    config.ssl_verification = false
+  end
+end

data/config/initializers/sidekiq.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+Rails.application.config.active_job.queue_adapter = Rails.env.test? ? :test : :sidekiq
+
+Sidekiq.default_worker_options = { backtrace: true }
+
+connection = proc {
+  Redis.new(RedisUrlParser.call(Settings.redis_url))
+}
+
+Sidekiq.configure_server do |config|
+  config.redis = ConnectionPool.new(size: 50, &connection)
+  config.logger.level = Logger::DEBUG
+
+  config.on(:startup) do
+    Sidekiq.schedule = YAML.load_file(File.expand_path("../sidekiq_scheduler.yml", __dir__))
+    SidekiqScheduler::Scheduler.instance.reload_schedule!
+  end
+end
+
+Sidekiq.configure_client do |config|
+  config.redis = ConnectionPool.new(size: 50, &connection)
+  config.logger.level = Logger::DEBUG
+end

data/config/initializers/wrap_parameters.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+# Be sure to restart your server when you modify this file.
+
+# This file contains settings for ActionController::ParamsWrapper which
+# is enabled by default.
+
+# Enable parameter wrapping for JSON. You can disable this by setting :format to an empty array.
+ActiveSupport.on_load(:action_controller) do
+  wrap_parameters format: [:json]
+end
+
+# To enable root element in JSON for ActiveRecord objects.
+# ActiveSupport.on_load(:active_record) do
+#   self.include_root_in_json = true
+# end

data/config/locales/en.yml

@@ -0,0 +1,33 @@
+# Files in the config/locales directory are used for internationalization
+# and are automatically loaded by Rails. If you want to use locales other
+# than English, add the necessary files in this directory.
+#
+# To use the locales, use `I18n.t`:
+#
+#     I18n.t 'hello'
+#
+# In views, this is aliased to just `t`:
+#
+#     <%= t('hello') %>
+#
+# To use a different locale, set it with `I18n.locale`:
+#
+#     I18n.locale = :es
+#
+# This would use the information in config/locales/es.yml.
+#
+# The following keys must be escaped otherwise they will not be retrieved by
+# the default I18n backend:
+#
+# true, false, on, off, yes, no
+#
+# Instead, surround them with single quotes.
+#
+# en:
+#   'true': 'foo'
+#
+# To learn more, please read the Rails Internationalization guide
+# available at http://guides.rubyonrails.org/i18n.html.
+
+en:
+  hello: "Hello world"

data/config/mongoid.yml

@@ -0,0 +1,10 @@
+test:
+  clients:
+    default:
+      database: container-broker_test
+      hosts:
+        - <%= ENV["MONGODB_URL"] || "localhost:27017" %>
+      options:
+        read:
+          mode: :primary
+        max_pool_size: 1

data/config/routes.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+ContainerBroker::Engine.routes.draw do
+  resources :status, only: :index do
+    collection do
+      get :nodes
+      get :tasks
+      get :tags
+      get :tag_values
+      get :task_statuses
+      post "retry_task/:uuid", action: :retry_task
+    end
+  end
+
+  resources :tasks, only: %i[create show], param: :uuid do
+    get :logs, on: :member
+
+    member do
+      put :mark_as_error
+      post :kill_container
+    end
+
+    collection do
+      delete :errors, action: :clear_errors
+      get :healthcheck, controller: :tasks_healthcheck, action: :index
+    end
+  end
+
+  resources :nodes, except: %i[edit new], param: :uuid do
+    member do
+      post :accept_new_tasks, :reject_new_tasks, :kill_containers
+    end
+
+    collection do
+      get :healthcheck, controller: :nodes_healthcheck, action: :index
+    end
+  end
+  # For details on the DSL available within this file, see http://guides.rubyonrails.org/routing.html
+
+  # root to: redirect(Settings.backstage_admin.url)
+
+  get "/healthcheck" => "healthcheck#index"
+end

data/config/secrets.yml

@@ -0,0 +1,35 @@
+# Be sure to restart your server when you modify this file.
+
+# Your secret key is used for verifying the integrity of signed cookies.
+# If you change this key, all old signed cookies will become invalid!
+
+# Make sure the secret is at least 30 characters and all random,
+# no regular words or you'll be exposed to dictionary attacks.
+# You can use `rails secret` to generate a secure secret key.
+
+# Make sure the secrets in this file are kept private
+# if you're sharing your code publicly.
+
+# Shared secrets are available across all environments.
+
+# shared:
+#   api_key: a1B2c3D4e5F6
+
+# Environmental secrets are only available for that specific environment.
+
+development:
+  secret_key_base: 41b8e60baf81a74ce740ca7115ea449c4b25bed764ed32f6a8453a2ced129cf229ee87e674b8aaad5a6f652129b3a29aabb6324df1f53ccde60b2cef665204a1
+
+test:
+  secret_key_base: 187f05efcc897d1d81fe968e825825053ff28fa492e352febc7e7957320c32b7a5359ec0c8aecb46b34ebb909c6809ef8fb133dd3844e1d0f699333972295e42
+
+# Do not keep production secrets in the unencrypted secrets file.
+# Instead, either read values from the environment.
+# Or, use `bin/rails secrets:setup` to configure encrypted secrets
+# and move the `production:` environment over there.
+
+dev:
+  secret_key_base: <%= ENV["SECRET_KEY_BASE"] %>
+
+production:
+  secret_key_base: <%= ENV["SECRET_KEY_BASE"] %>

data/config/settings.yml

@@ -0,0 +1,34 @@
+ignore_containers: []
+cors_origins: []
+kubernetes:
+  liveness_probe_seconds_interval: 30
+  requests:
+    memory: <%= ENV.fetch("KUBERNETES_REQUESTS_MEMORY", "4G") %>
+    cpu: <%= ENV.fetch("KUBERNETES_REQUESTS_CPU", 4) %>
+  limits:
+    memory: <%= ENV.fetch("KUBERNETES_LIMITS_MEMORY", "8G") %>
+    cpu: <%= ENV.fetch("KUBERNETES_LIMITS_CPU", 8) %>
+sentry:
+  enabled: false
+measures:
+  enabled: false
+run_container_as:
+  user_id: root
+  group_id: root
+timeout_tasks_after_hours: <%= 20.hours %>
+task_retry_count: 3
+redis_url: <%= ENV["DBAAS_SENTINEL_ENDPOINT"] %>
+delete_container_after_run: <%= ENV.fetch("DELETE_CONTAINER_AFTER_RUN", true) %>
+node_unavailable_after_seconds: <%= 2.minutes %>
+storage_mounts:
+  docker:
+    shared: "/opt/shared"
+    tmp: "/tmp"
+  kubernetes:
+    shared:
+      nfs:
+        server: "nfs-server.test"
+        path: "/shared"
+    tmp:
+      local:
+        path: "/tmp"

data/config/settings/test.yml

@@ -0,0 +1,27 @@
+task_retry_count: 3
+redis_url: <%= ENV["REDIS_URL"] || "redis://localhost:6379/0" %>
+delete_container_after_run: true
+node_unavailable_after_seconds: <%= 5.minutes %>
+timeout_tasks_after_hours: <%= 20.hours %>
+sentry:
+  enabled: false
+measures:
+  enabled: false
+backstage_admin:
+  url: admin.test
+run_container_as:
+  user_id: 1001
+  group_id: 1002
+storage_mounts:
+  docker:
+    shared_nfs: /mnt/nfs/node
+    temp: /tmp/node
+    ingest-nfs: /tmp/ingest_nfs
+  kubernetes:
+    shared_nfs:
+      nfs:
+        server: nfs.test
+        path: /mnt/nfs/node
+    temp:
+      local:
+        path: /mnt/local/node

data/config/sidekiq_scheduler.yml

@@ -0,0 +1,18 @@
+update_nodes_status:
+  cron: '*/5 * * * * *'
+  class: UpdateAllNodesStatusJob
+run_tasks:
+  in: '10s'
+  class: RunTasksForAllExecutionTypesJob
+monitor_unresponsive_nodes:
+  cron: '*/30 * * * * *'
+  class: MonitorUnresponsiveNodesJob
+remove_unreferenced_tags:
+  cron: '0 30 4 * * *'
+  class: RemoveUnusedTagsJob
+timeout_failed_tasks:
+  cron: '0 */15 * * * *'
+  class: TimeoutFailedTasksJob
+collect_load_metrics:
+  cron: '*/5 * * * * *'
+  class: CollectLoadMetricsJob

data/config/spring.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+%w[
+  .ruby-version
+  .rbenv-vars
+  tmp/restart.txt
+  tmp/caching-dev.txt
+].each { |path| Spring.watch(path) }

data/lib/constants.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Constants
+  class ExecutionType
+    FORMAT = /\A([a-z0-9])+(\-[a-z0-9]+)*\z/.freeze
+    INVALID_FORMAT_MESSAGE = "only allows lowercase letters, numbers and hyphen symbol"
+  end
+
+  class Runner
+    MAX_NAME_SIZE = 63
+  end
+end

data/lib/container_broker.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require "container_broker/engine"
+
+require "redis_url_parser"
+require "idempotent_request/callback"
+require "idempotent_request/policy"
+require "constants"
+require "current_thread_request_id"
+
+# Gems
+require "config"
+require "docker-api"
+require "active_model_serializers"
+require "config"
+require "idempotent-request"
+require "kubeclient"
+require "mongoid/uuid"
+require "mongoid_enumerable"
+require "redis-namespace"
+require "sentry-raven"
+require "sidekiq"
+require "sidekiq-failures"
+require "sidekiq-scheduler"
+require "docker-api"
+require "measures"
+
+module ContainerBroker
+  # Your code goes here...
+end

data/lib/container_broker/engine.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module ContainerBroker
+  class Engine < ::Rails::Engine
+  end
+end

data/lib/container_broker/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module ContainerBroker
+  VERSION = "1.0.1"
+end