kube-platform 3.3.1.gk.0

data/lib/kube-platform/handlers/dockerhub_secret_copy.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require_relative "secret_copy"
+
+module KubePlatform
+  module Handlers
+    class DockerhubSecretCopy < SecretCopy
+
+      def post_create(client)
+        update_service_account(client)
+      end
+
+      def post_update(client)
+        update_service_account(client)
+      end
+
+      private
+
+      def update_service_account(client)
+        wait_for_service_account(client)
+        api_client(client).patch_service_account(service_account_name, dockerhub_secret, namespace)
+      end
+
+      def api_client(client)
+        client.client_for_api
+      end
+
+      def dockerhub_secret
+        { imagePullSecrets: [{ name: secret_name }] }
+      end
+
+      def wait_for_service_account(client)
+        api_client(client).get_service_account(service_account_name, namespace)
+      rescue Kubeclient::ResourceNotFoundError
+        sleep(service_account_retry_delay)
+        retry # TODO: are infinite retries ok?
+      end
+
+      def service_account_name
+        config[:service_account_name]
+      end
+
+      def service_account_retry_delay
+        config[:service_account_retry_delay]
+      end
+
+      def namespace
+        resource.namespace
+      end
+    end
+  end
+end

data/lib/kube-platform/handlers/ebs_from_snapshot.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require "aws-sdk-ec2"
+require_relative "handler"
+require_relative "../logger"
+
+module KubePlatform
+  module Handlers
+    class EbsFromSnapshot < Handler
+      include Logger
+
+      VOLUME_TYPE = "gp2"
+      WAIT_DELAY = 5.seconds
+
+      def pre_create(_client)
+        logger.info("Creating EBS volume from snapshot ID #{snapshot_id}")
+        volume_id = create_volume
+        update_resource(volume_id)
+        logger.info("Created EBS volume #{volume_id} for use with #{resource.kind} #{resource.name}")
+      end
+
+      def pre_delete(client)
+        if resource_exists?(client)
+          volume_id = volume_id_from_resource(client)
+          logger.info("Waiting for EBS volume #{volume_id} to delete")
+          wait_until_deleted(volume_id)
+        end
+      end
+
+      private
+
+      def resource_exists?(client)
+        retrieve_resource(client) != nil
+      end
+
+      def volume_id_from_resource(client)
+        unwrapped = retrieve_resource(client).unwrap
+        unwrapped.spec.awsElasticBlockStore.volumeID
+      end
+
+      def retrieve_resource(client)
+        @retrieve_resource ||= client.get(resource)
+      end
+
+      def availability_zone
+        config[:availability_zone]
+      end
+
+      def tags
+        config[:tags]
+      end
+
+      def create_volume
+        volume = ec2_client.create_volume(
+          availability_zone: availability_zone,
+          volume_type: VOLUME_TYPE,
+          snapshot_id: snapshot_id
+        )
+
+        id = volume.volume_id
+        wait_until_available(id)
+        id
+      end
+
+      def wait_until_available(volume_id)
+        ec2_client.wait_until(:volume_available, { volume_ids: [volume_id] }, delay: WAIT_DELAY)
+      end
+
+      def wait_until_deleted(volume_id)
+        ec2_client.wait_until(:volume_deleted, { volume_ids: [volume_id] }, delay: WAIT_DELAY)
+      end
+
+      def update_resource(volume_id)
+        unwrapped = resource.unwrap
+        unwrapped.spec.awsElasticBlockStore.volumeID = volume_id
+      end
+
+      def snapshot_id
+        @snapshot_id ||= snapshot_search(tags)
+      end
+
+      def ec2_client
+        @ec2_client ||= Aws::EC2::Client.new
+      end
+
+      def owner_id
+        @owner_id ||= Aws::STS::Client.new.get_caller_identity[:account]
+      end
+
+      def snapshot_search(tags)
+        snapshots = ec2_client.describe_snapshots(owner_ids: [owner_id], filters: build_filters(tags)).snapshots
+        snapshots.min { |a, b| b.start_time <=> a.start_time }.snapshot_id
+      end
+
+      def build_filters(tags)
+        tag_hash_to_filter_format(tags) + [filter_format("status", "completed")]
+      end
+
+      def tag_hash_to_filter_format(tags)
+        tags.map { |tag_name, value| filter_format("tag:#{tag_name}", value) }
+      end
+
+      def filter_format(name, value)
+        { name: name, values: Array(value).map(&:to_s) }
+      end
+    end
+  end
+end

data/lib/kube-platform/handlers/handler.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module KubePlatform
+  module Handlers
+    class Handler
+      attr_reader :config
+      attr_accessor :resource # TODO: don't expose this mutator. Pass the resource into the constructor.
+
+      def initialize(config)
+        @config = config
+      end
+
+      def pre_create(_client)
+      end
+
+      def post_create(_client)
+      end
+
+      def pre_update(_client)
+      end
+
+      def post_update(_client)
+      end
+
+      def pre_delete(_client)
+      end
+
+      def post_delete(_client)
+      end
+
+      def ==(other)
+        self.class == other.class && config == other.config
+      end
+    end
+  end
+end

data/lib/kube-platform/handlers/recreate_resource.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module KubePlatform
+  module Handlers
+    class RecreateResource < Handler
+      def pre_create(client)
+        client.delete(resource)
+      end
+    end
+  end
+end

data/lib/kube-platform/handlers/secret_copy.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module KubePlatform
+  module Handlers
+    class SecretCopy < Handler
+      def pre_create(client)
+        copy_secret(client)
+      end
+
+      def pre_update(client)
+        copy_secret(client)
+      end
+
+      private
+
+      def copy_secret(client)
+        unwrapped = resource.unwrap
+        unwrapped.data = existing_secret(client)
+      end
+
+      def api_client(client)
+        client.client_for_api
+      end
+
+      def existing_secret(client)
+        secret_resource = api_client(client).get_secret(secret_name, source_namespace)
+        extract_secret(secret_resource)
+      end
+
+      def extract_secret(secret)
+        secret.data.to_hash
+      end
+
+      def secret_name
+        resource.name
+      end
+
+      def source_namespace
+        config[:source_namespace]
+      end
+    end
+  end
+end

data/lib/kube-platform/handlers/wait_for_job_completion.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require_relative "../logger"
+
+module KubePlatform
+  module Handlers
+    class WaitForJobCompletion < Handler
+      include Logger
+
+      DEFAULT_CONFIG = { polling_interval: 5, timeout: 300 }.freeze
+
+      def initialize(config)
+        super(config.apply_defaults(DEFAULT_CONFIG))
+      end
+
+      def post_create(client)
+        logger.info("Waiting for #{resource.kind} #{resource.name} to complete")
+        raise_if_job_does_not_complete!(client)
+      end
+
+      def timeout
+        config[:timeout]
+      end
+
+      def polling_interval
+        config[:polling_interval]
+      end
+
+      private
+
+      def raise_if_job_does_not_complete!(client)
+        attempts, interval = calculate_attempts_and_interval
+
+        job_complete_within_allotted_time?(client, attempts, interval) or
+          raise WaitTimeoutException, "#{resource.kind} #{resource.name} did not complete within #{timeout} seconds"
+      end
+
+      def calculate_attempts_and_interval
+        if polling_interval <= 0
+          logger.warn("Polling interval is set to #{polling_interval}.  Will retry once after #{timeout} seconds.")
+          attempts = 2
+          interval = timeout
+        else
+          interval = polling_interval
+          attempts = (timeout / interval).ceil + 1
+        end
+
+        [attempts, interval]
+      end
+
+      def job_complete_within_allotted_time?(client, attempts, interval)
+        Helpers::Retry.with_retries(attempts, interval) do
+          if job_complete?(client) # TODO: Move this logging into the caller?
+            logger.info("#{resource.kind} #{resource.name} is complete")
+            true
+          else
+            logger.debug("#{resource.kind} #{resource.name} is not ready. Sleeping #{interval} seconds.")
+            false
+          end
+        end
+      end
+
+      def job_complete?(client)
+        job = client.get(resource)
+        job.unwrap.spec.completions == job.unwrap&.status&.succeeded
+      end
+    end
+  end
+end

data/lib/kube-platform/handlers/wait_for_termination.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require_relative "../logger"
+require_relative "../helpers/retry"
+
+module KubePlatform
+  module Handlers
+    class WaitForTermination < Handler
+      include Logger
+
+      DEFAULT_CONFIG = { polling_interval: 5, timeout: 300 }.freeze
+
+      def initialize(config)
+        super(config.apply_defaults(DEFAULT_CONFIG))
+      end
+
+      def polling_interval
+        config[:polling_interval]
+      end
+
+      def timeout
+        config[:timeout]
+      end
+
+      def post_delete(client)
+        resource_terminated?(client) or
+          raise WaitTimeoutException, "Timeout of #{timeout} seconds reached while waiting for #{resource.kind} #{resource.name} to terminate"
+
+        logger.debug("#{resource.kind} #{resource.name} has terminated")
+      end
+
+      private
+
+      def resource_terminated?(client)
+        attempts = timeout / polling_interval
+        Helpers::Retry.with_retries(attempts, polling_interval) do
+          if client.exist?(resource)
+            logger.debug("#{resource.kind} #{resource.name} exists.  Sleeping for #{polling_interval} seconds")
+            false
+          else
+            true
+          end
+        end
+      end
+    end
+  end
+end

data/lib/kube-platform/health_check.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module KubePlatform
+  class HealthCheck
+    attr_reader :name, :config
+
+    def initialize(name, config)
+      @name = name
+      @config = config
+    end
+
+    class << self
+      def load(class_name:, name:, config:)
+        klass = "KubePlatform::HealthChecks::#{class_name}".constantize
+        klass.new(name, config)
+      end
+    end
+  end
+end

data/lib/kube-platform/health_checks/pods_ready.rb

@@ -0,0 +1,188 @@
+# frozen_string_literal: true
+
+require_relative '../health_check'
+require_relative '../logger'
+
+module KubePlatform
+  module HealthChecks
+    class PodsReady < HealthCheck
+      NotReadyContainerException = Class.new(KubePlatformException)
+
+      include Logger
+
+      DEFAULT_CONFIG = { attempts: 80, interval: 30 }.freeze
+
+      def initialize(name, config)
+        super(name, config.apply_defaults(DEFAULT_CONFIG))
+      end
+
+      # on success, returns true
+      # on failure, raises NotReadyContainerException
+      def run(client, _cluster_definition)
+        logger.info("Waiting for pods to become ready.  This may take a while.")
+        list_of_unready_pods = wait_for_pods_to_become_ready(client)
+
+        if list_of_unready_pods.empty?
+          logger.info("All pods are ready")
+          true
+        else
+          log_unhealthy(:error, list_of_unready_pods)
+
+          independent_unhealthy_pods = independently_failing_pods(list_of_unready_pods)
+          not_ready_container_info   = not_ready_containers(independent_unhealthy_pods)
+          not_ready_container_info.any? or raise ArgumentError, "not_ready_container_pairs is empty? #{independent_unhealthy_pods.inspect}"
+
+          raise NotReadyContainerException, not_ready_container_detail(not_ready_container_info, client)
+        end
+      end
+
+      private
+
+      def independently_failing_pods(unready_pods)
+        unready_pod_labels = Set.new(unready_pods.map { |pod| pod.metadata.labels.app || pod.metadata.labels.send(:"job-name") })
+
+        unready_pods.reject do |pod|
+          if (dependencies_csv = pod.metadata&.annotations&.platform_startup_dependencies)
+            dependencies_csv.split(/, */).any? { |pod_label| unready_pod_labels.include?(pod_label) }
+          end
+        end
+      end
+
+      # returns a hash of not ready pods
+      #   key is the pod name
+      #   value is a hash containing array of not ready containers and pod details
+      #
+      # Ex: { "pod-123" => { not_ready_containers: ["nginx"], pod_details: PodStruct } }
+      def not_ready_containers(unready_pods)
+        unready_pods.each_with_object({}) do |pod, result|
+          not_ready_containers = pod.status.containerStatuses&.reject(&:ready) || []
+          result[pod.metadata.name] = {
+            not_ready_containers: not_ready_containers.map(&:name),
+            details:              pod
+          }
+        end
+      end
+
+      # returns detailed text describing the root cause of failure for the given array of not_ready_container pairs
+      # suitable for framing in an exception body
+      def not_ready_container_detail(not_ready_containers, client)
+        not_ready_containers.map do |pod_name, pod|
+          not_ready_container_details_for_pod(pod_name, pod[:not_ready_containers], pod[:details], client)
+        end.flatten.join("\n").sub(/\s+\z/m, "")
+      end
+
+      def not_ready_container_details_for_pod(pod_name, not_ready_containers, pod_details, client)
+        if not_ready_containers.any?
+          not_ready_containers.map do |container_name|
+            not_ready_container_details_from_log(pod_name, container_name, pod_details, client)
+          end
+        else
+          "Pod #{pod_name} not ready but no containerStatus available"
+        end
+      end
+
+      def not_ready_container_details_from_log(pod_name, container_name, pod_details, client)
+        summary = ["Pod #{pod_name} container #{container_name} not ready:"]
+
+        log_lines = client.client_for_api.get_pod_log(pod_name, config.cluster_name, container: container_name).to_s.split("\n")
+
+        summary << if (exit_reason_line = last_log_occurrence(log_lines, '"exit_reason":'))
+                     exit_reason_yaml_from_log_line(exit_reason_line) || "JSON parse error or no 'app' key: #{exit_reason_line.inspect}"
+                   else
+                     "last 50 log lines:\n#{last_n_log_lines(log_lines, 50).join("\n")}"
+                   end
+
+        summary.join("\n")
+      rescue Kubeclient::HttpError => ex
+        ex.message.include?("ContainerCreating") or raise
+        summary << pod_details.status.containerStatuses.map { |status| status.to_hash.stringify_keys }.to_yaml
+        summary.join("\n")
+      end
+
+      def exit_reason_yaml_from_log_line(exit_reason_line)
+        if (exit_reason_hash = JSON.parse(exit_reason_line) rescue nil)
+          if (exit_reason_hash_app = exit_reason_hash['app'])
+            exit_reason_hash_app_simplified = exit_reason_hash_app.except("host", "pid", "tid", "logfile", "fiber", "exit_code", "timestamp", "progname", "log_tags")
+            exit_reason_hash_app_simplified.to_yaml
+          end
+        end
+      end
+
+      def last_log_occurrence(lines, pattern)
+        last_line = nil
+        lines.each do |line|
+          if line[pattern]
+            last_line = line
+          end
+        end
+        last_line
+      end
+
+      def last_n_log_lines(lines, n)
+        last_line_count = [lines.size, n].min
+        lines[-last_line_count..-1]
+      end
+
+      # returns the array of 0 or more unready pods
+      def wait_for_pods_to_become_ready(client)
+        unready = []
+        Helpers::Retry.with_retries(attempts, retry_interval) do
+          unready = unready_pods(client)
+          unready.empty? or begin
+            log_status_update(unready)
+            false
+          end
+        end
+
+        unready
+      end
+
+      def unready_pods(client)
+        client.client_for_api("v1").get_pods(namespace: namespace).reject { |pod| evicted?(pod) || pod_ready?(pod) }
+      end
+
+      def pod_ready?(pod)
+        if job?(pod)
+          pod.status.phase == "Succeeded"
+        else
+          pod.status&.containerStatuses&.all?(&:ready)
+        end
+      end
+
+      def evicted?(pod)
+        pod.status.phase == "Failed" && pod.status.reason == "Evicted"
+      end
+
+      def job?(pod)
+        pod.metadata.ownerReferences&.any? { |owner| owner.kind == "Job" }
+      end
+
+      def log_status_update(unready)
+        optional_detail =
+          if unready.size <= 5
+            ": " + unready.map do |pod|
+              pod.metadata.name.split('-', 2).first
+            end.join(", ")
+          end
+        logger.info("#{unready.size} pod#{unready.size > 1 ? 's are' : ' is'} not in a ready state#{optional_detail}")
+        log_unhealthy(:debug, unready)
+      end
+
+      def log_unhealthy(level, pods)
+        pods.each { |pod| logger.send(level, "Pod #{pod.metadata.name} is not in a healthy state") }
+      end
+
+      def namespace
+        config[:cluster_name]
+      end
+
+      def attempts
+        config[:attempts]
+      end
+
+      def retry_interval
+        config[:interval]
+      end
+    end
+  end
+end

data/lib/kube-platform/health_checks/r53_records.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require "aws-sdk-route53"
+require_relative "../logger"
+require_relative "../helpers/retry"
+
+module KubePlatform
+  module HealthChecks
+    class R53Records < HealthCheck
+      include Logger
+
+      DEFAULT_CONFIG = { attempts: 20, interval: 30 }.freeze
+
+      def initialize(name, config)
+        super(name, config.apply_defaults(DEFAULT_CONFIG))
+      end
+
+      def run(_client, _cluster_definition)
+        logger.info("Checking for Route53 DNS records")
+
+        missing = missing_records
+        if missing.empty?
+          logger.info("All Route53 records exist")
+          true
+        else
+          missing.each { |record| logger.error("Route53 record #{record} does not exist") }
+          false
+        end
+      end
+
+      private
+
+      def missing_records
+        missing_records = fully_qualified_names
+        Helpers::Retry.with_retries(attempts, retry_interval) do
+          missing_records.reject! { |record| record_exists?(record) }
+          missing_records.empty? and break
+        end
+
+        missing_records
+      end
+
+      def fully_qualified_names
+        @fully_qualified_names ||= r53_records.map { |record| record.end_with?(".") ? record : "#{record}." }
+      end
+
+      def record_exists?(record)
+        response = r53_client.list_resource_record_sets(
+          hosted_zone_id: r53_zone_id,
+          start_record_name: record,
+          start_record_type: "A",
+          max_items: 1
+        )
+        response.resource_record_sets.first&.name == record
+      end
+
+      def r53_client
+        @r53_client ||= Aws::Route53::Client.new(region: region)
+      end
+
+      def region
+        config[:region]
+      end
+
+      def r53_zone_id
+        config[:r53_zone_id]
+      end
+
+      def r53_records
+        @r53_records ||= config[:r53_records]
+      end
+
+      def attempts
+        config[:attempts]
+      end
+
+      def retry_interval
+        config[:interval]
+      end
+    end
+  end
+end

data/lib/kube-platform/helpers/retry.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module KubePlatform
+  module Helpers
+    module Retry
+      class << self
+        def with_retries(max_attempts, retry_interval)
+          try = 0
+          loop do
+            success = yield and break success
+
+            (try += 1) >= max_attempts and break false
+
+            sleep(retry_interval)
+          end
+        end
+      end
+    end
+  end
+end