kube-platform 3.3.1.gk.0

data/lib/kube-platform/images/descriptor.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require "active_support/core_ext/hash/indifferent_access"
+
+module KubePlatform
+  module Images
+    class Descriptor
+      DEFAULT_IMAGE_CONFIG = "../../../config/images.yaml"
+
+      attr_reader :name, :default_tag, :default_tag_from_key, :key, :description, :key_alias
+
+      def initialize(name:, default_tag:, default_tag_from_key:, key:, description:, key_alias: nil)
+        @name = name
+        @default_tag = default_tag
+        @default_tag_from_key = default_tag_from_key
+        @key = key.to_sym
+        @description = description
+        @key_alias = key_alias&.to_sym
+      end
+
+      class << self
+        def with_image_defaults
+          images_path = File.expand_path(File.join(__dir__, DEFAULT_IMAGE_CONFIG))
+          array_from_yaml_file(filename: images_path)
+        end
+
+        def array_from_yaml_file(filename:)
+          yaml_to_hash(filename).map do |item|
+            new(
+              name:                 item[:name],
+              default_tag:          item[:default_tag],
+              default_tag_from_key: item[:default_tag_from_key],
+              key:                  item[:key],
+              description:          item[:description],
+              key_alias:            item[:key_alias]
+            )
+          end
+        end
+
+        private
+
+        def yaml_to_hash(filename)
+          YAML.load_file(filename).with_indifferent_access[:images] or # TODO: get rid of indifferent access
+            raise ConfigException, "#{filename} does not contain a set of valid image descriptors"
+        end
+      end
+    end
+  end
+end

data/lib/kube-platform/images/docker_hub_image.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require "base64"
+
+module KubePlatform
+  module Images
+    class DockerHubImage
+      include Logger
+
+      attr_reader :image_name, :secret_provider
+
+      def initialize(image_name:, secret_provider:)
+        @image_name      = image_name
+        @secret_provider = secret_provider
+      end
+
+      def tag_exist?(tag)
+        response = RestClient.get(
+          "https://index.docker.io/v2/#{image_name}/manifests/#{tag}",
+          Authorization: "Bearer #{dockerhub_token}", Accept: "application/json"
+        )
+        response.code == 200
+      rescue RestClient::RequestFailed
+        false
+      end
+
+      private
+
+      def basic_auth_secret
+        secret_provider.secret
+      end
+
+      def dockerhub_token
+        @dockerhub_token ||= read_dockerhub_token
+      end
+
+      def read_dockerhub_token
+        response = RestClient.get(
+          "https://auth.docker.io/token?service=registry.docker.io&scope=repository:#{image_name}:pull",
+          Authorization: "Basic #{basic_auth_secret}", Accept: "application/json"
+        )
+
+        JSON.parse(response.body)["token"]
+      rescue RestClient::RequestFailed => e
+        raise KubePlatformException, "Could not retrieve Docker Hub token: #{e.message}"
+      end
+    end
+  end
+end

data/lib/kube-platform/images/dockerhub_image_factory.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module KubePlatform
+  module Images
+    class DockerHubImageFactory
+      def initialize(secret_provider:)
+        @secret_provider = secret_provider
+        @image_cache = {}
+        @tagged_image_cache = {}
+      end
+
+      def build(image_name:, tag:)
+        if fully_qualified_docker_image?(tag)
+          parsed_tag = parse_tag_for_full_docker_image(tag)
+          tagged_dockerhub_image(parsed_tag[:image_name], parsed_tag[:image_tag])
+        else
+          tagged_dockerhub_image(image_name, tag)
+        end
+      end
+
+      private
+
+      def fully_qualified_docker_image?(image_tag)
+        image_tag.split(":").count > 1
+      end
+
+      def parse_tag_for_full_docker_image(image_tag)
+        split_tag = image_tag.split(":")
+        { image_name: split_tag[0], image_tag:  split_tag[1] }
+      end
+
+      def tagged_dockerhub_image(image_name, image_tag)
+        retrieve_tagged_image(image_name, image_tag) || create_tagged_dockerhub_image(image_name, image_tag)
+      end
+
+      def retrieve_tagged_image(image_name, image_tag)
+        @tagged_image_cache[tagged_image_hash_key(image_name, image_tag)]
+      end
+
+      def tagged_image_hash_key(image_name, image_tag)
+        [image_name, image_tag]
+      end
+
+      def create_tagged_dockerhub_image(image_name, image_tag)
+        dockerhub_image = dockerhub_image(image_name)
+        @tagged_image_cache[tagged_image_hash_key(image_name, image_tag)] = KubePlatform::Images::TaggedDockerHubImage.new(
+          image: dockerhub_image, tag: image_tag
+        )
+      end
+
+      def dockerhub_image(image_name)
+        retrieve_cached_image(image_name) || create_dockerhub_image(image_name)
+      end
+
+      def retrieve_cached_image(image_name)
+        @image_cache[image_name]
+      end
+
+      def create_dockerhub_image(image_name)
+        @image_cache[image_name] = KubePlatform::Images::DockerHubImage.new(image_name: image_name, secret_provider: @secret_provider)
+      end
+    end
+  end
+end

data/lib/kube-platform/images/kubernetes_docker_hub_secret_provider.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module KubePlatform
+  module Images
+    class KubernetesDockerHubSecretProvider
+      include Logger
+
+      attr_reader :client, :secret_namespace, :secret_name
+
+      def initialize(client:, secret_name:, secret_namespace:)
+        @client = client
+        @secret_name = secret_name
+        @secret_namespace = secret_namespace
+      end
+
+      def secret
+        basic_auth_secret
+      end
+
+      private
+
+      def basic_auth_secret
+        resource = client.get(secret_resource)
+        if resource.nil?
+          message = "Could not retrieve Docker Hub secret from #{secret_namespace}/#{secret_name}"
+          logger.error(message) # TODO: don't double-log this
+          raise KubePlatform::KubePlatformException, message
+        end
+
+        decode_token(resource.unwrap.data[".dockercfg"])
+      end
+
+      def secret_resource
+        @secret_resource ||= KubePlatform::Resource.from_spec(apiVersion: "v1", kind: "Secret",
+                                                             metadata: { name: secret_name, namespace: secret_namespace })
+      end
+
+      def decode_token(data)
+        decoded = Base64.decode64(data)
+        JSON.parse(decoded).dig("https://index.docker.io/v1/", "auth") or raise TokenNotFound, "Token not found in #{decoded}"
+      end
+    end
+  end
+end

data/lib/kube-platform/images/repository.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module KubePlatform
+  module Images
+    class Repository
+      def initialize(image_descriptors:, image_tags:, image_factory:)
+        @image_descriptors = image_descriptors
+        @image_tags = image_tags.compact
+        @image_factory = image_factory
+      end
+
+      def get(key)
+        registry[key.to_sym]
+      end
+
+      def missing_images
+        registry.values.reject(&:exist?)
+      end
+
+      def key_tag_pairs
+        registry.each_with_object({}) { |(key, image), pairs| pairs[key] = image.tag }
+      end
+
+      def key_tag_pairs_without_defaults
+        @image_tags
+      end
+
+      def add_key_tag_pairs(additional_pairs)
+        @image_tags.merge!(additional_pairs) { |_key, old, _new| old }
+        rebuild_registry
+      end
+
+      def update_image_tag_defaults(key_tag_pairs)
+        (keys_with_default_tags & key_tag_pairs.keys).each do |key|
+          descriptor = descriptor_for_key(key)
+          registry[key] = @image_factory.build(image_name: descriptor.name, tag: key_tag_pairs[key])
+        end
+      end
+
+      private
+
+      def registry
+        @registry ||= build_registry
+      end
+
+      def build_registry
+        image_key_tag_map = associate_tags(@image_descriptors, @image_tags)
+
+        @image_descriptors.each_with_object({}) do |descriptor, registry|
+          key = descriptor.key
+          registry[key] = @image_factory.build(image_name: descriptor.name, tag: image_key_tag_map[key])
+        end
+      end
+
+      def rebuild_registry
+        @registry = nil
+        registry
+      end
+
+      def associate_tags(image_descriptors, image_tags)
+        TagAssociator.new(image_descriptors: image_descriptors, image_tags: image_tags).run
+      end
+
+      def keys_with_default_tags
+        registry.keys - @image_tags.keys
+      end
+
+      def descriptor_for_key(key)
+        key_descriptor_map[key]
+      end
+
+      def key_descriptor_map
+        @key_descriptor_map ||= @image_descriptors.each_with_object({}) { |descriptor, hash| hash[descriptor.key] = descriptor }
+      end
+    end
+  end
+end

data/lib/kube-platform/images/tag_associator.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module KubePlatform
+  module Images
+    class TagAssociator
+      DescriptorTagPair = Struct.new(:descriptor, :tag)
+
+      def initialize(image_descriptors:, image_tags:)
+        @image_descriptors = image_descriptors
+        @image_tags = image_tags
+        @associated = {}
+      end
+
+      def run
+        descriptors_with_image_tags, descriptors_requiring_defaults = @image_descriptors.partition { |d| @image_tags.key?(d.key) }
+        associate_descriptors_with_image_tags(descriptors_with_image_tags)
+        associate_descriptors_with_defaults(descriptors_requiring_defaults)
+        transform_descriptor_tag_map_to_image_key_tag_map
+      end
+
+      private
+
+      def associate_descriptors_with_image_tags(descriptors)
+        descriptors.each { |d| associate_tag_to_descriptor(d, @image_tags[d.key]) }
+      end
+
+      def associate_tag_to_descriptor(descriptor, tag)
+        @associated[descriptor.key] = DescriptorTagPair.new(descriptor, tag)
+      end
+
+      def associate_descriptors_with_defaults(descriptors)
+        remaining_untagged = associate_tags_to_descriptors_containing_default_tag_attributes(descriptors)
+        remaining_untagged = associate_tags_to_descriptors_containing_references_to_other_descriptors(remaining_untagged)
+        remaining_untagged.empty? or
+          raise ConfigException, "No tag could be found for image with key '#{remaining_untagged.first.key}'"
+      end
+
+      def associate_tags_to_descriptors_containing_default_tag_attributes(descriptors)
+        descriptors.reject do |d|
+          if (tag = d.default_tag)
+            associate_tag_to_descriptor(d, tag)
+            true
+          else
+            false
+          end
+        end
+      end
+
+      def associate_tags_to_descriptors_containing_references_to_other_descriptors(descriptors)
+        descriptors.reject do |d|
+          if (ref = d.default_tag_from_key)
+            resolve_descriptor_reference(d, ref)
+            true
+          else
+            false
+          end
+        end
+      end
+
+      def resolve_descriptor_reference(descriptor, ref)
+        descriptor_tag_pair = @associated[ref.to_sym] or
+            raise ConfigException,
+                  "#{descriptor.key} wants to use the image tag from the image with key '#{ref}', but that image does not have a tag"
+
+        if descriptor.name != descriptor_tag_pair.descriptor.name &&
+          ([descriptor.name, descriptor_tag_pair.descriptor.name] - ['invocaops/web_app', 'invocaops/ringswitch']).any?
+            raise ConfigException,
+                  "#{descriptor.key} is trying to use the default_tag_from_key directive against #{ref}, "\
+                  "but the image names don't match"
+        end
+
+        associate_tag_to_descriptor(descriptor, descriptor_tag_pair.tag)
+      end
+
+      def transform_descriptor_tag_map_to_image_key_tag_map
+        @associated.each_with_object({}) { |(key, descriptor_tag_pair), hash| hash[key] = descriptor_tag_pair.tag }
+      end
+    end
+  end
+end

data/lib/kube-platform/images/tagged_dockerhub_image.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module KubePlatform
+  module Images
+    class TaggedDockerHubImage
+      include Logger
+
+      attr_reader :tag
+
+      def initialize(image:, tag:)
+        @image = image
+        @tag = tag
+      end
+
+      def name
+        "#{@image.image_name}:#{@tag}"
+      end
+
+      def exist?
+        defined?(@exists) or @exists = log_success_fail(@image.tag_exist?(@tag))
+      end
+
+      private
+
+      def log_success_fail(success)
+        if success
+          logger.info("Successfully located Docker Hub image for #{name}")
+        else
+          logger.info("Could not locate Docker Hub image for #{name}")
+        end
+
+        success
+      end
+    end
+  end
+end

data/lib/kube-platform/logger.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require "logger"
+
+module KubePlatform
+  module Logger
+    ANSI_ESCAPE = "\x1B"
+    ANSI_GOTO_START_OF_LINE   = ANSI_ESCAPE + "[0F"
+    ANSI_CLEAR_TO_END_OF_LINE = ANSI_ESCAPE + "[J"
+
+    class << self
+      attr_accessor :verbose
+      attr_writer :logger
+
+      def logger
+        @logger ||= ::Logger.new(STDOUT).tap do |logger|
+          logger.formatter = -> (severity, time, programname, msg) do
+            if STDOUT.isatty && !verbose
+              ANSI_GOTO_START_OF_LINE + ANSI_CLEAR_TO_END_OF_LINE + "%5s  %s\n" % [severity, msg]
+            else
+              ::Logger::Formatter.new.call(severity, time, programname, msg)
+            end
+          end
+        end
+      end
+    end
+
+    def logger
+      Logger.logger
+    end
+  end
+end

data/lib/kube-platform/manifest.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require "active_support/core_ext/hash"
+require "active_support/core_ext/string"
+require "jsonnet"
+
+module KubePlatform
+  class Manifest
+    attr_reader :resource_dir, :filename, :config
+
+    FILENAME_ANNOTATION = "invoca.com/resource_filename"
+
+    def initialize(filename:, resource_dir:, config:)
+      @filename = filename
+      @resource_dir = resource_dir
+      @config = config
+    end
+
+    def cluster_definition_by_name(name)
+      raise_if_definition_not_found(name)
+      definitions[name]
+    end
+
+    def version
+      manifest[:version] or raise ManifestException, "The manifest does not contain a 'version' key"
+    end
+
+    private
+
+    def raise_if_definition_not_found(definition_name)
+      definitions[definition_name] or raise ManifestException, "A platform definition named #{definition_name} was not found in #{filename}"
+    end
+
+    def definitions
+      @definitions ||= manifest[:definitions].each_with_object({}) do |(name, definition), clusters|
+        clusters[name] = ClusterDefinition.new(name: name, definition: definition, resource_dir: resource_dir, config: config)
+      end
+    end
+
+    def manifest
+      @manifest ||= case filename
+                    when /\.jsonnet$/
+                      load_jsonnet_manifest
+                    else
+                      load_yaml_manifest
+                    end.with_indifferent_access
+    end
+
+    def load_yaml_manifest
+      template = File.read(filename)
+      renderer = ERB.new(template)
+      YAML.safe_load(renderer.result(config.binding), aliases: true)
+    end
+
+    def load_jsonnet_manifest
+      vm = Jsonnet::VM.new
+      vm.tla_code("vars", config.to_json)
+      JSON.parse(vm.evaluate_file(filename))
+    end
+  end
+end

data/lib/kube-platform/pre_checks/r53_records.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require "aws-sdk-route53"
+require_relative "../logger"
+require_relative "../helpers/retry"
+
+module KubePlatform
+  module PreChecks
+    class R53Records < PreCheck
+      include Logger
+
+      DEFAULT_CONFIG = { }.freeze
+
+      def initialize(name, config)
+        super(name, config.apply_defaults(DEFAULT_CONFIG))
+      end
+
+      def run(_client, _cluster_definition)
+        logger.info("Checking for Route53 DNS records")
+
+        exists = []
+
+        fully_qualified_names.each do |record|
+          if record_exists?(record)
+            exists << record
+            logger.error("Route53 record #{record} already exists!")
+          end
+        end
+
+        exists.length > 0 ? false : true
+      end
+
+      private
+
+      def fully_qualified_names
+        @fully_qualified_names ||= r53_records.map { |record| record.end_with?(".") ? record : "#{record}." }
+      end
+
+      def record_exists?(record)
+        response = r53_client.list_resource_record_sets(
+          hosted_zone_id: r53_zone_id,
+          start_record_name: record,
+          start_record_type: "A",
+          max_items: 1
+        )
+        response.resource_record_sets.first&.name == record
+      end
+
+      def r53_client
+        @r53_client ||= Aws::Route53::Client.new(region: region)
+      end
+
+      def region
+        config[:region]
+      end
+
+      def r53_zone_id
+        config[:r53_zone_id]
+      end
+
+      def r53_records
+        @r53_records ||= config[:r53_records]
+      end
+    end
+  end
+end

data/lib/kube-platform/pre_checks/valid_platform_dependencies.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require_relative "../health_check"
+require_relative "../logger"
+
+module KubePlatform
+  module PreChecks
+    class ValidPlatformDependencies < PreCheck
+      PlatformDependencyException = Class.new(KubePlatformException)
+
+      include Logger
+
+      def run(_client, cluster_definition)
+        pod_names = Set.new(cluster_definition.resources.map(&:name))
+        pods_with_invalid_dependencies = invalid_platform_startup_dependencies(cluster_definition, pod_names)
+        pods_with_invalid_dependencies.empty? or raise PlatformDependencyException, invalid_dependency_details(pods_with_invalid_dependencies)
+        logger.info("Platform startup dependencies are valid")
+        true
+      end
+
+      private
+
+      def invalid_platform_startup_dependencies(cluster_definition, pod_names)
+        cluster_definition.resources.map do |resource|
+          if (dependencies_csv = resource.pod_annotation(:platform_startup_dependencies))
+            unless valid_dependency_list?(dependencies_csv, pod_names)
+              { name: resource.name, dependencies: dependencies_csv }
+            end
+          end
+        end.compact
+      end
+
+      def valid_dependency_list?(dependencies_csv, expected_pod_names)
+        dependencies_csv.split(/, */).all? { |dependency| expected_pod_names.include?(dependency) }
+      end
+
+      def invalid_dependency_details(pods_with_invalid_dependencies)
+        if pods_with_invalid_dependencies.size > 5
+          <<~EOS.chomp
+            Found #{pods_with_invalid_dependencies.size} pod templates with startup dependencies that don't match existing pod names. Showing the first 5.
+            #{pods_with_invalid_dependencies.first(5).join("\n")}
+          EOS
+        else
+          <<~EOS.chomp
+            Found pod templates with startup dependencies that don't match existing pod names.
+            #{pods_with_invalid_dependencies.join("\n")}
+          EOS
+        end
+      end
+    end
+  end
+end

data/lib/kube-platform/pre_checks.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module KubePlatform
+  class PreCheck
+    attr_reader :name, :config
+
+    def initialize(name, config)
+      @name = name
+      @config = config
+    end
+
+    class << self
+      def load(class_name:, name:, config:)
+        klass = "KubePlatform::PreChecks::#{class_name}".constantize
+        klass.new(name, config)
+      end
+    end
+  end
+end