kube_deploy_tools 3.0.5

data/lib/kube_deploy_tools/image_registry/driver/aws.rb

@@ -0,0 +1,121 @@
+require 'json'
+
+require_relative 'base'
+require_relative '../image'
+require 'kube_deploy_tools/deploy_config_file/util'
+require 'kube_deploy_tools/shellrunner'
+
+module KubeDeployTools
+  class ImageRegistry::Driver::Aws < ImageRegistry::Driver::Base
+    include DeployConfigFileUtil
+
+    def initialize(h)
+      super(h)
+
+      check_and_err(
+        @registry.config&.has_key?('region'),
+        "Expected .image_registries['#{@name}'] to have .config.region to be set "\
+        "for the AWS image registry driver, but .config.region is not set "\
+        "e.g. .config.region = 'us-west-2'"
+      )
+    end
+
+    def push_image(image)
+      create_repository(image.repository) unless repository_exists?(image.repository)
+      super(image)
+    end
+
+    def authorize_command
+      login_cmd = get_docker_login
+      raise "Unexpected login command: #{login_cmd}" if login_cmd.first(2) != ['docker', 'login']
+      login_cmd
+    end
+
+    def unauthorize
+    end
+
+    def delete_image(image, dryrun)
+      # In the AWS driver, the 'delete many' primitive is the primary one.
+      delete_images([image], dryrun)
+    end
+
+    def delete_images(images, dryrun)
+      # Aggregate images by repository and call aws ecr batch-delete-image
+      # once per repository.
+      ids_by_repository = {}
+      images.each do |image|
+        repository, tag = split_full_image_id(image)
+        item = {'imageTag': tag}
+        if ids_by_repository[repository].nil?
+          ids_by_repository[repository] = [item]
+        else
+          ids_by_repository[repository].push(item)
+        end
+      end
+
+      # JSON format documented here:
+      # https://docs.aws.amazon.com/cli/latest/reference/ecr/batch-delete-image.html
+      ids_by_repository.each do |repository, image_ids|
+        # batch-delete-image has a threshold of 100 image_ids at a time
+        image_chunks = image_ids.each_slice(100).to_a
+
+        image_chunks.each do |images|
+          cmd = [
+            'aws', 'ecr', 'batch-delete-image',
+            '--repository-name', repository,
+            '--region', @registry.config.fetch('region'),
+            '--image-ids', images.to_json,
+          ]
+
+          if dryrun
+            Logger.info("Would run: #{cmd}")
+          else
+            Shellrunner.check_call(*cmd)
+          end
+        end
+      end
+    end
+
+    private
+    def get_docker_login
+      args = Shellrunner.check_call('aws', 'ecr', 'get-login', '--region', @registry.config.fetch('region')).split
+
+      # Remove '-e' and subsequent argument
+      # This compensates for --no-include-email not being recognized in the Ubuntu packaged awscli
+      # and not usable unless you upgrade
+      i = args.index('-e')
+      if !i.nil?
+        # delete '-e'
+        args.delete_at(i)
+        # delete whatever value is after (usually 'none')
+        args.delete_at(i)
+      end
+
+      args
+    end
+
+    def repository_exists?(repository)
+      _, _, status = Shellrunner.run_call('aws', 'ecr', 'describe-repositories', '--repository-names', repository, '--region', @registry.config.fetch('region'))
+      status.success?
+    end
+
+    def create_repository(repository)
+      Shellrunner.check_call('aws', 'ecr', 'create-repository', '--repository-name', repository, '--region', @registry.config.fetch('region'))
+    end
+
+    def split_full_image_id(image_id)
+      # Create syntax suitable for aws ecr subcommand.
+      # Example: 12345678.dkr.ecr.amazonaws.com/my_app:deadbeef-123
+      # splits into ('my_app', 'deadbeef-123') after verifying that the
+      # prefix is the expected one for this driver instance.
+      repo_with_prefix, tag = image_id.split(':', 2)
+      prefix, repository = repo_with_prefix.split('/', 2)
+
+      # Sanity check, as the resultant command line uses the region to specify
+      # the prefix implicitly.
+      raise "This driver can't delete images from #{prefix}, only #{@registry.prefix}" unless prefix == @registry.prefix
+
+      return repository, tag
+    end
+  end
+end

data/lib/kube_deploy_tools/image_registry/driver/base.rb

@@ -0,0 +1,50 @@
+require 'kube_deploy_tools/formatted_logger'
+require 'kube_deploy_tools/shellrunner'
+
+require 'kube_deploy_tools/image_registry/image'
+
+# Abstract Driver class that specific implementations inherit
+module KubeDeployTools
+  class ImageRegistry
+    module Driver
+      class Base
+        def initialize(registry:)
+          @registry = registry
+        end
+
+        def push_image(image)
+          Shellrunner.check_call('docker', 'push', image.full_tag)
+        end
+
+        def authorize
+          Logger.info "performing registry login for #{@registry.prefix}"
+          Shellrunner.check_call(*authorize_command, print_cmd: false)
+        end
+
+        def authorize_command
+          raise "#{self.class}#authorize_command needs explicit implementation"
+        end
+
+        def unauthorize
+          Logger.info "Performing registry unauthorization for #{@registry.prefix}"
+          Shellrunner.check_call(*unauthorize_command, print_cmd: false)
+        end
+
+        def unauthorize_command
+          raise "#{self.class}#unauthorize_command needs explicit implementation"
+        end
+
+        def delete_images(images, dryrun)
+          # Naive default implementation.
+          images.each do |image|
+            delete_image(image, dryrun)
+          end
+        end
+
+        def delete_image(image, dryrun)
+          raise "#{self.class}#delete_image needs explicit implementation"
+        end
+      end
+    end
+  end
+end

data/lib/kube_deploy_tools/image_registry/driver/gcp.rb

@@ -0,0 +1,71 @@
+require_relative 'base'
+require 'tmpdir'
+
+module KubeDeployTools
+  class ImageRegistry::Driver::Gcp < ImageRegistry::Driver::Base
+    def initialize(registry:)
+      super
+
+      @gcloud_config_dir = Dir.mktmpdir
+      @activated = false
+    end
+
+    def authorize
+      # Always prefer and activate a service account under a protected namespace if present.
+      if @activated
+        return
+      elsif ENV.member?('GOOGLE_APPLICATION_CREDENTIALS')
+        raise "Failed to activate service account" unless activate_service_account()[2].success?
+        @activated = true
+      else
+        user = current_user
+        if ! user.empty?
+          Logger.info "Skipping Google activation, using current user #{user}"
+          @activated = true
+        else
+          raise 'No usable Google authorization for pushing images; specify GOOGLE_APPLICATION_CREDENTIALS?'
+        end
+      end
+    end
+
+    # Delete temporary config dir for gcloud authentication
+    def unauthorize
+      Logger.info "Cleaning up authorization for #{@registry.prefix}"
+      FileUtils.rm_rf(@gcloud_config_dir) unless @gcloud_config_dir.nil?
+    end
+
+    def delete_image(image_id, dryrun)
+      # Need the id path to be [HOSTNAME]/[PROJECT-ID]/[IMAGE]
+      if dryrun
+        Logger.info("DRYRUN: delete gcp image #{image_id}")
+      else
+        # --quiet removes the user-input component
+        _, err, status = Shellrunner.run_call('gcloud', 'container', 'images', 'delete', '--quiet', image_id, '--force-delete-tags')
+        if !status.success?
+          # gcloud gives a deceptive error msg when the image does not exist
+          if err.include?('is not a valid name')
+            Logger.warn("Image #{image_id} does not exist, skipping")
+          else
+            raise "gcloud image deletion failed!"
+          end
+        end
+      end
+    end
+
+    private
+    # activate gcloud with svc json keys on Jenkins
+    def activate_service_account
+      keypath = ENV.fetch('GOOGLE_APPLICATION_CREDENTIALS')
+      Logger.info("Authorizing using temp directory #{@gcloud_config_dir} and credentials #{keypath}")
+
+      ENV['XDG_CONFIG_HOME'] = @gcloud_config_dir
+      ENV['CLOUDSDK_CONFIG']= File.join(@gcloud_config_dir, 'gcloud')
+
+      Shellrunner.run_call('gcloud', 'auth', 'activate-service-account', '--key-file', keypath)
+    end
+
+    def current_user
+      Shellrunner.run_call('gcloud', 'config', 'list', 'account', '--format', "value(core.account)")[0]
+    end
+  end
+end

data/lib/kube_deploy_tools/image_registry/driver/login.rb

@@ -0,0 +1,26 @@
+require_relative 'base'
+require_relative '../image'
+
+module KubeDeployTools
+  class ImageRegistry::Driver::Login < ImageRegistry::Driver::Base
+    # This driver expects the following to be set in the @registry hash:
+    # username_var: set to a string which is the env var containing the docker
+    # registry username
+    # password_var: set to a string which is the env var containing the docker
+    # registry password
+    # prefix: passed directly to docker login
+    def authorize_command
+      ['docker', 'login',
+       '--username', ENV.fetch(@registry.config.fetch('username_var')),
+       '--password', ENV.fetch(@registry.config.fetch('password_var')),
+       @registry.prefix]
+    end
+
+    def delete_image(image, dryrun)
+      raise 'not implemented'
+    end
+
+    def unauthorize
+    end
+  end
+end

data/lib/kube_deploy_tools/image_registry/driver/noop.rb

@@ -0,0 +1,15 @@
+require_relative 'base'
+
+# Noop driver, does nothing!
+module KubeDeployTools
+  class ImageRegistry::Driver::Noop < ImageRegistry::Driver::Base
+    def push_image(image)
+    end
+
+    def authorize
+    end
+
+    def unauthorize
+    end
+  end
+end

data/lib/kube_deploy_tools/image_registry/image.rb

@@ -0,0 +1,17 @@
+module KubeDeployTools
+  class Push
+    class Image
+      attr_accessor :registry, :repository, :tag
+      def initialize(registry, repository, tag)
+        registry += '/' unless registry.end_with?('/')
+        @registry = registry
+        @repository = repository
+        @tag = tag
+      end
+
+      def full_tag
+        "#{registry}#{repository}:#{tag}"
+      end
+    end
+  end
+end

data/lib/kube_deploy_tools/kdt.rb

@@ -0,0 +1,52 @@
+require 'kube_deploy_tools/version'
+require 'kube_deploy_tools/formatted_logger'
+
+module KubeDeployTools
+  class Kdt
+    DESCRIPTIONS = {
+      'deploy'   => 'Releases all Kubernetes resources in a deploy artifact with |kubectl apply|',
+      'push'     => 'Tags and pushes images to defined image registries',
+      'generate' => 'Generates artifacts based on templates in kubernetes/ and your deploy.yaml.',
+      'publish'  => 'Publishes generated artifacts to your artifact store.',
+      'upgrade'   => 'Upgrades a KDT 1.x deploy.yml to a KDT 2.x deploy.yaml',
+    }
+
+    def initialize(path, args)
+      KubeDeployTools::Logger.logger = KubeDeployTools::FormattedLogger.build
+
+      @path = path
+      @args = args
+    end
+
+    def bins_names
+      @bins ||= Dir["#{@path}/*"].map { |x| File.basename(x) } - ['kdt']
+    end
+
+    def display_bins
+      # Print full runtime version
+      version = Gem.loaded_specs["kube_deploy_tools"].version
+      puts "kube_deploy_tools #{version}"
+
+      bins_names.each do |bin|
+        spaces_count = 25 - bin.size
+        puts "-> #{bin}#{' ' * spaces_count}| #{DESCRIPTIONS[bin]}"
+      end
+    end
+
+    def execute!
+      bin = @args.first
+
+      raise "command '#{bin}' is not a valid command" unless valid_bin?(bin)
+      bin_with_path = "#{@path}/#{bin}"
+      bin_args = @args[1..-1]
+
+      # calling exec with multiple args will prevent shell expansion
+      # https://ruby-doc.org/core/Kernel.html#method-i-exec
+      exec bin_with_path, *bin_args
+    end
+
+    def valid_bin?(bin)
+      bins_names.include?(bin)
+    end
+  end
+end

data/lib/kube_deploy_tools/kubectl.rb

@@ -0,0 +1,25 @@
+require 'kube_deploy_tools/object'
+require 'kube_deploy_tools/shellrunner'
+
+module KubeDeployTools
+  class Kubectl
+    def initialize(
+      context:,
+      kubeconfig:)
+      @context = context
+      @kubeconfig = kubeconfig
+
+      raise ArgumentError, "context is required" if context.empty?
+    end
+
+    def run(*args, print_cmd: true, timeout: nil)
+      args = args.unshift("kubectl")
+      args.push("--context=#{@context}")
+      args.push("--kubeconfig=#{@kubeconfig}") if @kubeconfig.present?
+      args.push("--request-timeout=#{timeout}") if timeout.present?
+
+      Shellrunner.run_call(*args, print_cmd: print_cmd)
+    end
+
+  end
+end

data/lib/kube_deploy_tools/kubernetes_resource.rb

@@ -0,0 +1,57 @@
+require 'tempfile'
+
+module KubeDeployTools
+  class KubernetesResource
+    attr_accessor :definition,
+      :kind,
+      :name,
+      :namespace,
+      :annotations
+
+    def self.build(filepath: nil, definition:, kubectl:)
+      opts = { filepath: filepath, definition: definition, kubectl: kubectl }
+      # Find the corresponding class for the Kubernetes resource, if available
+      if ["Deployment"].include?(definition["kind"])
+        klass = KubeDeployTools.const_get(definition["kind"])
+        klass.new(**opts)
+      else
+        # Otherwise initialize here if no class exists for this Kubernetes
+        # resource kind
+        inst = new(**opts)
+        inst.kind = definition["kind"]
+        inst
+      end
+    end
+
+    def initialize(filepath:, definition:, kubectl:)
+      @filepath = filepath
+      @definition = definition
+      @kubectl = kubectl
+
+      @namespace = definition.dig('metadata', 'namespace')
+      @name = definition.dig('metadata', 'name')
+      @kind = definition['kind']
+      @annotations = definition.dig('metadata', 'annotations')
+    end
+
+    def filepath
+      @filepath ||= file.path
+    end
+
+    def file
+      @file ||= create_definition_tempfile
+    end
+
+    def create_definition_tempfile
+      file = Tempfile.new(["#{@namespace}-#{@kind}-#{@name}", ".yaml"])
+      file.write(YAML.dump(@definition))
+      file
+    ensure
+      file&.close
+    end
+
+    def sync
+      # unimplemented
+    end
+  end
+end

data/lib/kube_deploy_tools/kubernetes_resource/deployment.rb

@@ -0,0 +1,56 @@
+require 'json'
+
+require 'kube_deploy_tools/formatted_logger'
+require 'kube_deploy_tools/object'
+
+module KubeDeployTools
+  class Deployment < KubernetesResource
+    TIMEOUT = '60s'
+
+    attr_accessor :found,
+      :local_replicas,
+      :remote_replicas,
+      :recorded_replicas
+
+    def sync
+      @local_replicas = @definition["spec"]["replicas"]
+
+      raw_json, _err, st = @kubectl.run("get", "-f", filepath, "--output=json", print_cmd: false, timeout: TIMEOUT)
+      @found = st.success?
+
+      if st.success?
+        deployment_data = JSON.parse(raw_json)
+        @remote_replicas = deployment_data["spec"]["replicas"]
+      end
+
+      raw_json, _err, st = @kubectl.run("apply", "view-last-applied", "-f", filepath, "--output=json", print_cmd: false, timeout: TIMEOUT)
+      if st.success?
+        raw_json = fix_kubectl_apply_view_last_applied_output(raw_json)
+        deployment_data = JSON.parse(raw_json)
+        @recorded_replicas = deployment_data["spec"]["replicas"]
+      end
+    end
+
+    def warn_replicas_mismatch
+      if @found
+        if @local_replicas.present? && @local_replicas.to_i != @remote_replicas.to_i
+          warning = "Deployment replica count mismatch! Will scale deployment/#{@name} from #{@remote_replicas} to #{@local_replicas}"
+          Logger.warn(warning)
+        elsif @local_replicas.nil? && !@recorded_replicas.nil?
+          # Check if we're converting to a replicaless Deployment
+          warning = "Deployment replica count mismatch! Will scale deployment/#{@name} from #{@remote_replicas} to 1. Run `kubectl apply set-last-applied -f #{@filepath}` first."
+          Logger.warn(warning)
+        end
+      end
+    end
+  end
+end
+
+# In kubectl version <= 1.7, `kubectl apply view-last-applied` may
+# produces an invalid JSON output, which we must sanitize.
+#
+# The upstream fix is in kubect >= 1.8:
+# https://github.com/kubernetes/kubernetes/commit/7c656ab4d2ca41e07db9f90c99ee360b0d48c651
+def fix_kubectl_apply_view_last_applied_output(json)
+  json.sub(/\!\"\(MISSING\)/, '"')
+end