kubernetes-deploy 0.26.7 → 0.27.0

data/lib/kubernetes-deploy/duration_parser.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'active_support/duration'
+
 module KubernetesDeploy
   ##
   # This class is a less strict extension of ActiveSupport::Duration::ISO8601Parser.

data/lib/kubernetes-deploy/ejson_secret_provisioner.rb

@@ -17,10 +17,11 @@ module KubernetesDeploy
     EJSON_SECRETS_FILE = "secrets.ejson"
     EJSON_KEYS_SECRET = "ejson-keys"
 
-    def initialize(namespace:, context:, template_dir:, logger:, statsd_tags:, selector: nil)
+    def initialize(namespace:, context:, ejson_keys_secret:, ejson_file:, logger:, statsd_tags:, selector: nil)
       @namespace = namespace
       @context = context
-      @ejson_file = "#{template_dir}/#{EJSON_SECRETS_FILE}"
+      @ejson_keys_secret = ejson_keys_secret
+      @ejson_file = ejson_file
       @logger = logger
       @statsd_tags = statsd_tags
       @selector = selector
@@ -37,20 +38,12 @@ module KubernetesDeploy
       @resources ||= build_secrets
     end
 
-    def ejson_keys_secret
-      @ejson_keys_secret ||= begin
-        out, err, st = @kubectl.run("get", "secret", EJSON_KEYS_SECRET, output: "json",
-          raise_if_not_found: true, attempts: 3, output_is_sensitive: true, log_failure: true)
-        unless st.success?
-          raise EjsonSecretError, "Error retrieving Secret/#{EJSON_KEYS_SECRET}: #{err}"
-        end
-        JSON.parse(out)
-      end
-    end
-
     private
 
     def build_secrets
+      unless @ejson_keys_secret
+        raise EjsonSecretError, "Secret #{EJSON_KEYS_SECRET} not provided, cannot decrypt secrets"
+      end
       return [] unless File.exist?(@ejson_file)
       with_decrypted_ejson do |decrypted|
         secrets = decrypted[EJSON_SECRET_KEY]
@@ -153,7 +146,7 @@ module KubernetesDeploy
     end
 
     def fetch_private_key_from_secret
-      encoded_private_key = ejson_keys_secret["data"][public_key]
+      encoded_private_key = @ejson_keys_secret["data"][public_key]
       unless encoded_private_key
         raise EjsonSecretError, "Private key for #{public_key} not found in #{EJSON_KEYS_SECRET} secret"
       end

data/lib/kubernetes-deploy/errors.rb

@@ -30,12 +30,4 @@ module KubernetesDeploy
           "kubernetes-deploy will not continue since it is extremely unlikely that this secret should be pruned.")
     end
   end
-
-  module Errors
-    extend self
-    def server_version_warning(server_version)
-      "Minimum cluster version requirement of #{MIN_KUBE_VERSION} not met. "\
-      "Using #{server_version} could result in unexpected behavior as it is no longer tested against"
-    end
-  end
 end

data/lib/kubernetes-deploy/formatted_logger.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 require 'logger'
+require 'colorized_string'
 require 'kubernetes-deploy/deferred_summary_logging'
 
 module KubernetesDeploy

data/lib/kubernetes-deploy/kubeclient_builder.rb

@@ -106,11 +106,11 @@ module KubernetesDeploy
     def validate_config_files
       errors = []
       if @kubeconfig_files.empty?
-        errors << "Kube config file name(s) not set in $KUBECONFIG"
+        errors << "Kubeconfig file name(s) not set in $KUBECONFIG"
       else
         @kubeconfig_files.each do |f|
           # If any files in the list are not valid, we can't be sure the merged context list is what the user intended
-          errors << "Kube config not found at #{f}" unless File.file?(f)
+          errors << "Kubeconfig not found at #{f}" unless File.file?(f)
         end
       end
       errors

data/lib/kubernetes-deploy/kubectl.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+require 'open3'
 
 module KubernetesDeploy
   class Kubectl

data/lib/kubernetes-deploy/kubernetes_resource.rb

@@ -1,9 +1,11 @@
 # frozen_string_literal: true
 require 'json'
-require 'open3'
 require 'shellwords'
 
 require 'kubernetes-deploy/remote_logs'
+require 'kubernetes-deploy/duration_parser'
+require 'kubernetes-deploy/label_selector'
+require 'kubernetes-deploy/rollout_conditions'
 
 module KubernetesDeploy
   class KubernetesResource

data/lib/kubernetes-deploy/kubernetes_resource/custom_resource_definition.rb

@@ -1,6 +1,4 @@
 # frozen_string_literal: true
-require 'kubernetes-deploy/rollout_conditions'
-
 module KubernetesDeploy
   class CustomResourceDefinition < KubernetesResource
     TIMEOUT = 2.minutes

data/lib/kubernetes-deploy/kubernetes_resource/deployment.rb

@@ -1,4 +1,6 @@
 # frozen_string_literal: true
+require 'kubernetes-deploy/kubernetes_resource/replica_set'
+
 module KubernetesDeploy
   class Deployment < KubernetesResource
     TIMEOUT = 7.minutes

data/lib/kubernetes-deploy/kubernetes_resource/pod_set_base.rb

@@ -1,4 +1,6 @@
 # frozen_string_literal: true
+require 'kubernetes-deploy/kubernetes_resource/pod'
+
 module KubernetesDeploy
   class PodSetBase < KubernetesResource
     def failure_message

data/lib/kubernetes-deploy/kubernetes_resource/replica_set.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 require 'kubernetes-deploy/kubernetes_resource/pod_set_base'
+
 module KubernetesDeploy
   class ReplicaSet < PodSetBase
     TIMEOUT = 5.minutes

data/lib/kubernetes-deploy/kubernetes_resource/service.rb

@@ -1,4 +1,6 @@
 # frozen_string_literal: true
+require 'kubernetes-deploy/kubernetes_resource/pod'
+
 module KubernetesDeploy
   class Service < KubernetesResource
     TIMEOUT = 7.minutes
@@ -6,11 +8,11 @@ module KubernetesDeploy
     def sync(cache)
       super
       if exists? && selector.present?
-        @related_deployments = cache.get_all(Deployment.kind, selector)
         @related_pods = cache.get_all(Pod.kind, selector)
+        @related_workloads = fetch_related_workloads(cache)
       else
-        @related_deployments = []
         @related_pods = []
+        @related_workloads = []
       end
     end
 
@@ -30,7 +32,7 @@ module KubernetesDeploy
       return false unless exists?
       return exists? unless requires_endpoints?
       # We can't use endpoints if we want the service to be able to fail fast when the pods are down
-      exposes_zero_replica_deployment? || selects_some_pods?
+      exposes_zero_replica_workload? || selects_some_pods?
     end
 
     def deploy_failed?
@@ -38,18 +40,27 @@ module KubernetesDeploy
     end
 
     def timeout_message
-      "This service does not seem to select any pods. This means its spec.selector is probably incorrect."
+      "This service does not seem to select any pods and this is likely invalid. "\
+      "Please confirm the spec.selector is correct and the targeted workload is healthy."
     end
 
     private
 
-    def exposes_zero_replica_deployment?
+    def fetch_related_workloads(cache)
+      related_deployments = cache.get_all(Deployment.kind)
+      related_statefulsets = cache.get_all(StatefulSet.kind)
+      (related_deployments + related_statefulsets).select do |workload|
+        selector.all? { |k, v| workload['spec']['template']['metadata']['labels'][k] == v }
+      end
+    end
+
+    def exposes_zero_replica_workload?
       return false unless related_replica_count
       related_replica_count == 0
     end
 
     def requires_endpoints?
-      # service of type External don't have endpoints
+      # services of type External don't have endpoints
       return false if external_name_svc?
 
       # problem counting replicas - by default, assume endpoints are required
@@ -69,8 +80,10 @@ module KubernetesDeploy
 
     def related_replica_count
       return 0 unless selector.present?
-      return unless @related_deployments.length == 1
-      @related_deployments.first["spec"]["replicas"].to_i
+
+      if @related_workloads.present?
+        @related_workloads.inject(0) { |sum, d| sum + d["spec"]["replicas"].to_i }
+      end
     end
 
     def external_name_svc?

data/lib/kubernetes-deploy/options_helper.rb

@@ -6,30 +6,43 @@ module KubernetesDeploy
 
     STDIN_TEMP_FILE = "from_stdin.yml.erb"
     class << self
-      def with_validated_template_dir(template_dir)
-        if template_dir == '-'
+      def with_processed_template_paths(template_paths)
+        validated_paths = []
+        if template_paths.empty?
+          validated_paths << default_template_dir
+        else
+          template_paths.uniq!
+          template_paths.each do |template_path|
+            next if template_path == '-'
+            validated_paths << template_path
+          end
+        end
+
+        if template_paths.include?("-")
           Dir.mktmpdir("kubernetes-deploy") do |dir|
             template_dir_from_stdin(temp_dir: dir)
-            yield dir
+            validated_paths << dir
+            yield validated_paths
           end
-        elsif template_dir
-          yield template_dir
         else
-          yield default_template_dir(template_dir)
+          yield validated_paths
         end
       end
 
       private
 
-      def default_template_dir(template_dir)
-        if ENV.key?("ENVIRONMENT")
-          template_dir = File.join("config", "deploy", ENV['ENVIRONMENT'])
+      def default_template_dir
+        template_dir = if ENV.key?("ENVIRONMENT")
+          File.join("config", "deploy", ENV['ENVIRONMENT'])
         end
 
-        if !template_dir || template_dir.empty?
+        unless template_dir
           raise OptionsError, "Template directory is unknown. " \
-            "Either specify --template-dir argument or set $ENVIRONMENT to use config/deploy/$ENVIRONMENT " \
-            "as a default path."
+          "Either specify --template-dir argument or set $ENVIRONMENT to use config/deploy/$ENVIRONMENT " \
+          "as a default path."
+        end
+        unless Dir.exist?(template_dir)
+          raise OptionsError, "Template directory #{template_dir} does not exist."
         end
 
         template_dir

data/lib/kubernetes-deploy/render_task.rb

@@ -1,13 +1,13 @@
 # frozen_string_literal: true
 require 'tempfile'
 
+require 'kubernetes-deploy/common'
 require 'kubernetes-deploy/renderer'
-require 'kubernetes-deploy/template_discovery'
 
 module KubernetesDeploy
   class RenderTask
-    def initialize(logger:, current_sha:, template_dir:, bindings:)
-      @logger = logger
+    def initialize(logger: nil, current_sha:, template_dir:, bindings:)
+      @logger = logger || KubernetesDeploy::FormattedLogger.build
       @template_dir = template_dir
       @renderer = KubernetesDeploy::Renderer.new(
         current_sha: current_sha,
@@ -29,7 +29,7 @@ module KubernetesDeploy
       @logger.phase_heading("Initializing render task")
 
       filenames = if only_filenames.empty?
-        TemplateDiscovery.new(@template_dir).templates
+        Dir.foreach(@template_dir).select { |filename| filename.end_with?(".yml.erb", ".yml", ".yaml", ".yaml.erb") }
       else
         only_filenames
       end

data/lib/kubernetes-deploy/resource_watcher.rb

@@ -1,4 +1,8 @@
 # frozen_string_literal: true
+
+require 'kubernetes-deploy/concurrency'
+require 'kubernetes-deploy/resource_cache'
+
 module KubernetesDeploy
   class ResourceWatcher
     extend KubernetesDeploy::StatsD::MeasureMethods

data/lib/kubernetes-deploy/restart_task.rb

@@ -1,4 +1,7 @@
 # frozen_string_literal: true
+require 'kubernetes-deploy/common'
+require 'kubernetes-deploy/kubernetes_resource'
+require 'kubernetes-deploy/kubernetes_resource/deployment'
 require 'kubernetes-deploy/kubeclient_builder'
 require 'kubernetes-deploy/resource_watcher'
 require 'kubernetes-deploy/kubectl'
@@ -18,10 +21,11 @@ module KubernetesDeploy
     HTTP_OK_RANGE = 200..299
     ANNOTATION = "shipit.shopify.io/restart"
 
-    def initialize(context:, namespace:, logger:, max_watch_seconds: nil)
+    def initialize(context:, namespace:, logger: nil, max_watch_seconds: nil)
+      @logger = logger || KubernetesDeploy::FormattedLogger.build(namespace, context)
+      @task_config = KubernetesDeploy::TaskConfig.new(context, namespace, @logger)
       @context = context
       @namespace = namespace
-      @logger = logger
       @max_watch_seconds = max_watch_seconds
     end
 
@@ -37,11 +41,9 @@ module KubernetesDeploy
       @logger.reset
 
       @logger.phase_heading("Initializing restart")
-      verify_namespace
+      verify_config!
       deployments = identify_target_deployments(deployments_names, selector: selector)
-      if kubectl.server_version < Gem::Version.new(MIN_KUBE_VERSION)
-        @logger.warn(KubernetesDeploy::Errors.server_version_warning(kubectl.server_version))
-      end
+
       @logger.phase_heading("Triggering restart by touching ENV[RESTARTED_AT]")
       patch_kubeclient_deployments(deployments)
 
@@ -117,13 +119,6 @@ module KubernetesDeploy
       end
     end
 
-    def verify_namespace
-      kubeclient.get_namespace(@namespace)
-      @logger.info("Namespace #{@namespace} found in context #{@context}")
-    rescue Kubeclient::ResourceNotFoundError
-      raise NamespaceNotFoundError.new(@namespace, @context)
-    end
-
     def patch_deployment_with_restart(record)
       v1beta1_kubeclient.patch_deployment(
         record.metadata.name,
@@ -173,6 +168,15 @@ module KubernetesDeploy
       }
     end
 
+    def verify_config!
+      task_config_validator = TaskConfigValidator.new(@task_config, kubectl, kubeclient_builder)
+      unless task_config_validator.valid?
+        @logger.summary.add_action("Configuration invalid")
+        @logger.summary.add_paragraph(task_config_validator.errors.map { |err| "- #{err}" }.join("\n"))
+        raise KubernetesDeploy::TaskConfigurationError
+      end
+    end
+
     def kubeclient
       @kubeclient ||= kubeclient_builder.build_v1_kubeclient(@context)
     end

data/lib/kubernetes-deploy/runner_task.rb

@@ -1,8 +1,13 @@
 # frozen_string_literal: true
 require 'tempfile'
 
+require 'kubernetes-deploy/common'
 require 'kubernetes-deploy/kubeclient_builder'
 require 'kubernetes-deploy/kubectl'
+require 'kubernetes-deploy/resource_cache'
+require 'kubernetes-deploy/resource_watcher'
+require 'kubernetes-deploy/kubernetes_resource'
+require 'kubernetes-deploy/kubernetes_resource/pod'
 
 module KubernetesDeploy
   class RunnerTask
@@ -10,8 +15,9 @@ module KubernetesDeploy
 
     attr_reader :pod_name
 
-    def initialize(namespace:, context:, logger:, max_watch_seconds: nil)
-      @logger = logger
+    def initialize(namespace:, context:, logger: nil, max_watch_seconds: nil)
+      @logger = logger || KubernetesDeploy::FormattedLogger.build(namespace, context)
+      @task_config = KubernetesDeploy::TaskConfig.new(context, namespace, @logger)
       @namespace = namespace
       @context = context
       @max_watch_seconds = max_watch_seconds
@@ -135,9 +141,7 @@ module KubernetesDeploy
         raise TaskConfigurationError, "Configuration invalid: #{errors.join(', ')}"
       end
 
-      if kubectl.server_version < Gem::Version.new(MIN_KUBE_VERSION)
-        @logger.warn(KubernetesDeploy::Errors.server_version_warning(kubectl.server_version))
-      end
+      TaskConfigValidator.new(@task_config, kubectl, kubeclient_builder, only: [:validate_server_version]).valid?
     end
 
     def get_template(template_name)

data/lib/kubernetes-deploy/task_config.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+module KubernetesDeploy
+  class TaskConfig
+    attr_reader :context, :namespace
+
+    def initialize(context, namespace, logger = nil)
+      @context = context
+      @namespace = namespace
+      @logger = logger
+    end
+
+    def logger
+      @logger ||= KubernetesDeploy::FormattedLogger.build(@namespace, @context)
+    end
+  end
+end

data/lib/kubernetes-deploy/task_config_validator.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+module KubernetesDeploy
+  class TaskConfigValidator
+    DEFAULT_VALIDATIONS = %i(
+      validate_kubeconfig
+      validate_context_exists_in_kubeconfig
+      validate_context_reachable
+      validate_server_version
+      validate_namespace_exists
+    ).freeze
+
+    delegate :context, :namespace, :logger, to: :@task_config
+
+    def initialize(task_config, kubectl, kubeclient_builder, only: nil)
+      @task_config = task_config
+      @kubectl = kubectl
+      @kubeclient_builder = kubeclient_builder
+      @errors = nil
+      @validations = only || DEFAULT_VALIDATIONS
+    end
+
+    def valid?
+      @errors = []
+      @validations.each do |validator_name|
+        break if @errors.present?
+        send(validator_name)
+      end
+      @errors.empty?
+    end
+
+    def errors
+      valid?
+      @errors
+    end
+
+    private
+
+    def validate_kubeconfig
+      @errors += @kubeclient_builder.validate_config_files
+    end
+
+    def validate_context_exists_in_kubeconfig
+      unless context.present?
+        return @errors << "Context can not be blank"
+      end
+
+      _, err, st = @kubectl.run("config", "get-contexts", context, "-o", "name",
+        use_namespace: false, use_context: false, log_failure: false)
+
+      unless st.success?
+        @errors << if err.match("error: context #{context} not found")
+          "Context #{context} missing from your kubeconfig file(s)"
+        else
+          "Something went wrong. #{err} "
+        end
+      end
+    end
+
+    def validate_context_reachable
+      _, err, st = @kubectl.run("get", "namespaces", "-o", "name",
+        use_namespace: false, log_failure: false)
+
+      unless st.success?
+        @errors << "Something went wrong connecting to #{context}. #{err} "
+      end
+    end
+
+    def validate_namespace_exists
+      unless namespace.present?
+        return @errors << "Namespace can not be blank"
+      end
+
+      _, err, st = @kubectl.run("get", "namespace", "-o", "name", namespace,
+        use_namespace: false, log_failure: false)
+
+      unless st.success?
+        @errors << if err.match("Error from server [(]NotFound[)]: namespace")
+          "Could not find Namespace: #{namespace} in Context: #{context}"
+        else
+          "Could not connect to kubernetes cluster. #{err}"
+        end
+      end
+    end
+
+    def validate_server_version
+      if @kubectl.server_version < Gem::Version.new(MIN_KUBE_VERSION)
+        logger.warn(server_version_warning(@kubectl.server_version))
+      end
+    end
+
+    def server_version_warning(server_version)
+      "Minimum cluster version requirement of #{MIN_KUBE_VERSION} not met. "\
+      "Using #{server_version} could result in unexpected behavior as it is no longer tested against"
+    end
+  end
+end