kubernetes-deploy 0.25.0 → 0.26.0

data/dev.yml

@@ -1,14 +1,14 @@
 ---
 name: kubernetes-deploy
 up:
-  - ruby: 2.3.3
+  - ruby: 2.3.3 # Matches gemspec
   - bundler
   - homebrew:
     - Caskroom/cask/minikube
   - custom:
       name: Minikube Cluster
-      met?: test $(minikube status | grep Running | wc -l) -eq 2 && $(minikube status | grep -q 'Correctly Configured')
-      meet: minikube start --kubernetes-version=v1.10.6 --vm-driver=hyperkit
+      met?: test $(minikube status | grep Running | wc -l) -ge 2 && $(minikube status | grep -q 'Correctly Configured')
+      meet: minikube start --kubernetes-version=v1.11.6 --vm-driver=hyperkit
       down: minikube stop
 commands:
   reset-minikube: minikube delete && rm -rf ~/.minikube

data/exe/kubernetes-deploy

@@ -11,23 +11,29 @@ prune = true
 bindings = {}
 verbose_log_prefix = false
 max_watch_seconds = nil
+selector = nil
 
 ARGV.options do |opts|
+  parser = KubernetesDeploy::BindingsParser.new
   opts.on("--bindings=BINDINGS", "Expose additional variables to ERB templates " \
-    "(format: k1=v1,k2=v2, JSON string or file (JSON or YAML) path prefixed by '@')") do |binds|
-    bindings.merge!(KubernetesDeploy::BindingsParser.parse(binds))
-  end
+    "(format: k1=v1,k2=v2, JSON string or file (JSON or YAML) path prefixed by '@')") { |b| parser.add(b) }
 
   opts.on("--skip-wait", "Skip verification of non-priority-resource success (not recommended)") { skip_wait = true }
   prot_ns = KubernetesDeploy::DeployTask::PROTECTED_NAMESPACES.join(', ')
   opts.on("--allow-protected-ns", "Enable deploys to #{prot_ns}; requires --no-prune") { allow_protected_ns = true }
   opts.on("--no-prune", "Disable deletion of resources that do not appear in the template dir") { prune = false }
-  opts.on("--template-dir=DIR", "Set the template dir (default: config/deploy/$ENVIRONMENT)") { |v| template_dir = v }
+  opts.on("--template-dir=DIR", "Set the template dir (default: config/deploy/$ENVIRONMENT).") do |d|
+    template_dir = d
+  end
   opts.on("--verbose-log-prefix", "Add [context][namespace] to the log prefix") { verbose_log_prefix = true }
   opts.on("--max-watch-seconds=seconds",
     "Timeout error is raised if it takes longer than the specified number of seconds") do |t|
     max_watch_seconds = t.to_i
   end
+  opts.on("--selector=SELECTOR", "Ensure that all resources in your template dir match the given selector, " \
+    "and restrict pruning to deployed resources it selects.  (format: k1=v1,k2=v2)") do |s|
+    selector = KubernetesDeploy::LabelSelector.parse(s)
+  end
 
   opts.on_tail("-h", "--help", "Print this help") do
     puts opts
@@ -38,32 +44,37 @@ ARGV.options do |opts|
     exit
   end
   opts.parse!
+  bindings = parser.parse
 end
 
 namespace = ARGV[0]
 context = ARGV[1]
-template_dir = KubernetesDeploy::OptionsHelper.default_and_check_template_dir(template_dir)
-revision = KubernetesDeploy::OptionsHelper.revision_from_environment
 logger = KubernetesDeploy::FormattedLogger.build(namespace, context, verbose_prefix: verbose_log_prefix)
 
-runner = KubernetesDeploy::DeployTask.new(
-  namespace: namespace,
-  context: context,
-  current_sha: revision,
-  template_dir: template_dir,
-  bindings: bindings,
-  logger: logger,
-  max_watch_seconds: max_watch_seconds
-)
-
 begin
-  runner.run!(
-    verify_result: !skip_wait,
-    allow_protected_ns: allow_protected_ns,
-    prune: prune
-  )
+  KubernetesDeploy::OptionsHelper.with_validated_template_dir(template_dir) do |dir|
+    runner = KubernetesDeploy::DeployTask.new(
+      namespace: namespace,
+      context: context,
+      current_sha: ENV["REVISION"],
+      template_dir: dir,
+      bindings: bindings,
+      logger: logger,
+      max_watch_seconds: max_watch_seconds,
+      selector: selector,
+    )
+
+    runner.run!(
+      verify_result: !skip_wait,
+      allow_protected_ns: allow_protected_ns,
+      prune: prune
+    )
+  end
 rescue KubernetesDeploy::DeploymentTimeoutError
   exit(70)
 rescue KubernetesDeploy::FatalDeploymentError
   exit(1)
+rescue KubernetesDeploy::OptionsHelper::OptionsError => e
+  logger.error(e.message)
+  exit(1)
 end

data/exe/kubernetes-render

@@ -9,24 +9,32 @@ template_dir = nil
 bindings = {}
 
 ARGV.options do |opts|
+  parser = KubernetesDeploy::BindingsParser.new
   opts.on("--bindings=BINDINGS", "Expose additional variables to ERB templates " \
-    "(format: k1=v1,k2=v2, JSON string or file (JSON or YAML) path prefixed by '@')") do |binds|
-    bindings.merge!(KubernetesDeploy::BindingsParser.parse(binds))
+    "(format: k1=v1,k2=v2, JSON string or file (JSON or YAML) path prefixed by '@')") { |b| parser.add(b) }
+  opts.on("--template-dir=DIR", "Set the template dir (default: config/deploy/$ENVIRONMENT)." ) do |d|
+    template_dir = d
   end
-  opts.on("--template-dir=DIR", "Set the template dir (default: config/deploy/$ENVIRONMENT)") { |v| template_dir = v }
   opts.parse!
+  bindings = parser.parse
 end
 
 templates = ARGV
 logger = KubernetesDeploy::FormattedLogger.build(verbose_prefix: false)
-revision = KubernetesDeploy::OptionsHelper.revision_from_environment
 
-runner = KubernetesDeploy::RenderTask.new(
-  logger: logger,
-  current_sha: revision,
-  template_dir: template_dir,
-  bindings: bindings,
-)
+begin
+  KubernetesDeploy::OptionsHelper.with_validated_template_dir(template_dir) do |dir|
+    runner = KubernetesDeploy::RenderTask.new(
+      logger: logger,
+      current_sha: ENV["REVISION"],
+      template_dir: dir,
+      bindings: bindings,
+    )
 
-success = runner.run(STDOUT, templates)
-exit(1) unless success
+    success = runner.run(STDOUT, templates)
+    exit(1) unless success
+  end
+rescue KubernetesDeploy::OptionsHelper::OptionsError => e
+  logger.error(e.message)
+  exit(1)
+end

data/exe/kubernetes-restart

@@ -8,9 +8,13 @@ require 'kubernetes-deploy/restart_task'
 
 raw_deployments = nil
 max_watch_seconds = nil
+selector = nil
 ARGV.options do |opts|
   opts.on("--deployments=LIST") { |v| raw_deployments = v.split(",") }
   opts.on("--max-watch-seconds=seconds") { |t| max_watch_seconds = t.to_i }
+  opts.on("--selector=SELECTOR", "Restarts deployments matching selector (format: k1=v1,k2=v2)") do |s|
+    selector = KubernetesDeploy::LabelSelector.parse(s)
+  end
   opts.parse!
 end
 
@@ -21,7 +25,7 @@ logger = KubernetesDeploy::FormattedLogger.build(namespace, context)
 restart = KubernetesDeploy::RestartTask.new(namespace: namespace, context: context, logger: logger,
    max_watch_seconds: max_watch_seconds)
 begin
-  restart.perform!(raw_deployments)
+  restart.perform!(raw_deployments, selector: selector)
 rescue KubernetesDeploy::DeploymentTimeoutError
   exit(70)
 rescue KubernetesDeploy::FatalDeploymentError

data/lib/kubernetes-deploy.rb

@@ -21,6 +21,7 @@ require 'kubernetes-deploy/concurrency'
 require 'kubernetes-deploy/bindings_parser'
 require 'kubernetes-deploy/duration_parser'
 require 'kubernetes-deploy/resource_cache'
+require 'kubernetes-deploy/label_selector'
 
 module KubernetesDeploy
   MIN_KUBE_VERSION = '1.10.0'

data/lib/kubernetes-deploy/bindings_parser.rb

@@ -4,17 +4,29 @@ require 'yaml'
 require 'csv'
 
 module KubernetesDeploy
-  module BindingsParser
-    extend self
+  class BindingsParser
+    def self.parse(string)
+      new(string).parse
+    end
 
-    def parse(string)
-      bindings = parse_file(string) || parse_json(string) || parse_csv(string)
+    def initialize(initial_string = nil)
+      @raw_bindings = Array(initial_string)
+    end
 
-      unless bindings
-        raise ArgumentError, "Failed to parse bindings."
-      end
+    def add(string)
+      @raw_bindings << string
+    end
 
-      bindings
+    def parse
+      result = {}
+      @raw_bindings.each do |string|
+        bindings = parse_file(string) || parse_json(string) || parse_csv(string)
+        unless bindings
+          raise ArgumentError, "Failed to parse bindings."
+        end
+        result.deep_merge!(bindings)
+      end
+      result
     end
 
     private

data/lib/kubernetes-deploy/cluster_resource_discovery.rb

@@ -19,7 +19,7 @@ module KubernetesDeploy
     private
 
     def fetch_crds
-      raw_json, _, st = kubectl.run("get", "CustomResourceDefinition", "-a", "--output=json", attempts: 5)
+      raw_json, _, st = kubectl.run("get", "CustomResourceDefinition", "-a", output: "json", attempts: 5)
       if st.success?
         JSON.parse(raw_json)["items"]
       else

data/lib/kubernetes-deploy/container_logs.rb

@@ -13,6 +13,7 @@ module KubernetesDeploy
       @logger = logger
       @lines = []
       @next_print_index = 0
+      @printed_latest = false
     end
 
     def sync
@@ -33,12 +34,17 @@ module KubernetesDeploy
       end
 
       @next_print_index = lines.length
+      @printed_latest = true
     end
 
     def print_all
       lines.each { |line| @logger.info("\t#{line}") }
     end
 
+    def printing_started?
+      @printed_latest
+    end
+
     private
 
     def fetch_latest

data/lib/kubernetes-deploy/deploy_task.rb

@@ -14,6 +14,7 @@ require 'kubernetes-deploy/kubernetes_resource'
   persistent_volume_claim
   pod
   redis
+  network_policy
   memcached
   service
   pod_template
@@ -30,6 +31,7 @@ require 'kubernetes-deploy/kubernetes_resource'
   job
   custom_resource_definition
   horizontal_pod_autoscaler
+  secret
 ).each do |subresource|
   require "kubernetes-deploy/kubernetes_resource/#{subresource}"
 end
@@ -43,7 +45,6 @@ require 'kubernetes-deploy/template_discovery'
 
 module KubernetesDeploy
   class DeployTask
-    include KubeclientBuilder
     extend KubernetesDeploy::StatsD::MeasureMethods
 
     PROTECTED_NAMESPACES = %w(
@@ -58,11 +59,11 @@ module KubernetesDeploy
     # core/v1/PersistentVolumeClaim -- would delete data
     # core/v1/ReplicationController -- superseded by deployments/replicasets
     # extensions/v1beta1/ReplicaSet -- managed by deployments
-    # core/v1/Secret -- should not committed / managed by shipit
 
     def predeploy_sequence
       before_crs = %w(
         ResourceQuota
+        NetworkPolicy
       )
       after_crs = %w(
         ConfigMap
@@ -70,6 +71,7 @@ module KubernetesDeploy
         ServiceAccount
         Role
         RoleBinding
+        Secret
         Pod
       )
 
@@ -82,10 +84,12 @@ module KubernetesDeploy
         core/v1/Pod
         core/v1/Service
         core/v1/ResourceQuota
+        core/v1/Secret
         batch/v1/Job
         extensions/v1beta1/DaemonSet
         extensions/v1beta1/Deployment
         extensions/v1beta1/Ingress
+        networking.k8s.io/v1/NetworkPolicy
         apps/v1beta1/StatefulSet
         autoscaling/v1/HorizontalPodAutoscaler
         policy/v1beta1/PodDisruptionBudget
@@ -99,7 +103,7 @@ module KubernetesDeploy
     end
 
     def initialize(namespace:, context:, current_sha:, template_dir:, logger:, kubectl_instance: nil, bindings: {},
-      max_watch_seconds: nil)
+      max_watch_seconds: nil, selector: nil)
       @namespace = namespace
       @namespace_tags = []
       @context = context
@@ -114,6 +118,7 @@ module KubernetesDeploy
         logger: @logger,
         bindings: bindings,
       )
+      @selector = selector
     end
 
     def run(*args)
@@ -134,7 +139,6 @@ module KubernetesDeploy
 
       @logger.phase_heading("Checking initial resource statuses")
       check_initial_status(resources)
-      create_ejson_secrets(prune)
 
       if deploy_has_priority_resources?(resources)
         @logger.phase_heading("Predeploying priority resources")
@@ -187,6 +191,10 @@ module KubernetesDeploy
 
     private
 
+    def kubeclient_builder
+      @kubeclient_builder ||= KubeclientBuilder.new
+    end
+
     def cluster_resource_discoverer
       @cluster_resource_discoverer ||= ClusterResourceDiscovery.new(
         namespace: @namespace,
@@ -196,11 +204,27 @@ module KubernetesDeploy
       )
     end
 
+    def ejson_provisioner
+      @ejson_provisioner ||= EjsonSecretProvisioner.new(
+        namespace: @namespace,
+        context: @context,
+        template_dir: @template_dir,
+        logger: @logger,
+        statsd_tags: @namespace_tags,
+        selector: @selector,
+      )
+    end
+
     def deploy_has_priority_resources?(resources)
       resources.any? { |r| predeploy_sequence.include?(r.type) }
     end
 
     def predeploy_priority_resources(resource_list)
+      bare_pods = resource_list.select { |resource| resource.is_a?(Pod) }
+      if bare_pods.count == 1
+        bare_pods.first.stream_logs = true
+      end
+
       predeploy_sequence.each do |resource_type|
         matching_resources = resource_list.select { |r| r.type == resource_type }
         next if matching_resources.empty?
@@ -221,7 +245,9 @@ module KubernetesDeploy
     measure_method(:predeploy_priority_resources, 'priority_resources.duration')
 
     def validate_resources(resources)
-      KubernetesDeploy::Concurrency.split_across_threads(resources) { |r| r.validate_definition(kubectl) }
+      KubernetesDeploy::Concurrency.split_across_threads(resources) do |r|
+        r.validate_definition(kubectl, selector: @selector)
+      end
       failed_resources = resources.select(&:validation_failed?)
       return unless failed_resources.present?
 
@@ -240,20 +266,9 @@ module KubernetesDeploy
     end
     measure_method(:check_initial_status, "initial_status.duration")
 
-    def create_ejson_secrets(prune)
-      ejson = EjsonSecretProvisioner.new(
-        namespace: @namespace,
-        context: @context,
-        template_dir: @template_dir,
-        logger: @logger,
-        prune: prune,
-      )
-      return unless ejson.secret_changes_required?
-
-      @logger.phase_heading("Deploying kubernetes secrets from #{EjsonSecretProvisioner::EJSON_SECRETS_FILE}")
-      ejson.run
+    def secrets_from_ejson
+      ejson_provisioner.resources
     end
-    measure_method(:create_ejson_secrets)
 
     def discover_resources
       resources = []
@@ -269,11 +284,15 @@ module KubernetesDeploy
           @logger.info("  - #{r.id}")
         end
       end
+      secrets_from_ejson.each do |secret|
+        resources << secret
+        @logger.info("  - #{secret.id} (from ejson)")
+      end
       if (global = resources.select(&:global?).presence)
         @logger.warn("Detected non-namespaced #{'resource'.pluralize(global.count)} which will never be pruned:")
         global.each { |r| @logger.warn("  - #{r.id}") }
       end
-      resources
+      resources.sort
     end
     measure_method(:discover_resources)
 
@@ -297,30 +316,16 @@ module KubernetesDeploy
       raise FatalDeploymentError, "Failed to render and parse template"
     end
 
-    def record_invalid_template(err:, filename:, content:)
+    def record_invalid_template(err:, filename:, content: nil)
       debug_msg = ColorizedString.new("Invalid template: #{filename}\n").red
       debug_msg += "> Error message:\n#{FormattedLogger.indent_four(err)}"
-      debug_msg += "\n> Template content:\n#{FormattedLogger.indent_four(content)}"
+      debug_msg += "\n> Template content:\n#{FormattedLogger.indent_four(content)}" if content
       @logger.summary.add_paragraph(debug_msg)
     end
 
     def validate_configuration(allow_protected_ns:, prune:)
       errors = []
-      if ENV["KUBECONFIG"].blank?
-        errors << "$KUBECONFIG not set"
-      elsif config_files.empty?
-        errors << "Kube config file name(s) not set in $KUBECONFIG"
-      else
-        config_files.each do |f|
-          unless File.file?(f)
-            errors << "Kube config not found at #{f}"
-          end
-        end
-      end
-
-      if @current_sha.blank?
-        errors << "Current SHA must be specified"
-      end
+      errors += kubeclient_builder.validate_config_files
 
       if !File.directory?(@template_dir)
         errors << "Template directory `#{@template_dir}` doesn't exist"
@@ -354,6 +359,8 @@ module KubernetesDeploy
 
       confirm_context_exists
       confirm_namespace_exists
+      confirm_ejson_keys_not_prunable if prune
+      @logger.info("Using resource selector #{@selector}") if @selector
       @namespace_tags |= tags_from_namespace_labels
       @logger.info("All required parameters and files are present")
     end
@@ -365,6 +372,9 @@ module KubernetesDeploy
 
       if resources.length > 1
         @logger.info("Deploying resources:")
+        resources.each do |r|
+          @logger.info("- #{r.id} (#{r.pretty_timeout_type})")
+        end
       else
         resource = resources.first
         @logger.info("Deploying #{resource.id} (#{resource.pretty_timeout_type})")
@@ -377,7 +387,6 @@ module KubernetesDeploy
       applyables += individuals.select { |r| pruneable_types.include?(r.type) }
 
       individuals.each do |r|
-        @logger.info("- #{r.id} (#{r.pretty_timeout_type})") if resources.length > 1
         r.deploy_started_at = Time.now.utc
         case r.deploy_method
         when :replace
@@ -421,23 +430,28 @@ module KubernetesDeploy
 
       Dir.mktmpdir do |tmp_dir|
         resources.each do |r|
-          @logger.info("- #{r.id} (#{r.pretty_timeout_type})") if resources.length > 1
           FileUtils.symlink(r.file_path, tmp_dir)
           r.deploy_started_at = Time.now.utc
         end
         command.push("-f", tmp_dir)
 
         if prune
-          command.push("--prune", "--all")
+          command.push("--prune")
+          if @selector
+            command.push("--selector", @selector.to_s)
+          else
+            command.push("--all")
+          end
           prune_whitelist.each { |type| command.push("--prune-whitelist=#{type}") }
         end
 
-        out, err, st = kubectl.run(*command, log_failure: false)
+        output_is_sensitive = resources.any?(&:kubectl_output_is_sensitive?)
+        out, err, st = kubectl.run(*command, log_failure: false, output_is_sensitive: output_is_sensitive)
 
         if st.success?
           log_pruning(out) if prune
         else
-          record_apply_failure(err)
+          record_apply_failure(err, resources: resources)
           raise FatalDeploymentError, "Command failed: #{Shellwords.join(command)}"
         end
       end
@@ -452,22 +466,37 @@ module KubernetesDeploy
       @logger.summary.add_action("pruned #{pruned.length} #{'resource'.pluralize(pruned.length)}")
     end
 
-    def record_apply_failure(err)
+    def record_apply_failure(err, resources: [])
       warn_msg = "WARNING: Any resources not mentioned in the error(s) below were likely created/updated. " \
         "You may wish to roll back this deploy."
       @logger.summary.add_paragraph(ColorizedString.new(warn_msg).yellow)
 
       unidentified_errors = []
+      filenames_with_sensitive_content = resources
+        .select(&:kubectl_output_is_sensitive?)
+        .map { |r| File.basename(r.file_path) }
+
       err.each_line do |line|
         bad_files = find_bad_files_from_kubectl_output(line)
         if bad_files.present?
-          bad_files.each { |f| record_invalid_template(err: f[:err], filename: f[:filename], content: f[:content]) }
+          bad_files.each do |f|
+            if filenames_with_sensitive_content.include?(f[:filename])
+              # Hide the error and template contents in case it has senitive information
+              record_invalid_template(err: "SUPPRESSED FOR SECURITY", filename: f[:filename], content: nil)
+            else
+              record_invalid_template(err: f[:err], filename: f[:filename], content: f[:content])
+            end
+          end
         else
           unidentified_errors << line
         end
       end
 
-      if unidentified_errors.present?
+      if unidentified_errors.present? && filenames_with_sensitive_content.any?
+        warn_msg = "WARNING: There was an error applying some or all resources. The raw output may be sensitive and " \
+          "so cannot be displayed."
+        @logger.summary.add_paragraph(ColorizedString.new(warn_msg).yellow)
+      elsif unidentified_errors.present?
         heading = ColorizedString.new('Unidentified error(s):').red
         msg = FormattedLogger.indent_four(unidentified_errors.join)
         @logger.summary.add_paragraph("#{heading}\n#{msg}")
@@ -526,6 +555,18 @@ module KubernetesDeploy
       @logger.info("Namespace #{@namespace} found")
     end
 
+    # make sure to never prune the ejson-keys secret
+    def confirm_ejson_keys_not_prunable
+      secret = ejson_provisioner.ejson_keys_secret
+      return unless secret.dig("metadata", "annotations", KubernetesResource::LAST_APPLIED_ANNOTATION)
+
+      @logger.error("Deploy cannot proceed because protected resource " \
+        "Secret/#{EjsonSecretProvisioner::EJSON_KEYS_SECRET} would be pruned.")
+      raise EjsonPrunableError
+    rescue Kubectl::ResourceNotFoundError => e
+      @logger.debug("Secret/#{EjsonSecretProvisioner::EJSON_KEYS_SECRET} does not exist: #{e}")
+    end
+
     def tags_from_namespace_labels
       namespace_info = nil
       with_retries(2) do