kubernetes-deploy 0.20.6 → 0.21.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5f4739131c95e48be2df66d1ca820fc63fa92a0f
-  data.tar.gz: ef6afd733102c00d3ac9b42571da84a1edde0970
+  metadata.gz: 1155a58879313f8d0466b9d551a5ba9fb4a0ccab
+  data.tar.gz: a5856d1be46d1aef9a6393e4b5f95715e5832284
 SHA512:
-  metadata.gz: a025aaf9b1e4cb4689e8ad0e4101502f460146f5dbe36ed1da096a5b53679e24cf75dbb5f84cbe1ec0b739988a0eafebe20237c69a51614d130b551b8456fbfa
-  data.tar.gz: 4364997f2d82b1b80530ba1c5c52797dd009e5a6ab882570df0937479200cd17099fb8ac9d99527bc98dd0dfa4b8859ca6a9bd4e075d25d2fe055c32ea0fd40c
+  metadata.gz: 9eecadcbaa7709aae55ba3baac6e0afaf120277df59c1dae1a407bd49f6f07755757448de776b7f33ed30298cacad081276ba2afcee4927d2b56ae899ea9a196
+  data.tar.gz: 75cc0f78449088f47d621da287477c77a01ef33a0d1460fc5739676fee82d5a6652542412c483cd2ad80c708ed72a8350573a57515e848100b5c27cb6b9a2129

data/.buildkite/pipeline.nightly.yml

@@ -1,5 +1,19 @@
+shared: &shared
+  retry:
+     automatic:
+       - exit_status: "*"
+         limit: 1
 steps:
+  - name: 'Run Test Suite (:kubernetes: 1.11-latest)'
+    <<: *shared
+    command: bin/ci
+    agents:
+      queue: k8s-ci
+    env:
+      LOGGING_LEVEL: 4
+      KUBERNETES_VERSION: v1.11-latest
   - name: 'Run Test Suite (:kubernetes: 1.10-latest)'
+    <<: *shared
     command: bin/ci
     agents:
       queue: minikube-ci
@@ -7,6 +21,7 @@ steps:
       LOGGING_LEVEL: 4
       KUBERNETES_VERSION: v1.10-latest
   - name: 'Run Test Suite (:kubernetes: 1.9-latest)'
+    <<: *shared
     command: bin/ci
     agents:
       queue: minikube-ci
@@ -14,16 +29,10 @@ steps:
       LOGGING_LEVEL: 4
       KUBERNETES_VERSION: v1.9-latest
   - name: 'Run Test Suite (:kubernetes: 1.8-latest)'
+    <<: *shared
     command: bin/ci
     agents:
       queue: minikube-ci
     env:
       LOGGING_LEVEL: 4
       KUBERNETES_VERSION: v1.8-latest
-  - name: 'Run Test Suite (:kubernetes: 1.7-latest)'
-    command: bin/ci
-    agents:
-      queue: minikube-ci
-    env:
-      LOGGING_LEVEL: 4
-      KUBERNETES_VERSION: v1.7-latest

data/.buildkite/pipeline.yml

@@ -1,5 +1,19 @@
+shared: &shared
+  retry:
+     automatic:
+       - exit_status: "*"
+         limit: 1
 steps:
+  - name: 'Run Test Suite (:kubernetes: 1.11-latest)'
+    <<: *shared
+    command: bin/ci
+    agents:
+      queue: k8s-ci
+    env:
+      LOGGING_LEVEL: 4
+      KUBERNETES_VERSION: v1.11-latest
   - name: 'Run Test Suite (:kubernetes: 1.10-latest)'
+    <<: *shared
     command: bin/ci
     agents:
       queue: minikube-ci
@@ -7,6 +21,7 @@ steps:
       LOGGING_LEVEL: 4
       KUBERNETES_VERSION: v1.10-latest
   - name: 'Run Test Suite (:kubernetes: 1.9-latest)'
+    <<: *shared
     command: bin/ci
     agents:
       queue: minikube-ci
@@ -14,16 +29,10 @@ steps:
       LOGGING_LEVEL: 4
       KUBERNETES_VERSION: v1.9-latest
   - name: 'Run Test Suite (:kubernetes: 1.8-latest)'
+    <<: *shared
     command: bin/ci
     agents:
       queue: minikube-ci
     env:
       LOGGING_LEVEL: 4
       KUBERNETES_VERSION: v1.8-latest
-  - name: 'Run Test Suite (:kubernetes: 1.7-latest)'
-    command: bin/ci
-    agents:
-      queue: minikube-ci
-    env:
-      LOGGING_LEVEL: 4
-      KUBERNETES_VERSION: v1.7-latest

data/CHANGELOG.md

@@ -1,6 +1,11 @@
-### Master
+## 0.21.0
 
-*Enhancements*
+*Features*
+- **[Breaking change]** `kubernetes-run` now streams container logs and waits for the pod to succeed or fail **by default**. You can disable this using `--skip-wait`, or you can use `--max-watch-seconds=seconds` to set a time limit on the watch. ([#337](https://github.com/Shopify/kubernetes-deploy/pull/337))
+
+
+*Other*
+- Kubernetes 1.7 is no longer officially supported as of this version
 
 ## 0.20.6
 

data/README.md

@@ -69,19 +69,21 @@ This repo also includes related tools for [running tasks](#kubernetes-run) and [
 ## Prerequisites
 
 * Ruby 2.3+
-* Your cluster must be running Kubernetes v1.7.0 or higher<sup>1</sup>
+* Your cluster must be running Kubernetes v1.8.0 or higher<sup>1</sup>
 * Each app must have a deploy directory containing its Kubernetes templates (see [Templates](#using-templates-and-variables))
 * You must remove the` kubectl.kubernetes.io/last-applied-configuration` annotation from any resources in the namespace that are not included in your deploy directory. This annotation is added automatically when you create resources with `kubectl apply`. `kubernetes-deploy` will prune any resources that have this annotation and are not in the deploy directory.<sup>2</sup>
 * Each app managed by `kubernetes-deploy` must have its own exclusive Kubernetes namespace.
 
-<sup>1</sup> We run integration tests against these Kubernetes versions. Kubernetes v1.6 was officially supported in gem versions < 0.16. Kubernetes v1.5 was officially supported in gem versions < 0.12.
+<sup>1</sup> We run integration tests against these Kubernetes versions. Kubernetes v1.7 was officially
+supported in gem versions < 0.21. Kubernetes v1.6 was officially supported in gem versions < 0.16. Kubernetes
+v1.5 was officially supported in gem versions < 0.12.
 
 <sup>2</sup> This requirement can be bypassed with the `--no-prune` option, but it is not recommended.
 
 
 ## Installation
 
-1. [Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-kubectl-binary-via-curl) (requires v1.7.0 or higher) and make sure it is available in your $PATH
+1. [Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-kubectl-binary-via-curl) (requires v1.8.0 or higher) and make sure it is available in your $PATH
 2. Set up your [kubeconfig file](https://kubernetes.io/docs/tasks/access-application-cluster/authenticate-across-clusters-kubeconfig/) for access to your cluster(s).
 3. `gem install kubernetes-deploy`
 
@@ -352,12 +354,15 @@ Based on this specification `kubernetes-run` will create a new pod with the entr
 
 ## Usage
 
-`kubernetes-run <kube namespace> <kube context> <arguments> --entrypoint=/bin/bash`
+`kubernetes-run <kube namespace> <kube context> <arguments> --entrypoint=<entrypoint> --template=<template name>`
 
 *Options:*
 
 * `--template=TEMPLATE`:  Specifies the name of the PodTemplate to use (default is `task-runner-template` if this option is not set).
 * `--env-vars=ENV_VARS`: Accepts a comma separated list of environment variables to be added to the pod template. For example, `--env-vars="ENV=VAL,ENV2=VAL2"` will make `ENV` and `ENV2` available to the container.
+* `--entrypoint=ENTRYPOINT`: Specify the entrypoint to use to start the task runner container.
+* `--skip-wait`: Skip verification of pod success
+* `--max-watch-seconds=seconds`: Raise a timeout error if the pod runs for longer than the specified number of seconds
 
 
 
@@ -369,7 +374,7 @@ Based on this specification `kubernetes-run` will create a new pod with the entr
 
 If you work for Shopify, just run `dev up`, but otherwise:
 
-1. [Install kubectl version 1.7.0 or higher](https://kubernetes.io/docs/user-guide/prereqs/) and make sure it is in your path
+1. [Install kubectl version 1.8.0 or higher](https://kubernetes.io/docs/user-guide/prereqs/) and make sure it is in your path
 2. [Install minikube](https://kubernetes.io/docs/getting-started-guides/minikube/#installation) (required to run the test suite)
 3. Check out the repo
 4. Run `bin/setup` to install dependencies
@@ -423,9 +428,18 @@ Bug reports and pull requests are welcome on GitHub at https://github.com/Shopif
 
 Contributions to help us support additional resource types or increase the sophistication of our success heuristics for an existing type are especially encouraged! (See tips below)
 
+## Feature acceptance guidelines
 
+- This project's mission is to make it easy to ship changes to a Kubernetes namespace and understand the result. Features that introduce new classes of responsibility to the tool are not usually accepted.
+  - Deploys can be a very tempting place to cram features. Imagine a proposed feature actually fits better elsewhere—where might that be? (Examples: validator in CI, custom controller, initializer, pre-processing step in the CD pipeline, or even Kubernetes core)
+  - The basic ERB renderer included with the tool is intended as a convenience feature for a better out-of-the box experience. Providing complex rendering capabilities is out of scope of this project's mission, and enhancements in this area may be rejected.
+  - The deploy command does not officially support non-namespaced resource types.
+- This project strives to be composable with other tools in the ecosystem, such as renderers and validators. The deploy command must work with any Kubernetes templates provided to it, no matter how they were generated.
+- This project is open-source. Features tied to any specific organization (including Shopify) will be rejected.
+- The deploy command must remain performant when given several hundred resources at a time, generating 1000+ pods. (Technical note: This means only `sync` methods can make calls to the Kuberentes API server during result verification. This both limits the number of API calls made and ensures a consistent view of the world within each polling cycle.)
+- This tool must be able to run concurrent deploys to different targets safely, including when used as a library.
 
-### Contributing a new resource type
+## Contributing a new resource type
 
 The list of fully supported types is effectively the list of classes found in `lib/kubernetes-deploy/kubernetes_resource/`.
 
@@ -440,7 +454,7 @@ This gem uses subclasses of `KubernetesResource` to implement custom success/fai
 4. Add the a basic example of the type to the hello-cloud [fixture set](https://github.com/Shopify/kubernetes-deploy/tree/master/test/fixtures/hello-cloud) and appropriate assertions to `#assert_all_up` in [`hello_cloud.rb`](https://github.com/Shopify/kubernetes-deploy/blob/master/test/helpers/fixture_sets/hello_cloud.rb). This will get you coverage in several existing tests, such as `test_full_hello_cloud_set_deploy_succeeds`.
 5. Add tests for any edge cases you foresee.
 
-### Code of Conduct
+## Code of Conduct
 Everyone is expected to follow our [Code of Conduct](https://github.com/Shopify/kubernetes-deploy/blob/master/CODE_OF_CONDUCT.md).
 
 

data/bin/setup

@@ -9,8 +9,8 @@ if [ ! -x "$(which minikube)" ]; then
 fi
 
 if [ ! -x "$(which kubectl)" ]; then
-  echo -e "\n\033[0;33mPlease install kubectl version 1.7.0 or higher:\nhttps://kubernetes.io/docs/user-guide/prereqs/\033[0m"
+  echo -e "\n\033[0;33mPlease install kubectl version 1.8.0 or higher:\nhttps://kubernetes.io/docs/user-guide/prereqs/\033[0m"
 else
   KUBECTL_VERSION=$(kubectl version --short --client | grep -oe "v[[:digit:]\.]\+")
-  echo -e "\n\033[0;32mKubectl version $KUBECTL_VERSION is already installed. This gem requires version v1.7.0 or greater.\033[0m"
+  echo -e "\n\033[0;32mKubectl version $KUBECTL_VERSION is already installed. This gem requires version v1.8.0 or greater.\033[0m"
 fi

data/dev.yml

@@ -8,7 +8,7 @@ up:
   - custom:
       name: Minikube Cluster
       met?: test $(minikube status | grep Running | wc -l) -eq 2 && $(minikube status | grep -q 'Correctly Configured')
-      meet: minikube start --kubernetes-version=v1.10.0 --vm-driver=hyperkit
+      meet: minikube start --kubernetes-version=v1.10.6 --vm-driver=hyperkit
       down: minikube stop
 commands:
   reset-minikube: minikube delete && rm -rf ~/.minikube

data/exe/kubernetes-run

@@ -8,8 +8,15 @@ require 'optparse'
 template = "task-runner-template"
 entrypoint = nil
 env_vars = []
+verify_result = true
+max_watch_seconds = nil
 
 ARGV.options do |opts|
+  opts.on("--skip-wait", "Skip verification of pod success") { verify_result = false }
+  opts.on("--max-watch-seconds=seconds",
+    "Timeout error is raised if the pod runs for longer than the specified number of seconds") do |t|
+    max_watch_seconds = t.to_i
+  end
   opts.on("--template=TEMPLATE") { |n| template = n }
   opts.on("--env-vars=ENV_VARS") { |vars| env_vars = vars.split(",") }
   opts.on("--entrypoint=ENTRYPOINT") { |c| entrypoint = [c] }
@@ -23,10 +30,12 @@ logger = KubernetesDeploy::FormattedLogger.build(namespace, context)
 runner = KubernetesDeploy::RunnerTask.new(
   namespace: namespace,
   context: context,
-  logger: logger
+  logger: logger,
+  max_watch_seconds: max_watch_seconds
 )
 
 success = runner.run(
+  verify_result: verify_result,
   task_template: template,
   entrypoint: entrypoint,
   args: ARGV[2..-1],

data/lib/kubernetes-deploy.rb

@@ -22,6 +22,6 @@ require 'kubernetes-deploy/duration_parser'
 require 'kubernetes-deploy/sync_mediator'
 
 module KubernetesDeploy
-  MIN_KUBE_VERSION = '1.7.0'
+  MIN_KUBE_VERSION = '1.8.0'
   KubernetesDeploy::StatsD.build
 end

data/lib/kubernetes-deploy/container_logs.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+module KubernetesDeploy
+  class ContainerLogs
+    attr_reader :lines, :container_name
+
+    DEFAULT_LINE_LIMIT = 250
+
+    def initialize(parent_id:, container_name:, logger:)
+      @parent_id = parent_id
+      @container_name = container_name
+      @logger = logger
+      @lines = []
+      @next_print_index = 0
+    end
+
+    def sync(kubectl)
+      new_logs = fetch_latest(kubectl)
+      return unless new_logs.present?
+      @lines += deduplicate(new_logs)
+    end
+
+    def empty?
+      lines.empty?
+    end
+
+    def print_latest(prefix: false)
+      prefix_str = "[#{container_name}]  " if prefix
+
+      lines[@next_print_index..-1].each do |msg|
+        @logger.info "#{prefix_str}#{msg}"
+      end
+
+      @next_print_index = lines.length
+    end
+
+    def print_all
+      lines.each { |line| @logger.info("\t#{line}") }
+    end
+
+    private
+
+    def fetch_latest(kubectl)
+      cmd = ["logs", @parent_id, "--container=#{container_name}", "--timestamps"]
+      cmd << if @last_timestamp.present?
+        "--since-time=#{rfc3339_timestamp(@last_timestamp)}"
+      else
+        "--tail=#{DEFAULT_LINE_LIMIT}"
+      end
+      out, _err, _st = kubectl.run(*cmd, log_failure: false)
+      out.split("\n")
+    end
+
+    def rfc3339_timestamp(time)
+      time.strftime("%FT%T.%N%:z")
+    end
+
+    def deduplicate(logs)
+      deduped = []
+      check_for_duplicate = true
+
+      logs.each do |line|
+        timestamp, msg = split_timestamped_line(line)
+        next if check_for_duplicate && likely_duplicate?(timestamp)
+        check_for_duplicate = false # logs are ordered, so once we've seen a new one, assume all subsequent logs are new
+        @last_timestamp = timestamp if timestamp
+        deduped << msg
+      end
+
+      deduped
+    end
+
+    def split_timestamped_line(log_line)
+      timestamp, message = log_line.split(" ", 2)
+      [Time.parse(timestamp), message]
+    rescue ArgumentError
+      # If the log file can't be opened, k8s 1.8 writes an error message without a timestamp to stdout
+      [nil, log_line]
+    end
+
+    def likely_duplicate?(timestamp)
+      return false unless @last_timestamp && timestamp
+      # The --since-time granularity the API server supports is not adequate to prevent duplicates
+      # This comparison takes the fractional seconds into account
+      timestamp <= @last_timestamp
+    end
+  end
+end

data/lib/kubernetes-deploy/deferred_summary_logging.rb

@@ -47,11 +47,15 @@ module KubernetesDeploy
         level = :fatal
       end
 
-      public_send(level, summary.actions_sentence)
-      summary.paragraphs.each do |para|
+      if actions_sentence = summary.actions_sentence.presence
+        public_send(level, actions_sentence)
         blank_line(level)
+      end
+
+      summary.paragraphs.each do |para|
         msg_lines = para.split("\n")
         msg_lines.each { |line| public_send(level, line) }
+        blank_line(level) unless para == summary.paragraphs.last
       end
     end
 
@@ -64,11 +68,8 @@ module KubernetesDeploy
       end
 
       def actions_sentence
-        case @actions_taken.length
-        when 0 then "No actions taken"
-        else
-          @actions_taken.to_sentence.capitalize
-        end
+        return unless @actions_taken.present?
+        @actions_taken.to_sentence.capitalize
       end
 
       # Saves a sentence fragment to be displayed in the first sentence of the summary section

data/lib/kubernetes-deploy/deploy_task.rb

@@ -78,7 +78,6 @@ module KubernetesDeploy
         batch/v1/Job
         extensions/v1beta1/DaemonSet
         extensions/v1beta1/Deployment
-        apps/v1beta1/Deployment
         extensions/v1beta1/Ingress
         apps/v1beta1/StatefulSet
         autoscaling/v1/HorizontalPodAutoscaler

data/lib/kubernetes-deploy/errors.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 module KubernetesDeploy
   class FatalDeploymentError < StandardError; end
+  class FatalKubeAPIError < FatalDeploymentError; end
   class KubectlError < StandardError; end
 
   class InvalidTemplateError < FatalDeploymentError

data/lib/kubernetes-deploy/kubeclient_builder.rb

@@ -89,7 +89,11 @@ module KubernetesDeploy
         "#{kube_context.api_endpoint}#{endpoint_path}",
         api_version,
         ssl_options: kube_context.ssl_options,
-        auth_options: kube_context.auth_options
+        auth_options: kube_context.auth_options,
+        timeouts: {
+          open: KubernetesDeploy::Kubectl::DEFAULT_TIMEOUT,
+          read: KubernetesDeploy::Kubectl::DEFAULT_TIMEOUT
+        }
       )
       client.discover
       client

data/lib/kubernetes-deploy/kubectl.rb

@@ -2,7 +2,9 @@
 
 module KubernetesDeploy
   class Kubectl
-    def initialize(namespace:, context:, logger:, log_failure_by_default:, default_timeout: '30s',
+    DEFAULT_TIMEOUT = 30
+
+    def initialize(namespace:, context:, logger:, log_failure_by_default:, default_timeout: DEFAULT_TIMEOUT,
       output_is_sensitive: false)
       @namespace = namespace
       @context = context

data/lib/kubernetes-deploy/kubernetes_resource.rb

@@ -3,6 +3,8 @@ require 'json'
 require 'open3'
 require 'shellwords'
 
+require 'kubernetes-deploy/remote_logs'
+
 module KubernetesDeploy
   class KubernetesResource
     attr_reader :name, :namespace, :context
@@ -87,6 +89,10 @@ module KubernetesDeploy
       @instance_data = {}
     end
 
+    def to_kubeclient_resource
+      Kubeclient::Resource.new(@definition)
+    end
+
     def validate_definition(kubectl)
       @validation_errors = []
       validate_timeout_annotation
@@ -118,6 +124,9 @@ module KubernetesDeploy
       @instance_data = mediator.get_instance(kubectl_resource_type, name)
     end
 
+    def after_sync
+    end
+
     def deploy_failed?
       false
     end
@@ -173,9 +182,8 @@ module KubernetesDeploy
     end
 
     def sync_debug_info(kubectl)
-      @events = fetch_events(kubectl) unless ENV[DISABLE_FETCHING_EVENT_INFO]
-      @logs = fetch_logs(kubectl) if supports_logs? && !ENV[DISABLE_FETCHING_EVENT_INFO]
-      @debug_info_synced = true
+      @debug_events = fetch_events(kubectl) unless ENV[DISABLE_FETCHING_EVENT_INFO]
+      @debug_logs = fetch_debug_logs(kubectl) if print_debug_logs? && !ENV[DISABLE_FETCHING_LOG_INFO]
     end
 
     def debug_message(cause = nil, info_hash = {})
@@ -198,9 +206,9 @@ module KubernetesDeploy
       end
       helpful_info << "  - Final status: #{status}"
 
-      if @events.present?
+      if @debug_events.present?
         helpful_info << "  - Events (common success events excluded):"
-        @events.each do |identifier, event_hashes|
+        @debug_events.each do |identifier, event_hashes|
           event_hashes.each { |event| helpful_info << "      [#{identifier}]\t#{event}" }
         end
       elsif ENV[DISABLE_FETCHING_EVENT_INFO]
@@ -209,21 +217,24 @@ module KubernetesDeploy
         helpful_info << "  - Events: #{DEBUG_RESOURCE_NOT_FOUND_MESSAGE}"
       end
 
-      if supports_logs?
+      if print_debug_logs?
         if ENV[DISABLE_FETCHING_LOG_INFO]
           helpful_info << "  - Logs: #{DISABLED_LOG_INFO_MESSAGE}"
-        elsif @logs.blank? || @logs.values.all?(&:blank?)
+        elsif @debug_logs.blank?
           helpful_info << "  - Logs: #{DEBUG_RESOURCE_NOT_FOUND_MESSAGE}"
         else
-          sorted_logs = @logs.sort_by { |_, log_lines| log_lines.length }
-          sorted_logs.each do |identifier, log_lines|
-            if log_lines.empty?
-              helpful_info << "  - Logs from container '#{identifier}': #{DEBUG_RESOURCE_NOT_FOUND_MESSAGE}"
+          container_logs = @debug_logs.container_logs.sort_by { |c| c.lines.length }
+          container_logs.each do |logs|
+            if logs.empty?
+              helpful_info << "  - Logs from container '#{logs.container_name}': #{DEBUG_RESOURCE_NOT_FOUND_MESSAGE}"
               next
             end
 
-            helpful_info << "  - Logs from container '#{identifier}' (last #{LOG_LINE_COUNT} lines shown):"
-            log_lines.each do |line|
+            if logs.lines.length == ContainerLogs::DEFAULT_LINE_LIMIT
+              truncated = " (last #{ContainerLogs::DEFAULT_LINE_LIMIT} lines shown)"
+            end
+            helpful_info << "  - Logs from container '#{logs.container_name}'#{truncated}:"
+            logs.lines.each do |line|
               helpful_info << "      #{line}"
             end
           end
@@ -366,8 +377,8 @@ module KubernetesDeploy
       file&.close
     end
 
-    def supports_logs?
-      respond_to?(:fetch_logs)
+    def print_debug_logs?
+      false
     end
 
     def statsd_tags

data/lib/kubernetes-deploy/kubernetes_resource/daemon_set.rb

@@ -28,10 +28,13 @@ module KubernetesDeploy
       observed_generation == current_generation
     end
 
-    def fetch_logs(kubectl)
-      return {} unless pods.present?
+    def fetch_debug_logs(kubectl)
       most_useful_pod = pods.find(&:deploy_failed?) || pods.find(&:deploy_timed_out?) || pods.first
-      most_useful_pod.fetch_logs(kubectl)
+      most_useful_pod.fetch_debug_logs(kubectl)
+    end
+
+    def print_debug_logs?
+      pods.present? # the kubectl command times out if no pods exist
     end
 
     private

data/lib/kubernetes-deploy/kubernetes_resource/deployment.rb

@@ -24,9 +24,12 @@ module KubernetesDeploy
       own_events.merge(@latest_rs.fetch_events(kubectl))
     end
 
-    def fetch_logs(kubectl)
-      return {} unless @latest_rs.present?
-      @latest_rs.fetch_logs(kubectl)
+    def print_debug_logs?
+      @latest_rs.present?
+    end
+
+    def fetch_debug_logs(kubectl)
+      @latest_rs.fetch_debug_logs(kubectl)
     end
 
     def deploy_succeeded?
@@ -142,12 +145,6 @@ module KubernetesDeploy
     def deploy_failing_to_progress?
       return false unless progress_condition.present?
 
-      if @server_version < Gem::Version.new("1.7.7")
-        # Deployments were being updated prematurely with incorrect progress information
-        # https://github.com/kubernetes/kubernetes/issues/49637
-        return false unless Time.now.utc - @deploy_started_at >= progress_deadline.to_i
-      end
-
       # This assumes that when the controller bumps the observed generation, it also updates/clears all the status
       # conditions. Specifically, it assumes the progress condition is immediately set to True if a rollout is starting.
       deploy_started? &&

data/lib/kubernetes-deploy/kubernetes_resource/pod.rb

@@ -10,15 +10,17 @@ module KubernetesDeploy
     )
 
     def initialize(namespace:, context:, definition:, logger:,
-      statsd_tags: nil, parent: nil, deploy_started_at: nil)
+      statsd_tags: nil, parent: nil, deploy_started_at: nil, stream_logs: false)
       @parent = parent
       @deploy_started_at = deploy_started_at
+
       @containers = definition.fetch("spec", {}).fetch("containers", []).map { |c| Container.new(c) }
       unless @containers.present?
         logger.summary.add_paragraph("Rendered template content:\n#{definition.to_yaml}")
         raise FatalDeploymentError, "Template is missing required field spec.containers"
       end
       @containers += definition["spec"].fetch("initContainers", []).map { |c| Container.new(c, init_container: true) }
+      @stream_logs = stream_logs
       super(namespace: namespace, context: context, definition: definition,
             logger: logger, statsd_tags: statsd_tags)
     end
@@ -28,12 +30,19 @@ module KubernetesDeploy
       raise_predates_deploy_error if exists? && unmanaged? && !deploy_started?
 
       if exists?
+        logs.sync(mediator.kubectl) if unmanaged?
         update_container_statuses(@instance_data["status"])
       else # reset
         @containers.each(&:reset_status)
       end
+    end
 
-      display_logs(mediator) if unmanaged? && deploy_succeeded?
+    def after_sync
+      if @stream_logs
+        logs.print_latest
+      elsif unmanaged? && deploy_succeeded?
+        logs.print_all
+      end
     end
 
     def status
@@ -80,28 +89,25 @@ module KubernetesDeploy
       "#{phase_problem}#{container_problems}".presence
     end
 
-    # Returns a hash in the following format:
-    # {
-    #   "app" => ["array of log lines", "received from app container"],
-    #   "nginx" => ["array of log lines", "received from nginx container"]
-    # }
-    def fetch_logs(kubectl)
-      return {} unless exists? && @containers.present?
-      @containers.each_with_object({}) do |container, container_logs|
-        cmd = [
-          "logs",
-          @name,
-          "--container=#{container.name}",
-          "--since-time=#{@deploy_started_at.to_datetime.rfc3339}",
-        ]
-        cmd << "--tail=#{LOG_LINE_COUNT}" unless unmanaged?
-        out, _err, _st = kubectl.run(*cmd, log_failure: false)
-        container_logs[container.name] = out.split("\n")
-      end
+    def fetch_debug_logs(kubectl)
+      logs.sync(kubectl)
+      logs
+    end
+
+    def print_debug_logs?
+      exists? && !@stream_logs # don't print them a second time
     end
 
     private
 
+    def logs
+      @logs ||= KubernetesDeploy::RemoteLogs.new(
+        logger: @logger,
+        parent_id: id,
+        container_names: @containers.map(&:name)
+      )
+    end
+
     def phase
       @instance_data.dig("status", "phase") || "Unknown"
     end
@@ -138,29 +144,6 @@ module KubernetesDeploy
       @parent.blank?
     end
 
-    def display_logs(mediator)
-      return if @already_displayed
-      container_logs = fetch_logs(mediator.kubectl)
-
-      if container_logs.empty?
-        @logger.warn("No logs found for pod #{id}")
-        return
-      end
-
-      container_logs.each do |container_identifier, logs|
-        if logs.blank?
-          @logger.warn("No logs found for container '#{container_identifier}'")
-        else
-          @logger.blank_line
-          @logger.info("Logs from #{id} container '#{container_identifier}':")
-          logs.each { |line| @logger.info("\t#{line}") }
-          @logger.blank_line
-        end
-      end
-
-      @already_displayed = true
-    end
-
     def raise_predates_deploy_error
       example_color = :green
       msg = <<-STRING.strip_heredoc

data/lib/kubernetes-deploy/kubernetes_resource/pod_set_base.rb

@@ -16,19 +16,14 @@ module KubernetesDeploy
       own_events.merge(most_useful_pod.fetch_events(kubectl))
     end
 
-    def fetch_logs(kubectl)
-      return {} unless pods.present? # the kubectl command times out if no pods exist
-      container_names.each_with_object({}) do |container_name, container_logs|
-        out, _err, _st = kubectl.run(
-          "logs",
-          id,
-          "--container=#{container_name}",
-          "--since-time=#{@deploy_started_at.to_datetime.rfc3339}",
-          "--tail=#{LOG_LINE_COUNT}",
-          log_failure: false
-        )
-        container_logs[container_name] = out.split("\n")
-      end
+    def fetch_debug_logs(kubectl)
+      logs = KubernetesDeploy::RemoteLogs.new(logger: @logger, parent_id: id, container_names: container_names)
+      logs.sync(kubectl)
+      logs
+    end
+
+    def print_debug_logs?
+      pods.present? # the kubectl command times out if no pods exist
     end
 
     private

data/lib/kubernetes-deploy/kubernetes_resource/stateful_set.rb

@@ -25,7 +25,7 @@ module KubernetesDeploy
         unless @success_assumption_warning_shown
           @logger.warn("WARNING: Your StatefulSet's updateStrategy is set to OnDelete, "\
                        "which means updates will not be applied until its pods are deleted. "\
-                       "If you are using k8s 1.7+, consider switching to rollingUpdate.")
+                       "Consider switching to rollingUpdate.")
           @success_assumption_warning_shown = true
         end
         true
@@ -46,9 +46,7 @@ module KubernetesDeploy
     private
 
     def update_strategy
-      if @server_version < Gem::Version.new("1.7.0")
-        ONDELETE
-      elsif exists?
+      if exists?
         @instance_data['spec']['updateStrategy']['type']
       else
         'Unknown'

data/lib/kubernetes-deploy/remote_logs.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+require 'kubernetes-deploy/container_logs'
+
+module KubernetesDeploy
+  class RemoteLogs
+    attr_reader :container_logs
+
+    def initialize(logger:, parent_id:, container_names:)
+      @logger = logger
+      @parent_id = parent_id
+      @container_logs = container_names.map do |n|
+        ContainerLogs.new(logger: logger, container_name: n, parent_id: parent_id)
+      end
+    end
+
+    def empty?
+      @container_logs.all?(&:empty?)
+    end
+
+    def sync(kubectl)
+      @container_logs.each { |cl| cl.sync(kubectl) }
+    end
+
+    def print_latest
+      @container_logs.each { |cl| cl.print_latest(prefix: @container_logs.length > 1) }
+    end
+
+    def print_all(prevent_duplicate: true)
+      return if @already_displayed && prevent_duplicate
+
+      if @container_logs.all?(&:empty?)
+        @logger.warn("No logs found for #{@parent_id}")
+        return
+      end
+
+      @container_logs.each do |cl|
+        if cl.empty?
+          @logger.warn("No logs found for #{@parent_id} container '#{cl.container_name}'")
+        else
+          @logger.info("Logs from #{@parent_id} container '#{cl.container_name}':")
+          cl.print_all
+          @logger.blank_line
+        end
+      end
+
+      @already_displayed = true
+    end
+  end
+end

data/lib/kubernetes-deploy/resource_watcher.rb

@@ -17,19 +17,16 @@ module KubernetesDeploy
     end
 
     def run(delay_sync: 3.seconds, reminder_interval: 30.seconds, record_summary: true)
-      delay_sync_until = last_message_logged_at = monitoring_started = Time.now.utc
+      last_message_logged_at = monitoring_started = Time.now.utc
       remainder = @resources.dup
 
       while remainder.present?
-        if @timeout && (Time.now.utc - monitoring_started > @timeout)
-          report_and_give_up(remainder)
-        end
-        if (sleep_duration = delay_sync_until - Time.now.utc) > 0
-          sleep(sleep_duration)
-        end
-        delay_sync_until = Time.now.utc + delay_sync # don't pummel the API if the sync is fast
+        report_and_give_up(remainder) if global_timeout?(monitoring_started)
+        sleep_until_next_sync(delay_sync)
 
         @sync_mediator.sync(remainder)
+        remainder.each(&:after_sync)
+
         new_successes, remainder = remainder.partition(&:deploy_succeeded?)
         new_failures, remainder = remainder.partition(&:deploy_failed?)
         new_timeouts, remainder = remainder.partition(&:deploy_timed_out?)
@@ -48,6 +45,18 @@ module KubernetesDeploy
 
     private
 
+    def global_timeout?(started_at)
+      @timeout && (Time.now.utc - started_at > @timeout)
+    end
+
+    def sleep_until_next_sync(min_interval)
+      @next_sync_time ||= Time.now.utc
+      if (sleep_duration = @next_sync_time - Time.now.utc) > 0
+        sleep(sleep_duration)
+      end
+      @next_sync_time = Time.now.utc + min_interval
+    end
+
     def report_what_just_happened(new_successes, new_failures, new_timeouts)
       watch_time = (Time.now.utc - @deploy_started_at).round(1)
       new_failures.each do |resource|
@@ -62,7 +71,7 @@ module KubernetesDeploy
 
       if new_successes.present?
         new_successes.each { |r| r.report_status_to_statsd(watch_time) }
-        success_string = ColorizedString.new("Successfully #{@operation_name}ed in #{watch_time}s:").green
+        success_string = ColorizedString.new("Successfully #{past_tense_operation} in #{watch_time}s:").green
         @logger.info("#{success_string} #{new_successes.map(&:id).join(', ')}")
       end
     end
@@ -120,7 +129,7 @@ module KubernetesDeploy
     def record_success_statuses(successful_resources)
       success_count = successful_resources.length
       if success_count > 0
-        @logger.summary.add_action("successfully #{@operation_name}ed #{success_count} "\
+        @logger.summary.add_action("successfully #{past_tense_operation} #{success_count} "\
           "#{'resource'.pluralize(success_count)}")
         final_statuses = successful_resources.map(&:pretty_status).join("\n")
         @logger.summary.add_paragraph("#{ColorizedString.new('Successful resources').green}\n#{final_statuses}")
@@ -130,5 +139,9 @@ module KubernetesDeploy
     def due_for_reminder?(last_message_logged_at, reminder_interval)
       (last_message_logged_at.to_f + reminder_interval.to_f) <= Time.now.utc.to_f
     end
+
+    def past_tense_operation
+      @operation_name == "run" ? "ran" : "#{@operation_name}ed"
+    end
   end
 end

data/lib/kubernetes-deploy/runner_task.rb

@@ -8,51 +8,99 @@ module KubernetesDeploy
   class RunnerTask
     include KubeclientBuilder
 
-    class FatalTaskRunError < FatalDeploymentError; end
-    class TaskTemplateMissingError < FatalDeploymentError
-      def initialize(task_template, namespace, context)
-        super("Pod template `#{task_template}` cannot be found in namespace: `#{namespace}`, context: `#{context}`")
-      end
-    end
+    class TaskConfigurationError < FatalDeploymentError; end
+    class TaskTemplateMissingError < TaskConfigurationError; end
+
+    attr_reader :pod_name
 
-    def initialize(namespace:, context:, logger:)
+    def initialize(namespace:, context:, logger:, max_watch_seconds: nil)
       @logger = logger
       @namespace = namespace
-      @kubeclient = build_v1_kubeclient(context)
       @context = context
+      @max_watch_seconds = max_watch_seconds
     end
 
     def run(*args)
       run!(*args)
       true
-    rescue FatalDeploymentError => error
-      @logger.fatal "#{error.class}: #{error.message}"
+    rescue DeploymentTimeoutError, FatalDeploymentError
       false
     end
 
-    def run!(task_template:, entrypoint:, args:, env_vars: [])
+    def run!(task_template:, entrypoint:, args:, env_vars: [], verify_result: true)
       @logger.reset
-      @logger.phase_heading("Validating configuration")
+
+      @logger.phase_heading("Initializing task")
       validate_configuration(task_template, args)
-      if kubectl.server_version < Gem::Version.new(MIN_KUBE_VERSION)
-        @logger.warn(KubernetesDeploy::Errors.server_version_warning(kubectl.server_version))
+      pod = build_pod(task_template, entrypoint, args, env_vars, verify_result)
+      validate_pod(pod)
+
+      @logger.phase_heading("Running pod")
+      create_pod(pod)
+
+      if verify_result
+        @logger.phase_heading("Streaming logs")
+        watch_pod(pod)
+      else
+        record_status_once(pod)
       end
-      @logger.phase_heading("Fetching task template")
-      raw_template = get_template(task_template)
+      @logger.print_summary(:success)
+    rescue DeploymentTimeoutError
+      @logger.print_summary(:timed_out)
+      raise
+    rescue FatalDeploymentError
+      @logger.print_summary(:failure)
+      raise
+    end
 
-      @logger.phase_heading("Constructing final pod specification")
-      rendered_template = build_pod_template(raw_template, entrypoint, args, env_vars)
+    private
 
-      validate_pod_spec(rendered_template)
+    def create_pod(pod)
+      @logger.info "Creating pod '#{pod.name}'"
+      pod.deploy_started_at = Time.now.utc
+      kubeclient.create_pod(pod.to_kubeclient_resource)
+      @pod_name = pod.name
+      @logger.info("Pod creation succeeded")
+    rescue KubeException => e
+      msg = "Failed to create pod: #{e.class.name}: #{e.message}"
+      @logger.summary.add_paragraph(msg)
+      raise FatalDeploymentError, msg
+    end
 
-      @logger.phase_heading("Creating pod")
-      @logger.info("Starting task runner pod: '#{rendered_template.metadata.name}'")
-      @kubeclient.create_pod(rendered_template)
+    def build_pod(template_name, entrypoint, args, env_vars, verify_result)
+      task_template = get_template(template_name)
+      @logger.info("Using template '#{template_name}'")
+      pod_template = build_pod_definition(task_template)
+      set_container_overrides!(pod_template, entrypoint, args, env_vars)
+      ensure_valid_restart_policy!(pod_template, verify_result)
+      Pod.new(namespace: @namespace, context: @context, logger: @logger, stream_logs: true,
+                    definition: pod_template.to_hash.deep_stringify_keys, statsd_tags: [])
     end
 
-    private
+    def validate_pod(pod)
+      pod.validate_definition(kubectl)
+    end
+
+    def watch_pod(pod)
+      rw = ResourceWatcher.new(resources: [pod], logger: @logger, timeout: @max_watch_seconds,
+        sync_mediator: sync_mediator, operation_name: "run")
+      rw.run(delay_sync: 1, reminder_interval: 30.seconds)
+      raise DeploymentTimeoutError if pod.deploy_timed_out?
+      raise FatalDeploymentError if pod.deploy_failed?
+    end
+
+    def record_status_once(pod)
+      pod.sync(sync_mediator)
+      warning = <<~STRING
+        #{ColorizedString.new('Result verification is disabled for this task.').yellow}
+        The following status was observed immediately after pod creation:
+        #{pod.pretty_status}
+      STRING
+      @logger.summary.add_paragraph(warning)
+    end
 
     def validate_configuration(task_template, args)
+      @logger.info("Validating configuration")
       errors = []
 
       if task_template.blank?
@@ -68,71 +116,74 @@ module KubernetesDeploy
       end
 
       begin
-        @kubeclient.get_namespace(@namespace) if @namespace.present?
+        kubeclient.get_namespace(@namespace) if @namespace.present?
+        @logger.info "Using namespace '#{@namespace}' in context '#{@context}'"
       rescue KubeException => e
         msg = e.error_code == 404 ? "Namespace was not found" : "Could not connect to kubernetes cluster"
         errors << msg
       end
 
-      raise FatalTaskRunError, "Configuration invalid: #{errors.join(', ')}" unless errors.empty?
+      unless errors.empty?
+        @logger.summary.add_action("Configuration invalid")
+        @logger.summary.add_paragraph(errors.map { |err| "- #{err}" }.join("\n"))
+        raise TaskConfigurationError, "Configuration invalid: #{errors.join(', ')}"
+      end
+
+      if kubectl.server_version < Gem::Version.new(MIN_KUBE_VERSION)
+        @logger.warn(KubernetesDeploy::Errors.server_version_warning(kubectl.server_version))
+      end
     end
 
     def get_template(template_name)
-      @logger.info(
-        "Fetching task runner pod template: '#{template_name}' in namespace: '#{@namespace}'"
-      )
-
-      pod_template = @kubeclient.get_pod_template(template_name, @namespace)
+      pod_template = kubeclient.get_pod_template(template_name, @namespace)
 
       pod_template.template
     rescue KubeException => error
       if error.error_code == 404
-        raise TaskTemplateMissingError.new(template_name, @namespace, @context)
+        msg = "Pod template `#{template_name}` not found in namespace `#{@namespace}`, context `#{@context}`"
+        @logger.summary.add_paragraph(msg)
+        raise TaskTemplateMissingError, msg
       else
-        raise
+        raise FatalKubeAPIError, "Error retrieving pod template: #{error.class.name}: #{error.message}"
       end
     end
 
-    def build_pod_template(base_template, entrypoint, args, env_vars)
-      @logger.info("Rendering template for task runner pod")
+    def build_pod_definition(base_template)
+      pod_definition = base_template.dup
+      pod_definition.kind = 'Pod'
+      pod_definition.apiVersion = 'v1'
+      pod_definition.metadata.namespace = @namespace
 
-      rendered_template = base_template.dup
-      rendered_template.kind = 'Pod'
-      rendered_template.apiVersion = 'v1'
+      unique_name = pod_definition.metadata.name + "-" + SecureRandom.hex(8)
+      @logger.warn("Name is too long, using '#{unique_name[0..62]}'") if unique_name.length > 63
+      pod_definition.metadata.name = unique_name[0..62]
 
-      container = rendered_template.spec.containers.find { |cont| cont.name == 'task-runner' }
+      pod_definition
+    end
 
-      raise FatalTaskRunError, "Pod spec does not contain a template container called 'task-runner'" if container.nil?
+    def set_container_overrides!(pod_definition, entrypoint, args, env_vars)
+      container = pod_definition.spec.containers.find { |cont| cont.name == 'task-runner' }
+      if container.nil?
+        raise TaskConfigurationError, "Pod spec does not contain a template container called 'task-runner'"
+      end
 
       container.command = entrypoint
       container.args = args
-      container.env ||= []
 
       env_args = env_vars.map do |env|
         key, value = env.split('=', 2)
         { name: key, value: value }
       end
-
+      container.env ||= []
       container.env = container.env.map(&:to_h) + env_args
-
-      unique_name = rendered_template.metadata.name + "-" + SecureRandom.hex(8)
-
-      @logger.warn("Name is too long, using '#{unique_name[0..62]}'") if unique_name.length > 63
-      rendered_template.metadata.name = unique_name[0..62]
-      rendered_template.metadata.namespace = @namespace
-
-      rendered_template
     end
 
-    def validate_pod_spec(pod)
-      f = Tempfile.new(pod.metadata.name)
-      f.write recursive_to_h(pod).to_json
-      f.close
-
-      _out, err, status = kubectl.run("apply", "--dry-run", "-f", f.path)
-
-      unless status.success?
-        raise FatalTaskRunError, "Invalid pod spec: #{err}"
+    def ensure_valid_restart_policy!(template, verify)
+      restart_policy = template.spec.restartPolicy
+      if verify && restart_policy != "Never"
+        @logger.warn("Changed Pod RestartPolicy from '#{restart_policy}' to 'Never'. Disable "\
+          "result verification to use '#{restart_policy}'.")
+        template.spec.restartPolicy = "Never"
       end
     end
 
@@ -140,18 +191,12 @@ module KubernetesDeploy
       @kubectl ||= Kubectl.new(namespace: @namespace, context: @context, logger: @logger, log_failure_by_default: true)
     end
 
-    def recursive_to_h(struct)
-      if struct.is_a?(Array)
-        return struct.map { |v| v.is_a?(OpenStruct) || v.is_a?(Array) || v.is_a?(Hash) ? recursive_to_h(v) : v }
-      end
-
-      hash = {}
+    def sync_mediator
+      @sync_mediator ||= SyncMediator.new(namespace: @namespace, context: @context, logger: @logger)
+    end
 
-      struct.each_pair do |k, v|
-        recursive_val = v.is_a?(OpenStruct) || v.is_a?(Array) || v.is_a?(Hash)
-        hash[k] = recursive_val ? recursive_to_h(v) : v
-      end
-      hash
+    def kubeclient
+      @kubeclient ||= build_v1_kubeclient(@context)
     end
   end
 end

data/lib/kubernetes-deploy/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module KubernetesDeploy
-  VERSION = "0.20.6"
+  VERSION = "0.21.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: kubernetes-deploy
 version: !ruby/object:Gem::Version
-  version: 0.20.6
+  version: 0.21.0
 platform: ruby
 authors:
 - Katrina Verey
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-09-18 00:00:00.000000000 Z
+date: 2018-10-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -213,6 +213,7 @@ files:
 - lib/kubernetes-deploy.rb
 - lib/kubernetes-deploy/bindings_parser.rb
 - lib/kubernetes-deploy/concurrency.rb
+- lib/kubernetes-deploy/container_logs.rb
 - lib/kubernetes-deploy/deferred_summary_logging.rb
 - lib/kubernetes-deploy/deploy_task.rb
 - lib/kubernetes-deploy/duration_parser.rb
@@ -248,6 +249,7 @@ files:
 - lib/kubernetes-deploy/kubernetes_resource/stateful_set.rb
 - lib/kubernetes-deploy/kubernetes_resource/statefulservice.rb
 - lib/kubernetes-deploy/kubernetes_resource/topic.rb
+- lib/kubernetes-deploy/remote_logs.rb
 - lib/kubernetes-deploy/renderer.rb
 - lib/kubernetes-deploy/resource_discovery.rb
 - lib/kubernetes-deploy/resource_watcher.rb