kubernetes-deploy 0.21.1 → 0.22.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b7b6b27e2d19b9194f53092f3037a6597d0c098b
-  data.tar.gz: 5c1e656293fdb2f0e193bdb90419bd7629f683fd
+  metadata.gz: 93fa4f3f00a9ff96a2dfc20fa26ead2ccf77d5e2
+  data.tar.gz: 6b1e6a305c64c94ffb1cb38b5f7d722ba2e11fd6
 SHA512:
-  metadata.gz: f6968be66492775b5edac4e2a30fe30c168aafd1db5766f5a97eae84c2fd42dd5603c608960d6b0aadc5dec6f717842c41b9dbbd6cd7f6dad69716b1106b33e3
-  data.tar.gz: 751eb781b35bdf1081c41d4f182b84cc97a897b7c199c5020ecd2691c1478fb48174a1527d9ad0e3a58103a4dff734f149b261682d52a923fdb516239db0f8f0
+  metadata.gz: 88e0586648309f18d49b7d7e27650edd273bc0fdef8ae9bb0a0aa639ed6e4016fb0d7c6d2a1c25caede0774bb5fdc30b79d100e0ade49fcd6dd1a3823d20dc3a
+  data.tar.gz: a2ae7f6b727838b2c090c4208d0d321dee080902f8a50057c62f53691888294121211b863225945d5178ceea24876299e49ebdee1bc70f911e66eafd129e28fe

data/.buildkite/pipeline.nightly.yml

@@ -4,6 +4,14 @@ shared: &shared
        - exit_status: "*"
          limit: 1
 steps:
+  - name: 'Run Test Suite (:kubernetes: 1.12-latest)'
+    <<: *shared
+    command: bin/ci
+    agents:
+      queue: k8s-ci
+    env:
+      LOGGING_LEVEL: 4
+      KUBERNETES_VERSION: v1.12-latest
   - name: 'Run Test Suite (:kubernetes: 1.11-latest)'
     <<: *shared
     command: bin/ci
@@ -28,11 +36,3 @@ steps:
     env:
       LOGGING_LEVEL: 4
       KUBERNETES_VERSION: v1.9-latest
-  - name: 'Run Test Suite (:kubernetes: 1.8-latest)'
-    <<: *shared
-    command: bin/ci
-    agents:
-      queue: minikube-ci
-    env:
-      LOGGING_LEVEL: 4
-      KUBERNETES_VERSION: v1.8-latest

data/.buildkite/pipeline.yml

@@ -4,6 +4,14 @@ shared: &shared
        - exit_status: "*"
          limit: 1
 steps:
+  - name: 'Run Test Suite (:kubernetes: 1.12-latest)'
+    <<: *shared
+    command: bin/ci
+    agents:
+      queue: k8s-ci
+    env:
+      LOGGING_LEVEL: 4
+      KUBERNETES_VERSION: v1.12-latest
   - name: 'Run Test Suite (:kubernetes: 1.11-latest)'
     <<: *shared
     command: bin/ci
@@ -28,11 +36,3 @@ steps:
     env:
       LOGGING_LEVEL: 4
       KUBERNETES_VERSION: v1.9-latest
-  - name: 'Run Test Suite (:kubernetes: 1.8-latest)'
-    <<: *shared
-    command: bin/ci
-    agents:
-      queue: minikube-ci
-    env:
-      LOGGING_LEVEL: 4
-      KUBERNETES_VERSION: v1.8-latest

data/.gitignore

@@ -17,3 +17,4 @@ coverage/*
 dev/flamegraph.svg
 dev/profile
 dev/flamegraph.pl
+.local-context

data/CHANGELOG.md

@@ -1,3 +1,30 @@
+## 0.22.0
+
+*Features*
+- **[Breaking change]** `kubernetes-restart` now produces StatsD `distribution` instead of `metric`.
+Dashboards that used these metrics will need to be updated. ([#374](https://github.com/Shopify/kubernetes-deploy/pull/374))
+- `kubernetes-run` now produces StatsD `distribution` to aid in tracking usage ([#374](https://github.com/Shopify/kubernetes-deploy/pull/374))
+
+*Enhancements*
+- Predeploy RoleBinding before unmanaged pods ([#354](https://github.com/Shopify/kubernetes-deploy/pull/354))
+
+*Bug Fixes*
+- Fixed bug in `kubernetes-restart` that caused "Pod spec does not contain a template
+ container called 'task-runner'" error message to not be printed
+ ([#371](https://github.com/Shopify/kubernetes-deploy/pull/371))
+
+*Other*
+- Kubernetes 1.8 is no longer officially supported as of this version
+
+## 0.21.1
+
+*Enhancements*
+- Improved failure detection for job resources. ([#355](https://github.com/Shopify/kubernetes-deploy/pull/355))
+- Unmanaged pods are now immediately identified as failed if they are evicted, preempted or deleted out of band. This is especially important to `kubernetes-run`. ([#353](https://github.com/Shopify/kubernetes-deploy/pull/353))
+
+*Other*
+- Relaxed our `googleauth` dependency. ([#333](https://github.com/Shopify/kubernetes-deploy/pull/333))
+
 ## 0.21.0
 
 *Features*
@@ -13,7 +40,6 @@
 - All resources marked as prunable will now be added to the prune whitelist ([#326](https://github.com/Shopify/kubernetes-deploy/pull/326))
 - Improve deploy status detection by ensuring we examine the correct generation ([#325](https://github.com/Shopify/kubernetes-deploy/pull/325))
 
-*Bug Fixes*
 
 ## 0.20.5
 *Enhancements*

data/README.md

@@ -69,21 +69,26 @@ This repo also includes related tools for [running tasks](#kubernetes-run) and [
 ## Prerequisites
 
 * Ruby 2.3+
-* Your cluster must be running Kubernetes v1.8.0 or higher<sup>1</sup>
+* Your cluster must be running Kubernetes v1.9.0 or higher<sup>1</sup>
 * Each app must have a deploy directory containing its Kubernetes templates (see [Templates](#using-templates-and-variables))
 * You must remove the` kubectl.kubernetes.io/last-applied-configuration` annotation from any resources in the namespace that are not included in your deploy directory. This annotation is added automatically when you create resources with `kubectl apply`. `kubernetes-deploy` will prune any resources that have this annotation and are not in the deploy directory.<sup>2</sup>
 * Each app managed by `kubernetes-deploy` must have its own exclusive Kubernetes namespace.
 
-<sup>1</sup> We run integration tests against these Kubernetes versions. Kubernetes v1.7 was officially
-supported in gem versions < 0.21. Kubernetes v1.6 was officially supported in gem versions < 0.16. Kubernetes
-v1.5 was officially supported in gem versions < 0.12.
+<sup>1</sup> We run integration tests against these Kubernetes versions. You can find our
+offical compatibility chart below.
 
 <sup>2</sup> This requirement can be bypassed with the `--no-prune` option, but it is not recommended.
 
+| Kubernetes version | Last officially supported in gem version |
+| :----------------: | :-------------------: |
+|        1.5         |        0.11.2         |
+|        1.6         |        0.15.2         |
+|        1.7         |        0.20.6         |
+|        1.8         |        0.21.1         |
 
 ## Installation
 
-1. [Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-kubectl-binary-via-curl) (requires v1.8.0 or higher) and make sure it is available in your $PATH
+1. [Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-kubectl-binary-via-curl) (requires v1.9.0 or higher) and make sure it is available in your $PATH
 2. Set up your [kubeconfig file](https://kubernetes.io/docs/tasks/access-application-cluster/authenticate-across-clusters-kubeconfig/) for access to your cluster(s).
 3. `gem install kubernetes-deploy`
 
@@ -374,7 +379,7 @@ Based on this specification `kubernetes-run` will create a new pod with the entr
 
 If you work for Shopify, just run `dev up`, but otherwise:
 
-1. [Install kubectl version 1.8.0 or higher](https://kubernetes.io/docs/user-guide/prereqs/) and make sure it is in your path
+1. [Install kubectl version 1.9.0 or higher](https://kubernetes.io/docs/user-guide/prereqs/) and make sure it is in your path
 2. [Install minikube](https://kubernetes.io/docs/getting-started-guides/minikube/#installation) (required to run the test suite)
 3. Check out the repo
 4. Run `bin/setup` to install dependencies
@@ -385,16 +390,23 @@ To install this gem onto your local machine, run `bundle exec rake install`.
 
 ## Running the test suite locally
 
-1. Start [minikube](https://kubernetes.io/docs/getting-started-guides/minikube/#installation) (`minikube start [options]`)
-2. Make sure you have a context named "minikube" in your kubeconfig. Minikube adds this context for you when you run `minikube start`; please do not rename it. You can check for it using `kubectl config get-contexts`.
-3. Run `bundle exec rake test`
+Using minikube:
 
+1. Start [minikube](https://kubernetes.io/docs/getting-started-guides/minikube/#installation) (`minikube start [options]`).
+2. Make sure you have a context named "minikube" in your kubeconfig. Minikube adds this context for you when you run `minikube start`. You can check for it using `kubectl config get-contexts`.
+3. Run `bundle exec rake test` (or `dev test` if you work for Shopify).
 
+Using another local cluster:
 
-To see the full-color output of a specific integration test, you can use `PRINT_LOGS=1 bundle exec ruby -I test test/integration/kubernetes_deploy_test.rb -n/test_name/`.
+1. Start your cluster.
+2. Put the name of the context you want to use in a file named `.local-context` in the root of this project. For example: `echo "dind" > .local-context`.
+3. Run `bundle exec rake test` (or `dev test` if you work for Shopify).
 
 To make StatsD log what it would have emitted, run a test with `STATSD_DEV=1`.
 
+To see the full-color output of a specific integration test, you can use `PRINT_LOGS=1`. For example: `PRINT_LOGS=1 bundle exec ruby -I test test/integration/kubernetes_deploy_test.rb -n/test_name/`.
+
+
 
 
 ![test-output](screenshots/test-output.png)

data/bin/setup

@@ -9,8 +9,8 @@ if [ ! -x "$(which minikube)" ]; then
 fi
 
 if [ ! -x "$(which kubectl)" ]; then
-  echo -e "\n\033[0;33mPlease install kubectl version 1.8.0 or higher:\nhttps://kubernetes.io/docs/user-guide/prereqs/\033[0m"
+  echo -e "\n\033[0;33mPlease install kubectl version 1.9.0 or higher:\nhttps://kubernetes.io/docs/user-guide/prereqs/\033[0m"
 else
   KUBECTL_VERSION=$(kubectl version --short --client | grep -oe "v[[:digit:]\.]\+")
-  echo -e "\n\033[0;32mKubectl version $KUBECTL_VERSION is already installed. This gem requires version v1.8.0 or greater.\033[0m"
+  echo -e "\n\033[0;32mKubectl version $KUBECTL_VERSION is already installed. This gem requires version v1.9.0 or greater.\033[0m"
 fi

data/kubernetes-deploy.gemspec

@@ -28,7 +28,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "googleauth", "~> 0.6.6" # https://github.com/google/google-auth-library-ruby/issues/153
   spec.add_dependency "ejson", "~> 1.0"
   spec.add_dependency "colorize", "~> 0.8"
-  spec.add_dependency "statsd-instrument", "~> 2.2"
+  spec.add_dependency "statsd-instrument", "~> 2.3"
 
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake", "~> 10.0"

data/lib/kubernetes-deploy.rb

@@ -1,4 +1,3 @@
-# rubocop:disable Naming/FileName
 # frozen_string_literal: true
 
 require 'active_support/core_ext/object/blank'
@@ -22,6 +21,6 @@ require 'kubernetes-deploy/duration_parser'
 require 'kubernetes-deploy/sync_mediator'
 
 module KubernetesDeploy
-  MIN_KUBE_VERSION = '1.8.0'
+  MIN_KUBE_VERSION = '1.9.0'
   KubernetesDeploy::StatsD.build
 end

data/lib/kubernetes-deploy/container_logs.rb

@@ -73,7 +73,7 @@ module KubernetesDeploy
       timestamp, message = log_line.split(" ", 2)
       [Time.parse(timestamp), message]
     rescue ArgumentError
-      # If the log file can't be opened, k8s 1.8 writes an error message without a timestamp to stdout
+      # Don't fail on unparsable timestamp
       [nil, log_line]
     end
 

data/lib/kubernetes-deploy/deploy_task.rb

@@ -52,6 +52,7 @@ module KubernetesDeploy
       ConfigMap
       PersistentVolumeClaim
       ServiceAccount
+      RoleBinding
       Pod
     )
 
@@ -82,10 +83,8 @@ module KubernetesDeploy
         apps/v1beta1/StatefulSet
         autoscaling/v1/HorizontalPodAutoscaler
         policy/v1beta1/PodDisruptionBudget
+        batch/v1beta1/CronJob
       )
-      if server_version >= Gem::Version.new('1.8.0')
-        wl << "batch/v1beta1/CronJob"
-      end
       wl + cluster_resource_discoverer.crds(@sync_mediator).select(&:prunable?).map(&:group_version_kind)
     end
 

data/lib/kubernetes-deploy/kubeclient_builder.rb

@@ -76,6 +76,14 @@ module KubernetesDeploy
       )
     end
 
+    def build_rbac_v1_kubeclient(context)
+      _build_kubeclient(
+        api_version: "v1",
+        context: context,
+        endpoint_path: "/apis/rbac.authorization.k8s.io"
+      )
+    end
+
     def _build_kubeclient(api_version:, context:, endpoint_path: nil)
       # Find a context defined in kube conf files that matches the input context by name
       friendly_configs = config_files.map { |f| GoogleFriendlyConfig.read(f) }

data/lib/kubernetes-deploy/kubernetes_resource/deployment.rb

@@ -10,7 +10,6 @@ module KubernetesDeploy
     def sync(mediator)
       super
       @latest_rs = exists? ? find_latest_rs(mediator) : nil
-      @server_version ||= mediator.kubectl.server_version
     end
 
     def status

data/lib/kubernetes-deploy/kubernetes_resource/pod.rb

@@ -210,10 +210,6 @@ module KubernetesDeploy
           limbo_message.match(/(?:not found)|(?:back-off)/i)
           "Failed to pull image #{@image}. "\
           "Did you wait for it to be built and pushed to the registry before deploying?"
-        elsif limbo_message == "Generate Container Config Failed"
-          # reason/message are backwards in <1.8.0 (next condition used by 1.8.0+)
-          # Fixed by https://github.com/kubernetes/kubernetes/commit/df41787b1a3f51b73fb6db8a2203f0a7c7c92931
-          "Failed to generate container configuration: #{limbo_reason}"
         elsif limbo_reason == "CreateContainerConfigError"
           "Failed to generate container configuration: #{limbo_message}"
         end

data/lib/kubernetes-deploy/kubernetes_resource/role_binding.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+module KubernetesDeploy
+  class RoleBinding < KubernetesResource
+    TIMEOUT = 30.seconds
+
+    def status
+      exists? ? "Created" : "Unknown"
+    end
+
+    def deploy_succeeded?
+      exists?
+    end
+
+    def deploy_failed?
+      false
+    end
+
+    def timeout_message
+      UNUSUAL_FAILURE_MESSAGE
+    end
+  end
+end

data/lib/kubernetes-deploy/kubernetes_resource/stateful_set.rb

@@ -10,7 +10,6 @@ module KubernetesDeploy
     def sync(mediator)
       super
       @pods = exists? ? find_pods(mediator) : []
-      @server_version ||= mediator.kubectl.server_version
     end
 
     def status

data/lib/kubernetes-deploy/restart_task.rb

@@ -58,14 +58,14 @@ module KubernetesDeploy
         raise DeploymentTimeoutError
       end
       raise FatalDeploymentError unless success
-      ::StatsD.measure('restart.duration', StatsD.duration(start), tags: tags('success', deployments))
+      ::StatsD.distribution('restart.duration', StatsD.duration(start), tags: tags('success', deployments))
       @logger.print_summary(:success)
     rescue DeploymentTimeoutError
-      ::StatsD.measure('restart.duration', StatsD.duration(start), tags: tags('timeout', deployments))
+      ::StatsD.distribution('restart.duration', StatsD.duration(start), tags: tags('timeout', deployments))
       @logger.print_summary(:timed_out)
       raise
     rescue FatalDeploymentError => error
-      ::StatsD.measure('restart.duration', StatsD.duration(start), tags: tags('failure', deployments))
+      ::StatsD.distribution('restart.duration', StatsD.duration(start), tags: tags('failure', deployments))
       @logger.summary.add_action(error.message) if error.message != error.class.to_s
       @logger.print_summary(:failure)
       raise

data/lib/kubernetes-deploy/runner_task.rb

@@ -28,6 +28,7 @@ module KubernetesDeploy
     end
 
     def run!(task_template:, entrypoint:, args:, env_vars: [], verify_result: true)
+      start = Time.now.utc
       @logger.reset
 
       @logger.phase_heading("Initializing task")
@@ -44,11 +45,14 @@ module KubernetesDeploy
       else
         record_status_once(pod)
       end
+      ::StatsD.distribution('task_runner.duration', StatsD.duration(start), tags: statsd_tags('success'))
       @logger.print_summary(:success)
     rescue DeploymentTimeoutError
+      ::StatsD.distribution('task_runner.duration', StatsD.duration(start), tags: statsd_tags('timeout'))
       @logger.print_summary(:timed_out)
       raise
     rescue FatalDeploymentError
+      ::StatsD.distribution('task_runner.duration', StatsD.duration(start), tags: statsd_tags('failure'))
       @logger.print_summary(:failure)
       raise
     end
@@ -164,7 +168,9 @@ module KubernetesDeploy
     def set_container_overrides!(pod_definition, entrypoint, args, env_vars)
       container = pod_definition.spec.containers.find { |cont| cont.name == 'task-runner' }
       if container.nil?
-        raise TaskConfigurationError, "Pod spec does not contain a template container called 'task-runner'"
+        message = "Pod spec does not contain a template container called 'task-runner'"
+        @logger.summary.add_paragraph(message)
+        raise TaskConfigurationError, message
       end
 
       container.command = entrypoint
@@ -198,5 +204,9 @@ module KubernetesDeploy
     def kubeclient
       @kubeclient ||= build_v1_kubeclient(@context)
     end
+
+    def statsd_tags(status)
+      %W(namespace:#{@namespace} context:#{@context} status:#{status})
+    end
   end
 end

data/lib/kubernetes-deploy/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module KubernetesDeploy
-  VERSION = "0.21.1"
+  VERSION = "0.22.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: kubernetes-deploy
 version: !ruby/object:Gem::Version
-  version: 0.21.1
+  version: 0.22.0
 platform: ruby
 authors:
 - Katrina Verey
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-10-25 00:00:00.000000000 Z
+date: 2018-11-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -87,14 +87,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.2'
+        version: '2.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.2'
+        version: '2.3'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -244,6 +244,7 @@ files:
 - lib/kubernetes-deploy/kubernetes_resource/redis.rb
 - lib/kubernetes-deploy/kubernetes_resource/replica_set.rb
 - lib/kubernetes-deploy/kubernetes_resource/resource_quota.rb
+- lib/kubernetes-deploy/kubernetes_resource/role_binding.rb
 - lib/kubernetes-deploy/kubernetes_resource/service.rb
 - lib/kubernetes-deploy/kubernetes_resource/service_account.rb
 - lib/kubernetes-deploy/kubernetes_resource/stateful_set.rb