kubernetes-deploy 0.15.2 → 0.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c48f0cc4be05113a752768e19118a03d26c3505a
-  data.tar.gz: 2d51f6e48f927cbbf44a277e600bd728f09a1025
+  metadata.gz: 6184836f627f154591e094a5d5311614b1a35d71
+  data.tar.gz: e17050ecc7698c759e9669e0b52327ff67c038c5
 SHA512:
-  metadata.gz: d87392236bf5d4ceea11ad27ede7248abc611869b983f4eff8ecf8c43c4d09dc6164f36c5ce1ef7efa6022eaddee3791898e2f4b09e113ea63dc9c89b47ccd8a
-  data.tar.gz: f31edab5e0c2afd6e89cc81df85dba6e271040ffb932a83e79117da109b7353b95e17dbfe36c158d6a37ea42bc245a13af7eda92c3cd882c1766bc32c9619950
+  metadata.gz: f43cef2d6cf0aa8454ae2c46f47a9095c65f9faf13c235a0cd29e74d23518cb5575bb927bfba084190af55d9db1e325259ce6bce9596a9218a29ee7b74bee721
+  data.tar.gz: bdbac63fceea6418db1e30dc12e115782c26ffb17ce04437ec5992d3c3695ad6fc8cca10e7ae33cd22c9f95f91789732b290627446dd3f52be89536fbaf42895

data/.buildkite/pipeline.nightly.yml

@@ -26,10 +26,3 @@
     env:
       LOGGING_LEVEL: 4
       KUBERNETES_VERSION: v1.7-latest
-  - name: 'Run Test Suite (:kubernetes: 1.6.4)'
-    command: bin/ci
-    agents:
-      queue: minikube-ci
-    env:
-      LOGGING_LEVEL: 4
-      KUBERNETES_VERSION: v1.6.4

data/.buildkite/pipeline.yml

@@ -1,21 +1,21 @@
-  - name: 'Run Test Suite (:kubernetes: 1.8-latest)'
+  - name: 'Run Test Suite (:kubernetes: 1.9-latest)'
     command: bin/ci
     agents:
       queue: minikube-ci
     env:
       LOGGING_LEVEL: 4
-      KUBERNETES_VERSION: v1.8-latest
-  - name: 'Run Test Suite (:kubernetes: 1.7-latest)'
+      KUBERNETES_VERSION: v1.9-latest
+  - name: 'Run Test Suite (:kubernetes: 1.8-latest)'
     command: bin/ci
     agents:
       queue: minikube-ci
     env:
       LOGGING_LEVEL: 4
-      KUBERNETES_VERSION: v1.7-latest
-  - name: 'Run Test Suite (:kubernetes: 1.6.4)'
+      KUBERNETES_VERSION: v1.8-latest
+  - name: 'Run Test Suite (:kubernetes: 1.7-latest)'
     command: bin/ci
     agents:
       queue: minikube-ci
     env:
       LOGGING_LEVEL: 4
-      KUBERNETES_VERSION: v1.6.4
+      KUBERNETES_VERSION: v1.7-latest

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+### 0.16.0
+**Important:** This release changes the officially supported Kubernetes versions to v1.7 through v1.9. Other versions may continue to work, but we are no longer running our test suite against them.
+
+*Features*
+- Support partials to reduce duplication in yaml files ([#207](https://github.com/Shopify/kubernetes-deploy/pull/207))
+
+*Bug Fixes*
+- Handle podless deamon sets properly ([#242](https://github.com/Shopify/kubernetes-deploy/pull/242))
+
 ### 0.15.2
 *Enhancements*
 - Print warnings if kubernetes server version is not supported ([#237](https://github.com/Shopify/kubernetes-deploy/pull/237)).

data/Gemfile

@@ -12,3 +12,4 @@ gem 'byebug'
 gem 'codecov', require: false
 gem 'ruby-prof', require: false
 gem 'ruby-prof-flamegraph', require: false
+gem 'minitest-reporters'

data/README.md

@@ -68,17 +68,19 @@ This repo also includes related tools for [running tasks](#kubernetes-run) and [
 ## Prerequisites
 
 * Ruby 2.3+
-* Your cluster must be running Kubernetes v1.6.0 or higher
+* Your cluster must be running Kubernetes v1.7.0 or higher<sup>1</sup>
 * Each app must have a deploy directory containing its Kubernetes templates (see [Templates](#templates))
 * You must remove the` kubectl.kubernetes.io/last-applied-configuration` annotation from any resources in the namespace that are not included in your deploy directory. This annotation is added automatically when you create resources with `kubectl apply`. `kubernetes-deploy` will prune any resources that have this annotation and are not in the deploy directory.**
-* Each app managed by `kubernetes-deploy` must have its own exclusive Kubernetes namespace.**
+* Each app managed by `kubernetes-deploy` must have its own exclusive Kubernetes namespace.<sup>2</sup>
 
+<sup>1</sup> We run integration tests against these Kubernetes versions. Kubernetes v1.6 was officially supported in gem versions < 0.16. Kubernetes v1.5 was officially supported in gem versions < 0.12.
+
+<sup>2</sup> This requirement can be bypassed with the `--no-prune` option, but it is not recommended.
 
-**This requirement can be bypassed with the `--no-prune` option, but it is not recommended.
 
 ## Installation
 
-1. [Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-kubectl-binary-via-curl) (requires v1.6.0 or higher) and make sure it is available in your $PATH
+1. [Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-kubectl-binary-via-curl) (requires v1.7.0 or higher) and make sure it is available in your $PATH
 2. Set up your [kubeconfig file](https://kubernetes.io/docs/tasks/access-application-cluster/authenticate-across-clusters-kubeconfig/) for access to your cluster(s).
 3. `gem install kubernetes-deploy`
 
@@ -120,6 +122,92 @@ All templates must be YAML formatted. You can also use ERB. The following local
 
 You can add additional variables using the `--bindings=BINDINGS` option. For example, `kubernetes-deploy my-app cluster1 --bindings=color=blue,size=large` will expose `color` and `size` in your templates.
 
+#### Using partials
+
+`kubernetes-deploy` supports composing templates from so called partials in order to reduce duplication in Kubernetes YAML files. Given a template directory `DIR`, partials are searched for in `DIR/partials`and in 'DIR/../partials', in that order. They can be embedded in other ERB templates using the helper method `partial`. For example, let's assume an application needs a number of different CronJob resources, one could place a template called `cron` in one of those directories and then use it in the main deployment.yaml.erb like so:
+
+```yaml
+<%= partial "cron", name: "cleanup",   schedule: "0 0 * * *", args: %w(cleanup),    cpu: "100m", memory: "100Mi" %>
+<%= partial "cron", name: "send-mail", schedule: "0 0 * * *", args: %w(send-mails), cpu: "200m", memory: "256Mi" %>
+```
+
+Inside a partial, parameters can be accessed as normal variables, or via a hash called `locals`. Thus, the `cron` template could like this:
+
+```yaml
+---
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: cron-<%= name %>
+spec:
+  schedule: <%= schedule %>
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: cron-<%= name %>
+            image: ...
+            args: <%= args %>
+            resources:
+              requests:
+                cpu: "<%= cpu %>"
+                memory: <%= memory %>
+          restartPolicy: OnFailure
+```
+
+Both `.yaml.erb` and `.yml.erb` file extensions are supported. Templates must refer to the bare filename (e.g. use `partial: 'cron'` to reference `cron.yaml.erb`).
+
+##### Limitations when using partials
+
+Partials can be included almost everywhere in ERB templates, with one notable exception: you cannot use a partial to define a subset of fields. For example, given a partial `p` defining two fields 'a' and 'b',
+
+```yaml
+a: 1
+b: 2
+```
+
+you cannot do this:
+
+```yaml
+x: yz
+<%= partial 'p' %>
+```
+
+hoping to get
+
+```yaml
+x: yz
+a: 1
+b: 2
+```
+
+but you can do:
+
+```yaml
+x:
+  <%= partial 'p' %>
+```
+
+or even
+
+```yaml
+x: <%= partial 'p' %>
+```
+
+which both will result in
+
+```yaml
+x:
+  a: 1
+  b: 2
+```
+
+This is a limitation of the current implementation.
+
 
 ### Customizing behaviour with annotations
 - `kubernetes-deploy.shopify.io/timeout-override`: Override the tool's hard timeout for one specific resource. Both full ISO8601 durations and the time portion of ISO8601 durations are valid. Value must be between 1 second and 24 hours.
@@ -137,7 +225,6 @@ before the deployment is considered successful.
   - Percent (e.g. 90%): The deploy is successful when the number of new pods that are ready is equal to
   `spec.replicas` * Percent.
 
-
 ### Running tasks at the beginning of a deploy
 
 To run a task in your cluster at the beginning of every deploy, simply include a `Pod` template in your deploy directory. `kubernetes-deploy` will first deploy any `ConfigMap` and `PersistentVolumeClaim` resources in your template set, followed by any such pods. If the command run by one of these pods fails (i.e. exits with a non-zero status), the overall deploy will fail at this step (no other resources will be deployed).
@@ -249,7 +336,7 @@ With this done, you can use the following command to restart all of them:
 
 ## Prerequisites
 
-* You've already deployed a [`PodTemplate`](https://kubernetes.io/docs/api-reference/v1.6/#podtemplate-v1-core) object with field `template` containing a `Pod` specification that does not include the `apiVersion` or `kind` parameters. An example is provided in this repo in `test/fixtures/hello-cloud/template-runner.yml`.
+* You've already deployed a [`PodTemplate`](https://kubernetes.io/docs/api-reference/v1.9/#podtemplate-v1-core) object with field `template` containing a `Pod` specification that does not include the `apiVersion` or `kind` parameters. An example is provided in this repo in `test/fixtures/hello-cloud/template-runner.yml`.
 * The `Pod` specification in that template has a container named `task-runner`.
 
 Based on this specification `kubernetes-run` will create a new pod with the entrypoint of the `task-runner ` container overridden with the supplied arguments.
@@ -275,7 +362,7 @@ Based on this specification `kubernetes-run` will create a new pod with the entr
 
 If you work for Shopify, just run `dev up`, but otherwise:
 
-1. [Install kubectl version 1.6.0 or higher](https://kubernetes.io/docs/user-guide/prereqs/) and make sure it is in your path
+1. [Install kubectl version 1.7.0 or higher](https://kubernetes.io/docs/user-guide/prereqs/) and make sure it is in your path
 2. [Install minikube](https://kubernetes.io/docs/getting-started-guides/minikube/#installation) (required to run the test suite)
 3. Check out the repo
 4. Run `bin/setup` to install dependencies

data/Rakefile

@@ -2,10 +2,28 @@
 require "bundler/gem_tasks"
 require "rake/testtask"
 
-Rake::TestTask.new(:test) do |t|
+desc "Run integration tests that can be run in parallel"
+Rake::TestTask.new(:integration_test) do |t|
   t.libs << "test"
   t.libs << "lib"
-  t.test_files = FileList['test/**/*_test.rb']
+  t.test_files = FileList['test/integration/**/*_test.rb']
 end
 
+desc "Run integration tests that CANNOT be run in parallel"
+Rake::TestTask.new(:serial_integration_test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList['test/integration-serial/**/*_test.rb']
+end
+
+desc "Run unit tests"
+Rake::TestTask.new(:unit_test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList['test/unit/**/*_test.rb']
+end
+
+desc "Run all tests"
+task test: %w(unit_test serial_integration_test integration_test)
+
 task default: :test

data/bin/ci

@@ -14,6 +14,8 @@ docker run --rm \
     -e CI=1 \
     -e CODECOV_TOKEN=$CODECOV_TOKEN \
     -e COVERAGE=1 \
+    -e VERBOSE=1 \
+    -e PARALLELISM=$PARALLELISM \
     -w /usr/src/app \
     ruby:2.3 \
     bin/test

data/bin/setup

@@ -9,8 +9,8 @@ if [ ! -x "$(which minikube)" ]; then
 fi
 
 if [ ! -x "$(which kubectl)" ]; then
-  echo -e "\n\033[0;33mPlease install kubectl version 1.6.0 or higher:\nhttps://kubernetes.io/docs/user-guide/prereqs/\033[0m"
+  echo -e "\n\033[0;33mPlease install kubectl version 1.7.0 or higher:\nhttps://kubernetes.io/docs/user-guide/prereqs/\033[0m"
 else
   KUBECTL_VERSION=$(kubectl version --short --client | grep -oe "v[[:digit:]\.]\+")
-  echo -e "\n\033[0;32mKubectl version $KUBECTL_VERSION is already installed. This gem requires version v1.6.0 or greater.\033[0m"
+  echo -e "\n\033[0;32mKubectl version $KUBECTL_VERSION is already installed. This gem requires version v1.7.0 or greater.\033[0m"
 fi

data/bin/test

@@ -1,7 +1,44 @@
 #!/bin/bash
-set -euo pipefail
+set -uo pipefail
 
-echo "--- :ruby: Bundle Install"
-bundle install --jobs 4 
-echo "+++ :kubernetes: Run Tests"
-bundle exec rake test TESTOPTS='-v'
+err=0
+trap 'err=1' ERR
+trap 'exit 1' SIGINT
+
+function print_header() {
+  if [[ ${CI:="0"} == "1" ]]; then
+    printf "+++ :kubernetes: %s\n" "${1}"
+  else
+    printf "\n\n\033[0;35m⎈   %s\033[0m\n" "${1}"
+  fi
+}
+
+if [[ ${PARALLELISM:=0} -lt 1 ]]; then
+  if [[ $(uname) == "Darwin" ]]; then
+    num_cpus=$(sysctl -n hw.ncpu)
+  else
+    num_cpus=$(nproc --all)
+  fi
+
+  if [[ $num_cpus -le 2 ]]; then
+    PARALLELISM=1
+  else
+    (( PARALLELISM=num_cpus/2 ))
+  fi
+fi
+
+if [[ ${CI:="0"} == "1" ]]; then
+  echo "--- :ruby: Bundle Install"
+  bundle install --jobs 4
+fi
+
+print_header "Run Unit Tests"
+bundle exec rake unit_test
+
+print_header "Run Non-Parallel Integration Tests"
+bundle exec rake serial_integration_test
+
+print_header "Run Parallel Integration Tests (N=$PARALLELISM)"
+PARALLELIZE_ME=1 N=$PARALLELISM bundle exec rake integration_test
+
+test $err -eq 0

data/dev.yml

@@ -13,7 +13,7 @@ up:
 commands:
   reset-minikube: minikube delete && rm -rf ~/.minikube
   test:
-    run: bundle exec rake test
+    run: bin/test
   tophat:
     run: PRINT_LOGS=1 bundle exec ruby -I test test/integration/kubernetes_deploy_test.rb -n/${1}/
     desc: Tophat a change by running a test scenario with logging output enabled.

data/lib/kubernetes-deploy.rb

@@ -21,6 +21,6 @@ require 'kubernetes-deploy/bindings_parser'
 require 'kubernetes-deploy/duration_parser'
 
 module KubernetesDeploy
-  MIN_KUBE_VERSION = '1.6.0'
+  MIN_KUBE_VERSION = '1.7.0'
   KubernetesDeploy::StatsD.build
 end

data/lib/kubernetes-deploy/deploy_task.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 require 'open3'
-require 'securerandom'
-require 'erb'
 require 'yaml'
 require 'shellwords'
 require 'tempfile'
@@ -36,6 +34,7 @@ require 'kubernetes-deploy/resource_watcher'
 require 'kubernetes-deploy/kubectl'
 require 'kubernetes-deploy/kubeclient_builder'
 require 'kubernetes-deploy/ejson_secret_provisioner'
+require 'kubernetes-deploy/renderer'
 
 module KubernetesDeploy
   class DeployTask
@@ -98,9 +97,12 @@ module KubernetesDeploy
       @template_dir = File.expand_path(template_dir)
       @logger = logger
       @kubectl = kubectl_instance
-      @bindings = bindings
-      # Max length of podname is only 63chars so try to save some room by truncating sha to 8 chars
-      @id = current_sha[0...8] + "-#{SecureRandom.hex(4)}" if current_sha
+      @renderer = KubernetesDeploy::Renderer.new(
+        current_sha: @current_sha,
+        template_dir: @template_dir,
+        logger: @logger,
+        bindings: bindings,
+      )
     end
 
     def run(verify_result: true, allow_protected_ns: false, prune: true)
@@ -167,13 +169,6 @@ module KubernetesDeploy
       success
     end
 
-    def template_variables
-      {
-        'current_sha' => @current_sha,
-        'deployment_id' => @id,
-      }.merge(@bindings)
-    end
-
     private
 
     # Inspect the file referenced in the kubectl stderr
@@ -234,7 +229,7 @@ module KubernetesDeploy
 
     def split_templates(filename)
       file_content = File.read(File.join(@template_dir, filename))
-      rendered_content = render_template(filename, file_content)
+      rendered_content = @renderer.render_template(filename, file_content)
       YAML.load_stream(rendered_content) do |doc|
         yield doc unless doc.blank?
       end
@@ -271,20 +266,6 @@ module KubernetesDeploy
       "    " + str.gsub("\n", "\n    ")
     end
 
-    def render_template(filename, raw_template)
-      return raw_template unless File.extname(filename) == ".erb"
-
-      erb_template = ERB.new(raw_template)
-      erb_binding = binding
-      template_variables.each do |var_name, value|
-        erb_binding.local_variable_set(var_name, value)
-      end
-      erb_template.result(erb_binding)
-    rescue NameError => e
-      @logger.summary.add_paragraph("Error from renderer:\n  #{e.message.tr("\n", ' ')}")
-      raise FatalDeploymentError, "Template '#{filename}' cannot be rendered"
-    end
-
     def validate_configuration(allow_protected_ns:, prune:)
       errors = []
       if ENV["KUBECONFIG"].blank?

data/lib/kubernetes-deploy/kubernetes_resource/daemon_set.rb

@@ -37,6 +37,7 @@ module KubernetesDeploy
     end
 
     def fetch_logs
+      return {} unless @pods.present?
       most_useful_pod = @pods.find(&:deploy_failed?) || @pods.find(&:deploy_timed_out?) || @pods.first
       most_useful_pod.fetch_logs
     end

data/lib/kubernetes-deploy/renderer.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+require 'erb'
+require 'securerandom'
+require 'yaml'
+require 'json'
+
+module KubernetesDeploy
+  class Renderer
+    class InvalidPartialError < FatalDeploymentError
+      attr_reader :parents
+      def initialize(msg, parents = [])
+        @parents = parents
+        super(msg)
+      end
+    end
+    class PartialNotFound < InvalidPartialError; end
+
+    def initialize(current_sha:, template_dir:, logger:, bindings: {})
+      @current_sha = current_sha
+      @template_dir = template_dir
+      @partials_dirs =
+        %w(partials ../partials).map { |d| File.expand_path(File.join(@template_dir, d)) }
+      @logger = logger
+      @bindings = bindings
+      # Max length of podname is only 63chars so try to save some room by truncating sha to 8 chars
+      @id = current_sha[0...8] + "-#{SecureRandom.hex(4)}" if current_sha
+    end
+
+    def render_template(filename, raw_template)
+      return raw_template unless File.extname(filename) == ".erb"
+
+      erb_binding = TemplateContext.new(self).template_binding
+      bind_template_variables(erb_binding, template_variables)
+
+      ERB.new(raw_template, nil, '-').result(erb_binding)
+    rescue InvalidPartialError => err
+      all_parents = err.parents.dup.unshift(filename)
+      raise FatalDeploymentError, "#{err.message} (included from: #{all_parents.join(' -> ')})"
+    rescue StandardError => err
+      report_template_invalid(err.message, raw_template)
+      raise FatalDeploymentError, "Template '#{filename}' cannot be rendered"
+    end
+
+    def render_partial(partial, locals)
+      variables = template_variables.merge(locals)
+      erb_binding = TemplateContext.new(self).template_binding
+      bind_template_variables(erb_binding, variables)
+      erb_binding.local_variable_set("locals", locals)
+
+      partial_path = find_partial(partial)
+      template = File.read(partial_path)
+      expanded_template = ERB.new(template, nil, '-').result(erb_binding)
+
+      docs = Psych.parse_stream(expanded_template)
+      # If the partial contains multiple documents or has an explicit document header,
+      # we know it cannot validly be indented in the parent, so return it immediately.
+      return expanded_template unless docs.children.one? && docs.children.first.implicit
+      # Make sure indentation isn't a problem by producing a single line of parseable YAML.
+      # Note that JSON is a subset of YAML.
+      JSON.generate(docs.children.first.to_ruby)
+    rescue PartialNotFound => err
+      raise InvalidPartialError, err.message
+    rescue InvalidPartialError => err
+      raise InvalidPartialError.new(err.message, err.parents.dup.unshift(File.basename(partial_path)))
+    rescue StandardError => err
+      report_template_invalid(err.message, expanded_template)
+      raise InvalidPartialError, "Template '#{partial_path}' cannot be rendered"
+    end
+
+    private
+
+    def template_variables
+      {
+        'current_sha' => @current_sha,
+        'deployment_id' => @id,
+      }.merge(@bindings)
+    end
+
+    def bind_template_variables(erb_binding, variables)
+      variables.each do |var_name, value|
+        erb_binding.local_variable_set(var_name, value)
+      end
+    end
+
+    def find_partial(name)
+      partial_names = [name + '.yaml.erb', name + '.yml.erb']
+      @partials_dirs.each do |dir|
+        partial_names.each do |partial_name|
+          partial_path = File.join(dir, partial_name)
+          return partial_path if File.exist?(partial_path)
+        end
+      end
+      raise PartialNotFound, "Could not find partial '#{name}' in any of #{@partials_dirs.join(':')}"
+    end
+
+    def report_template_invalid(message, content)
+      @logger.summary.add_paragraph("Error from renderer:\n  #{message.tr("\n", ' ')}")
+      @logger.summary.add_paragraph("Rendered template content:\n#{content}")
+    end
+
+    class TemplateContext
+      def initialize(renderer)
+        @_renderer = renderer
+      end
+
+      def template_binding
+        binding
+      end
+
+      def partial(partial, locals = {})
+        @_renderer.render_partial(partial, locals)
+      end
+    end
+  end
+end

data/lib/kubernetes-deploy/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module KubernetesDeploy
-  VERSION = "0.15.2"
+  VERSION = "0.16.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: kubernetes-deploy
 version: !ruby/object:Gem::Version
-  version: 0.15.2
+  version: 0.16.0
 platform: ruby
 authors:
 - Katrina Verey
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-01-19 00:00:00.000000000 Z
+date: 2018-02-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -258,6 +258,7 @@ files:
 - lib/kubernetes-deploy/kubernetes_resource/stateful_set.rb
 - lib/kubernetes-deploy/kubernetes_resource/statefulservice.rb
 - lib/kubernetes-deploy/kubernetes_resource/topic.rb
+- lib/kubernetes-deploy/renderer.rb
 - lib/kubernetes-deploy/resource_watcher.rb
 - lib/kubernetes-deploy/restart_task.rb
 - lib/kubernetes-deploy/runner_task.rb