kubernetes-deploy 0.23.0 → 0.24.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 01ddead06a28a399d7364d5789c2bcb1cee0e0aa4ae471a4af1a90c32973050b
-  data.tar.gz: ebcedef932871bd542db69bc0b48fb6f47f2482c712aa9380a79cb872b6c3e82
+  metadata.gz: e8e6aa3c405f95c085c79e274915275e1d2a225803c4abf21813c11543bdfa8f
+  data.tar.gz: ae315a1ce265d03205bd39972303d54d015a32210b43151c8f5d541377946143
 SHA512:
-  metadata.gz: 837dcaf96f89beae9f9a57ceb207595d602f1e039217007750112828a8076d0337c17bb6a75bcc2d5a5a8c531f8b0db363a19293d5dc6ce011cdd70440c888a7
-  data.tar.gz: e8e6484dbfae1b867542a02c39f7c96ed4434ab3f900f49fea72523466ea9be23ce2d1f2209d9fb609b7e71af63d1c047ab6f98f756ec28eef8bb47245fb9180
+  metadata.gz: 0111a5ab0e4959ff3e135311ed879f284cb9db70b5a510f0fa0e667385f35ab2c0c0323167f3ac30378d032e2daddeeab2f3afeacf6c36b5f0d3e7a5f5a21754
+  data.tar.gz: 182840922280570f41aa6bff6918b24f8c0a34da631d01fe97c3b078a3ef376346118c0026ef3d4f6f1c96cc1d96e9789f483b00fb0ea35d19044129ea0830e2

data/.buildkite/pipeline.nightly.yml

@@ -4,6 +4,14 @@ shared: &shared
        - exit_status: "*"
          limit: 1
 steps:
+  - name: 'Run Test Suite (:kubernetes: 1.13-latest)'
+    <<: *shared
+    command: bin/ci
+    agents:
+      queue: k8s-ci
+    env:
+      LOGGING_LEVEL: 4
+      KUBERNETES_VERSION: v1.13-latest
   - name: 'Run Test Suite (:kubernetes: 1.12-latest)'
     <<: *shared
     command: bin/ci
@@ -28,11 +36,3 @@ steps:
     env:
       LOGGING_LEVEL: 4
       KUBERNETES_VERSION: v1.10-latest
-  - name: 'Run Test Suite (:kubernetes: 1.9-latest)'
-    <<: *shared
-    command: bin/ci
-    agents:
-      queue: minikube-ci
-    env:
-      LOGGING_LEVEL: 4
-      KUBERNETES_VERSION: v1.9-latest

data/.buildkite/pipeline.yml

@@ -4,6 +4,14 @@ shared: &shared
        - exit_status: "*"
          limit: 1
 steps:
+  - name: 'Run Test Suite (:kubernetes: 1.13-latest)'
+    <<: *shared
+    command: bin/ci
+    agents:
+      queue: k8s-ci
+    env:
+      LOGGING_LEVEL: 4
+      KUBERNETES_VERSION: v1.13-latest
   - name: 'Run Test Suite (:kubernetes: 1.12-latest)'
     <<: *shared
     command: bin/ci
@@ -28,11 +36,3 @@ steps:
     env:
       LOGGING_LEVEL: 4
       KUBERNETES_VERSION: v1.10-latest
-  - name: 'Run Test Suite (:kubernetes: 1.9-latest)'
-    <<: *shared
-    command: bin/ci
-    agents:
-      queue: minikube-ci
-    env:
-      LOGGING_LEVEL: 4
-      KUBERNETES_VERSION: v1.9-latest

data/.rubocop.yml

@@ -4,15 +4,6 @@ inherit_from:
 AllCops:
   TargetRubyVersion: 2.3
 
-Style/TrailingCommaInArrayLiteral:
-  Enabled: false
-
-Style/TrailingCommaInHashLiteral:
-  Enabled: false
-
-Style/MethodCallWithArgsParentheses:
-  Enabled: false
-
 Naming/FileName:
   Enabled: true
   Exclude:

data/CHANGELOG.md

@@ -1,3 +1,20 @@
+## next
+
+## 0.24.0
+
+*Features*
+- Add support for specifying pass/fail conditions of Custom Resources ([#376](https://github.com/Shopify/kubernetes-deploy/pull/376)).
+- Add support for custom timeouts for Custom Resources([#376](https://github.com/Shopify/kubernetes-deploy/pull/376))
+
+*Enhancements*
+- Officially support Kubernetes 1.13 ([#409](https://github.com/Shopify/kubernetes-deploy/pull/409))
+
+*Bug fixes*
+- Fixed bug that caused `NameError: wrong constant name` if custom resources had kind with a lowercase first letter. ([#413](https://github.com/Shopify/kubernetes-deploy/pull/413))
+
+*Other*
+- Kubernetes 1.9 is no longer officially supported as of this version
+
 ## 0.23.0
 
 *Features*

data/README.md

@@ -42,6 +42,7 @@ This repo also includes related tools for [running tasks](#kubernetes-run) and [
   * [Customizing behaviour with annotations](#customizing-behaviour-with-annotations)
   * [Running tasks at the beginning of a deploy](#running-tasks-at-the-beginning-of-a-deploy)
   * [Deploying Kubernetes secrets (from EJSON)](#deploying-kubernetes-secrets-from-ejson)
+  * [Deploying custom resources](#deploying-custom-resources)
 
 **KUBERNETES-RESTART**
 * [Usage](#usage-1)
@@ -73,7 +74,7 @@ This repo also includes related tools for [running tasks](#kubernetes-run) and [
 ## Prerequisites
 
 * Ruby 2.3+
-* Your cluster must be running Kubernetes v1.9.0 or higher<sup>1</sup>
+* Your cluster must be running Kubernetes v1.10.0 or higher<sup>1</sup>
 * Each app must have a deploy directory containing its Kubernetes templates (see [Templates](#using-templates-and-variables))
 * You must remove the` kubectl.kubernetes.io/last-applied-configuration` annotation from any resources in the namespace that are not included in your deploy directory. This annotation is added automatically when you create resources with `kubectl apply`. `kubernetes-deploy` will prune any resources that have this annotation and are not in the deploy directory.<sup>2</sup>
 * Each app managed by `kubernetes-deploy` must have its own exclusive Kubernetes namespace.
@@ -92,7 +93,7 @@ offical compatibility chart below.
 
 ## Installation
 
-1. [Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-kubectl-binary-via-curl) (requires v1.9.0 or higher) and make sure it is available in your $PATH
+1. [Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-kubectl-binary-via-curl) (requires v1.10.0 or higher) and make sure it is available in your $PATH
 2. Set up your [kubeconfig file](https://kubernetes.io/docs/tasks/access-application-cluster/authenticate-across-clusters-kubeconfig/) for access to your cluster(s).
 3. `gem install kubernetes-deploy`
 
@@ -312,7 +313,95 @@ Since their data is only base64 encoded, Kubernetes secrets should not be commit
   }
 ```
 
+### Deploying custom resources
 
+By default, kubernetes-deploy does not check the status of custom resources; it simply assumes that they deployed successfully. In order to meaningfully monitor the rollout of custom resources, kubernetes-deploy supports configuring pass/fail conditions using annotations on CustomResourceDefinitions (CRDs).
+
+>Note:
+This feature is only available on clusters running Kubernetes 1.11+ since it relies on the `metadata.generation` field being updated when custom resource specs are changed.
+
+*Requirements:*
+
+* The custom resource must expose a `status` subresource with an `observedGeneration` field.
+* The `kubernetes-deploy.shopify.io/instance-rollout-conditions` annotation must be present on the CRD that defines the custom resource.
+* (optional) The `kubernetes-deploy.shopify.io/instance-timeout` annotation can be added to the CRD that defines the custom resource to override the global default timeout for all instances of that resource. This annotation can use ISO8601 format or unprefixed ISO8601 time components (e.g. '1H', '60S').
+
+#### Specifying pass/fail conditions
+
+The presence of a valid `kubernetes-deploy.shopify.io/instance-rollout-conditions` annotation on a CRD will cause kubernetes-deploy to monitor the rollout of all instances of that custom resource. Its value can either be `"true"` (giving you the defaults described in the next section) or a valid JSON string with the following format:
+```
+'{
+  "success_conditions": [
+    { "path": <JsonPath expression>, "value": <target value> }
+    ... more success conditions
+  ],
+  "failure_conditions": [
+    { "path": <JsonPath expression>, "value": <target value> }
+    ... more failure conditions
+  ]
+}'
+```
+
+For all conditions, `path` must be a valid JsonPath expression that points to a field in the custom resource's status. `value` is the value that must be present at `path` in order to fulfill a condition. For a deployment to be successful, _all_ `success_conditions` must be fulfilled. Conversely, the deploy will be marked as failed if _any one of_ `failure_conditions` is fulfilled. `success_conditions` are mandatory, but `failure_conditions` can be omitted (the resource will simply time out if it never reaches a successful state).
+
+In addition to `path` and `value`, a failure condition can also contain `error_msg_path` or `custom_error_msg`. `error_msg_path` is a JsonPath expression that points to a field you want to surface when a failure condition is fulfilled. For example, a status condition may expose a `message` field that contains a description of the problem it encountered. `custom_error_msg` is a string that can be used if your custom resource doesn't contain sufficient information to warrant using `error_msg_path`. Note that `custom_error_msg` has higher precedence than `error_msg_path` so it will be used in favor of `error_msg_path` when both fields are present.
+
+**Warning:**
+
+You **must** ensure that your custom resource controller sets `.status.observedGeneration` to match the observed `.metadata.generation` of the monitored resource once its sync is complete. If this does not happen, kubernetes-deploy will not check success or failure conditions and the deploy will time out.
+
+#### Example
+
+As an example, the following is the default configuration that will be used if you set `kubernetes-deploy.shopify.io/instance-rollout-conditions: "true"` on the CRD that defines the custom resources you wish to monitor:
+
+```
+'{
+  "success_conditions": [
+    {
+      "path": "$.status.conditions[?(@.type == \"Ready\")].status",
+      "value": "True",
+    },
+  ],
+  "failure_conditions": [
+    {
+      "path": '$.status.conditions[?(@.type == \"Failed\")].status',
+      "value": "True",
+      "error_msg_path": '$.status.conditions[?(@.type == \"Failed\")].message',
+    },
+  ],
+}'
+```
+
+The paths defined here are based on the [typical status properties](https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md#typical-status-properties) as defined by the Kubernetes community. It expects the `status` subresource to contain a `conditions` array whose entries minimally specify `type`, `status`, and `message` fields.
+
+You can see how these conditions relate to the following resource:
+
+```
+apiVersion: stable.shopify.io/v1
+kind: Example
+metadata:
+  generation: 2
+  name: example
+  namespace: namespace
+spec:
+  ...
+status:
+  observedGeneration: 2
+  conditions:
+  - type: "Ready"
+    status: "False"
+    reason: "exampleNotReady"
+    message: "resource is not ready"
+  - type: "Failed"
+    status: "True"
+    reason: "exampleFailed"
+    message: "resource is failed"
+```
+
+- `observedGeneration == metadata.generation`, so kubernetes-deploy will check this resource's success and failure conditions.
+- Since `$.status.conditions[?(@.type == "Ready")].status == "False"`, the resource is not considered successful yet.
+- `$.status.conditions[?(@.type == "Failed")].status == "True"` means that a failure condition has been fulfilled and the resource is considered failed.
+- Since `error_msg_path` is specified, kubernetes-deploy will log the contents of `$.status.conditions[?(@.type == "Failed")].message`, which in this case is: `resource is failed`.
 
 # kubernetes-restart
 
@@ -354,7 +443,7 @@ With this done, you can use the following command to restart all of them:
 
 ## Prerequisites
 
-* You've already deployed a [`PodTemplate`](https://kubernetes.io/docs/api-reference/v1.9/#podtemplate-v1-core) object with field `template` containing a `Pod` specification that does not include the `apiVersion` or `kind` parameters. An example is provided in this repo in `test/fixtures/hello-cloud/template-runner.yml`.
+* You've already deployed a [`PodTemplate`](https://v1-10.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#podtemplate-v1-core) object with field `template` containing a `Pod` specification that does not include the `apiVersion` or `kind` parameters. An example is provided in this repo in `test/fixtures/hello-cloud/template-runner.yml`.
 * The `Pod` specification in that template has a container named `task-runner`.
 
 Based on this specification `kubernetes-run` will create a new pod with the entrypoint of the `task-runner ` container overridden with the supplied arguments.
@@ -411,7 +500,7 @@ kubernetes-render --template-dir=./path/to/template/dir this-template.yaml.erb t
 
 If you work for Shopify, just run `dev up`, but otherwise:
 
-1. [Install kubectl version 1.9.0 or higher](https://kubernetes.io/docs/user-guide/prereqs/) and make sure it is in your path
+1. [Install kubectl version 1.10.0 or higher](https://kubernetes.io/docs/user-guide/prereqs/) and make sure it is in your path
 2. [Install minikube](https://kubernetes.io/docs/getting-started-guides/minikube/#installation) (required to run the test suite)
 3. Check out the repo
 4. Run `bin/setup` to install dependencies

data/Rakefile

@@ -2,28 +2,28 @@
 require "bundler/gem_tasks"
 require "rake/testtask"
 
-desc "Run integration tests that can be run in parallel"
+desc("Run integration tests that can be run in parallel")
 Rake::TestTask.new(:integration_test) do |t|
   t.libs << "test"
   t.libs << "lib"
   t.test_files = FileList['test/integration/**/*_test.rb']
 end
 
-desc "Run integration tests that CANNOT be run in parallel"
+desc("Run integration tests that CANNOT be run in parallel")
 Rake::TestTask.new(:serial_integration_test) do |t|
   t.libs << "test"
   t.libs << "lib"
   t.test_files = FileList['test/integration-serial/**/*_test.rb']
 end
 
-desc "Run unit tests"
+desc("Run unit tests")
 Rake::TestTask.new(:unit_test) do |t|
   t.libs << "test"
   t.libs << "lib"
   t.test_files = FileList['test/unit/**/*_test.rb']
 end
 
-desc "Run all tests"
-task test: %w(unit_test serial_integration_test integration_test)
+desc("Run all tests")
+task(test: %w(unit_test serial_integration_test integration_test))
 
-task default: :test
+task(default: :test)

data/bin/setup

@@ -9,8 +9,8 @@ if [ ! -x "$(which minikube)" ]; then
 fi
 
 if [ ! -x "$(which kubectl)" ]; then
-  echo -e "\n\033[0;33mPlease install kubectl version 1.9.0 or higher:\nhttps://kubernetes.io/docs/user-guide/prereqs/\033[0m"
+  echo -e "\n\033[0;33mPlease install kubectl version 1.10.0 or higher:\nhttps://kubernetes.io/docs/user-guide/prereqs/\033[0m"
 else
   KUBECTL_VERSION=$(kubectl version --short --client | grep -oe "v[[:digit:]\.]\+")
-  echo -e "\n\033[0;32mKubectl version $KUBECTL_VERSION is already installed. This gem requires version v1.9.0 or greater.\033[0m"
+  echo -e "\n\033[0;32mKubectl version $KUBECTL_VERSION is already installed. This gem requires version v1.10.0 or greater.\033[0m"
 fi

data/exe/kubernetes-deploy

@@ -63,7 +63,7 @@ begin
     prune: prune
   )
 rescue KubernetesDeploy::DeploymentTimeoutError
-  exit 70
+  exit(70)
 rescue KubernetesDeploy::FatalDeploymentError
-  exit 1
+  exit(1)
 end

data/exe/kubernetes-render

@@ -29,4 +29,4 @@ runner = KubernetesDeploy::RenderTask.new(
 )
 
 success = runner.run(STDOUT, templates)
-exit 1 unless success
+exit(1) unless success

data/exe/kubernetes-restart

@@ -23,7 +23,7 @@ restart = KubernetesDeploy::RestartTask.new(namespace: namespace, context: conte
 begin
   restart.perform!(raw_deployments)
 rescue KubernetesDeploy::DeploymentTimeoutError
-  exit 70
+  exit(70)
 rescue KubernetesDeploy::FatalDeploymentError
-  exit 1
+  exit(1)
 end

data/exe/kubernetes-run

@@ -41,4 +41,4 @@ success = runner.run(
   args: ARGV[2..-1],
   env_vars: env_vars
 )
-exit 1 unless success
+exit(1) unless success

data/kubernetes-deploy.gemspec

@@ -23,19 +23,20 @@ Gem::Specification.new do |spec|
   spec.require_paths = %w(lib)
 
   spec.required_ruby_version = '>= 2.3.0'
-  spec.add_dependency "activesupport", ">= 5.0"
-  spec.add_dependency "kubeclient", "~> 3.0"
-  spec.add_dependency "googleauth", "~> 0.6.6" # https://github.com/google/google-auth-library-ruby/issues/153
-  spec.add_dependency "ejson", "~> 1.0"
-  spec.add_dependency "colorize", "~> 0.8"
-  spec.add_dependency "statsd-instrument", '~> 2.3', '>= 2.3.2'
-  spec.add_dependency "oj", "~> 3.7"
-  spec.add_dependency "concurrent-ruby", "~> 1.1"
+  spec.add_dependency("activesupport", ">= 5.0")
+  spec.add_dependency("kubeclient", "~> 3.0")
+  spec.add_dependency("googleauth", "~> 0.6.6") # https://github.com/google/google-auth-library-ruby/issues/153
+  spec.add_dependency("ejson", "~> 1.0")
+  spec.add_dependency("colorize", "~> 0.8")
+  spec.add_dependency("statsd-instrument", '~> 2.3', '>= 2.3.2')
+  spec.add_dependency("oj", "~> 3.7")
+  spec.add_dependency("concurrent-ruby", "~> 1.1")
+  spec.add_dependency("jsonpath", "~> 0.9.6")
 
-  spec.add_development_dependency "bundler"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "minitest", "~> 5.0"
-  spec.add_development_dependency "minitest-stub-const", "~> 0.6"
-  spec.add_development_dependency "webmock", "~> 3.0"
-  spec.add_development_dependency "mocha", "~> 1.5"
+  spec.add_development_dependency("bundler")
+  spec.add_development_dependency("rake", "~> 10.0")
+  spec.add_development_dependency("minitest", "~> 5.0")
+  spec.add_development_dependency("minitest-stub-const", "~> 0.6")
+  spec.add_development_dependency("webmock", "~> 3.0")
+  spec.add_development_dependency("mocha", "~> 1.5")
 end

data/lib/kubernetes-deploy.rb

@@ -23,6 +23,6 @@ require 'kubernetes-deploy/duration_parser'
 require 'kubernetes-deploy/resource_cache'
 
 module KubernetesDeploy
-  MIN_KUBE_VERSION = '1.9.0'
+  MIN_KUBE_VERSION = '1.10.0'
   StatsD.build
 end

data/lib/kubernetes-deploy/container_logs.rb

@@ -29,7 +29,7 @@ module KubernetesDeploy
       prefix_str = "[#{container_name}]  " if prefix
 
       lines[@next_print_index..-1].each do |msg|
-        @logger.info "#{prefix_str}#{msg}"
+        @logger.info("#{prefix_str}#{msg}")
       end
 
       @next_print_index = lines.length

data/lib/kubernetes-deploy/deploy_task.rb

@@ -6,6 +6,7 @@ require 'tempfile'
 require 'fileutils'
 require 'kubernetes-deploy/kubernetes_resource'
 %w(
+  custom_resource
   cloudsql
   config_map
   deployment
@@ -24,7 +25,6 @@ require 'kubernetes-deploy/kubernetes_resource'
   elasticsearch
   statefulservice
   topic
-  bucket
   stateful_set
   cron_job
   job
@@ -46,19 +46,6 @@ module KubernetesDeploy
     include KubeclientBuilder
     extend KubernetesDeploy::StatsD::MeasureMethods
 
-    PREDEPLOY_SEQUENCE = %w(
-      ResourceQuota
-      Cloudsql
-      Redis
-      Memcached
-      ConfigMap
-      PersistentVolumeClaim
-      ServiceAccount
-      Role
-      RoleBinding
-      Pod
-    )
-
     PROTECTED_NAMESPACES = %w(
       default
       kube-system
@@ -73,6 +60,22 @@ module KubernetesDeploy
     # extensions/v1beta1/ReplicaSet -- managed by deployments
     # core/v1/Secret -- should not committed / managed by shipit
 
+    def predeploy_sequence
+      before_crs = %w(
+        ResourceQuota
+      )
+      after_crs = %w(
+        ConfigMap
+        PersistentVolumeClaim
+        ServiceAccount
+        Role
+        RoleBinding
+        Pod
+      )
+
+      before_crs + cluster_resource_discoverer.crds.map(&:kind) + after_crs
+    end
+
     def prune_whitelist
       wl = %w(
         core/v1/ConfigMap
@@ -194,11 +197,11 @@ module KubernetesDeploy
     end
 
     def deploy_has_priority_resources?(resources)
-      resources.any? { |r| PREDEPLOY_SEQUENCE.include?(r.type) }
+      resources.any? { |r| predeploy_sequence.include?(r.type) }
     end
 
     def predeploy_priority_resources(resource_list)
-      PREDEPLOY_SEQUENCE.each do |resource_type|
+      predeploy_sequence.each do |resource_type|
         matching_resources = resource_list.select { |r| r.type == resource_type }
         next if matching_resources.empty?
         deploy_resources(matching_resources, verify: true, record_summary: false)
@@ -254,14 +257,16 @@ module KubernetesDeploy
 
     def discover_resources
       resources = []
+      crds = cluster_resource_discoverer.crds.group_by(&:kind)
       @logger.info("Discovering templates:")
 
       TemplateDiscovery.new(@template_dir).templates.each do |filename|
         split_templates(filename) do |r_def|
-          r = KubernetesResource.build(namespace: @namespace, context: @context, logger: @logger,
-                                       definition: r_def, statsd_tags: @namespace_tags)
+          crd = crds[r_def["kind"]]&.first
+          r = KubernetesResource.build(namespace: @namespace, context: @context, logger: @logger, definition: r_def,
+            statsd_tags: @namespace_tags, crd: crd)
           resources << r
-          @logger.info "  - #{r.id}"
+          @logger.info("  - #{r.id}")
         end
       end
       if (global = resources.select(&:global?).presence)

data/lib/kubernetes-deploy/ejson_secret_provisioner.rb

@@ -145,9 +145,9 @@ module KubernetesDeploy
           "name" => secret_name,
           "labels" => { "name" => secret_name },
           "namespace" => @namespace,
-          "annotations" => { MANAGEMENT_ANNOTATION => "true" }
+          "annotations" => { MANAGEMENT_ANNOTATION => "true" },
         },
-        "data" => encoded_data
+        "data" => encoded_data,
       }
       secret.to_yaml
     end

data/lib/kubernetes-deploy/kubeclient_builder.rb

@@ -100,7 +100,7 @@ module KubernetesDeploy
         auth_options: kube_context.auth_options,
         timeouts: {
           open: KubernetesDeploy::Kubectl::DEFAULT_TIMEOUT,
-          read: KubernetesDeploy::Kubectl::DEFAULT_TIMEOUT
+          read: KubernetesDeploy::Kubectl::DEFAULT_TIMEOUT,
         }
       )
       client.discover

data/lib/kubernetes-deploy/kubectl.rb

@@ -30,7 +30,7 @@ module KubernetesDeploy
       out, err, st = nil
 
       (1..attempts).to_a.each do |attempt|
-        @logger.debug "Running command (attempt #{attempt}): #{args.join(' ')}"
+        @logger.debug("Running command (attempt #{attempt}): #{args.join(' ')}")
         out, err, st = Open3.capture3(*args)
         @logger.debug("Kubectl out: " + out.gsub(/\s+/, ' ')) unless output_is_sensitive?
 
@@ -47,7 +47,7 @@ module KubernetesDeploy
           @logger.debug("Kubectl err: #{err}") unless output_is_sensitive?
           StatsD.increment('kubectl.error', 1, tags: { context: @context, namespace: @namespace, cmd: args[1] })
         end
-        sleep retry_delay(attempt) unless attempt == attempts
+        sleep(retry_delay(attempt)) unless attempt == attempts
       end
 
       [out.chomp, err.chomp, st]

data/lib/kubernetes-deploy/kubernetes_resource.rb

@@ -31,14 +31,21 @@ module KubernetesDeploy
     TIMEOUT_OVERRIDE_ANNOTATION = "kubernetes-deploy.shopify.io/timeout-override"
 
     class << self
-      def build(namespace:, context:, definition:, logger:, statsd_tags:)
+      def build(namespace:, context:, definition:, logger:, statsd_tags:, crd: nil)
         opts = { namespace: namespace, context: context, definition: definition, logger: logger,
                  statsd_tags: statsd_tags }
         if definition["kind"].blank?
           raise InvalidTemplateError.new("Template missing 'Kind'", content: definition.to_yaml)
-        elsif KubernetesDeploy.const_defined?(definition["kind"])
-          klass = KubernetesDeploy.const_get(definition["kind"])
-          klass.new(**opts)
+        end
+        begin
+          if KubernetesDeploy.const_defined?(definition["kind"])
+            klass = KubernetesDeploy.const_get(definition["kind"])
+            return klass.new(**opts)
+          end
+        rescue NameError
+        end
+        if crd
+          CustomResource.new(crd: crd, **opts)
         else
           inst = new(**opts)
           inst.type = definition["kind"]
@@ -162,13 +169,13 @@ module KubernetesDeploy
 
     def current_generation
       return -1 unless exists? # must be different default than observed_generation
-      @instance_data["metadata"]["generation"]
+      @instance_data.dig("metadata", "generation")
     end
 
     def observed_generation
       return -2 unless exists?
       # populating this is a best practice, but not all controllers actually do it
-      @instance_data["status"]["observedGeneration"]
+      @instance_data.dig('status', 'observedGeneration')
     end
 
     def status
@@ -319,7 +326,7 @@ module KubernetesDeploy
           '(ne .reason "SuccessfulCreate")',
           '(ne .reason "Scheduled")',
           '(ne .reason "Pulling")',
-          '(ne .reason "Pulled")'
+          '(ne .reason "Pulled")',
         ]
         condition_start = "{{if and #{and_conditions.join(' ')}}}"
         field_part = FIELDS.map { |f| "{{#{f}}}" }.join(%({{print "#{FIELD_SEPARATOR}"}}))

data/lib/kubernetes-deploy/kubernetes_resource/custom_resource.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+require 'jsonpath'
+
+module KubernetesDeploy
+  class CustomResource < KubernetesResource
+    TIMEOUT_MESSAGE_DIFFERENT_GENERATIONS = <<~MSG
+      This resource's status could not be used to determine rollout success because it is not up-to-date
+      (.metadata.generation != .status.observedGeneration).
+    MSG
+
+    def initialize(namespace:, context:, definition:, logger:, statsd_tags: [], crd:)
+      super(namespace: namespace, context: context, definition: definition,
+            logger: logger, statsd_tags: statsd_tags)
+      @crd = crd
+    end
+
+    def timeout
+      timeout_override || @crd.timeout_for_instance || TIMEOUT
+    end
+
+    def deploy_succeeded?
+      return super unless rollout_conditions
+      return false unless observed_generation == current_generation
+
+      rollout_conditions.rollout_successful?(@instance_data)
+    end
+
+    def deploy_failed?
+      return super unless rollout_conditions
+      return false unless observed_generation == current_generation
+
+      rollout_conditions.rollout_failed?(@instance_data)
+    end
+
+    def failure_message
+      return super unless rollout_conditions
+      messages = rollout_conditions.failure_messages(@instance_data)
+      messages.join("\n") if messages.present?
+    end
+
+    def timeout_message
+      if rollout_conditions && current_generation != observed_generation
+        TIMEOUT_MESSAGE_DIFFERENT_GENERATIONS
+      else
+        super
+      end
+    end
+
+    def status
+      if !exists? || rollout_conditions.nil?
+        super
+      elsif deploy_succeeded?
+        "Healthy"
+      elsif deploy_failed?
+        "Unhealthy"
+      else
+        "Unknown"
+      end
+    end
+
+    def type
+      kind
+    end
+
+    def validate_definition(kubectl)
+      super
+
+      @crd.validate_rollout_conditions
+    rescue RolloutConditionsError => e
+      @validation_errors << "The CRD that specifies this resource is using invalid rollout conditions. " \
+      "Kubernetes-deploy will not be able to continue until those rollout conditions are fixed.\n" \
+      "Rollout conditions can be found on the CRD that defines this resource (#{@crd.name}), " \
+      "under the annotation #{CustomResourceDefinition::ROLLOUT_CONDITIONS_ANNOTATION}.\n" \
+      "Validation failed with: #{e}"
+    end
+
+    private
+
+    def kind
+      @definition["kind"]
+    end
+
+    def rollout_conditions
+      @crd.rollout_conditions
+    end
+  end
+end

data/lib/kubernetes-deploy/kubernetes_resource/custom_resource_definition.rb

@@ -1,7 +1,11 @@
 # frozen_string_literal: true
+require 'kubernetes-deploy/rollout_conditions'
+
 module KubernetesDeploy
   class CustomResourceDefinition < KubernetesResource
     TIMEOUT = 2.minutes
+    ROLLOUT_CONDITIONS_ANNOTATION = "kubernetes-deploy.shopify.io/instance-rollout-conditions"
+    TIMEOUT_FOR_INSTANCE_ANNOTATION = "kubernetes-deploy.shopify.io/instance-timeout"
     GLOBAL = true
 
     def deploy_succeeded?
@@ -16,6 +20,13 @@ module KubernetesDeploy
       "The names this CRD is attempting to register were neither accepted nor rejected in time"
     end
 
+    def timeout_for_instance
+      timeout = @definition.dig("metadata", "annotations", TIMEOUT_FOR_INSTANCE_ANNOTATION)
+      DurationParser.new(timeout).parse!.to_i
+    rescue DurationParser::ParsingError
+      nil
+    end
+
     def status
       if !exists?
         super
@@ -36,11 +47,42 @@ module KubernetesDeploy
       @definition.dig("spec", "names", "kind")
     end
 
+    def name
+      @definition.dig("metadata", "name")
+    end
+
     def prunable?
       prunable = @definition.dig("metadata", "annotations", "kubernetes-deploy.shopify.io/prunable")
       prunable == "true"
     end
 
+    def rollout_conditions
+      return @rollout_conditions if defined?(@rollout_conditions)
+
+      @rollout_conditions = if rollout_conditions_annotation
+        RolloutConditions.from_annotation(rollout_conditions_annotation)
+      end
+    rescue RolloutConditionsError
+      @rollout_conditions = nil
+    end
+
+    def validate_definition(_)
+      super
+
+      validate_rollout_conditions
+    rescue RolloutConditionsError => e
+      @validation_errors << "Annotation #{ROLLOUT_CONDITIONS_ANNOTATION} on #{name} is invalid: #{e}"
+    end
+
+    def validate_rollout_conditions
+      if rollout_conditions_annotation && @rollout_conditions_validated.nil?
+        conditions = RolloutConditions.from_annotation(rollout_conditions_annotation)
+        conditions.validate!
+      end
+
+      @rollout_conditions_validated = true
+    end
+
     private
 
     def names_accepted_condition
@@ -51,5 +93,9 @@ module KubernetesDeploy
     def names_accepted_status
       names_accepted_condition["status"]
     end
+
+    def rollout_conditions_annotation
+      @definition.dig("metadata", "annotations", ROLLOUT_CONDITIONS_ANNOTATION)
+    end
   end
 end

data/lib/kubernetes-deploy/options_helper.rb

@@ -11,7 +11,7 @@ module KubernetesDeploy
         puts "Template directory is unknown. " \
           "Either specify --template-dir argument or set $ENVIRONMENT to use config/deploy/$ENVIRONMENT " \
         + "as a default path."
-        exit 1
+        exit(1)
       end
 
       template_dir

data/lib/kubernetes-deploy/resource_watcher.rb

@@ -59,7 +59,7 @@ module KubernetesDeploy
       {
         namespace: @namespace,
         context: @context,
-        sha: @sha
+        sha: @sha,
       }
     end
 

data/lib/kubernetes-deploy/restart_task.rb

@@ -132,7 +132,7 @@ module KubernetesDeploy
       deployments.each do |record|
         begin
           patch_deployment_with_restart(record)
-          @logger.info "Triggered `#{record.metadata.name}` restart"
+          @logger.info("Triggered `#{record.metadata.name}` restart")
         rescue Kubeclient::ResourceNotFoundError, Kubeclient::HttpError => e
           raise RestartAPIError.new(record.metadata.name, e.message)
         end
@@ -164,12 +164,12 @@ module KubernetesDeploy
               containers: containers.map do |container|
                 {
                   name: container.name,
-                  env: [{ name: "RESTARTED_AT", value: Time.now.to_i.to_s }]
+                  env: [{ name: "RESTARTED_AT", value: Time.now.to_i.to_s }],
                 }
-              end
-            }
-          }
-        }
+              end,
+            },
+          },
+        },
       }
     end
 

data/lib/kubernetes-deploy/rollout_conditions.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+module KubernetesDeploy
+  class RolloutConditionsError < StandardError
+  end
+
+  class RolloutConditions
+    VALID_FAILURE_CONDITION_KEYS = [:path, :value, :error_msg_path, :custom_error_msg]
+    VALID_SUCCESS_CONDITION_KEYS = [:path, :value]
+
+    class << self
+      def from_annotation(conditions_string)
+        return new(default_conditions) if conditions_string.downcase.strip == "true"
+
+        conditions = JSON.parse(conditions_string).slice('success_conditions', 'failure_conditions')
+        conditions.deep_symbolize_keys!
+
+        # Create JsonPath objects
+        conditions[:success_conditions]&.each do |query|
+          query.slice!(*VALID_SUCCESS_CONDITION_KEYS)
+          query[:path] = JsonPath.new(query[:path]) if query.key?(:path)
+        end
+        conditions[:failure_conditions]&.each do |query|
+          query.slice!(*VALID_FAILURE_CONDITION_KEYS)
+          query[:path] = JsonPath.new(query[:path]) if query.key?(:path)
+          query[:error_msg_path] = JsonPath.new(query[:error_msg_path]) if query.key?(:error_msg_path)
+        end
+
+        new(conditions)
+      rescue JSON::ParserError => e
+        raise RolloutConditionsError, "Rollout conditions are not valid JSON: #{e}"
+      rescue StandardError => e
+        raise RolloutConditionsError,
+          "Error parsing rollout conditions. " \
+          "This is most likely caused by an invalid JsonPath expression. Failed with: #{e}"
+      end
+
+      def default_conditions
+        {
+          success_conditions: [
+            {
+              path: JsonPath.new('$.status.conditions[?(@.type == "Ready")].status'),
+              value: "True",
+            },
+          ],
+          failure_conditions: [
+            {
+              path: JsonPath.new('$.status.conditions[?(@.type == "Failed")].status'),
+              value: "True",
+              error_msg_path: JsonPath.new('$.status.conditions[?(@.type == "Failed")].message'),
+            },
+          ],
+        }
+      end
+    end
+
+    def initialize(conditions)
+      @success_conditions = conditions.fetch(:success_conditions, [])
+      @failure_conditions = conditions.fetch(:failure_conditions, [])
+    end
+
+    def rollout_successful?(instance_data)
+      @success_conditions.all? do |query|
+        query[:path].first(instance_data) == query[:value]
+      end
+    end
+
+    def rollout_failed?(instance_data)
+      @failure_conditions.any? do |query|
+        query[:path].first(instance_data) == query[:value]
+      end
+    end
+
+    def failure_messages(instance_data)
+      @failure_conditions.map do |query|
+        next unless query[:path].first(instance_data) == query[:value]
+        query[:custom_error_msg].presence || query[:error_msg_path]&.first(instance_data)
+      end.compact
+    end
+
+    def validate!
+      errors = validate_conditions(@success_conditions, 'success_conditions')
+      errors += validate_conditions(@failure_conditions, 'failure_conditions', required: false)
+      raise RolloutConditionsError, errors.join(", ") unless errors.empty?
+    end
+
+    private
+
+    def validate_conditions(conditions, source_key, required: true)
+      return [] unless conditions.present? || required
+      errors = []
+      errors << "#{source_key} should be Array but found #{conditions.class}" unless conditions.is_a?(Array)
+      return errors if errors.present?
+      errors << "#{source_key} must contain at least one entry" if conditions.empty?
+      return errors if errors.present?
+
+      conditions.each do |query|
+        missing = [:path, :value].reject { |k| query.key?(k) }
+        errors << "Missing required key(s) for #{source_key.singularize}: #{missing}" if missing.present?
+      end
+      errors
+    end
+  end
+end

data/lib/kubernetes-deploy/runner_task.rb

@@ -59,7 +59,7 @@ module KubernetesDeploy
     private
 
     def create_pod(pod)
-      @logger.info "Creating pod '#{pod.name}'"
+      @logger.info("Creating pod '#{pod.name}'")
       pod.deploy_started_at = Time.now.utc
       kubeclient.create_pod(pod.to_kubeclient_resource)
       @pod_name = pod.name
@@ -121,7 +121,7 @@ module KubernetesDeploy
 
       begin
         kubeclient.get_namespace(@namespace) if @namespace.present?
-        @logger.info "Using namespace '#{@namespace}' in context '#{@context}'"
+        @logger.info("Using namespace '#{@namespace}' in context '#{@context}'")
       rescue KubeException => e
         msg = e.error_code == 404 ? "Namespace was not found" : "Could not connect to kubernetes cluster"
         errors << msg

data/lib/kubernetes-deploy/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module KubernetesDeploy
-  VERSION = "0.23.0"
+  VERSION = "0.24.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: kubernetes-deploy
 version: !ruby/object:Gem::Version
-  version: 0.23.0
+  version: 0.24.0
 platform: ruby
 authors:
 - Katrina Verey
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-12-14 00:00:00.000000000 Z
+date: 2019-01-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -129,6 +129,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: jsonpath
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.6
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -261,10 +275,10 @@ files:
 - lib/kubernetes-deploy/kubeclient_builder/google_friendly_config.rb
 - lib/kubernetes-deploy/kubectl.rb
 - lib/kubernetes-deploy/kubernetes_resource.rb
-- lib/kubernetes-deploy/kubernetes_resource/bucket.rb
 - lib/kubernetes-deploy/kubernetes_resource/cloudsql.rb
 - lib/kubernetes-deploy/kubernetes_resource/config_map.rb
 - lib/kubernetes-deploy/kubernetes_resource/cron_job.rb
+- lib/kubernetes-deploy/kubernetes_resource/custom_resource.rb
 - lib/kubernetes-deploy/kubernetes_resource/custom_resource_definition.rb
 - lib/kubernetes-deploy/kubernetes_resource/daemon_set.rb
 - lib/kubernetes-deploy/kubernetes_resource/deployment.rb
@@ -296,6 +310,7 @@ files:
 - lib/kubernetes-deploy/resource_cache.rb
 - lib/kubernetes-deploy/resource_watcher.rb
 - lib/kubernetes-deploy/restart_task.rb
+- lib/kubernetes-deploy/rollout_conditions.rb
 - lib/kubernetes-deploy/runner_task.rb
 - lib/kubernetes-deploy/statsd.rb
 - lib/kubernetes-deploy/template_discovery.rb

data/lib/kubernetes-deploy/kubernetes_resource/bucket.rb

@@ -1,22 +0,0 @@
-# frozen_string_literal: true
-module KubernetesDeploy
-  class Bucket < KubernetesResource
-    def deploy_succeeded?
-      return false unless deploy_started?
-
-      unless @success_assumption_warning_shown
-        @logger.warn("Don't know how to monitor resources of type #{type}. Assuming #{id} deployed successfully.")
-        @success_assumption_warning_shown = true
-      end
-      true
-    end
-
-    def status
-      exists? ? "Available" : "Unknown"
-    end
-
-    def deploy_failed?
-      false
-    end
-  end
-end