kubernetes-deploy 0.12.12 → 0.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1824eeb996c6d82344af3296b3000706ea6f3516
-  data.tar.gz: bd6786d3454f52ac5ac8edf332c62a3a5f7f7164
+  metadata.gz: 6f9ccce3d57089a13afd20e62c5bba0b49c08b92
+  data.tar.gz: 58f63b9dbda402ec6cee2833ec0beb209aa3d51c
 SHA512:
-  metadata.gz: 07442fae42d0982d81f9d48a684afb59330124a62680ccb00e9b51b93422f60340e8a48b799f43b7b048dcb084f5d3a8daba93229818e00e9d71f7bafb765731
-  data.tar.gz: f3acaaa7d676ee0c286d800232b96f6232e04fbb2ee2187a31156f217008705a7b87d6152a030a6d524e548905d3dd7bb93d012232dc375137c1e2b0d27936a2
+  metadata.gz: 7969d60b7979ef48b82cb717a22a153203ce495bd8c94c6c8fb1a0c711ed229b7bb32de359b7f71d0b812c64c53caf7afaafab362295bcf8ace6bc090649ef7b
+  data.tar.gz: 214dfbc251bcb0a121ec49ca8e37a803793e8f6aa7bcc1fdac8b5069d149abbdd0154b24f58324e5c9eb3c4cb6e43b705946862db5963b523887285aeb5d31ad

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+### 0.13.0
+*Features*
+- Added support for StatefulSets for kubernetes 1.7+ using RollingUpdate
+
+*Bug Fixes*
+- Explicitly require the minimum rest-client version required by kubeclient ([#202](https://github.com/Shopify/kubernetes-deploy/pull/202))
+
+*Enhancements*
+- Begin official support for Kubernetes v1.8 ([#198](https://github.com/Shopify/kubernetes-deploy/pull/198), [#200](https://github.com/Shopify/kubernetes-deploy/pull/200))
+
 ### 0.12.12
 *Bug Fixes*
 - Fix an issue deploying Shopify's internal custom resources.

data/README.md

@@ -298,9 +298,13 @@ If you push your commit and the tag separately, Shipit usually fails with `You n
 
 ## CI (External contributors)
 
-Please make sure you run the tests locally before submitting your PR (see [Running the test suite locally](#running-the-test-suite-locally)). After reviewing your PR, a Shopify employee will trigger CI for you from the [Buildkite UI](https://buildkite.com/shopify/kubernetes-deploy-gem) (just specify the branch; SHA is not required).
+Please make sure you run the tests locally before submitting your PR (see [Running the test suite locally](#running-the-test-suite-locally)). After reviewing your PR, a Shopify employee will trigger CI for you.
 
-<img width="464" alt="screen shot 2017-02-21 at 10 55 33" src="https://cloud.githubusercontent.com/assets/522155/23172610/52771a3a-f824-11e6-8c8e-3d59c45e7ff8.png">
+#### Employees: Triggering CI for a contributed PR
+
+Go to the [kubernetes-deploy-gem pipeline](https://buildkite.com/shopify/kubernetes-deploy-gem) and click "New Build". Use branch `external_contrib_ci` and the specific sha of the commit you want to build. Add `BUILDKITE_REFSPEC="refs/pull/${PR_NUM}/head"` in the Environment Variables section.
+
+<img width="350" alt="build external contrib PR" src="https://screenshot.click/2017-11-07--163728_7ovek-wrpwq.png">
 
 # Contributing
 

data/kubernetes-deploy.gemspec

@@ -25,6 +25,7 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = '>= 2.3.0'
   spec.add_dependency "activesupport", ">= 4.2"
   spec.add_dependency "kubeclient", "~> 2.4"
+  spec.add_dependency "rest-client", ">= 1.7" # Minimum required by kubeclient. Remove when kubeclient releases v3.0.
   spec.add_dependency "googleauth", ">= 0.5"
   spec.add_dependency "ejson", "1.0.1"
   spec.add_dependency "colorize", "~> 0.8"

data/lib/kubernetes-deploy/kubeclient_builder.rb

@@ -35,6 +35,14 @@ module KubernetesDeploy
       )
     end
 
+    def build_apps_v1beta1_kubeclient(context)
+      _build_kubeclient(
+        api_version: "v1beta1",
+        context: context,
+        endpoint_path: "/apis/apps"
+      )
+    end
+
     def _build_kubeclient(api_version:, context:, endpoint_path: nil)
       config = GoogleFriendlyConfig.read(ENV.fetch("KUBECONFIG"))
       unless config.contexts.include?(context)

data/lib/kubernetes-deploy/kubectl.rb

@@ -31,5 +31,35 @@ module KubernetesDeploy
       end
       [out.chomp, err.chomp, st]
     end
+
+    def version_info
+      @version_info ||=
+        begin
+          response, _, status = run("version", use_namespace: false, log_failure: true)
+          raise KubectlError, "Could not retrieve kubectl version info" unless status.success?
+          extract_version_info_from_kubectl_response(response)
+        end
+    end
+
+    def client_version
+      version_info[:client]
+    end
+
+    def server_version
+      version_info[:server]
+    end
+
+    private
+
+    def extract_version_info_from_kubectl_response(response)
+      info = {}
+      response.each_line do |l|
+        match = l.match(/^(?<kind>Client|Server).* GitVersion:"v(?<version>\d+\.\d+\.\d+)/)
+        if match
+          info[match[:kind].downcase.to_sym] = Gem::Version.new(match[:version])
+        end
+      end
+      info
+    end
   end
 end

data/lib/kubernetes-deploy/kubernetes_resource/daemon_set.rb

@@ -1,7 +1,9 @@
 # frozen_string_literal: true
+require 'kubernetes-deploy/kubernetes_resource/pod_set_base'
 module KubernetesDeploy
-  class DaemonSet < KubernetesResource
+  class DaemonSet < PodSetBase
     TIMEOUT = 5.minutes
+    attr_reader :pods
 
     def sync
       raw_json, _err, st = kubectl.run("get", type, @name, "--output=json")
@@ -31,66 +33,24 @@ module KubernetesDeploy
     end
 
     def deploy_failed?
-      @pods.present? && @pods.any?(&:deploy_failed?)
+      pods.present? && pods.any?(&:deploy_failed?)
     end
 
-    def failure_message
-      @pods.map(&:failure_message).compact.uniq.join("\n")
-    end
-
-    def timeout_message
-      @pods.map(&:timeout_message).compact.uniq.join("\n")
-    end
-
-    def deploy_timed_out?
-      super || @pods.present? && @pods.any?(&:deploy_timed_out?)
+    def fetch_logs
+      most_useful_pod = @pods.find(&:deploy_failed?) || @pods.find(&:deploy_timed_out?) || @pods.first
+      most_useful_pod.fetch_logs
     end
 
     def exists?
       @found
     end
 
-    def fetch_events
-      own_events = super
-      return own_events unless @pods.present?
-      most_useful_pod = @pods.find(&:deploy_failed?) || @pods.find(&:deploy_timed_out?) || @pods.first
-      own_events.merge(most_useful_pod.fetch_events)
-    end
-
-    def fetch_logs
-      most_useful_pod = @pods.find(&:deploy_failed?) || @pods.find(&:deploy_timed_out?) || @pods.first
-      most_useful_pod.fetch_logs
-    end
-
     private
 
-    def find_pods(ds_data)
-      label_string = ds_data["spec"]["selector"]["matchLabels"].map { |k, v| "#{k}=#{v}" }.join(",")
-      raw_json, _err, st = kubectl.run("get", "pods", "-a", "--output=json", "--selector=#{label_string}")
-      return [] unless st.success?
-
-      all_pods = JSON.parse(raw_json)["items"]
-      template_generation = ds_data["spec"]["templateGeneration"]
-
-      latest_pods = all_pods.find_all do |pod|
-        next unless owners = pod.dig("metadata", "ownerReferences")
-        owners.any? { |ref| ref["uid"] == ds_data["metadata"]["uid"] } &&
-        pod["metadata"]["labels"]["pod-template-generation"].to_i == template_generation.to_i
-      end
-      return [] unless latest_pods.present?
-
-      latest_pods.each_with_object([]) do |pod_data, relevant_pods|
-        pod = Pod.new(
-          namespace: namespace,
-          context: context,
-          definition: pod_data,
-          logger: @logger,
-          parent: "#{@name.capitalize} daemon set",
-          deploy_started: @deploy_started
-        )
-        pod.sync(pod_data)
-        relevant_pods << pod
-      end
+    def parent_of_pod?(set_data, pod_data)
+      return false unless pod_data.dig("metadata", "ownerReferences")
+      pod_data["metadata"]["ownerReferences"].any? { |ref| ref["uid"] == set_data["metadata"]["uid"] } &&
+      pod_data["metadata"]["labels"]["pod-template-generation"].to_i == set_data["spec"]["templateGeneration"].to_i
     end
   end
 end

data/lib/kubernetes-deploy/kubernetes_resource/pod_set_base.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+module KubernetesDeploy
+  class PodSetBase < KubernetesResource
+    def failure_message
+      pods.map(&:failure_message).compact.uniq.join("\n")
+    end
+
+    def timeout_message
+      pods.map(&:timeout_message).compact.uniq.join("\n")
+    end
+
+    def fetch_events
+      own_events = super
+      return own_events unless pods.present?
+      most_useful_pod = pods.find(&:deploy_failed?) || pods.find(&:deploy_timed_out?) || pods.first
+      own_events.merge(most_useful_pod.fetch_events)
+    end
+
+    def fetch_logs
+      return {} unless pods.present? # the kubectl command times out if no pods exist
+      container_names.each_with_object({}) do |container_name, container_logs|
+        out, _err, _st = kubectl.run(
+          "logs",
+          id,
+          "--container=#{container_name}",
+          "--since-time=#{@deploy_started.to_datetime.rfc3339}",
+          "--tail=#{LOG_LINE_COUNT}"
+        )
+        container_logs[container_name] = out.split("\n")
+      end
+    end
+
+    private
+
+    def pods
+      raise NotImplementedError, "Subclasses must define a `pods` accessor"
+    end
+
+    def parent_of_pod?(_, _)
+      raise NotImplementedError, "Subclasses must define a `parent_of_pod?` method"
+    end
+
+    def container_names
+      regular_containers = @definition["spec"]["template"]["spec"]["containers"].map { |c| c["name"] }
+      init_containers = @definition["spec"]["template"]["spec"].fetch("initContainers", {}).map { |c| c["name"] }
+      regular_containers + init_containers
+    end
+
+    def find_pods(pod_controller_data)
+      label_string = pod_controller_data["spec"]["selector"]["matchLabels"].map { |k, v| "#{k}=#{v}" }.join(",")
+      raw_json, _err, st = kubectl.run("get", "pods", "-a", "--output=json", "--selector=#{label_string}")
+      return [] unless st.success?
+
+      all_pods = JSON.parse(raw_json)["items"]
+      all_pods.each_with_object([]) do |pod_data, relevant_pods|
+        next unless parent_of_pod?(pod_controller_data, pod_data)
+        pod = Pod.new(
+          namespace: namespace,
+          context: context,
+          definition: pod_data,
+          logger: @logger,
+          parent: "#{name.capitalize} #{self.class.name}",
+          deploy_started: @deploy_started
+        )
+        pod.sync(pod_data)
+        relevant_pods << pod
+      end
+    end
+  end
+end

data/lib/kubernetes-deploy/kubernetes_resource/replica_set.rb

@@ -1,8 +1,9 @@
 # frozen_string_literal: true
+require 'kubernetes-deploy/kubernetes_resource/pod_set_base'
 module KubernetesDeploy
-  class ReplicaSet < KubernetesResource
+  class ReplicaSet < PodSetBase
     TIMEOUT = 5.minutes
-    attr_reader :desired_replicas
+    attr_reader :desired_replicas, :pods
 
     def initialize(namespace:, context:, definition:, logger:, parent: nil, deploy_started: nil)
       @parent = parent
@@ -39,74 +40,22 @@ module KubernetesDeploy
     end
 
     def deploy_failed?
-      @pods.present? && @pods.all?(&:deploy_failed?)
-    end
-
-    def failure_message
-      @pods.map(&:failure_message).compact.uniq.join("\n")
-    end
-
-    def timeout_message
-      @pods.map(&:timeout_message).compact.uniq.join("\n")
+      pods.present? && pods.all?(&:deploy_failed?)
     end
 
     def exists?
       @found
     end
 
-    def fetch_events
-      own_events = super
-      return own_events unless @pods.present?
-      most_useful_pod = @pods.find(&:deploy_failed?) || @pods.find(&:deploy_timed_out?) || @pods.first
-      own_events.merge(most_useful_pod.fetch_events)
-    end
+    private
 
-    def fetch_logs
-      return {} unless @pods.present? # the kubectl command times out if no pods exist
-      container_names.each_with_object({}) do |container_name, container_logs|
-        out, _err, _st = kubectl.run(
-          "logs",
-          id,
-          "--container=#{container_name}",
-          "--since-time=#{@deploy_started.to_datetime.rfc3339}",
-          "--tail=#{LOG_LINE_COUNT}"
-        )
-        container_logs[container_name] = out.split("\n")
-      end
+    def parent_of_pod?(set_data, pod_data)
+      return false unless pod_data.dig("metadata", "ownerReferences")
+      pod_data["metadata"]["ownerReferences"].any? { |ref| ref["uid"] == set_data["metadata"]["uid"] }
     end
 
-    private
-
     def unmanaged?
       @parent.blank?
     end
-
-    def container_names
-      regular_containers = @definition["spec"]["template"]["spec"]["containers"].map { |c| c["name"] }
-      init_containers = @definition["spec"]["template"]["spec"].fetch("initContainers", []).map { |c| c["name"] }
-      regular_containers + init_containers
-    end
-
-    def find_pods(rs_data)
-      label_string = rs_data["spec"]["selector"]["matchLabels"].map { |k, v| "#{k}=#{v}" }.join(",")
-      raw_json, _err, st = kubectl.run("get", "pods", "-a", "--output=json", "--selector=#{label_string}")
-      return [] unless st.success?
-
-      all_pods = JSON.parse(raw_json)["items"]
-      all_pods.each_with_object([]) do |pod_data, relevant_pods|
-        next unless owners = pod_data.dig("metadata", "ownerReferences")
-        next unless owners.any? { |ref| ref["uid"] == rs_data["metadata"]["uid"] }
-        pod = Pod.new(
-          namespace: namespace,
-          context: context,
-          definition: pod_data,
-          logger: @logger,
-          parent: "#{@name.capitalize} replica set",
-          deploy_started: @deploy_started
-        )
-        pod.sync(pod_data)
-        relevant_pods << pod
-      end
-    end
   end
 end

data/lib/kubernetes-deploy/kubernetes_resource/stateful_set.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+require 'kubernetes-deploy/kubernetes_resource/pod_set_base'
+module KubernetesDeploy
+  class StatefulSet < PodSetBase
+    TIMEOUT = 10.minutes
+    ONDELETE = 'OnDelete'
+    attr_reader :pods
+
+    def sync
+      raw_json, _err, st = kubectl.run("get", type, @name, "--output=json")
+      @found = st.success?
+
+      if @found
+        stateful_data = JSON.parse(raw_json)
+        @desired_replicas = stateful_data["spec"]["replicas"].to_i
+        @status_data = stateful_data["status"]
+        rollout_data = stateful_data["status"].slice("replicas", "readyReplicas", "currentReplicas")
+        @update_strategy = if kubectl.server_version < Gem::Version.new("1.7.0")
+          ONDELETE
+        else
+          stateful_data['spec']['updateStrategy']['type']
+        end
+        @status = rollout_data.map { |state_replicas, num| "#{num} #{state_replicas.chop.pluralize(num)}" }.join(", ")
+        @pods = find_pods(stateful_data)
+      else # reset
+        @status_data = { 'readyReplicas' => '-1', 'currentReplicas' => '-2' }
+        @status = nil
+        @pods = []
+      end
+    end
+
+    def deploy_succeeded?
+      if @update_strategy == ONDELETE
+        # Gem cannot monitor update since it doesn't occur until delete
+        unless @success_assumption_warning_shown
+          @logger.warn("WARNING: Your StatefulSet's updateStrategy is set to OnDelete, "\
+                       "which means updates will not be applied until its pods are deleted. "\
+                       "If you are using k8s 1.7+, consider switching to rollingUpdate.")
+          @success_assumption_warning_shown = true
+        end
+        true
+      else
+        @status_data['currentRevision'] == @status_data['updateRevision'] &&
+        @desired_replicas == @status_data['readyReplicas'].to_i &&
+        @desired_replicas == @status_data['currentReplicas'].to_i
+      end
+    end
+
+    def deploy_failed?
+      return false if @update_strategy == ONDELETE
+      pods.present? && pods.any?(&:deploy_failed?)
+    end
+
+    def exists?
+      @found
+    end
+
+    private
+
+    def parent_of_pod?(set_data, pod_data)
+      return false unless pod_data.dig("metadata", "ownerReferences")
+      pod_data["metadata"]["ownerReferences"].any? { |ref| ref["uid"] == set_data["metadata"]["uid"] } &&
+      set_data["status"]["currentRevision"] == pod_data["metadata"]["labels"]["controller-revision-hash"]
+    end
+  end
+end

data/lib/kubernetes-deploy/runner.rb

@@ -27,6 +27,7 @@ require 'kubernetes-deploy/kubernetes_resource'
   statefulservice
   topic
   bucket
+  stateful_set
 ).each do |subresource|
   require "kubernetes-deploy/kubernetes_resource/#{subresource}"
 end
@@ -415,10 +416,12 @@ module KubernetesDeploy
     def confirm_cluster_reachable
       success = false
       with_retries(2) do
-        _, _, st = kubectl.run("version", use_namespace: false, log_failure: true)
-        success = st.success?
+        begin
+          success = kubectl.version_info
+        rescue KubectlError
+          success = false
+        end
       end
-
       raise FatalDeploymentError, "Failed to reach server for #{@context}" unless success
     end
 

data/lib/kubernetes-deploy/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module KubernetesDeploy
-  VERSION = "0.12.12"
+  VERSION = "0.13.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: kubernetes-deploy
 version: !ruby/object:Gem::Version
-  version: 0.12.12
+  version: 0.13.0
 platform: ruby
 authors:
 - Katrina Verey
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-11-02 00:00:00.000000000 Z
+date: 2017-11-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -39,6 +39,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.4'
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.7'
 - !ruby/object:Gem::Dependency
   name: googleauth
   requirement: !ruby/object:Gem::Requirement
@@ -229,12 +243,14 @@ files:
 - lib/kubernetes-deploy/kubernetes_resource/persistent_volume_claim.rb
 - lib/kubernetes-deploy/kubernetes_resource/pod.rb
 - lib/kubernetes-deploy/kubernetes_resource/pod_disruption_budget.rb
+- lib/kubernetes-deploy/kubernetes_resource/pod_set_base.rb
 - lib/kubernetes-deploy/kubernetes_resource/pod_template.rb
 - lib/kubernetes-deploy/kubernetes_resource/redis.rb
 - lib/kubernetes-deploy/kubernetes_resource/replica_set.rb
 - lib/kubernetes-deploy/kubernetes_resource/resource_quota.rb
 - lib/kubernetes-deploy/kubernetes_resource/service.rb
 - lib/kubernetes-deploy/kubernetes_resource/service_account.rb
+- lib/kubernetes-deploy/kubernetes_resource/stateful_set.rb
 - lib/kubernetes-deploy/kubernetes_resource/statefulservice.rb
 - lib/kubernetes-deploy/kubernetes_resource/topic.rb
 - lib/kubernetes-deploy/resource_watcher.rb