kubernetes-deploy 0.10.1 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cfefc3bd897ba4b3a56a3ad94309571bb2a5ba8f
-  data.tar.gz: 959b6beaf80885a427c3e35b8b18ddd897784bd6
+  metadata.gz: 1fcb28df3ae43f68d38b30177c5131f59c45c602
+  data.tar.gz: 26f3440846640400f726ef6fe9acc95460ddef3d
 SHA512:
-  metadata.gz: 867e36156d825450b7812e164d167facd2a6a4e150235eaec4bfe5ed1a8a1cb8e07a8ece2d74a5e82af240a93b833367599918104c1dffd8768e7d90c43724d9
-  data.tar.gz: abcc6c7d04d0cadfb4c63ffdb28ffe17adb639bdd362835a8a4c3a7badbf34b056e43ce8e31915032d925a9227e800a729493abbbd9b79882674141a978cc24d
+  metadata.gz: 6e6d10b5aea6ffacfacf466d7d3255e7f22709432ea49924cee5ef173f371b13b9f7d04cfb86b9182610e7cb3207e2f526fefca37cd864bc83ffdd1fc559869a
+  data.tar.gz: 7909b9c4475a6151b4b1d286601b49188d4797e0d0174f9526ade8afa67134ecfff07e793b42392d004c2eb145b075b986556dd7f35e1f812e9414737c89924f

data/lib/kubernetes-deploy.rb

@@ -15,6 +15,7 @@ require 'kubernetes-deploy/errors'
 require 'kubernetes-deploy/formatted_logger'
 require 'kubernetes-deploy/runner'
 require 'kubernetes-deploy/statsd'
+require 'kubernetes-deploy/concurrency'
 
 module KubernetesDeploy
   KubernetesDeploy::StatsD.build

data/lib/kubernetes-deploy/concurrency.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+module KubernetesDeploy
+  module Concurrency
+    MAX_THREADS = 8
+
+    def self.split_across_threads(all_work, &block)
+      return if all_work.empty?
+      raise ArgumentError, "Block of work is required" unless block_given?
+
+      slice_size = ((all_work.length + MAX_THREADS - 1) / MAX_THREADS)
+      threads = []
+      all_work.each_slice(slice_size) do |work_group|
+        threads << Thread.new { work_group.each(&block) }
+      end
+      threads.each(&:join)
+    end
+  end
+end

data/lib/kubernetes-deploy/kubernetes_resource.rb

@@ -6,7 +6,7 @@ require 'kubernetes-deploy/kubectl'
 
 module KubernetesDeploy
   class KubernetesResource
-    attr_reader :name, :namespace, :file, :context
+    attr_reader :name, :namespace, :file, :context, :validation_error_msg
     attr_writer :type, :deploy_started
 
     TIMEOUT = 5.minutes
@@ -55,6 +55,20 @@ module KubernetesDeploy
       @logger = logger
       @definition = definition
       @statsd_report_done = false
+      @validation_error_msg = nil
+    end
+
+    def validate_definition
+      @validation_error_msg = nil
+      command = ["create", "-f", file_path, "--dry-run", "--output=name"]
+      _, err, st = kubectl.run(*command, log_failure: false)
+      return true if st.success?
+      @validation_error_msg = err
+      false
+    end
+
+    def validation_failed?
+      @validation_error_msg.present?
     end
 
     def id

data/lib/kubernetes-deploy/kubernetes_resource/deployment.rb

@@ -14,7 +14,8 @@ module KubernetesDeploy
         @rollout_data = { "replicas" => 0 }.merge(deployment_data["status"]
           .slice("replicas", "updatedReplicas", "availableReplicas", "unavailableReplicas"))
         @status = @rollout_data.map { |state_replicas, num| "#{num} #{state_replicas.chop.pluralize(num)}" }.join(", ")
-        @progress = deployment_data["status"]["conditions"].find { |condition| condition['type'] == 'Progressing' }
+        conditions = deployment_data.fetch("status", {}).fetch("conditions", [])
+        @progress = conditions.find { |condition| condition['type'] == 'Progressing' }
       else # reset
         @latest_rs = nil
         @rollout_data = { "replicas" => 0 }

data/lib/kubernetes-deploy/resource_watcher.rb

@@ -21,7 +21,7 @@ module KubernetesDeploy
         end
         delay_sync_until = Time.now.utc + delay_sync # don't pummel the API if the sync is fast
 
-        @resources.each(&:sync)
+        KubernetesDeploy::Concurrency.split_across_threads(@resources, &:sync)
         newly_finished_resources, @resources = @resources.partition(&:deploy_finished?)
 
         if newly_finished_resources.present?

data/lib/kubernetes-deploy/runner.rb

@@ -20,7 +20,6 @@ require 'kubernetes-deploy/kubernetes_resource'
   pod_disruption_budget
   replica_set
   service_account
-  daemon_set
 ).each do |subresource|
   require "kubernetes-deploy/kubernetes_resource/#{subresource}"
 end
@@ -89,9 +88,10 @@ module KubernetesDeploy
       confirm_context_exists
       confirm_namespace_exists
       resources = discover_resources
+      validate_definitions(resources)
 
       @logger.phase_heading("Checking initial resource statuses")
-      resources.each(&:sync)
+      KubernetesDeploy::Concurrency.split_across_threads(resources, &:sync)
       resources.each { |r| @logger.info(r.pretty_status) }
 
       ejson = EjsonSecretProvisioner.new(
@@ -192,7 +192,7 @@ module KubernetesDeploy
       # stderr often contains one or more lines like the following, from which we can extract the file path(s):
       # Error from server (TypeOfError): error when creating "/path/to/service-gqq5oh.yml": Service "web" is invalid:
       matches = stderr.scan(%r{"(/\S+\.ya?ml\S*)"})
-      matches.flatten if matches
+      matches&.flatten
     end
 
     def deploy_has_priority_resources?(resources)
@@ -215,15 +215,26 @@ module KubernetesDeploy
       end
     end
 
+    def validate_definitions(resources)
+      KubernetesDeploy::Concurrency.split_across_threads(resources, &:validate_definition)
+      failed_resources = resources.select(&:validation_failed?)
+      return unless failed_resources.present?
+
+      failed_resources.each do |r|
+        record_invalid_template(r.validation_error_msg, file_paths: [r.file_path])
+      end
+      raise FatalDeploymentError, "Template validation failed"
+    end
+
     def discover_resources
       resources = []
       @logger.info("Discovering templates:")
+
       Dir.foreach(@template_dir) do |filename|
         next unless filename.end_with?(".yml.erb", ".yml", ".yaml", ".yaml.erb")
 
         split_templates(filename) do |r_def|
           r = KubernetesResource.build(namespace: @namespace, context: @context, logger: @logger, definition: r_def)
-          validate_template_via_dry_run(r.file_path, filename)
           resources << r
           @logger.info "  - #{r.id}"
         end
@@ -231,14 +242,6 @@ module KubernetesDeploy
       resources
     end
 
-    def validate_template_via_dry_run(file_path, original_filename)
-      command = ["create", "-f", file_path, "--dry-run", "--output=name"]
-      _, err, st = kubectl.run(*command, log_failure: false)
-      return if st.success?
-      record_invalid_template(err, file_paths: [file_path], original_filenames: [original_filename])
-      raise FatalDeploymentError, "Template validation failed (command: #{Shellwords.join(command)})"
-    end
-
     def split_templates(filename)
       file_content = File.read(File.join(@template_dir, filename))
       rendered_content = render_template(filename, file_content)
@@ -271,7 +274,6 @@ module KubernetesDeploy
       if file_content.present?
         debug_msg += "\n> Rendered template content:\n#{indent_four(file_content)}"
       end
-
       @logger.summary.add_paragraph(debug_msg)
     end
 

data/lib/kubernetes-deploy/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module KubernetesDeploy
-  VERSION = "0.10.1"
+  VERSION = "0.11.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: kubernetes-deploy
 version: !ruby/object:Gem::Version
-  version: 0.10.1
+  version: 0.11.0
 platform: ruby
 authors:
 - Katrina Verey
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-07-26 00:00:00.000000000 Z
+date: 2017-07-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -202,6 +202,7 @@ files:
 - exe/kubernetes-run
 - kubernetes-deploy.gemspec
 - lib/kubernetes-deploy.rb
+- lib/kubernetes-deploy/concurrency.rb
 - lib/kubernetes-deploy/deferred_summary_logging.rb
 - lib/kubernetes-deploy/ejson_secret_provisioner.rb
 - lib/kubernetes-deploy/errors.rb
@@ -213,7 +214,6 @@ files:
 - lib/kubernetes-deploy/kubernetes_resource/bugsnag.rb
 - lib/kubernetes-deploy/kubernetes_resource/cloudsql.rb
 - lib/kubernetes-deploy/kubernetes_resource/config_map.rb
-- lib/kubernetes-deploy/kubernetes_resource/daemon_set.rb
 - lib/kubernetes-deploy/kubernetes_resource/deployment.rb
 - lib/kubernetes-deploy/kubernetes_resource/ingress.rb
 - lib/kubernetes-deploy/kubernetes_resource/persistent_volume_claim.rb

data/lib/kubernetes-deploy/kubernetes_resource/daemon_set.rb

@@ -1,95 +0,0 @@
-# frozen_string_literal: true
-module KubernetesDeploy
-  class DaemonSet < KubernetesResource
-    TIMEOUT = 5.minutes
-
-    def sync
-      raw_json, _err, st = kubectl.run("get", type, @name, "--output=json")
-      @found = st.success?
-
-      if @found
-        daemonset_data = JSON.parse(raw_json)
-        @current_generation = daemonset_data["metadata"]["generation"]
-        @observed_generation = daemonset_data["status"]["observedGeneration"]
-        @rollout_data = daemonset_data["status"]
-          .slice("currentNumberScheduled", "desiredNumberScheduled", "numberReady", "numberAvailable")
-        @status = @rollout_data.map { |state_replicas, num| "#{num} #{state_replicas}" }.join(", ")
-        @pods = find_pods(daemonset_data)
-      else # reset
-        @rollout_data = { "currentNumberScheduled" => 0 }
-        @current_generation = 1 # to make sure the current and observed generations are different
-        @observed_generation = 0
-        @status = nil
-        @pods = []
-      end
-    end
-
-    def deploy_succeeded?
-      @rollout_data["desiredNumberScheduled"].to_i == @rollout_data["currentNumberScheduled"].to_i &&
-      @rollout_data["desiredNumberScheduled"].to_i == @rollout_data["numberAvailable"].to_i &&
-      @current_generation == @observed_generation
-    end
-
-    def deploy_failed?
-      @pods.present? && @pods.any?(&:deploy_failed?)
-    end
-
-    def failure_message
-      @pods.map(&:failure_message).compact.uniq.join("\n")
-    end
-
-    def timeout_message
-      @pods.map(&:timeout_message).compact.uniq.join("\n")
-    end
-
-    def deploy_timed_out?
-      super || @pods.present? && @pods.any?(&:deploy_timed_out?)
-    end
-
-    def exists?
-      @found
-    end
-
-    def fetch_events
-      own_events = super
-      return own_events unless @pods.present?
-      most_useful_pod = @pods.find(&:deploy_failed?) || @pods.find(&:deploy_timed_out?) || @pods.first
-      own_events.merge(most_useful_pod.fetch_events)
-    end
-
-    def fetch_logs
-      most_useful_pod = @pods.find(&:deploy_failed?) || @pods.find(&:deploy_timed_out?) || @pods.first
-      most_useful_pod.fetch_logs
-    end
-
-    private
-
-    def find_pods(ds_data)
-      label_string = ds_data["spec"]["selector"]["matchLabels"].map { |k, v| "#{k}=#{v}" }.join(",")
-      raw_json, _err, st = kubectl.run("get", "pods", "-a", "--output=json", "--selector=#{label_string}")
-      return [] unless st.success?
-
-      all_pods = JSON.parse(raw_json)["items"]
-      current_generation = ds_data["metadata"]["generation"]
-
-      latest_pods = all_pods.find_all do |pods|
-        pods["metadata"]["ownerReferences"].any? { |ref| ref["uid"] == ds_data["metadata"]["uid"] } &&
-        pods["metadata"]["labels"]["pod-template-generation"].to_i == current_generation.to_i
-      end
-      return unless latest_pods.present?
-
-      latest_pods.each_with_object([]) do |pod_data, relevant_pods|
-        pod = Pod.new(
-          namespace: namespace,
-          context: context,
-          definition: pod_data,
-          logger: @logger,
-          parent: "#{@name.capitalize} daemon set",
-          deploy_started: @deploy_started
-        )
-        pod.sync(pod_data)
-        relevant_pods << pod
-      end
-    end
-  end
-end