krane 1.1.2 → 2.1.1

data/lib/krane/psych_k8s_compatibility.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'psych'
+
+module PsychK8sCompatibility
+  def self.massage_node(n)
+    if n.is_a?(Psych::Nodes::Scalar) &&
+        (n.style == Psych::Nodes::Scalar::PLAIN) &&
+        n.value.is_a?(String) &&
+        n.value =~ /\A[+-]?\d+(?:\.\d+)?[eE][+-]?\d+\z/
+      n.style = Psych::Nodes::Scalar::DOUBLE_QUOTED
+    end
+  end
+
+  refine Psych.singleton_class do
+    def dump(o, io = nil, options = {})
+      if io.is_a?(Hash)
+        options = io
+        io = nil
+      end
+      visitor = Psych::Visitors::YAMLTree.create(options)
+      visitor << o
+      visitor.tree.each { |n| PsychK8sCompatibility.massage_node(n) }
+      visitor.tree.yaml(io, options)
+    end
+
+    def dump_stream(*objects)
+      visitor = Psych::Visitors::YAMLTree.create({})
+      objects.each do |o|
+        visitor << o
+      end
+      visitor.tree.each { |n| PsychK8sCompatibility.massage_node(n) }
+      visitor.tree.yaml
+    end
+  end
+end

data/lib/krane/render_task.rb

@@ -24,8 +24,8 @@ module Krane
     # Runs the task, returning a boolean representing success or failure
     #
     # @return [Boolean]
-    def run(*args)
-      run!(*args)
+    def run(**args)
+      run!(**args)
       true
     rescue Krane::FatalDeploymentError
       false

data/lib/krane/renderer.rb

@@ -28,6 +28,8 @@ module Krane
         ENV["TASK_ID"]
       elsif current_sha
         current_sha[0...8] + "-#{SecureRandom.hex(4)}"
+      else
+        SecureRandom.hex(8)
       end
     end
 

data/lib/krane/resource_deployer.rb

@@ -46,8 +46,13 @@ module Krane
         bare_pods.first.stream_logs = true
       end
 
-      predeploy_sequence.each do |resource_type|
-        matching_resources = resource_list.select { |r| r.type == resource_type }
+      predeploy_sequence.each do |resource_type, attributes|
+        matching_resources = resource_list.select do |r|
+          r.type == resource_type &&
+          (!attributes[:group] || r.group == attributes[:group])
+        end
+        StatsD.client.gauge('priority_resources.count', matching_resources.size, tags: statsd_tags)
+
         next if matching_resources.empty?
         deploy_resources(matching_resources, verify: true, record_summary: false)
 
@@ -90,11 +95,10 @@ module Krane
       applyables, individuals = resources.partition { |r| r.deploy_method == :apply }
       # Prunable resources should also applied so that they can  be pruned
       pruneable_types = @prune_whitelist.map { |t| t.split("/").last }
-      applyables += individuals.select { |r| pruneable_types.include?(r.type) }
+      applyables += individuals.select { |r| pruneable_types.include?(r.type) && !r.deploy_method_override }
 
       individuals.each do |individual_resource|
         individual_resource.deploy_started_at = Time.now.utc
-
         case individual_resource.deploy_method
         when :create
           err, status = create_resource(individual_resource)
@@ -148,7 +152,7 @@ module Krane
         output_is_sensitive = resources.any?(&:sensitive_template_content?)
         global_mode = resources.all?(&:global?)
         out, err, st = kubectl.run(*command, log_failure: false, output_is_sensitive: output_is_sensitive,
-          use_namespace: !global_mode)
+          attempts: 2, use_namespace: !global_mode)
 
         if st.success?
           log_pruning(out) if prune

data/lib/krane/restart_task.rb

@@ -22,15 +22,19 @@ module Krane
     HTTP_OK_RANGE = 200..299
     ANNOTATION = "shipit.shopify.io/restart"
 
+    attr_reader :task_config
+
+    delegate :kubeclient_builder, to: :task_config
+
     # Initializes the restart task
     #
     # @param context [String] Kubernetes context / cluster (*required*)
     # @param namespace [String] Kubernetes namespace (*required*)
     # @param logger [Object] Logger object (defaults to an instance of Krane::FormattedLogger)
     # @param global_timeout [Integer] Timeout in seconds
-    def initialize(context:, namespace:, logger: nil, global_timeout: nil)
+    def initialize(context:, namespace:, logger: nil, global_timeout: nil, kubeconfig: nil)
       @logger = logger || Krane::FormattedLogger.build(namespace, context)
-      @task_config = Krane::TaskConfig.new(context, namespace, @logger)
+      @task_config = Krane::TaskConfig.new(context, namespace, @logger, kubeconfig)
       @context = context
       @namespace = namespace
       @global_timeout = global_timeout
@@ -39,8 +43,8 @@ module Krane
     # Runs the task, returning a boolean representing success or failure
     #
     # @return [Boolean]
-    def run(*args)
-      perform!(*args)
+    def run(**args)
+      perform!(**args)
       true
     rescue FatalDeploymentError
       false
@@ -220,9 +224,5 @@ module Krane
     def v1beta1_kubeclient
       @v1beta1_kubeclient ||= kubeclient_builder.build_v1beta1_kubeclient(@context)
     end
-
-    def kubeclient_builder
-      @kubeclient_builder ||= KubeclientBuilder.new
-    end
   end
 end

data/lib/krane/runner_task.rb

@@ -9,13 +9,16 @@ require 'krane/resource_watcher'
 require 'krane/kubernetes_resource'
 require 'krane/kubernetes_resource/pod'
 require 'krane/runner_task_config_validator'
+require 'krane/container_overrides'
 
 module Krane
   # Run a pod that exits upon completing a task
   class RunnerTask
     class TaskTemplateMissingError < TaskConfigurationError; end
 
-    attr_reader :pod_name
+    attr_reader :pod_name, :task_config
+
+    delegate :kubeclient_builder, to: :task_config
 
     # Initializes the runner task
     #
@@ -23,9 +26,9 @@ module Krane
     # @param context [String] Kubernetes context / cluster (*required*)
     # @param logger [Object] Logger object (defaults to an instance of Krane::FormattedLogger)
     # @param global_timeout [Integer] Timeout in seconds
-    def initialize(namespace:, context:, logger: nil, global_timeout: nil)
+    def initialize(namespace:, context:, logger: nil, global_timeout: nil, kubeconfig: nil)
       @logger = logger || Krane::FormattedLogger.build(namespace, context)
-      @task_config = Krane::TaskConfig.new(context, namespace, @logger)
+      @task_config = Krane::TaskConfig.new(context, namespace, @logger, kubeconfig)
       @namespace = namespace
       @context = context
       @global_timeout = global_timeout
@@ -34,8 +37,8 @@ module Krane
     # Runs the task, returning a boolean representing success or failure
     #
     # @return [Boolean]
-    def run(*args)
-      run!(*args)
+    def run(**args)
+      run!(**args)
       true
     rescue DeploymentTimeoutError, FatalDeploymentError
       false
@@ -50,7 +53,7 @@ module Krane
     # @param verify_result [Boolean] Wait for completion and verify pod success
     #
     # @return [nil]
-    def run!(template:, command:, arguments:, env_vars: [], verify_result: true)
+    def run!(template:, command:, arguments:, env_vars: [], image_tag: nil, verify_result: true)
       start = Time.now.utc
       @logger.reset
 
@@ -59,8 +62,13 @@ module Krane
       @logger.info("Validating configuration")
       verify_config!(template)
       @logger.info("Using namespace '#{@namespace}' in context '#{@context}'")
-
-      pod = build_pod(template, command, arguments, env_vars, verify_result)
+      container_overrides = ContainerOverrides.new(
+        command: command,
+        arguments: arguments,
+        env_vars: env_vars,
+        image_tag: image_tag
+      )
+      pod = build_pod(template, container_overrides, verify_result)
       validate_pod(pod)
 
       @logger.phase_heading("Running pod")
@@ -98,11 +106,12 @@ module Krane
       raise FatalDeploymentError, msg
     end
 
-    def build_pod(template_name, command, args, env_vars, verify_result)
+    def build_pod(template_name, container_overrides, verify_result)
       task_template = get_template(template_name)
       @logger.info("Using template '#{template_name}'")
       pod_template = build_pod_definition(task_template)
-      set_container_overrides!(pod_template, command, args, env_vars)
+      container = extract_task_runner_container(pod_template)
+      container_overrides.apply!(container)
       ensure_valid_restart_policy!(pod_template, verify_result)
       Pod.new(namespace: @namespace, context: @context, logger: @logger, stream_logs: true,
                     definition: pod_template.to_hash.deep_stringify_keys, statsd_tags: [])
@@ -165,7 +174,7 @@ module Krane
       pod_definition
     end
 
-    def set_container_overrides!(pod_definition, command, args, env_vars)
+    def extract_task_runner_container(pod_definition)
       container = pod_definition.spec.containers.find { |cont| cont.name == 'task-runner' }
       if container.nil?
         message = "Pod spec does not contain a template container called 'task-runner'"
@@ -173,15 +182,7 @@ module Krane
         raise TaskConfigurationError, message
       end
 
-      container.command = command if command
-      container.args = args if args
-
-      env_args = env_vars.map do |env|
-        key, value = env.split('=', 2)
-        { name: key, value: value }
-      end
-      container.env ||= []
-      container.env = container.env.map(&:to_h) + env_args
+      container
     end
 
     def ensure_valid_restart_policy!(template, verify)
@@ -201,10 +202,6 @@ module Krane
       @kubeclient ||= kubeclient_builder.build_v1_kubeclient(@context)
     end
 
-    def kubeclient_builder
-      @kubeclient_builder ||= KubeclientBuilder.new
-    end
-
     def statsd_tags(status)
       %W(namespace:#{@namespace} context:#{@context} status:#{status})
     end

data/lib/krane/statsd.rb

@@ -35,10 +35,10 @@ module Krane
         end
 
         metric ||= "#{method_name}.duration"
-        self::InstrumentationProxy.send(:define_method, method_name) do |*args, &block|
+        self::InstrumentationProxy.send(:define_method, method_name) do |*args, **kwargs, &block|
           begin
             start_time = Time.now.utc
-            super(*args, &block)
+            super(*args, **kwargs, &block)
           rescue
             error = true
             raise

data/lib/krane/task_config.rb

@@ -4,12 +4,13 @@ require 'krane/cluster_resource_discovery'
 
 module Krane
   class TaskConfig
-    attr_reader :context, :namespace, :logger
+    attr_reader :context, :namespace, :logger, :kubeconfig
 
-    def initialize(context, namespace, logger = nil)
+    def initialize(context, namespace, logger = nil, kubeconfig = nil)
       @context = context
       @namespace = namespace
       @logger = logger || FormattedLogger.build(@namespace, @context)
+      @kubeconfig = kubeconfig || ENV['KUBECONFIG']
     end
 
     def global_kinds
@@ -18,5 +19,9 @@ module Krane
         cluster_resource_discoverer.fetch_resources(namespaced: false).map { |g| g["kind"] }
       end
     end
+
+    def kubeclient_builder
+      @kubeclient_builder ||= KubeclientBuilder.new(kubeconfig: kubeconfig)
+    end
   end
 end

data/lib/krane/task_config_validator.rb

@@ -45,7 +45,7 @@ module Krane
       end
 
       _, err, st = @kubectl.run("config", "get-contexts", context, "-o", "name",
-        use_namespace: false, use_context: false, log_failure: false)
+        use_namespace: false, use_context: false, log_failure: false, attempts: 2)
 
       unless st.success?
         @errors << if err.match("error: context #{context} not found")
@@ -58,7 +58,7 @@ module Krane
 
     def validate_context_reachable
       _, err, st = @kubectl.run("get", "namespaces", "-o", "name",
-        use_namespace: false, log_failure: false)
+        use_namespace: false, log_failure: false, attempts: 2)
 
       unless st.success?
         @errors << "Something went wrong connecting to #{context}. #{err} "
@@ -71,7 +71,7 @@ module Krane
       end
 
       _, err, st = @kubectl.run("get", "namespace", "-o", "name", namespace,
-        use_namespace: false, log_failure: false)
+        use_namespace: false, log_failure: false, attempts: 3)
 
       unless st.success?
         @errors << if err.match("Error from server [(]NotFound[)]: namespace")

data/lib/krane/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Krane
-  VERSION = "1.1.2"
+  VERSION = "2.1.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: krane
 version: !ruby/object:Gem::Version
-  version: 1.1.2
+  version: 2.1.1
 platform: ruby
 authors:
 - Katrina Verey
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-02-25 00:00:00.000000000 Z
+date: 2020-10-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -366,14 +366,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.78.0
+        version: 0.89.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.78.0
+        version: 0.89.1
 - !ruby/object:Gem::Dependency
   name: codecov
   requirement: !ruby/object:Gem::Requirement
@@ -423,6 +423,7 @@ files:
 - exe/krane
 - krane.gemspec
 - lib/krane.rb
+- lib/krane/annotation.rb
 - lib/krane/bindings_parser.rb
 - lib/krane/cli/deploy_command.rb
 - lib/krane/cli/global_deploy_command.rb
@@ -436,6 +437,7 @@ files:
 - lib/krane/concerns/template_reporting.rb
 - lib/krane/concurrency.rb
 - lib/krane/container_logs.rb
+- lib/krane/container_overrides.rb
 - lib/krane/deferred_summary_logging.rb
 - lib/krane/delayed_exceptions.rb
 - lib/krane/deploy_task.rb
@@ -475,6 +477,7 @@ files:
 - lib/krane/label_selector.rb
 - lib/krane/oj.rb
 - lib/krane/options_helper.rb
+- lib/krane/psych_k8s_compatibility.rb
 - lib/krane/remote_logs.rb
 - lib/krane/render_task.rb
 - lib/krane/renderer.rb
@@ -498,7 +501,8 @@ files:
 homepage: https://github.com/Shopify/krane
 licenses:
 - MIT
-metadata: {}
+metadata:
+  allowed_push_host: https://rubygems.org
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -507,7 +511,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.4.0
+      version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="