krane 1.1.0 → 2.0.0

data/lib/krane/kubernetes_resource/custom_resource.rb

@@ -62,7 +62,7 @@ module Krane
       kind
     end
 
-    def validate_definition(*)
+    def validate_definition(*, **)
       super
 
       @crd.validate_rollout_conditions
@@ -70,7 +70,7 @@ module Krane
       @validation_errors << "The CRD that specifies this resource is using invalid rollout conditions. " \
       "Krane will not be able to continue until those rollout conditions are fixed.\n" \
       "Rollout conditions can be found on the CRD that defines this resource (#{@crd.name}), " \
-      "under the annotation #{CustomResourceDefinition::ROLLOUT_CONDITIONS_ANNOTATION}.\n" \
+      "under the annotation #{Annotation.for(CustomResourceDefinition::ROLLOUT_CONDITIONS_ANNOTATION)}.\n" \
       "Validation failed with: #{e}"
     end
 

data/lib/krane/kubernetes_resource/custom_resource_definition.rb

@@ -2,9 +2,8 @@
 module Krane
   class CustomResourceDefinition < KubernetesResource
     TIMEOUT = 2.minutes
-    ROLLOUT_CONDITIONS_ANNOTATION_SUFFIX = "instance-rollout-conditions"
-    ROLLOUT_CONDITIONS_ANNOTATION = "krane.shopify.io/#{ROLLOUT_CONDITIONS_ANNOTATION_SUFFIX}"
-    TIMEOUT_FOR_INSTANCE_ANNOTATION = "krane.shopify.io/instance-timeout"
+    ROLLOUT_CONDITIONS_ANNOTATION = "instance-rollout-conditions"
+    TIMEOUT_FOR_INSTANCE_ANNOTATION = "instance-timeout"
     GLOBAL = true
 
     def deploy_succeeded?
@@ -20,7 +19,7 @@ module Krane
     end
 
     def timeout_for_instance
-      timeout = krane_annotation_value("instance-timeout")
+      timeout = krane_annotation_value(TIMEOUT_FOR_INSTANCE_ANNOTATION)
       DurationParser.new(timeout).parse!.to_i
     rescue DurationParser::ParsingError
       nil
@@ -46,6 +45,10 @@ module Krane
       @definition.dig("spec", "names", "kind")
     end
 
+    def group
+      @definition.dig("spec", "group")
+    end
+
     def prunable?
       prunable = krane_annotation_value("prunable")
       prunable == "true"
@@ -59,25 +62,25 @@ module Krane
     def rollout_conditions
       return @rollout_conditions if defined?(@rollout_conditions)
 
-      @rollout_conditions = if krane_annotation_value(ROLLOUT_CONDITIONS_ANNOTATION_SUFFIX)
-        RolloutConditions.from_annotation(krane_annotation_value(ROLLOUT_CONDITIONS_ANNOTATION_SUFFIX))
+      @rollout_conditions = if krane_annotation_value(ROLLOUT_CONDITIONS_ANNOTATION)
+        RolloutConditions.from_annotation(krane_annotation_value(ROLLOUT_CONDITIONS_ANNOTATION))
       end
     rescue RolloutConditionsError
       @rollout_conditions = nil
     end
 
-    def validate_definition(*)
+    def validate_definition(*, **)
       super
 
       validate_rollout_conditions
     rescue RolloutConditionsError => e
-      @validation_errors << "Annotation #{krane_annotation_key(ROLLOUT_CONDITIONS_ANNOTATION_SUFFIX)} "\
+      @validation_errors << "Annotation #{Annotation.for(ROLLOUT_CONDITIONS_ANNOTATION)} " \
         "on #{name} is invalid: #{e}"
     end
 
     def validate_rollout_conditions
-      if krane_annotation_value(ROLLOUT_CONDITIONS_ANNOTATION_SUFFIX) && @rollout_conditions_validated.nil?
-        conditions = RolloutConditions.from_annotation(krane_annotation_value(ROLLOUT_CONDITIONS_ANNOTATION_SUFFIX))
+      if krane_annotation_value(ROLLOUT_CONDITIONS_ANNOTATION) && @rollout_conditions_validated.nil?
+        conditions = RolloutConditions.from_annotation(krane_annotation_value(ROLLOUT_CONDITIONS_ANNOTATION))
         conditions.validate!
       end
 

data/lib/krane/kubernetes_resource/deployment.rb

@@ -5,9 +5,7 @@ module Krane
   class Deployment < KubernetesResource
     TIMEOUT = 7.minutes
     SYNC_DEPENDENCIES = %w(Pod ReplicaSet)
-    REQUIRED_ROLLOUT_ANNOTATION_SUFFIX = "required-rollout"
-    REQUIRED_ROLLOUT_ANNOTATION_DEPRECATED = "kubernetes-deploy.shopify.io/#{REQUIRED_ROLLOUT_ANNOTATION_SUFFIX}"
-    REQUIRED_ROLLOUT_ANNOTATION = "krane.shopify.io/#{REQUIRED_ROLLOUT_ANNOTATION_SUFFIX}"
+    REQUIRED_ROLLOUT_ANNOTATION = "required-rollout"
     REQUIRED_ROLLOUT_TYPES = %w(maxUnavailable full none).freeze
     DEFAULT_REQUIRED_ROLLOUT = 'full'
 
@@ -97,7 +95,7 @@ module Krane
       progress_condition.present? ? deploy_failing_to_progress? : super
     end
 
-    def validate_definition(*)
+    def validate_definition(*, **)
       super
 
       unless REQUIRED_ROLLOUT_TYPES.include?(required_rollout) || percent?(required_rollout)
@@ -106,7 +104,7 @@ module Krane
 
       strategy = @definition.dig('spec', 'strategy', 'type').to_s
       if required_rollout.downcase == 'maxunavailable' && strategy.present? && strategy.downcase != 'rollingupdate'
-        @validation_errors << "'#{krane_annotation_key(REQUIRED_ROLLOUT_ANNOTATION_SUFFIX)}: #{required_rollout}' "\
+        @validation_errors << "'#{Annotation.for(REQUIRED_ROLLOUT_ANNOTATION)}: #{required_rollout}' " \
           "is incompatible with strategy '#{strategy}'"
       end
 
@@ -151,7 +149,7 @@ module Krane
     end
 
     def rollout_annotation_err_msg
-      "'#{krane_annotation_key(REQUIRED_ROLLOUT_ANNOTATION_SUFFIX)}: #{required_rollout}' is invalid. "\
+      "'#{Annotation.for(REQUIRED_ROLLOUT_ANNOTATION)}: #{required_rollout}' is invalid. " \
         "Acceptable values: #{REQUIRED_ROLLOUT_TYPES.join(', ')}"
     end
 
@@ -191,7 +189,7 @@ module Krane
     def min_available_replicas
       if percent?(required_rollout)
         (desired_replicas * required_rollout.to_i / 100.0).ceil
-      elsif max_unavailable =~ /%/
+      elsif max_unavailable.is_a?(String) && max_unavailable =~ /%/
         (desired_replicas * (100 - max_unavailable.to_i) / 100.0).ceil
       else
         desired_replicas - max_unavailable.to_i

data/lib/krane/kubernetes_resource/pod.rb

@@ -219,14 +219,7 @@ module Krane
         limbo_reason = @status.dig("state", "waiting", "reason")
         limbo_message = @status.dig("state", "waiting", "message")
 
-        if @status.dig("lastState", "terminated", "reason") == "ContainerCannotRun"
-          # ref: https://github.com/kubernetes/kubernetes/blob/562e721ece8a16e05c7e7d6bdd6334c910733ab2/pkg/kubelet/dockershim/docker_container.go#L353
-          exit_code = @status.dig('lastState', 'terminated', 'exitCode')
-          "Failed to start (exit #{exit_code}): #{@status.dig('lastState', 'terminated', 'message')}"
-        elsif @status.dig("state", "terminated", "reason") == "ContainerCannotRun"
-          exit_code = @status.dig('state', 'terminated', 'exitCode')
-          "Failed to start (exit #{exit_code}): #{@status.dig('state', 'terminated', 'message')}"
-        elsif limbo_reason == "CrashLoopBackOff"
+        if limbo_reason == "CrashLoopBackOff"
           exit_code = @status.dig('lastState', 'terminated', 'exitCode')
           "Crashing repeatedly (exit #{exit_code}). See logs for more information."
         elsif limbo_reason == "ErrImagePull" && limbo_message.match(/not found/i)
@@ -234,6 +227,17 @@ module Krane
           "Did you wait for it to be built and pushed to the registry before deploying?"
         elsif limbo_reason == "CreateContainerConfigError"
           "Failed to generate container configuration: #{limbo_message}"
+        elsif @status.dig("lastState", "terminated", "reason") == "ContainerCannotRun"
+          # ref: https://github.com/kubernetes/kubernetes/blob/562e721ece8a16e05c7e7d6bdd6334c910733ab2/pkg/kubelet/dockershim/docker_container.go#L353
+          exit_code = @status.dig('lastState', 'terminated', 'exitCode')
+          # We've observed failures here that are actually issues with the node or kube infra, and not with the
+          # container. These issues have been transient and result in a 128 exit code, so do not treat these as fatal.
+          return if exit_code == 128
+          "Failed to start (exit #{exit_code}): #{@status.dig('lastState', 'terminated', 'message')}"
+        elsif @status.dig("state", "terminated", "reason") == "ContainerCannotRun"
+          exit_code = @status.dig('state', 'terminated', 'exitCode')
+          return if exit_code == 128
+          "Failed to start (exit #{exit_code}): #{@status.dig('state', 'terminated', 'message')}"
         end
       end
 

data/lib/krane/psych_k8s_compatibility.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'psych'
+
+module PsychK8sCompatibility
+  def self.massage_node(n)
+    if n.is_a?(Psych::Nodes::Scalar) &&
+        (n.style == Psych::Nodes::Scalar::PLAIN) &&
+        n.value.is_a?(String) &&
+        n.value =~ /\A[+-]?\d+(?:\.\d+)?[eE][+-]?\d+\z/
+      n.style = Psych::Nodes::Scalar::DOUBLE_QUOTED
+    end
+  end
+
+  refine Psych.singleton_class do
+    def dump(o, io = nil, options = {})
+      if io.is_a?(Hash)
+        options = io
+        io = nil
+      end
+      visitor = Psych::Visitors::YAMLTree.create(options)
+      visitor << o
+      visitor.tree.each { |n| PsychK8sCompatibility.massage_node(n) }
+      visitor.tree.yaml(io, options)
+    end
+
+    def dump_stream(*objects)
+      visitor = Psych::Visitors::YAMLTree.create({})
+      objects.each do |o|
+        visitor << o
+      end
+      visitor.tree.each { |n| PsychK8sCompatibility.massage_node(n) }
+      visitor.tree.yaml
+    end
+  end
+end

data/lib/krane/render_task.rb

@@ -24,8 +24,8 @@ module Krane
     # Runs the task, returning a boolean representing success or failure
     #
     # @return [Boolean]
-    def run(*args)
-      run!(*args)
+    def run(**args)
+      run!(**args)
       true
     rescue Krane::FatalDeploymentError
       false

data/lib/krane/renderer.rb

@@ -28,6 +28,8 @@ module Krane
         ENV["TASK_ID"]
       elsif current_sha
         current_sha[0...8] + "-#{SecureRandom.hex(4)}"
+      else
+        SecureRandom.hex(8)
       end
     end
 

data/lib/krane/resource_deployer.rb

@@ -46,8 +46,13 @@ module Krane
         bare_pods.first.stream_logs = true
       end
 
-      predeploy_sequence.each do |resource_type|
-        matching_resources = resource_list.select { |r| r.type == resource_type }
+      predeploy_sequence.each do |resource_type, attributes|
+        matching_resources = resource_list.select do |r|
+          r.type == resource_type &&
+          (!attributes[:group] || r.group == attributes[:group])
+        end
+        StatsD.client.gauge('priority_resources.count', matching_resources.size, tags: statsd_tags)
+
         next if matching_resources.empty?
         deploy_resources(matching_resources, verify: true, record_summary: false)
 

data/lib/krane/resource_watcher.rb

@@ -68,7 +68,7 @@ module Krane
     end
 
     def global_timeout?(started_at)
-      @timeout && (Time.now.utc - started_at > @timeout)
+      @timeout && (Time.now.utc - started_at >= @timeout)
     end
 
     def sleep_until_next_sync(min_interval)

data/lib/krane/restart_task.rb

@@ -39,8 +39,8 @@ module Krane
     # Runs the task, returning a boolean representing success or failure
     #
     # @return [Boolean]
-    def run(*args)
-      perform!(*args)
+    def run(**args)
+      perform!(**args)
       true
     rescue FatalDeploymentError
       false

data/lib/krane/runner_task.rb

@@ -9,6 +9,7 @@ require 'krane/resource_watcher'
 require 'krane/kubernetes_resource'
 require 'krane/kubernetes_resource/pod'
 require 'krane/runner_task_config_validator'
+require 'krane/container_overrides'
 
 module Krane
   # Run a pod that exits upon completing a task
@@ -34,8 +35,8 @@ module Krane
     # Runs the task, returning a boolean representing success or failure
     #
     # @return [Boolean]
-    def run(*args)
-      run!(*args)
+    def run(**args)
+      run!(**args)
       true
     rescue DeploymentTimeoutError, FatalDeploymentError
       false
@@ -50,7 +51,7 @@ module Krane
     # @param verify_result [Boolean] Wait for completion and verify pod success
     #
     # @return [nil]
-    def run!(template:, command:, arguments:, env_vars: [], verify_result: true)
+    def run!(template:, command:, arguments:, env_vars: [], image_tag: nil, verify_result: true)
       start = Time.now.utc
       @logger.reset
 
@@ -59,8 +60,13 @@ module Krane
       @logger.info("Validating configuration")
       verify_config!(template)
       @logger.info("Using namespace '#{@namespace}' in context '#{@context}'")
-
-      pod = build_pod(template, command, arguments, env_vars, verify_result)
+      container_overrides = ContainerOverrides.new(
+        command: command,
+        arguments: arguments,
+        env_vars: env_vars,
+        image_tag: image_tag
+      )
+      pod = build_pod(template, container_overrides, verify_result)
       validate_pod(pod)
 
       @logger.phase_heading("Running pod")
@@ -98,11 +104,12 @@ module Krane
       raise FatalDeploymentError, msg
     end
 
-    def build_pod(template_name, command, args, env_vars, verify_result)
+    def build_pod(template_name, container_overrides, verify_result)
       task_template = get_template(template_name)
       @logger.info("Using template '#{template_name}'")
       pod_template = build_pod_definition(task_template)
-      set_container_overrides!(pod_template, command, args, env_vars)
+      container = extract_task_runner_container(pod_template)
+      container_overrides.apply!(container)
       ensure_valid_restart_policy!(pod_template, verify_result)
       Pod.new(namespace: @namespace, context: @context, logger: @logger, stream_logs: true,
                     definition: pod_template.to_hash.deep_stringify_keys, statsd_tags: [])
@@ -165,7 +172,7 @@ module Krane
       pod_definition
     end
 
-    def set_container_overrides!(pod_definition, command, args, env_vars)
+    def extract_task_runner_container(pod_definition)
       container = pod_definition.spec.containers.find { |cont| cont.name == 'task-runner' }
       if container.nil?
         message = "Pod spec does not contain a template container called 'task-runner'"
@@ -173,15 +180,7 @@ module Krane
         raise TaskConfigurationError, message
       end
 
-      container.command = command if command
-      container.args = args if args
-
-      env_args = env_vars.map do |env|
-        key, value = env.split('=', 2)
-        { name: key, value: value }
-      end
-      container.env ||= []
-      container.env = container.env.map(&:to_h) + env_args
+      container
     end
 
     def ensure_valid_restart_policy!(template, verify)

data/lib/krane/statsd.rb

@@ -35,10 +35,10 @@ module Krane
         end
 
         metric ||= "#{method_name}.duration"
-        self::InstrumentationProxy.send(:define_method, method_name) do |*args, &block|
+        self::InstrumentationProxy.send(:define_method, method_name) do |*args, **kwargs, &block|
           begin
             start_time = Time.now.utc
-            super(*args, &block)
+            super(*args, **kwargs, &block)
           rescue
             error = true
             raise

data/lib/krane/template_sets.rb

@@ -150,7 +150,7 @@ module Krane
 
       if rendering_erb_disabled? && deploying_with_erb_files?
         errors << "ERB template discovered with rendering disabled. If you were trying to render ERB and " \
-          "deploy the result, try piping the output of `krane render` to `krane-deploy` with the --stdin flag"
+          "deploy the result, try piping the output of `krane render` to `krane-deploy -f -`"
       end
 
       errors

data/lib/krane/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Krane
-  VERSION = "1.1.0"
+  VERSION = "2.0.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: krane
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 2.0.0
 platform: ruby
 authors:
 - Katrina Verey
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-12-12 00:00:00.000000000 Z
+date: 2020-08-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -46,14 +46,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.0
+        version: '0.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.0
+        version: '0.8'
 - !ruby/object:Gem::Dependency
   name: ejson
   requirement: !ruby/object:Gem::Requirement
@@ -148,16 +148,22 @@ dependencies:
   name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 0.20.3
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.20.3
+        version: '1.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -206,14 +212,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '5.12'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '5.12'
 - !ruby/object:Gem::Dependency
   name: minitest-stub-const
   requirement: !ruby/object:Gem::Requirement
@@ -360,14 +366,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.76.0
+        version: 0.88.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.76.0
+        version: 0.88.0
 - !ruby/object:Gem::Dependency
   name: codecov
   requirement: !ruby/object:Gem::Requirement
@@ -392,7 +398,10 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".buildkite/pipeline.nightly.yml"
+- ".github/CODEOWNERS"
+- ".github/ISSUE_TEMPLATE.md"
 - ".github/probots.yml"
+- ".github/pull_request_template.md"
 - ".gitignore"
 - ".rubocop-http---shopify-github-io-ruby-style-guide-rubocop-yml"
 - ".rubocop.yml"
@@ -403,7 +412,6 @@ files:
 - CODE_OF_CONDUCT.md
 - CONTRIBUTING.md
 - Gemfile
-- ISSUE_TEMPLATE.md
 - LICENSE.txt
 - README.md
 - Rakefile
@@ -415,6 +423,7 @@ files:
 - exe/krane
 - krane.gemspec
 - lib/krane.rb
+- lib/krane/annotation.rb
 - lib/krane/bindings_parser.rb
 - lib/krane/cli/deploy_command.rb
 - lib/krane/cli/global_deploy_command.rb
@@ -428,6 +437,7 @@ files:
 - lib/krane/concerns/template_reporting.rb
 - lib/krane/concurrency.rb
 - lib/krane/container_logs.rb
+- lib/krane/container_overrides.rb
 - lib/krane/deferred_summary_logging.rb
 - lib/krane/delayed_exceptions.rb
 - lib/krane/deploy_task.rb
@@ -441,7 +451,6 @@ files:
 - lib/krane/kubeclient_builder.rb
 - lib/krane/kubectl.rb
 - lib/krane/kubernetes_resource.rb
-- lib/krane/kubernetes_resource/cloudsql.rb
 - lib/krane/kubernetes_resource/config_map.rb
 - lib/krane/kubernetes_resource/cron_job.rb
 - lib/krane/kubernetes_resource/custom_resource.rb
@@ -468,6 +477,7 @@ files:
 - lib/krane/label_selector.rb
 - lib/krane/oj.rb
 - lib/krane/options_helper.rb
+- lib/krane/psych_k8s_compatibility.rb
 - lib/krane/remote_logs.rb
 - lib/krane/render_task.rb
 - lib/krane/renderer.rb
@@ -483,7 +493,6 @@ files:
 - lib/krane/task_config_validator.rb
 - lib/krane/template_sets.rb
 - lib/krane/version.rb
-- pull_request_template.md
 - screenshots/deploy-demo.gif
 - screenshots/migrate-logs.png
 - screenshots/missing-secret-fail.png
@@ -492,7 +501,8 @@ files:
 homepage: https://github.com/Shopify/krane
 licenses:
 - MIT
-metadata: {}
+metadata:
+  allowed_push_host: https://rubygems.org
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -501,7 +511,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.4.0
+      version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="