krane 1.1.1 → 2.1.0

data/lib/krane/renderer.rb

@@ -28,6 +28,8 @@ module Krane
         ENV["TASK_ID"]
       elsif current_sha
         current_sha[0...8] + "-#{SecureRandom.hex(4)}"
+      else
+        SecureRandom.hex(8)
       end
     end
 

data/lib/krane/resource_deployer.rb

@@ -46,8 +46,13 @@ module Krane
         bare_pods.first.stream_logs = true
       end
 
-      predeploy_sequence.each do |resource_type|
-        matching_resources = resource_list.select { |r| r.type == resource_type }
+      predeploy_sequence.each do |resource_type, attributes|
+        matching_resources = resource_list.select do |r|
+          r.type == resource_type &&
+          (!attributes[:group] || r.group == attributes[:group])
+        end
+        StatsD.client.gauge('priority_resources.count', matching_resources.size, tags: statsd_tags)
+
         next if matching_resources.empty?
         deploy_resources(matching_resources, verify: true, record_summary: false)
 
@@ -90,11 +95,10 @@ module Krane
       applyables, individuals = resources.partition { |r| r.deploy_method == :apply }
       # Prunable resources should also applied so that they can  be pruned
       pruneable_types = @prune_whitelist.map { |t| t.split("/").last }
-      applyables += individuals.select { |r| pruneable_types.include?(r.type) }
+      applyables += individuals.select { |r| pruneable_types.include?(r.type) && !r.deploy_method_override }
 
       individuals.each do |individual_resource|
         individual_resource.deploy_started_at = Time.now.utc
-
         case individual_resource.deploy_method
         when :create
           err, status = create_resource(individual_resource)
@@ -148,7 +152,7 @@ module Krane
         output_is_sensitive = resources.any?(&:sensitive_template_content?)
         global_mode = resources.all?(&:global?)
         out, err, st = kubectl.run(*command, log_failure: false, output_is_sensitive: output_is_sensitive,
-          use_namespace: !global_mode)
+          attempts: 2, use_namespace: !global_mode)
 
         if st.success?
           log_pruning(out) if prune

data/lib/krane/resource_watcher.rb

@@ -68,7 +68,7 @@ module Krane
     end
 
     def global_timeout?(started_at)
-      @timeout && (Time.now.utc - started_at > @timeout)
+      @timeout && (Time.now.utc - started_at >= @timeout)
     end
 
     def sleep_until_next_sync(min_interval)

data/lib/krane/restart_task.rb

@@ -22,15 +22,19 @@ module Krane
     HTTP_OK_RANGE = 200..299
     ANNOTATION = "shipit.shopify.io/restart"
 
+    attr_reader :task_config
+
+    delegate :kubeclient_builder, to: :task_config
+
     # Initializes the restart task
     #
     # @param context [String] Kubernetes context / cluster (*required*)
     # @param namespace [String] Kubernetes namespace (*required*)
     # @param logger [Object] Logger object (defaults to an instance of Krane::FormattedLogger)
     # @param global_timeout [Integer] Timeout in seconds
-    def initialize(context:, namespace:, logger: nil, global_timeout: nil)
+    def initialize(context:, namespace:, logger: nil, global_timeout: nil, kubeconfig: nil)
       @logger = logger || Krane::FormattedLogger.build(namespace, context)
-      @task_config = Krane::TaskConfig.new(context, namespace, @logger)
+      @task_config = Krane::TaskConfig.new(context, namespace, @logger, kubeconfig)
       @context = context
       @namespace = namespace
       @global_timeout = global_timeout
@@ -39,8 +43,8 @@ module Krane
     # Runs the task, returning a boolean representing success or failure
     #
     # @return [Boolean]
-    def run(*args)
-      perform!(*args)
+    def run(**args)
+      perform!(**args)
       true
     rescue FatalDeploymentError
       false
@@ -220,9 +224,5 @@ module Krane
     def v1beta1_kubeclient
       @v1beta1_kubeclient ||= kubeclient_builder.build_v1beta1_kubeclient(@context)
     end
-
-    def kubeclient_builder
-      @kubeclient_builder ||= KubeclientBuilder.new
-    end
   end
 end

data/lib/krane/runner_task.rb

@@ -9,13 +9,16 @@ require 'krane/resource_watcher'
 require 'krane/kubernetes_resource'
 require 'krane/kubernetes_resource/pod'
 require 'krane/runner_task_config_validator'
+require 'krane/container_overrides'
 
 module Krane
   # Run a pod that exits upon completing a task
   class RunnerTask
     class TaskTemplateMissingError < TaskConfigurationError; end
 
-    attr_reader :pod_name
+    attr_reader :pod_name, :task_config
+
+    delegate :kubeclient_builder, to: :task_config
 
     # Initializes the runner task
     #
@@ -23,9 +26,9 @@ module Krane
     # @param context [String] Kubernetes context / cluster (*required*)
     # @param logger [Object] Logger object (defaults to an instance of Krane::FormattedLogger)
     # @param global_timeout [Integer] Timeout in seconds
-    def initialize(namespace:, context:, logger: nil, global_timeout: nil)
+    def initialize(namespace:, context:, logger: nil, global_timeout: nil, kubeconfig: nil)
       @logger = logger || Krane::FormattedLogger.build(namespace, context)
-      @task_config = Krane::TaskConfig.new(context, namespace, @logger)
+      @task_config = Krane::TaskConfig.new(context, namespace, @logger, kubeconfig)
       @namespace = namespace
       @context = context
       @global_timeout = global_timeout
@@ -34,8 +37,8 @@ module Krane
     # Runs the task, returning a boolean representing success or failure
     #
     # @return [Boolean]
-    def run(*args)
-      run!(*args)
+    def run(**args)
+      run!(**args)
       true
     rescue DeploymentTimeoutError, FatalDeploymentError
       false
@@ -50,7 +53,7 @@ module Krane
     # @param verify_result [Boolean] Wait for completion and verify pod success
     #
     # @return [nil]
-    def run!(template:, command:, arguments:, env_vars: [], verify_result: true)
+    def run!(template:, command:, arguments:, env_vars: [], image_tag: nil, verify_result: true)
       start = Time.now.utc
       @logger.reset
 
@@ -59,8 +62,13 @@ module Krane
       @logger.info("Validating configuration")
       verify_config!(template)
       @logger.info("Using namespace '#{@namespace}' in context '#{@context}'")
-
-      pod = build_pod(template, command, arguments, env_vars, verify_result)
+      container_overrides = ContainerOverrides.new(
+        command: command,
+        arguments: arguments,
+        env_vars: env_vars,
+        image_tag: image_tag
+      )
+      pod = build_pod(template, container_overrides, verify_result)
       validate_pod(pod)
 
       @logger.phase_heading("Running pod")
@@ -98,11 +106,12 @@ module Krane
       raise FatalDeploymentError, msg
     end
 
-    def build_pod(template_name, command, args, env_vars, verify_result)
+    def build_pod(template_name, container_overrides, verify_result)
       task_template = get_template(template_name)
       @logger.info("Using template '#{template_name}'")
       pod_template = build_pod_definition(task_template)
-      set_container_overrides!(pod_template, command, args, env_vars)
+      container = extract_task_runner_container(pod_template)
+      container_overrides.apply!(container)
       ensure_valid_restart_policy!(pod_template, verify_result)
       Pod.new(namespace: @namespace, context: @context, logger: @logger, stream_logs: true,
                     definition: pod_template.to_hash.deep_stringify_keys, statsd_tags: [])
@@ -165,7 +174,7 @@ module Krane
       pod_definition
     end
 
-    def set_container_overrides!(pod_definition, command, args, env_vars)
+    def extract_task_runner_container(pod_definition)
       container = pod_definition.spec.containers.find { |cont| cont.name == 'task-runner' }
       if container.nil?
         message = "Pod spec does not contain a template container called 'task-runner'"
@@ -173,15 +182,7 @@ module Krane
         raise TaskConfigurationError, message
       end
 
-      container.command = command if command
-      container.args = args if args
-
-      env_args = env_vars.map do |env|
-        key, value = env.split('=', 2)
-        { name: key, value: value }
-      end
-      container.env ||= []
-      container.env = container.env.map(&:to_h) + env_args
+      container
     end
 
     def ensure_valid_restart_policy!(template, verify)
@@ -201,10 +202,6 @@ module Krane
       @kubeclient ||= kubeclient_builder.build_v1_kubeclient(@context)
     end
 
-    def kubeclient_builder
-      @kubeclient_builder ||= KubeclientBuilder.new
-    end
-
     def statsd_tags(status)
       %W(namespace:#{@namespace} context:#{@context} status:#{status})
     end

data/lib/krane/statsd.rb

@@ -35,10 +35,10 @@ module Krane
         end
 
         metric ||= "#{method_name}.duration"
-        self::InstrumentationProxy.send(:define_method, method_name) do |*args, &block|
+        self::InstrumentationProxy.send(:define_method, method_name) do |*args, **kwargs, &block|
           begin
             start_time = Time.now.utc
-            super(*args, &block)
+            super(*args, **kwargs, &block)
           rescue
             error = true
             raise

data/lib/krane/task_config.rb

@@ -4,12 +4,13 @@ require 'krane/cluster_resource_discovery'
 
 module Krane
   class TaskConfig
-    attr_reader :context, :namespace, :logger
+    attr_reader :context, :namespace, :logger, :kubeconfig
 
-    def initialize(context, namespace, logger = nil)
+    def initialize(context, namespace, logger = nil, kubeconfig = nil)
       @context = context
       @namespace = namespace
       @logger = logger || FormattedLogger.build(@namespace, @context)
+      @kubeconfig = kubeconfig || ENV['KUBECONFIG']
     end
 
     def global_kinds
@@ -18,5 +19,9 @@ module Krane
         cluster_resource_discoverer.fetch_resources(namespaced: false).map { |g| g["kind"] }
       end
     end
+
+    def kubeclient_builder
+      @kubeclient_builder ||= KubeclientBuilder.new(kubeconfig: kubeconfig)
+    end
   end
 end

data/lib/krane/task_config_validator.rb

@@ -45,7 +45,7 @@ module Krane
       end
 
       _, err, st = @kubectl.run("config", "get-contexts", context, "-o", "name",
-        use_namespace: false, use_context: false, log_failure: false)
+        use_namespace: false, use_context: false, log_failure: false, attempts: 2)
 
       unless st.success?
         @errors << if err.match("error: context #{context} not found")
@@ -58,7 +58,7 @@ module Krane
 
     def validate_context_reachable
       _, err, st = @kubectl.run("get", "namespaces", "-o", "name",
-        use_namespace: false, log_failure: false)
+        use_namespace: false, log_failure: false, attempts: 2)
 
       unless st.success?
         @errors << "Something went wrong connecting to #{context}. #{err} "
@@ -71,7 +71,7 @@ module Krane
       end
 
       _, err, st = @kubectl.run("get", "namespace", "-o", "name", namespace,
-        use_namespace: false, log_failure: false)
+        use_namespace: false, log_failure: false, attempts: 3)
 
       unless st.success?
         @errors << if err.match("Error from server [(]NotFound[)]: namespace")

data/lib/krane/template_sets.rb

@@ -150,7 +150,7 @@ module Krane
 
       if rendering_erb_disabled? && deploying_with_erb_files?
         errors << "ERB template discovered with rendering disabled. If you were trying to render ERB and " \
-          "deploy the result, try piping the output of `krane render` to `krane-deploy` with the --stdin flag"
+          "deploy the result, try piping the output of `krane render` to `krane-deploy -f -`"
       end
 
       errors

data/lib/krane/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Krane
-  VERSION = "1.1.1"
+  VERSION = "2.1.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: krane
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 2.1.0
 platform: ruby
 authors:
 - Katrina Verey
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-01-22 00:00:00.000000000 Z
+date: 2020-10-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -150,7 +150,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.20.3
+        version: '1.0'
     - - "<"
       - !ruby/object:Gem::Version
         version: '2.0'
@@ -160,7 +160,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.20.3
+        version: '1.0'
     - - "<"
       - !ruby/object:Gem::Version
         version: '2.0'
@@ -212,14 +212,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '5.12'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '5.12'
 - !ruby/object:Gem::Dependency
   name: minitest-stub-const
   requirement: !ruby/object:Gem::Requirement
@@ -366,14 +366,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.76.0
+        version: 0.89.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.76.0
+        version: 0.89.1
 - !ruby/object:Gem::Dependency
   name: codecov
   requirement: !ruby/object:Gem::Requirement
@@ -398,7 +398,10 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".buildkite/pipeline.nightly.yml"
+- ".github/CODEOWNERS"
+- ".github/ISSUE_TEMPLATE.md"
 - ".github/probots.yml"
+- ".github/pull_request_template.md"
 - ".gitignore"
 - ".rubocop-http---shopify-github-io-ruby-style-guide-rubocop-yml"
 - ".rubocop.yml"
@@ -409,7 +412,6 @@ files:
 - CODE_OF_CONDUCT.md
 - CONTRIBUTING.md
 - Gemfile
-- ISSUE_TEMPLATE.md
 - LICENSE.txt
 - README.md
 - Rakefile
@@ -421,6 +423,7 @@ files:
 - exe/krane
 - krane.gemspec
 - lib/krane.rb
+- lib/krane/annotation.rb
 - lib/krane/bindings_parser.rb
 - lib/krane/cli/deploy_command.rb
 - lib/krane/cli/global_deploy_command.rb
@@ -434,6 +437,7 @@ files:
 - lib/krane/concerns/template_reporting.rb
 - lib/krane/concurrency.rb
 - lib/krane/container_logs.rb
+- lib/krane/container_overrides.rb
 - lib/krane/deferred_summary_logging.rb
 - lib/krane/delayed_exceptions.rb
 - lib/krane/deploy_task.rb
@@ -447,7 +451,6 @@ files:
 - lib/krane/kubeclient_builder.rb
 - lib/krane/kubectl.rb
 - lib/krane/kubernetes_resource.rb
-- lib/krane/kubernetes_resource/cloudsql.rb
 - lib/krane/kubernetes_resource/config_map.rb
 - lib/krane/kubernetes_resource/cron_job.rb
 - lib/krane/kubernetes_resource/custom_resource.rb
@@ -474,6 +477,7 @@ files:
 - lib/krane/label_selector.rb
 - lib/krane/oj.rb
 - lib/krane/options_helper.rb
+- lib/krane/psych_k8s_compatibility.rb
 - lib/krane/remote_logs.rb
 - lib/krane/render_task.rb
 - lib/krane/renderer.rb
@@ -489,7 +493,6 @@ files:
 - lib/krane/task_config_validator.rb
 - lib/krane/template_sets.rb
 - lib/krane/version.rb
-- pull_request_template.md
 - screenshots/deploy-demo.gif
 - screenshots/migrate-logs.png
 - screenshots/missing-secret-fail.png
@@ -498,7 +501,8 @@ files:
 homepage: https://github.com/Shopify/krane
 licenses:
 - MIT
-metadata: {}
+metadata:
+  allowed_push_host: https://rubygems.org
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -507,7 +511,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.4.0
+      version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="

data/lib/krane/kubernetes_resource/cloudsql.rb

@@ -1,44 +0,0 @@
-# frozen_string_literal: true
-module Krane
-  class Cloudsql < KubernetesResource
-    TIMEOUT = 10.minutes
-    SYNC_DEPENDENCIES = %w(Deployment Service)
-
-    def sync(cache)
-      super
-      @proxy_deployment = cache.get_instance(Deployment.kind, "cloudsql-#{cloudsql_resource_uuid}")
-      @proxy_service = cache.get_instance(Service.kind, "cloudsql-#{@name}")
-    end
-
-    def status
-      deploy_succeeded? ? "Provisioned" : "Unknown"
-    end
-
-    def deploy_succeeded?
-      proxy_deployment_ready? && proxy_service_ready?
-    end
-
-    def deploy_failed?
-      false
-    end
-
-    private
-
-    def proxy_deployment_ready?
-      return false unless (status = @proxy_deployment["status"])
-      # all cloudsql-proxy pods are running
-      status.fetch("availableReplicas", -1) == status.fetch("replicas", 0)
-    end
-
-    def proxy_service_ready?
-      return false unless @proxy_service.present?
-      # the service has an assigned cluster IP and is therefore functioning
-      @proxy_service.dig("spec", "clusterIP").present?
-    end
-
-    def cloudsql_resource_uuid
-      return unless @instance_data
-      @instance_data.dig("metadata", "uid")
-    end
-  end
-end