krane 1.1.1 → 2.1.0

data/bin/ci

@@ -17,5 +17,5 @@ docker run --rm \
     -e VERBOSE=1 \
     -e PARALLELISM=$PARALLELISM \
     -w /usr/src/app \
-    ruby:2.4 \
+    ruby:"${RUBY_VERSION:-2.5}" \
     bin/test

data/bin/test

@@ -41,7 +41,7 @@ bundle exec rake unit_test
 print_header "Run Non-Parallel Integration Tests"
 bundle exec rake serial_integration_test
 
-print_header "Run Parallel Integration Tests (N=$PARALLELISM)"
-PARALLELIZE_ME=1 N=$PARALLELISM bundle exec rake integration_test
+print_header "Run Parallel Integration Tests (MT_CPU=$PARALLELISM)"
+PARALLELIZE_ME=1 MT_CPU=$PARALLELISM bundle exec rake integration_test
 
 test $err -eq 0

data/dev.yml

@@ -1,10 +1,11 @@
 ---
 name: krane
 up:
-  - ruby: 2.4.6 # Matches gemspec
+  - ruby: 2.5.7 # Matches gemspec
   - bundler
   - homebrew:
-    - Caskroom/cask/minikube
+    - homebrew/cask/minikube
+    - hyperkit
   - custom:
       name: Install the minikube fork of driver-hyperkit
       met?: command -v docker-machine-driver-hyperkit

data/krane.gemspec

@@ -23,7 +23,9 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = %w(lib)
 
-  spec.required_ruby_version = '>= 2.4.0'
+  spec.metadata['allowed_push_host'] = "https://rubygems.org"
+
+  spec.required_ruby_version = '>= 2.5.0'
   spec.add_dependency("activesupport", ">= 5.0")
   spec.add_dependency("kubeclient", "~> 4.3")
   spec.add_dependency("googleauth", "~> 0.8")
@@ -33,7 +35,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency("oj", "~> 3.0")
   spec.add_dependency("concurrent-ruby", "~> 1.1")
   spec.add_dependency("jsonpath", "~> 0.9.6")
-  spec.add_dependency("thor", ">= 0.20.3", "< 2.0")
+  spec.add_dependency("thor", ">= 1.0", "< 2.0")
 
   # Basics
   spec.add_development_dependency("bundler")
@@ -41,7 +43,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency("yard")
 
   # Test framework
-  spec.add_development_dependency("minitest", "~> 5.0")
+  spec.add_development_dependency("minitest", "~> 5.12")
   spec.add_development_dependency("minitest-stub-const", "~> 0.6")
   spec.add_development_dependency("minitest-reporters")
   spec.add_development_dependency("mocha", "~> 1.5")
@@ -54,6 +56,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency("byebug")
   spec.add_development_dependency("ruby-prof")
   spec.add_development_dependency("ruby-prof-flamegraph")
-  spec.add_development_dependency("rubocop", "~> 0.76.0")
+  spec.add_development_dependency("rubocop", "~> 0.89.1")
   spec.add_development_dependency("codecov")
 end

data/lib/krane/annotation.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Krane
+  module Annotation
+    class << self
+      def for(suffix)
+        "krane.shopify.io/#{suffix}"
+      end
+    end
+  end
+end

data/lib/krane/cli/deploy_command.rb

@@ -14,7 +14,7 @@ module Krane
                          aliases: :f, required: false, default: [],
                          desc: "Directories and files that contains the configuration to apply" },
         "stdin" => { type: :boolean, default: false,
-                     desc: "Read resources from stdin" },
+                     desc: "[DEPRECATED] Read resources from stdin" },
         "global-timeout" => { type: :string, banner: "duration", default: DEFAULT_DEPLOY_TIMEOUT,
                               desc: "Max duration to monitor workloads correctly deployed" },
         "protected-namespaces" => { type: :array, banner: "namespace1 namespace2 namespaceN",
@@ -46,11 +46,10 @@ module Krane
           protected_namespaces = []
         end
 
-        # never mutate options directly
         filenames = options[:filenames].dup
         filenames << "-" if options[:stdin]
         if filenames.empty?
-          raise Thor::RequiredArgumentMissingError, 'At least one of --filenames or --stdin must be set'
+          raise(Thor::RequiredArgumentMissingError, '--filenames must be set and not empty')
         end
 
         ::Krane::OptionsHelper.with_processed_template_paths(filenames) do |paths|

data/lib/krane/cli/global_deploy_command.rb

@@ -8,7 +8,8 @@ module Krane
         "filenames" => { type: :array, banner: 'config/deploy/production config/deploy/my-extra-resource.yml',
                          aliases: :f, required: false, default: [],
                          desc: "Directories and files that contains the configuration to apply" },
-        "stdin" => { type: :boolean, default: false, desc: "Read resources from stdin" },
+        "stdin" => { type: :boolean, default: false,
+                     desc: "[DEPRECATED] Read resources from stdin" },
         "global-timeout" => { type: :string, banner: "duration", default: DEFAULT_DEPLOY_TIMEOUT,
                               desc: "Max duration to monitor workloads correctly deployed" },
         "verify-result" => { type: :boolean, default: true,
@@ -28,11 +29,10 @@ module Krane
 
         selector = ::Krane::LabelSelector.parse(options[:selector])
 
-        # never mutate options directly
         filenames = options[:filenames].dup
         filenames << "-" if options[:stdin]
         if filenames.empty?
-          raise Thor::RequiredArgumentMissingError, 'At least one of --filenames or --stdin must be set'
+          raise(Thor::RequiredArgumentMissingError, '--filenames must be set and not empty')
         end
 
         ::Krane::OptionsHelper.with_processed_template_paths(filenames) do |paths|

data/lib/krane/cli/render_command.rb

@@ -7,7 +7,8 @@ module Krane
         "bindings" => { type: :array, banner: "foo=bar abc=def", desc: 'Bindings for erb' },
         "filenames" => { type: :array, banner: 'config/deploy/production config/deploy/my-extra-resource.yml',
                          required: false, default: [], aliases: 'f', desc: 'Directories and files to render' },
-        "stdin" => { type: :boolean, desc: "Read resources from stdin", default: false },
+        "stdin" => { type: :boolean, default: false,
+                     desc: "[DEPRECATED] Read resources from stdin" },
         "current-sha" => { type: :string, banner: "SHA", desc: "Expose SHA `current_sha` in ERB bindings",
                            lazy_default: '' },
       }
@@ -20,11 +21,10 @@ module Krane
         bindings_parser = ::Krane::BindingsParser.new
         options[:bindings]&.each { |b| bindings_parser.add(b) }
 
-        # never mutate options directly
         filenames = options[:filenames].dup
         filenames << "-" if options[:stdin]
         if filenames.empty?
-          raise Thor::RequiredArgumentMissingError, 'At least one of --filenames or --stdin must be set'
+          raise(Thor::RequiredArgumentMissingError, '--filenames must be set and not empty')
         end
 
         ::Krane::OptionsHelper.with_processed_template_paths(filenames, render_erb: true) do |paths|

data/lib/krane/cluster_resource_discovery.rb

@@ -37,7 +37,7 @@ module Krane
     def fetch_resources(namespaced: false)
       command = %w(api-resources)
       command << "--namespaced=#{namespaced}"
-      raw, _, st = kubectl.run(*command, output: "wide", attempts: 5,
+      raw, err, st = kubectl.run(*command, output: "wide", attempts: 5,
         use_namespace: false)
       if st.success?
         rows = raw.split("\n")
@@ -59,7 +59,7 @@ module Krane
           resource
         end
       else
-        []
+        raise FatalKubeAPIError, "Error retrieving api-resources: #{err}"
       end
     end
 
@@ -68,7 +68,7 @@ module Krane
     # kubectl api-versions returns a list of group/version strings e.g. autoscaling/v2beta2
     # A kind may not exist in all versions of the group.
     def fetch_api_versions
-      raw, _, st = kubectl.run("api-versions", attempts: 5, use_namespace: false)
+      raw, err, st = kubectl.run("api-versions", attempts: 5, use_namespace: false)
       # The "core" group is represented by an empty string
       versions = { "" => %w(v1) }
       if st.success?
@@ -78,6 +78,8 @@ module Krane
           versions[group] ||= []
           versions[group] << version
         end
+      else
+        raise FatalKubeAPIError, "Error retrieving api-versions: #{err}"
       end
       versions
     end
@@ -85,7 +87,9 @@ module Krane
     def version_for_kind(versions, kind)
       # Override list for kinds that don't appear in the lastest version of a group
       version_override = { "CronJob" => "v1beta1", "VolumeAttachment" => "v1beta1",
-                           "CSIDriver" => "v1beta1", "Ingress" => "v1beta1", "CSINode" => "v1beta1" }
+                           "CSIDriver" => "v1beta1", "Ingress" => "v1beta1",
+                           "CSINode" => "v1beta1", "Job" => "v1",
+                           "IngressClass" => "v1beta1" }
 
       pattern = /v(?<major>\d+)(?<pre>alpha|beta)?(?<minor>\d+)?/
       latest = versions.sort_by do |version|
@@ -97,12 +101,12 @@ module Krane
     end
 
     def fetch_crds
-      raw_json, _, st = kubectl.run("get", "CustomResourceDefinition", output: "json", attempts: 5,
+      raw_json, err, st = kubectl.run("get", "CustomResourceDefinition", output: "json", attempts: 5,
         use_namespace: false)
       if st.success?
         JSON.parse(raw_json)["items"]
       else
-        []
+        raise FatalKubeAPIError, "Error retrieving CustomResourceDefinition: #{err}"
       end
     end
 

data/lib/krane/concerns/template_reporting.rb

@@ -15,12 +15,6 @@ module Krane
       logger.summary.add_paragraph(debug_msg)
     end
 
-    def record_warnings(logger:, warning:, filename:)
-      warn_msg = "Template warning: #{filename}\n"
-      warn_msg += "> Warning message:\n#{Krane::FormattedLogger.indent_four(warning)}"
-      logger.summary.add_paragraph(ColorizedString.new(warn_msg).yellow)
-    end
-
     def add_para_from_list(logger:, action:, enum:)
       logger.summary.add_action(action)
       logger.summary.add_paragraph(enum.map { |e| "- #{e}" }.join("\n"))

data/lib/krane/container_logs.rb

@@ -3,7 +3,7 @@ module Krane
   class ContainerLogs
     attr_reader :lines, :container_name
 
-    DEFAULT_LINE_LIMIT = 250
+    DEFAULT_LINE_LIMIT = 25
 
     def initialize(parent_id:, container_name:, namespace:, context:, logger:)
       @parent_id = parent_id

data/lib/krane/container_overrides.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+module Krane
+  class ContainerOverrides
+    attr_reader :command, :arguments, :env_vars, :image_tag
+
+    def initialize(command: nil, arguments: nil, env_vars: [], image_tag: nil)
+      @command = command
+      @arguments = arguments
+      @env_vars = env_vars
+      @image_tag = image_tag
+    end
+
+    def apply!(container)
+      container.command = command if command
+      container.args = arguments if arguments
+
+      if image_tag
+        image = container.image
+        base_image, _old_tag = image.split(':')
+        new_image = "#{base_image}:#{image_tag}"
+
+        container.image = new_image
+      end
+
+      env_args = env_vars.map do |env|
+        key, value = env.split('=', 2)
+        { name: key, value: value }
+      end
+      container.env ||= []
+      container.env = container.env.map(&:to_h) + env_args
+    end
+  end
+end

data/lib/krane/deploy_task.rb

@@ -4,13 +4,13 @@ require 'shellwords'
 require 'tempfile'
 require 'fileutils'
 
+require 'krane/annotation'
 require 'krane/common'
 require 'krane/concurrency'
 require 'krane/resource_cache'
 require 'krane/kubernetes_resource'
 %w(
   custom_resource
-  cloudsql
   config_map
   deployment
   ingress
@@ -57,6 +57,7 @@ module Krane
     )
 
     def predeploy_sequence
+      default_group = { group: nil }
       before_crs = %w(
         ResourceQuota
         NetworkPolicy
@@ -66,12 +67,14 @@ module Krane
         Role
         RoleBinding
         Secret
-      )
+      ).map { |r| [r, default_group] }
+
       after_crs = %w(
         Pod
-      )
+      ).map { |r| [r, default_group] }
 
-      before_crs + cluster_resource_discoverer.crds.select(&:predeployed?).map(&:kind) + after_crs
+      crs = cluster_resource_discoverer.crds.select(&:predeployed?).map { |cr| [cr.kind, { group: cr.group }] }
+      Hash[before_crs + crs + after_crs]
     end
 
     def prune_whitelist
@@ -82,6 +85,10 @@ module Krane
       kubectl.server_version
     end
 
+    attr_reader :task_config
+
+    delegate :kubeclient_builder, to: :task_config
+
     # Initializes the deploy task
     #
     # @param namespace [String] Kubernetes namespace (*required*)
@@ -98,10 +105,10 @@ module Krane
     # @param render_erb [Boolean] Enable ERB rendering
     def initialize(namespace:, context:, current_sha: nil, logger: nil, kubectl_instance: nil, bindings: {},
       global_timeout: nil, selector: nil, filenames: [], protected_namespaces: nil,
-      render_erb: false)
+      render_erb: false, kubeconfig: nil)
       @logger = logger || Krane::FormattedLogger.build(namespace, context)
       @template_sets = TemplateSets.from_dirs_and_files(paths: filenames, logger: @logger, render_erb: render_erb)
-      @task_config = Krane::TaskConfig.new(context, namespace, @logger)
+      @task_config = Krane::TaskConfig.new(context, namespace, @logger, kubeconfig)
       @bindings = bindings
       @namespace = namespace
       @namespace_tags = []
@@ -117,8 +124,8 @@ module Krane
     # Runs the task, returning a boolean representing success or failure
     #
     # @return [Boolean]
-    def run(*args)
-      run!(*args)
+    def run(**args)
+      run!(**args)
       true
     rescue FatalDeploymentError
       false
@@ -187,10 +194,6 @@ module Krane
         selector: @selector, statsd_tags: statsd_tags, current_sha: @current_sha)
     end
 
-    def kubeclient_builder
-      @kubeclient_builder ||= KubeclientBuilder.new
-    end
-
     def cluster_resource_discoverer
       @cluster_resource_discoverer ||= ClusterResourceDiscovery.new(
         task_config: @task_config,
@@ -211,7 +214,10 @@ module Krane
     end
 
     def deploy_has_priority_resources?(resources)
-      resources.any? { |r| predeploy_sequence.include?(r.type) }
+      resources.any? do |r|
+        next unless (pr = predeploy_sequence[r.type])
+        !pr[:group] || pr[:group] == r.group
+      end
     end
 
     def check_initial_status(resources)
@@ -243,6 +249,8 @@ module Krane
         @logger.info("  - #{secret.id} (from ejson)")
       end
 
+      StatsD.client.gauge('discover_resources.count', resources.size, tags: statsd_tags)
+
       resources.sort
     rescue InvalidTemplateError => e
       record_invalid_template(logger: @logger, err: e.message, filename: e.filename,
@@ -275,11 +283,6 @@ module Krane
         r.validate_definition(kubectl, selector: @selector)
       end
 
-      resources.select(&:has_warnings?).each do |resource|
-        record_warnings(logger: @logger, warning: resource.validation_warning_msg,
-          filename: File.basename(resource.file_path))
-      end
-
       failed_resources = resources.select(&:validation_failed?)
       if failed_resources.present?
 

data/lib/krane/ejson_secret_provisioner.rb

@@ -12,10 +12,10 @@ module Krane
   end
 
   class EjsonSecretProvisioner
-    EJSON_SECRET_ANNOTATION = "kubernetes-deploy.shopify.io/ejson-secret"
     EJSON_SECRET_KEY = "kubernetes_secrets"
     EJSON_SECRETS_FILE = "secrets.ejson"
     EJSON_KEYS_SECRET = "ejson-keys"
+
     delegate :namespace, :context, :logger, to: :@task_config
 
     def initialize(task_config:, ejson_keys_secret:, ejson_file:, statsd_tags:, selector: nil)
@@ -106,7 +106,6 @@ module Krane
           "name" => secret_name,
           "labels" => labels,
           "namespace" => namespace,
-          "annotations" => { EJSON_SECRET_ANNOTATION => "true" },
         },
         "data" => encoded_data,
       }

data/lib/krane/global_deploy_task.rb

@@ -25,7 +25,8 @@ module Krane
   class GlobalDeployTask
     extend Krane::StatsD::MeasureMethods
     include TemplateReporting
-    delegate :context, :logger, :global_kinds, to: :@task_config
+    delegate :context, :logger, :global_kinds, :kubeclient_builder, to: :@task_config
+    attr_reader :task_config
 
     # Initializes the deploy task
     #
@@ -33,10 +34,10 @@ module Krane
     # @param global_timeout [Integer] Timeout in seconds
     # @param selector [Hash] Selector(s) parsed by Krane::LabelSelector (*required*)
     # @param filenames [Array<String>] An array of filenames and/or directories containing templates (*required*)
-    def initialize(context:, global_timeout: nil, selector: nil, filenames: [], logger: nil)
+    def initialize(context:, global_timeout: nil, selector: nil, filenames: [], logger: nil, kubeconfig: nil)
       template_paths = filenames.map { |path| File.expand_path(path) }
 
-      @task_config = TaskConfig.new(context, nil, logger)
+      @task_config = TaskConfig.new(context, nil, logger, kubeconfig)
       @template_sets = TemplateSets.from_dirs_and_files(paths: template_paths,
         logger: @task_config.logger, render_erb: false)
       @global_timeout = global_timeout
@@ -46,8 +47,8 @@ module Krane
     # Runs the task, returning a boolean representing success or failure
     #
     # @return [Boolean]
-    def run(*args)
-      run!(*args)
+    def run(**args)
+      run!(**args)
       true
     rescue FatalDeploymentError
       false
@@ -133,11 +134,6 @@ module Krane
         r.validate_definition(@kubectl, selector: @selector)
       end
 
-      resources.select(&:has_warnings?).each do |resource|
-        record_warnings(logger: logger, warning: resource.validation_warning_msg,
-          filename: File.basename(resource.file_path))
-      end
-
       failed_resources = resources.select(&:validation_failed?)
       if failed_resources.present?
         failed_resources.each do |r|
@@ -173,6 +169,8 @@ module Krane
         logger.info("  - #{r.id}")
       end
 
+      StatsD.client.gauge('discover_resources.count', resources.size, tags: statsd_tags)
+
       resources.sort
     rescue InvalidTemplateError => e
       record_invalid_template(logger: logger, err: e.message, filename: e.filename, content: e.content)
@@ -192,10 +190,6 @@ module Krane
       @kubectl ||= Kubectl.new(task_config: @task_config, log_failure_by_default: true)
     end
 
-    def kubeclient_builder
-      @kubeclient_builder ||= KubeclientBuilder.new
-    end
-
     def prune_whitelist
       cluster_resource_discoverer.prunable_resources(namespaced: false)
     end