krane 2.1.6 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d496112ecd7ff05ab81ab6cad622632a7d169b38d1ad0340c5dc2088aa061677
-  data.tar.gz: 16c47fa820e244a7d42080e3fdf81065eda9167dda898e12412ab5c72d5bd4be
+  metadata.gz: 46134385171fa351edf5611c3ecd248c299339d2134cf4ff64f6227d68470b96
+  data.tar.gz: c30ad94176b02c7c402fa50ec5b73aa0c2c289a64401eefcc95494feb64354b3
 SHA512:
-  metadata.gz: 97eeceba4a6f132ec19bd40bfdcf9fb7ad94f0a791f784d6b3ec0e01eb78eddca2e6d5e6809fc2feb15d6f5a4aba9c7eee295e4660998d595d267700566d7ad5
-  data.tar.gz: 1c364bd503c83eb902e548d68d06aed5235e1ef33c8e72e3fe2fd90871da76d8580be288b822ac768fb75a1dd7ed0c1c801634a0d7068a18d74b8263670b0f7c
+  metadata.gz: '0792b68c262c3c310b7c51801c206f5422a2b65832eda020a2c0d58c804bb14adc755f9db5ff335b084d982303fbd376b846c5ff6af46d489530ffe6cfe2ae4b'
+  data.tar.gz: 185a56c603ccbc5dfcdbd7d247d6b7bec512da59a69f2a3e873f41243e79a70f3281596d5346829a4cd26ac1453c8d458cf45f41f498d8e61f1c5cf4e77a13ad

data/.rubocop.yml

@@ -1,5 +1,5 @@
-inherit_from:
-  - http://shopify.github.io/ruby-style-guide/rubocop.yml
+inherit_gem:
+  rubocop-shopify: rubocop.yml
 
 AllCops:
   TargetRubyVersion: 2.4

data/.shopify-build/krane.yml

@@ -8,6 +8,14 @@ steps:
     run:
       - bundle: ~
       - bundle exec rubocop
+  - label: 'Run Test Suite (:kubernetes: 1.21-latest :ruby: 3.0)'
+    command: bin/ci
+    agents:
+      queue: k8s-ci
+    env:
+      LOGGING_LEVEL: "4"
+      KUBERNETES_VERSION: v1.21-latest
+      RUBY_VERSION: "3.0"
   - label: 'Run Test Suite (:kubernetes: 1.20-latest :ruby: 3.0)'
     command: bin/ci
     agents:
@@ -38,17 +46,3 @@ steps:
     env:
       LOGGING_LEVEL: "4"
       KUBERNETES_VERSION: v1.17-latest
-  - label: 'Run Test Suite (:kubernetes: 1.16-latest)'
-    command: bin/ci
-    agents:
-      queue: k8s-ci
-    env:
-      LOGGING_LEVEL: "4"
-      KUBERNETES_VERSION: v1.16-latest
-  - label: 'Run Test Suite (:kubernetes: 1.15-latest)'
-    command: bin/ci
-    agents:
-      queue: k8s-ci
-    env:
-      LOGGING_LEVEL: "4"
-      KUBERNETES_VERSION: v1.15-latest

data/CHANGELOG.md

@@ -1,5 +1,44 @@
 ## next
 
+## 2.2.0
+
+*Enhancements*
+
+- Add a new option `--selector-as-filter` to command `krane deploy` and `krane global-deploy` [#831](https://github.com/Shopify/krane/pull/831)
+
+## 2.1.10
+
+*Bug Fixes*
+
+- Don't gather prunable resources by calling uniq only on `kind`: use `group` as well. Otherwise certain resources may not be added to the prune whitelist if the same kind exists across multiple groups [#825](https://github.com/Shopify/krane/pull/825)
+- Fix resource discovery failures when API paths are not located at the root of the API server (this occurs, for example, when using Rancher proxy) [#827](https://github.com/Shopify/krane/pull/827)
+
+*Other*
+
+- Fix ERB deprecation of positional arguments [#828](https://github.com/Shopify/krane/pull/828)
+
+## 2.1.9
+
+*Other*
+
+- Don't package screenshots in the built gem to reduce size [#817](https://github.com/Shopify/krane/pull/817)
+
+## 2.1.8
+
+*Other*
+
+- Change `statsd-instrument` dependency constraint to `< 4` [#815](https://github.com/Shopify/krane/pull/815)
+
+## 2.1.7
+
+*Enhancements*
+- ENV["KRANE_LOG_LINE_LIMIT"] allows the number of container logs printed for failures to be configurable from the 25 line default [#803](https://github.com/Shopify/krane/pull/803).
+
+*Other*
+- Remove the overly tight timeout on cluster resource discovery, which was causing too many timeouts in high latency environments [#813](https://github.com/Shopify/krane/pull/813)
+
+## 2.1.6
+
 *Enhancements*
 - Remove the need for a hard coded GVK overide list via improvements to cluster discovery [#778](https://github.com/Shopify/krane/pull/778)
 

data/README.md

@@ -117,6 +117,7 @@ Refer to `krane help` for the authoritative set of options.
 - `--global-timeout=duration`: Raise a timeout error if it takes longer than _duration_ for any
 resource to deploy.
 - `--selector`: Instructs krane to only prune resources which match the specified label selector, such as `environment=staging`. If you use this option, all resource templates must specify matching labels. See [Sharing a namespace](#sharing-a-namespace) below.
+- `--selector-as-filter`: Instructs krane to only deploy resources that are filtered by the specified labels in `--selector`. The deploy will not fail if not all resources match the labels. This is useful if you only want to deploy a subset of resources within a given YAML file. See [Sharing a namespace](#sharing-a-namespace) below.
 - `--no-verify-result`: Skip verification that workloads correctly deployed.
 - `--protected-namespaces=default kube-system kube-public`: Fail validation if a deploy is targeted at a protected namespace.
 - `--verbose-log-prefix`: Add [context][namespace] to the log prefix
@@ -132,6 +133,8 @@ If you need to, you may specify `--no-prune` to disable all pruning behaviour, b
 
 If you need to share a namespace with resources which are managed by other tools or indeed other krane deployments, you can supply the `--selector` option, such that only resources with labels matching the selector are considered for pruning.
 
+If you need to share a namespace with different set of resources using the same YAML file, you can supply the `--selector` and `--selector-as-filter` options, such that only the resources that match with the labels will be deployed. In each run of deploy, you can use different labels in `--selector` to deploy a different set of resources. Only the deployed resources in each run are considered for pruning.
+
 ### Using templates
 
 All templates must be YAML formatted.
@@ -441,6 +444,7 @@ Refer to `krane global-deploy help` for the authoritative set of options.
 - `--filenames / -f [PATHS]`: Accepts a list of directories and/or filenames to specify the set of directories/files that will be deployed. Use `-` to specify STDIN.
 - `--no-prune`: Skips pruning of resources that are no longer in your Kubernetes template set. Not recommended, as it allows your namespace to accumulate cruft that is not reflected in your deploy directory.
 - `--selector`: Instructs krane to only prune resources which match the specified label selector, such as `environment=staging`. By using this option, all resource templates must specify matching labels. See [Sharing a namespace](#sharing-a-namespace) below.
+- `--selector-as-filter`: Instructs krane to only deploy resources that are filtered by the specified labels in `--selector`. The deploy will not fail if not all resources match the labels. This is useful if you only want to deploy a subset of resources within a given YAML file. See [Sharing a namespace](#sharing-a-namespace) below.
 - `--global-timeout=duration`: Raise a timeout error if it takes longer than _duration_ for any
 resource to deploy.
 - `--no-verify-result`: Skip verification that resources correctly deployed.

data/dev.yml

@@ -4,7 +4,7 @@ up:
   - ruby: 2.6.6 # Matches gemspec
   - bundler
   - homebrew:
-    - homebrew/cask/minikube
+    - minikube
     - hyperkit
   - custom:
       name: Install the minikube fork of driver-hyperkit
@@ -13,7 +13,7 @@ up:
   - custom:
       name: Minikube Cluster
       met?: test $(minikube status | grep Running | wc -l) -ge 2 && $(minikube status | grep -q 'Configured')
-      meet: minikube start --kubernetes-version=v1.15.12 --vm-driver=hyperkit
+      meet: minikube start --kubernetes-version=v1.18.18 --vm-driver=hyperkit
       down: minikube stop
 commands:
   reset-minikube: minikube delete && rm -rf ~/.minikube

data/krane.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |spec|
   spec.license       = "MIT"
 
   spec.files         = %x(git ls-files -z).split("\x0").reject do |f|
-    f.match(%r{^(test|spec|features)/})
+    f.match(%r{^(test|spec|features|screenshots)/})
   end
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency("googleauth", "~> 0.8")
   spec.add_dependency("ejson", "~> 1.0")
   spec.add_dependency("colorize", "~> 0.8")
-  spec.add_dependency("statsd-instrument", ['>= 2.8', "< 3.1"])
+  spec.add_dependency("statsd-instrument", ['>= 2.8', "< 4"])
   spec.add_dependency("oj", "~> 3.0")
   spec.add_dependency("concurrent-ruby", "~> 1.1")
   spec.add_dependency("jsonpath", "~> 0.9.6")
@@ -57,5 +57,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency("ruby-prof")
   spec.add_development_dependency("ruby-prof-flamegraph")
   spec.add_development_dependency("rubocop", "~> 0.89.1")
+  spec.add_development_dependency("rubocop-shopify", "~> 1.0.5")
   spec.add_development_dependency("simplecov")
 end

data/lib/krane/cli/deploy_command.rb

@@ -25,6 +25,10 @@ module Krane
                      default: true },
         "selector" => { type: :string, banner: "'label=value'",
                         desc: "Select workloads by selector(s)" },
+        "selector-as-filter" => { type: :boolean,
+                                  desc: "Use --selector as a label filter to deploy only a subset "\
+                                    "of the provided resources",
+                                  default: false },
         "verbose-log-prefix" => { type: :boolean, desc: "Add [context][namespace] to the log prefix",
                                   default: false },
         "verify-result" => { type: :boolean, default: true,
@@ -37,6 +41,11 @@ module Krane
         require 'krane/label_selector'
 
         selector = ::Krane::LabelSelector.parse(options[:selector]) if options[:selector]
+        selector_as_filter = options['selector-as-filter']
+
+        if selector_as_filter && !selector
+          raise(Thor::RequiredArgumentMissingError, '--selector must be set when --selector-as-filter is set')
+        end
 
         logger = ::Krane::FormattedLogger.build(namespace, context,
           verbose_prefix: options['verbose-log-prefix'])
@@ -60,6 +69,7 @@ module Krane
             logger: logger,
             global_timeout: ::Krane::DurationParser.new(options["global-timeout"]).parse!.to_i,
             selector: selector,
+            selector_as_filter: selector_as_filter,
             protected_namespaces: protected_namespaces,
           )
 

data/lib/krane/cli/global_deploy_command.rb

@@ -16,6 +16,10 @@ module Krane
                              desc: "Verify workloads correctly deployed" },
         "selector" => { type: :string, banner: "'label=value'", required: true,
                         desc: "Select workloads owned by selector(s)" },
+        "selector-as-filter" => { type: :boolean,
+                                  desc: "Use --selector as a label filter to deploy only a subset "\
+                                    "of the provided resources",
+                                  default: false },
         "prune" => { type: :boolean, desc: "Enable deletion of resources that match"\
                      " the provided selector and do not appear in the provided templates",
                      default: true },
@@ -28,6 +32,11 @@ module Krane
         require 'krane/duration_parser'
 
         selector = ::Krane::LabelSelector.parse(options[:selector])
+        selector_as_filter = options['selector-as-filter']
+
+        if selector_as_filter && !selector
+          raise(Thor::RequiredArgumentMissingError, '--selector must be set when --selector-as-filter is set')
+        end
 
         filenames = options[:filenames].dup
         filenames << "-" if options[:stdin]
@@ -41,6 +50,7 @@ module Krane
             filenames: paths,
             global_timeout: ::Krane::DurationParser.new(options["global-timeout"]).parse!.to_i,
             selector: selector,
+            selector_as_filter: selector_as_filter,
           )
 
           deploy.run!(

data/lib/krane/cluster_resource_discovery.rb

@@ -34,7 +34,7 @@ module Krane
       end
       responses.flat_map do |path, resources|
         resources.map { |r| resource_hash(path, namespaced, r) }
-      end.compact.uniq { |r| r["kind"] }
+      end.compact.uniq { |r| "#{r['apigroup']}/#{r['kind']}" }
     end
 
     def fetch_mutating_webhook_configurations
@@ -52,9 +52,20 @@ module Krane
 
     private
 
+    # During discovery, the api paths may not actually be at the root, so we must programatically find it.
+    def base_api_path
+      @base_api_path ||= begin
+        raw_response, err, st = kubectl.run("config", "view", "--minify", "--output",
+          "jsonpath={.clusters[*].cluster.server}", attempts: 5, use_namespace: false)
+        raise FatalKubeAPIError, "Error retrieving cluster url: #{err}" unless st.success?
+
+        URI(raw_response).path.blank? ? "/" : URI(raw_response).path
+      end
+    end
+
     def api_paths
       @api_path_cache["/"] ||= begin
-        raw_json, err, st = kubectl.run("get", "--raw", "/", attempts: 5, use_namespace: false)
+        raw_json, err, st = kubectl.run("get", "--raw", base_api_path, attempts: 5, use_namespace: false)
         paths = if st.success?
           JSON.parse(raw_json)["paths"]
         else
@@ -101,7 +112,7 @@ module Krane
     end
 
     def kubectl
-      @kubectl ||= Kubectl.new(task_config: @task_config, log_failure_by_default: true, default_timeout: 1)
+      @kubectl ||= Kubectl.new(task_config: @task_config, log_failure_by_default: true)
     end
   end
 end

data/lib/krane/container_logs.rb

@@ -3,7 +3,7 @@ module Krane
   class ContainerLogs
     attr_reader :lines, :container_name
 
-    DEFAULT_LINE_LIMIT = 25
+    DEFAULT_LINE_LIMIT = Integer(ENV.fetch('KRANE_LOG_LINE_LIMIT', 25))
 
     def initialize(parent_id:, container_name:, namespace:, context:, logger:)
       @parent_id = parent_id

data/lib/krane/deploy_task.rb

@@ -100,12 +100,13 @@ module Krane
     # @param bindings [Hash] Bindings parsed by Krane::BindingsParser
     # @param global_timeout [Integer] Timeout in seconds
     # @param selector [Hash] Selector(s) parsed by Krane::LabelSelector
+    # @param selector_as_filter [Boolean] Allow selecting a subset of Kubernetes resource templates to deploy
     # @param filenames [Array<String>] An array of filenames and/or directories containing templates (*required*)
     # @param protected_namespaces [Array<String>] Array of protected Kubernetes namespaces (defaults
     #   to Krane::DeployTask::PROTECTED_NAMESPACES)
     # @param render_erb [Boolean] Enable ERB rendering
     def initialize(namespace:, context:, current_sha: nil, logger: nil, kubectl_instance: nil, bindings: {},
-      global_timeout: nil, selector: nil, filenames: [], protected_namespaces: nil,
+      global_timeout: nil, selector: nil, selector_as_filter: false, filenames: [], protected_namespaces: nil,
       render_erb: false, kubeconfig: nil)
       @logger = logger || Krane::FormattedLogger.build(namespace, context)
       @template_sets = TemplateSets.from_dirs_and_files(paths: filenames, logger: @logger, render_erb: render_erb)
@@ -118,6 +119,7 @@ module Krane
       @kubectl = kubectl_instance
       @global_timeout = global_timeout
       @selector = selector
+      @selector_as_filter = selector_as_filter
       @protected_namespaces = protected_namespaces || PROTECTED_NAMESPACES
       @render_erb = render_erb
     end
@@ -273,6 +275,7 @@ module Krane
 
       confirm_ejson_keys_not_prunable if prune
       @logger.info("Using resource selector #{@selector}") if @selector
+      @logger.info("Only deploying resources filtered by labels in selector") if @selector && @selector_as_filter
       @namespace_tags |= tags_from_namespace_labels
       @logger.info("All required parameters and files are present")
     end
@@ -295,6 +298,7 @@ module Krane
       batchable_resources, individuals = partition_dry_run_resources(resources.dup)
       batch_dry_run_success = kubectl.server_dry_run_enabled? && validate_dry_run(batchable_resources)
       individuals += batchable_resources unless batch_dry_run_success
+      resources.select! { |r| r.selected?(@selector) } if @selector_as_filter
       Krane::Concurrency.split_across_threads(resources) do |r|
         r.validate_definition(kubectl: kubectl, selector: @selector, dry_run: individuals.include?(r))
       end

data/lib/krane/global_deploy_task.rb

@@ -33,8 +33,10 @@ module Krane
     # @param context [String] Kubernetes context (*required*)
     # @param global_timeout [Integer] Timeout in seconds
     # @param selector [Hash] Selector(s) parsed by Krane::LabelSelector (*required*)
+    # @param selector_as_filter [Boolean] Allow selecting a subset of Kubernetes resource templates to deploy
     # @param filenames [Array<String>] An array of filenames and/or directories containing templates (*required*)
-    def initialize(context:, global_timeout: nil, selector: nil, filenames: [], logger: nil, kubeconfig: nil)
+    def initialize(context:, global_timeout: nil, selector: nil, selector_as_filter: false,
+      filenames: [], logger: nil, kubeconfig: nil)
       template_paths = filenames.map { |path| File.expand_path(path) }
 
       @task_config = TaskConfig.new(context, nil, logger, kubeconfig)
@@ -42,6 +44,7 @@ module Krane
         logger: @task_config.logger, render_erb: false)
       @global_timeout = global_timeout
       @selector = selector
+      @selector_as_filter = selector_as_filter
     end
 
     # Runs the task, returning a boolean representing success or failure
@@ -130,6 +133,7 @@ module Krane
     def validate_resources(resources)
       validate_globals(resources)
 
+      resources.select! { |r| r.selected?(@selector) } if @selector_as_filter
       Concurrency.split_across_threads(resources) do |r|
         r.validate_definition(kubectl: @kubectl, selector: @selector)
       end

data/lib/krane/kubernetes_resource.rb

@@ -499,6 +499,10 @@ module Krane
       @global || self.class::GLOBAL
     end
 
+    def selected?(selector)
+      selector.nil? || selector.to_h <= labels
+    end
+
     private
 
     def validate_timeout_annotation

data/lib/krane/renderer.rb

@@ -39,7 +39,7 @@ module Krane
       erb_binding = TemplateContext.new(self).template_binding
       bind_template_variables(erb_binding, template_variables)
 
-      ERB.new(raw_template, nil, '-').result(erb_binding)
+      ERB.new(raw_template, trim_mode: '-').result(erb_binding)
     rescue InvalidPartialError => err
       err.parents = err.parents.dup.unshift(filename)
       err.filename = "#{err.filename} (partial included from: #{err.parents.join(' -> ')})"
@@ -56,7 +56,7 @@ module Krane
 
       partial_path = find_partial(partial)
       template = File.read(partial_path)
-      expanded_template = ERB.new(template, nil, '-').result(erb_binding)
+      expanded_template = ERB.new(template, trim_mode: '-').result(erb_binding)
 
       docs = Psych.parse_stream(expanded_template, partial_path)
       # If the partial contains multiple documents or has an explicit document header,

data/lib/krane/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Krane
-  VERSION = "2.1.6"
+  VERSION = "2.2.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: krane
 version: !ruby/object:Gem::Version
-  version: 2.1.6
+  version: 2.2.0
 platform: ruby
 authors:
 - Katrina Verey
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-02-11 00:00:00.000000000 Z
+date: 2021-06-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -91,7 +91,7 @@ dependencies:
         version: '2.8'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '3.1'
+        version: '4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -101,7 +101,7 @@ dependencies:
         version: '2.8'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '3.1'
+        version: '4'
 - !ruby/object:Gem::Dependency
   name: oj
   requirement: !ruby/object:Gem::Requirement
@@ -374,6 +374,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.89.1
+- !ruby/object:Gem::Dependency
+  name: rubocop-shopify
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.5
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.5
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -494,11 +508,6 @@ files:
 - lib/krane/task_config_validator.rb
 - lib/krane/template_sets.rb
 - lib/krane/version.rb
-- screenshots/deploy-demo.gif
-- screenshots/migrate-logs.png
-- screenshots/missing-secret-fail.png
-- screenshots/success.png
-- screenshots/test-output.png
 homepage: https://github.com/Shopify/krane
 licenses:
 - MIT
@@ -519,7 +528,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.2.17
 signing_key: 
 specification_version: 4
 summary: A command line tool that helps you ship changes to a Kubernetes namespace