kubernetes-deploy 0.29.0 → 1.0.0.pre.2

data/exe/kubernetes-render

@@ -2,32 +2,34 @@
 # frozen_string_literal: true
 
 require 'kubernetes-deploy/render_task'
-require 'kubernetes-deploy/options_helper'
-require 'kubernetes-deploy/bindings_parser'
+require 'krane/options_helper'
+require 'krane/bindings_parser'
 
 require 'optparse'
 
 template_dir = []
 bindings = {}
+current_sha = ENV["REVISION"]
 
 ARGV.options do |opts|
-  parser = KubernetesDeploy::BindingsParser.new
+  parser = Krane::BindingsParser.new
   opts.on("--bindings=BINDINGS", "Expose additional variables to ERB templates " \
     "(format: k1=v1,k2=v2, JSON string or file (JSON or YAML) path prefixed by '@')") { |b| parser.add(b) }
   opts.on("--template-dir=DIR", "Set the template dir (default: config/deploy/$ENVIRONMENT).") do |d|
     template_dir = [d]
   end
+  opts.on("--current-sha=CURRENT_SHA", "Expose SHA `current_sha` in ERB bindings") { |r| current_sha = r }
   opts.parse!
   bindings = parser.parse
 end
 
 templates = ARGV
-logger = KubernetesDeploy::FormattedLogger.build(verbose_prefix: false)
+logger = Krane::FormattedLogger.build(verbose_prefix: false)
 
 begin
-  KubernetesDeploy::OptionsHelper.with_processed_template_paths(template_dir) do |dir|
+  Krane::OptionsHelper.with_processed_template_paths(template_dir) do |dir|
     runner = KubernetesDeploy::RenderTask.new(
-      current_sha: ENV["REVISION"],
+      current_sha: current_sha,
       template_dir: dir.first,
       bindings: bindings,
     )
@@ -35,7 +37,7 @@ begin
     success = runner.run(STDOUT, templates)
     exit(1) unless success
   end
-rescue KubernetesDeploy::OptionsHelper::OptionsError => e
+rescue Krane::OptionsHelper::OptionsError => e
   logger.error(e.message)
   exit(1)
 end

data/exe/kubernetes-restart

@@ -4,8 +4,8 @@
 require 'optparse'
 
 require 'kubernetes-deploy/restart_task'
-require 'kubernetes-deploy/options_helper'
-require 'kubernetes-deploy/label_selector'
+require 'krane/options_helper'
+require 'krane/label_selector'
 
 raw_deployments = nil
 max_watch_seconds = nil
@@ -14,7 +14,7 @@ ARGV.options do |opts|
   opts.on("--deployments=LIST") { |v| raw_deployments = v.split(",") }
   opts.on("--max-watch-seconds=seconds") { |t| max_watch_seconds = t.to_i }
   opts.on("--selector=SELECTOR", "Restarts deployments matching selector (format: k1=v1,k2=v2)") do |s|
-    selector = KubernetesDeploy::LabelSelector.parse(s)
+    selector = Krane::LabelSelector.parse(s)
   end
   opts.parse!
 end

data/exe/kubernetes-run

@@ -2,7 +2,7 @@
 # frozen_string_literal: true
 
 require 'kubernetes-deploy/runner_task'
-require 'kubernetes-deploy/options_helper'
+require 'krane/options_helper'
 require 'optparse'
 
 template = "task-runner-template"

data/kubernetes-deploy.gemspec

@@ -2,11 +2,11 @@
 # frozen_string_literal: true
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'kubernetes-deploy/version'
+require 'krane/version'
 
 Gem::Specification.new do |spec|
   spec.name          = "kubernetes-deploy"
-  spec.version       = KubernetesDeploy::VERSION
+  spec.version       = Krane::VERSION
   spec.authors       = ["Katrina Verey", "Kir Shatrov"]
   spec.email         = ["ops-accounts+shipit@shopify.com"]
 
@@ -25,14 +25,14 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = '>= 2.4.0'
   spec.add_dependency("activesupport", ">= 5.0")
   spec.add_dependency("kubeclient", "~> 4.3")
-  spec.add_dependency("googleauth", "~> 0.8.0")
+  spec.add_dependency("googleauth", "~> 0.8")
   spec.add_dependency("ejson", "~> 1.0")
   spec.add_dependency("colorize", "~> 0.8")
-  spec.add_dependency("statsd-instrument", '~> 2.3.2')
+  spec.add_dependency("statsd-instrument", ['>= 2.8', "< 3.1"])
   spec.add_dependency("oj", "~> 3.0")
   spec.add_dependency("concurrent-ruby", "~> 1.1")
   spec.add_dependency("jsonpath", "~> 0.9.6")
-  spec.add_dependency("thor", "~>  0.20.3")
+  spec.add_dependency("thor", [">= 0.20", "< 2.0"])
 
   spec.add_development_dependency("bundler")
   spec.add_development_dependency("rake", "~> 10.0")

data/lib/krane.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
-require 'kubernetes-deploy/common'
 
-module Krane
-end
+require 'krane/deploy_task'
+require 'krane/render_task'
+require 'krane/restart_task'
+require 'krane/runner_task'
+require 'krane/global_deploy_task'

data/lib/{kubernetes-deploy → krane}/bindings_parser.rb

@@ -3,7 +3,7 @@ require 'json'
 require 'yaml'
 require 'csv'
 
-module KubernetesDeploy
+module Krane
   class BindingsParser
     def self.parse(string)
       new(string).parse

data/lib/krane/cli/deploy_command.rb

@@ -13,8 +13,11 @@ module Krane
         "bindings" => { type: :array, banner: "foo=bar abc=def",
                         desc: "Expose additional variables to ERB templates (format: k1=v1 k2=v2, JSON string or file "\
                           "(JSON or YAML) path prefixed by '@')" },
-        "filenames" => { type: :string, banner: '/tmp/my-resource.yml', aliases: :f, required: true,
-                         desc: "Path to file or directory that contains the configuration to apply" },
+        "filenames" => { type: :array, banner: 'config/deploy/production config/deploy/my-extra-resource.yml',
+                         aliases: :f, required: false, default: [],
+                         desc: "Directories and files that contains the configuration to apply" },
+        "stdin" => { type: :boolean, default: false,
+                     desc: "Read resources from stdin" },
         "global-timeout" => { type: :string, banner: "duration", default: DEFAULT_DEPLOY_TIMEOUT,
                               desc: "Max duration to monitor workloads correctly deployed" },
         "protected-namespaces" => { type: :array, banner: "namespace1 namespace2 namespaceN",
@@ -30,20 +33,22 @@ module Krane
                                   default: true },
         "verify-result" => { type: :boolean, default: true,
                              desc: "Verify workloads correctly deployed" },
+        "current-sha" => { type: :string, banner: "SHA", desc: "Expose SHA `current_sha` in ERB bindings" },
+
       }
 
       def self.from_options(namespace, context, options)
-        require 'kubernetes-deploy/deploy_task'
-        require 'kubernetes-deploy/options_helper'
-        require 'kubernetes-deploy/bindings_parser'
-        require 'kubernetes-deploy/label_selector'
+        require 'krane/deploy_task'
+        require 'krane/options_helper'
+        require 'krane/bindings_parser'
+        require 'krane/label_selector'
 
-        bindings_parser = KubernetesDeploy::BindingsParser.new
+        bindings_parser = ::Krane::BindingsParser.new
         options[:bindings]&.each { |binding_pair| bindings_parser.add(binding_pair) }
 
-        selector = KubernetesDeploy::LabelSelector.parse(options[:selector]) if options[:selector]
+        selector = ::Krane::LabelSelector.parse(options[:selector]) if options[:selector]
 
-        logger = KubernetesDeploy::FormattedLogger.build(namespace, context,
+        logger = ::Krane::FormattedLogger.build(namespace, context,
           verbose_prefix: options['verbose-log-prefix'])
 
         protected_namespaces = options['protected-namespaces']
@@ -51,16 +56,23 @@ module Krane
           protected_namespaces = []
         end
 
-        KubernetesDeploy::OptionsHelper.with_processed_template_paths([options[:filenames]],
+        # never mutate options directly
+        filenames = options[:filenames].dup
+        filenames << "-" if options[:stdin]
+        if filenames.empty?
+          raise Thor::RequiredArgumentMissingError, 'At least one of --filenames or --stdin must be set'
+        end
+
+        ::Krane::OptionsHelper.with_processed_template_paths(filenames,
           require_explicit_path: true) do |paths|
-          deploy = KubernetesDeploy::DeployTask.new(
+          deploy = ::Krane::DeployTask.new(
             namespace: namespace,
             context: context,
-            current_sha: ENV["REVISION"],
+            current_sha: options['current-sha'],
             template_paths: paths,
             bindings: bindings_parser.parse,
             logger: logger,
-            max_watch_seconds: KubernetesDeploy::DurationParser.new(options["global-timeout"]).parse!.to_i,
+            max_watch_seconds: ::Krane::DurationParser.new(options["global-timeout"]).parse!.to_i,
             selector: selector,
             protected_namespaces: protected_namespaces,
           )

data/lib/krane/cli/global_deploy_command.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Krane
+  module CLI
+    class GlobalDeployCommand
+      DEFAULT_DEPLOY_TIMEOUT = '300s'
+      OPTIONS = {
+        "filenames" => { type: :array, banner: 'config/deploy/production config/deploy/my-extra-resource.yml',
+                         aliases: :f, required: false, default: [],
+                         desc: "Directories and files that contains the configuration to apply" },
+        "stdin" => { type: :boolean, default: false, desc: "Read resources from stdin" },
+        "global-timeout" => { type: :string, banner: "duration", default: DEFAULT_DEPLOY_TIMEOUT,
+                              desc: "Max duration to monitor workloads correctly deployed" },
+        "verify-result" => { type: :boolean, default: true,
+                             desc: "Verify workloads correctly deployed" },
+        "selector" => { type: :string, banner: "'label=value'", required: true,
+                        desc: "Select workloads owned by selector(s)" },
+        "prune" => { type: :boolean, desc: "Enable deletion of resources that match"\
+                     " the provided selector and do not appear in the provided templates",
+                     default: true },
+      }
+
+      def self.from_options(context, options)
+        require 'krane/global_deploy_task'
+        require 'krane/options_helper'
+        require 'krane/label_selector'
+        require 'krane/duration_parser'
+
+        selector = ::Krane::LabelSelector.parse(options[:selector])
+
+        # never mutate options directly
+        filenames = options[:filenames].dup
+        filenames << "-" if options[:stdin]
+        if filenames.empty?
+          raise Thor::RequiredArgumentMissingError, 'At least one of --filenames or --stdin must be set'
+        end
+
+        ::Krane::OptionsHelper.with_processed_template_paths(filenames,
+          require_explicit_path: true) do |paths|
+          deploy = ::Krane::GlobalDeployTask.new(
+            context: context,
+            filenames: paths,
+            global_timeout: ::Krane::DurationParser.new(options["global-timeout"]).parse!.to_i,
+            selector: selector,
+          )
+
+          deploy.run!(
+            verify_result: options["verify-result"],
+            prune: options[:prune],
+          )
+        end
+      end
+    end
+  end
+end

data/lib/krane/cli/krane.rb

@@ -7,6 +7,7 @@ require 'krane/cli/restart_command'
 require 'krane/cli/run_command'
 require 'krane/cli/render_command'
 require 'krane/cli/deploy_command'
+require 'krane/cli/global_deploy_command'
 
 module Krane
   module CLI
@@ -58,6 +59,14 @@ module Krane
         end
       end
 
+      desc("global-deploy CONTEXT", "Ship non-namespaced resources to a cluster")
+      expand_options(GlobalDeployCommand::OPTIONS)
+      def global_deploy(context)
+        rescue_and_exit do
+          GlobalDeployCommand.from_options(context, options)
+        end
+      end
+
       def self.exit_on_failure?
         true
       end
@@ -66,11 +75,11 @@ module Krane
 
       def rescue_and_exit
         yield
-      rescue KubernetesDeploy::DeploymentTimeoutError
+      rescue ::Krane::DeploymentTimeoutError
         exit(TIMEOUT_EXIT_CODE)
-      rescue KubernetesDeploy::FatalDeploymentError
+      rescue ::Krane::FatalDeploymentError
         exit(FAILURE_EXIT_CODE)
-      rescue KubernetesDeploy::DurationParser::ParsingError => e
+      rescue ::Krane::DurationParser::ParsingError => e
         STDERR.puts(<<~ERROR_MESSAGE)
           Error parsing duration
           #{e.message}. Duration must be a full ISO8601 duration or time value (e.g. 300s, 10m, 1h)

data/lib/krane/cli/render_command.rb

@@ -4,21 +4,31 @@ module Krane
   module CLI
     class RenderCommand
       OPTIONS = {
-        bindings: { type: :array, desc: 'Bindings for erb' },
-        filenames: { type: :array, required: true, aliases: 'f', desc: 'Directories and files to render' },
+        "bindings" => { type: :array, banner: "foo=bar abc=def", desc: 'Bindings for erb' },
+        "filenames" => { type: :array, banner: 'config/deploy/production config/deploy/my-extra-resource.yml',
+                         required: false, default: [], aliases: 'f', desc: 'Directories and files to render' },
+        "stdin" => { type: :boolean, desc: "Read resources from stdin", default: false },
+        "current-sha" => { type: :string, banner: "SHA", desc: "Expose SHA `current_sha` in ERB bindings" },
       }
 
       def self.from_options(options)
-        require 'kubernetes-deploy/render_task'
-        require 'kubernetes-deploy/bindings_parser'
-        require 'kubernetes-deploy/options_helper'
+        require 'krane/render_task'
+        require 'krane/bindings_parser'
+        require 'krane/options_helper'
 
-        bindings_parser = KubernetesDeploy::BindingsParser.new
+        bindings_parser = ::Krane::BindingsParser.new
         options[:bindings]&.each { |b| bindings_parser.add(b) }
 
-        KubernetesDeploy::OptionsHelper.with_processed_template_paths(options[:filenames]) do |paths|
-          runner = KubernetesDeploy::RenderTask.new(
-            current_sha: ENV["REVISION"],
+        # never mutate options directly
+        filenames = options[:filenames].dup
+        filenames << "-" if options[:stdin]
+        if filenames.empty?
+          raise Thor::RequiredArgumentMissingError, 'At least one of --filenames or --stdin must be set'
+        end
+
+        ::Krane::OptionsHelper.with_processed_template_paths(filenames) do |paths|
+          runner = ::Krane::RenderTask.new(
+            current_sha: options['current-sha'],
             template_paths: paths,
             bindings: bindings_parser.parse,
           )

data/lib/krane/cli/restart_command.rb

@@ -16,12 +16,12 @@ module Krane
       }
 
       def self.from_options(namespace, context, options)
-        require 'kubernetes-deploy/restart_task'
-        selector = KubernetesDeploy::LabelSelector.parse(options[:selector]) if options[:selector]
-        restart = KubernetesDeploy::RestartTask.new(
+        require 'krane/restart_task'
+        selector = ::Krane::LabelSelector.parse(options[:selector]) if options[:selector]
+        restart = ::Krane::RestartTask.new(
           namespace: namespace,
           context: context,
-          max_watch_seconds: KubernetesDeploy::DurationParser.new(options["global-timeout"]).parse!.to_i,
+          max_watch_seconds: ::Krane::DurationParser.new(options["global-timeout"]).parse!.to_i,
         )
         restart.run!(
           options[:deployments],

data/lib/krane/cli/run_command.rb

@@ -22,7 +22,7 @@ module Krane
         "template" => {
           type: :string,
           desc: "The template file you'll be rendering",
-          default: 'task-runner-template',
+          required: true,
         },
         "env-vars" => {
           type: :string,
@@ -33,11 +33,11 @@ module Krane
       }
 
       def self.from_options(namespace, context, options)
-        require "kubernetes-deploy/runner_task"
-        runner = KubernetesDeploy::RunnerTask.new(
+        require "krane/runner_task"
+        runner = ::Krane::RunnerTask.new(
           namespace: namespace,
           context: context,
-          max_watch_seconds: KubernetesDeploy::DurationParser.new(options["global-timeout"]).parse!.to_i,
+          max_watch_seconds: ::Krane::DurationParser.new(options["global-timeout"]).parse!.to_i,
         )
 
         runner.run!(

data/lib/krane/cli/version_command.rb

@@ -6,7 +6,7 @@ module Krane
       OPTIONS = {}
 
       def self.from_options(_)
-        puts("krane #{KubernetesDeploy::VERSION}")
+        puts("krane #{::Krane::VERSION}")
       end
     end
   end

data/lib/krane/cluster_resource_discovery.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+module Krane
+  class ClusterResourceDiscovery
+    delegate :namespace, :context, :logger, to: :@task_config
+
+    def initialize(task_config:, namespace_tags: [])
+      @task_config = task_config
+      @namespace_tags = namespace_tags
+    end
+
+    def crds
+      @crds ||= fetch_crds.map do |cr_def|
+        CustomResourceDefinition.new(namespace: namespace, context: context, logger: logger,
+          definition: cr_def, statsd_tags: @namespace_tags)
+      end
+    end
+
+    def prunable_resources(namespaced:)
+      black_list = %w(Namespace Node ControllerRevision)
+      api_versions = fetch_api_versions
+
+      fetch_resources(namespaced: namespaced).map do |resource|
+        next unless resource['verbs'].one? { |v| v == "delete" }
+        next if black_list.include?(resource['kind'])
+        group_versions = api_versions[resource['apigroup'].to_s]
+        version = version_for_kind(group_versions, resource['kind'])
+        [resource['apigroup'], version, resource['kind']].compact.join("/")
+      end.compact
+    end
+
+    # kubectl api-resources -o wide returns 5 columns
+    # NAME SHORTNAMES APIGROUP NAMESPACED KIND VERBS
+    # SHORTNAMES and APIGROUP may be blank
+    # VERBS is an array
+    # serviceaccounts sa <blank> true ServiceAccount [create delete deletecollection get list patch update watch]
+    def fetch_resources(namespaced: false)
+      command = %w(api-resources)
+      command << "--namespaced=#{namespaced}"
+      raw, _, st = kubectl.run(*command, output: "wide", attempts: 5,
+        use_namespace: false)
+      if st.success?
+        rows = raw.split("\n")
+        header = rows[0]
+        resources = rows[1..-1]
+        full_width_field_names = header.downcase.scan(/[a-z]+[\W]*/)
+        cursor = 0
+        fields = full_width_field_names.each_with_object({}) do |name, hash|
+          start = cursor
+          cursor = start + name.length
+          # Last field should consume the remainder of the line
+          cursor = 0 if full_width_field_names.last == name.strip
+          hash[name.strip] = [start, cursor - 1]
+        end
+        resources.map do |resource|
+          resource = fields.map { |k, (s, e)| [k.strip, resource[s..e].strip] }.to_h
+          # Manually parse verbs: "[get list]" into %w(get list)
+          resource["verbs"] = resource["verbs"][1..-2].split
+          resource
+        end
+      else
+        []
+      end
+    end
+
+    private
+
+    # kubectl api-versions returns a list of group/version strings e.g. autoscaling/v2beta2
+    # A kind may not exist in all versions of the group.
+    def fetch_api_versions
+      raw, _, st = kubectl.run("api-versions", attempts: 5, use_namespace: false)
+      # The "core" group is represented by an empty string
+      versions = { "" => %w(v1) }
+      if st.success?
+        rows = raw.split("\n")
+        rows.each do |group_version|
+          group, version = group_version.split("/")
+          versions[group] ||= []
+          versions[group] << version
+        end
+      end
+      versions
+    end
+
+    def version_for_kind(versions, kind)
+      # Override list for kinds that don't appear in the lastest version of a group
+      version_override = { "CronJob" => "v1beta1", "VolumeAttachment" => "v1beta1",
+                           "CSIDriver" => "v1beta1", "Ingress" => "v1beta1", "CSINode" => "v1beta1" }
+
+      pattern = /v(?<major>\d+)(?<pre>alpha|beta)?(?<minor>\d+)?/
+      latest = versions.sort_by do |version|
+        match = version.match(pattern)
+        pre = { "alpha" => 0, "beta" => 1, nil => 2 }.fetch(match[:pre])
+        [match[:major].to_i, pre, match[:minor].to_i]
+      end.last
+      version_override.fetch(kind, latest)
+    end
+
+    def fetch_crds
+      raw_json, _, st = kubectl.run("get", "CustomResourceDefinition", output: "json", attempts: 5,
+        use_namespace: false)
+      if st.success?
+        JSON.parse(raw_json)["items"]
+      else
+        []
+      end
+    end
+
+    def kubectl
+      @kubectl ||= Kubectl.new(task_config: @task_config, log_failure_by_default: true)
+    end
+  end
+end