tobsch-krane 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. checksums.yaml +7 -0
  2. data/.buildkite/pipeline.nightly.yml +43 -0
  3. data/.github/probots.yml +2 -0
  4. data/.gitignore +20 -0
  5. data/.rubocop.yml +17 -0
  6. data/.shopify-build/VERSION +1 -0
  7. data/.shopify-build/kubernetes-deploy.yml +53 -0
  8. data/1.0-Upgrade.md +185 -0
  9. data/CHANGELOG.md +431 -0
  10. data/CODE_OF_CONDUCT.md +46 -0
  11. data/CONTRIBUTING.md +164 -0
  12. data/Gemfile +16 -0
  13. data/ISSUE_TEMPLATE.md +25 -0
  14. data/LICENSE.txt +21 -0
  15. data/README.md +655 -0
  16. data/Rakefile +36 -0
  17. data/bin/ci +21 -0
  18. data/bin/setup +16 -0
  19. data/bin/test +47 -0
  20. data/dev.yml +28 -0
  21. data/dev/flamegraph-from-tests +35 -0
  22. data/exe/krane +5 -0
  23. data/krane.gemspec +44 -0
  24. data/lib/krane.rb +7 -0
  25. data/lib/krane/bindings_parser.rb +88 -0
  26. data/lib/krane/cli/deploy_command.rb +75 -0
  27. data/lib/krane/cli/global_deploy_command.rb +54 -0
  28. data/lib/krane/cli/krane.rb +91 -0
  29. data/lib/krane/cli/render_command.rb +41 -0
  30. data/lib/krane/cli/restart_command.rb +34 -0
  31. data/lib/krane/cli/run_command.rb +54 -0
  32. data/lib/krane/cli/version_command.rb +13 -0
  33. data/lib/krane/cluster_resource_discovery.rb +113 -0
  34. data/lib/krane/common.rb +23 -0
  35. data/lib/krane/concerns/template_reporting.rb +29 -0
  36. data/lib/krane/concurrency.rb +18 -0
  37. data/lib/krane/container_logs.rb +106 -0
  38. data/lib/krane/deferred_summary_logging.rb +95 -0
  39. data/lib/krane/delayed_exceptions.rb +14 -0
  40. data/lib/krane/deploy_task.rb +363 -0
  41. data/lib/krane/deploy_task_config_validator.rb +29 -0
  42. data/lib/krane/duration_parser.rb +27 -0
  43. data/lib/krane/ejson_secret_provisioner.rb +154 -0
  44. data/lib/krane/errors.rb +28 -0
  45. data/lib/krane/formatted_logger.rb +57 -0
  46. data/lib/krane/global_deploy_task.rb +210 -0
  47. data/lib/krane/global_deploy_task_config_validator.rb +12 -0
  48. data/lib/krane/kubeclient_builder.rb +156 -0
  49. data/lib/krane/kubectl.rb +120 -0
  50. data/lib/krane/kubernetes_resource.rb +621 -0
  51. data/lib/krane/kubernetes_resource/cloudsql.rb +43 -0
  52. data/lib/krane/kubernetes_resource/config_map.rb +22 -0
  53. data/lib/krane/kubernetes_resource/cron_job.rb +18 -0
  54. data/lib/krane/kubernetes_resource/custom_resource.rb +87 -0
  55. data/lib/krane/kubernetes_resource/custom_resource_definition.rb +98 -0
  56. data/lib/krane/kubernetes_resource/daemon_set.rb +90 -0
  57. data/lib/krane/kubernetes_resource/deployment.rb +213 -0
  58. data/lib/krane/kubernetes_resource/horizontal_pod_autoscaler.rb +65 -0
  59. data/lib/krane/kubernetes_resource/ingress.rb +18 -0
  60. data/lib/krane/kubernetes_resource/job.rb +60 -0
  61. data/lib/krane/kubernetes_resource/network_policy.rb +22 -0
  62. data/lib/krane/kubernetes_resource/persistent_volume_claim.rb +80 -0
  63. data/lib/krane/kubernetes_resource/pod.rb +269 -0
  64. data/lib/krane/kubernetes_resource/pod_disruption_budget.rb +23 -0
  65. data/lib/krane/kubernetes_resource/pod_set_base.rb +71 -0
  66. data/lib/krane/kubernetes_resource/pod_template.rb +20 -0
  67. data/lib/krane/kubernetes_resource/replica_set.rb +92 -0
  68. data/lib/krane/kubernetes_resource/resource_quota.rb +22 -0
  69. data/lib/krane/kubernetes_resource/role.rb +22 -0
  70. data/lib/krane/kubernetes_resource/role_binding.rb +22 -0
  71. data/lib/krane/kubernetes_resource/secret.rb +24 -0
  72. data/lib/krane/kubernetes_resource/service.rb +104 -0
  73. data/lib/krane/kubernetes_resource/service_account.rb +22 -0
  74. data/lib/krane/kubernetes_resource/stateful_set.rb +70 -0
  75. data/lib/krane/label_selector.rb +42 -0
  76. data/lib/krane/oj.rb +4 -0
  77. data/lib/krane/options_helper.rb +39 -0
  78. data/lib/krane/remote_logs.rb +60 -0
  79. data/lib/krane/render_task.rb +118 -0
  80. data/lib/krane/renderer.rb +118 -0
  81. data/lib/krane/resource_cache.rb +68 -0
  82. data/lib/krane/resource_deployer.rb +265 -0
  83. data/lib/krane/resource_watcher.rb +171 -0
  84. data/lib/krane/restart_task.rb +228 -0
  85. data/lib/krane/rollout_conditions.rb +103 -0
  86. data/lib/krane/runner_task.rb +212 -0
  87. data/lib/krane/runner_task_config_validator.rb +18 -0
  88. data/lib/krane/statsd.rb +65 -0
  89. data/lib/krane/task_config.rb +22 -0
  90. data/lib/krane/task_config_validator.rb +96 -0
  91. data/lib/krane/template_sets.rb +173 -0
  92. data/lib/krane/version.rb +4 -0
  93. data/pull_request_template.md +8 -0
  94. data/screenshots/deploy-demo.gif +0 -0
  95. data/screenshots/migrate-logs.png +0 -0
  96. data/screenshots/missing-secret-fail.png +0 -0
  97. data/screenshots/success.png +0 -0
  98. data/screenshots/test-output.png +0 -0
  99. metadata +375 -0
@@ -0,0 +1,42 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Krane
4
+ class LabelSelector
5
+ def self.parse(string)
6
+ selector = {}
7
+
8
+ string.split(',').each do |kvp|
9
+ key, value = kvp.split('=', 2)
10
+
11
+ if key.blank?
12
+ raise ArgumentError, "key is blank"
13
+ end
14
+
15
+ if key.end_with?("!")
16
+ raise ArgumentError, "!= selectors are not supported"
17
+ end
18
+
19
+ if value&.start_with?("=")
20
+ raise ArgumentError, "== selectors are not supported"
21
+ end
22
+
23
+ selector[key] = value
24
+ end
25
+
26
+ new(selector)
27
+ end
28
+
29
+ def initialize(hash)
30
+ @selector = hash
31
+ end
32
+
33
+ def to_h
34
+ @selector
35
+ end
36
+
37
+ def to_s
38
+ return "" if @selector.nil?
39
+ @selector.map { |k, v| "#{k}=#{v}" }.join(",")
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,4 @@
1
+ # frozen_string_literal: true
2
+ require 'oj'
3
+
4
+ Oj.mimic_JSON
@@ -0,0 +1,39 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Krane
4
+ module OptionsHelper
5
+ class OptionsError < StandardError; end
6
+
7
+ STDIN_TEMP_FILE = "from_stdin.yml"
8
+ class << self
9
+ def with_processed_template_paths(template_paths, render_erb: false)
10
+ validated_paths = []
11
+ template_paths.uniq!
12
+ template_paths.each do |template_path|
13
+ next if template_path == '-'
14
+ validated_paths << template_path
15
+ end
16
+
17
+ if template_paths.include?("-")
18
+ Dir.mktmpdir("krane") do |dir|
19
+ template_dir_from_stdin(temp_dir: dir, render_erb: render_erb)
20
+ validated_paths << dir
21
+ yield validated_paths
22
+ end
23
+ else
24
+ yield validated_paths
25
+ end
26
+ end
27
+
28
+ private
29
+
30
+ def template_dir_from_stdin(temp_dir:, render_erb:)
31
+ tempfile = STDIN_TEMP_FILE
32
+ tempfile += ".erb" if render_erb
33
+ File.open(File.join(temp_dir, tempfile), 'w+') { |f| f.print($stdin.read) }
34
+ rescue IOError, Errno::ENOENT => e
35
+ raise OptionsError, e.message
36
+ end
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,60 @@
1
+ # frozen_string_literal: true
2
+ require 'krane/container_logs'
3
+
4
+ module Krane
5
+ class RemoteLogs
6
+ attr_reader :container_logs
7
+
8
+ def initialize(logger:, parent_id:, container_names:, namespace:, context:)
9
+ @logger = logger
10
+ @parent_id = parent_id
11
+ @container_logs = container_names.map do |n|
12
+ ContainerLogs.new(
13
+ logger: logger,
14
+ container_name: n,
15
+ parent_id: parent_id,
16
+ namespace: namespace,
17
+ context: context
18
+ )
19
+ end
20
+ end
21
+
22
+ def empty?
23
+ @container_logs.all?(&:empty?)
24
+ end
25
+
26
+ def sync
27
+ @container_logs.each(&:sync)
28
+ end
29
+
30
+ def print_latest
31
+ @container_logs.each do |cl|
32
+ unless cl.printing_started?
33
+ @logger.info("Streaming logs from #{@parent_id} container '#{cl.container_name}':")
34
+ end
35
+ cl.print_latest(prefix: @container_logs.length > 1)
36
+ end
37
+ end
38
+
39
+ def print_all(prevent_duplicate: true)
40
+ return if @already_displayed && prevent_duplicate
41
+
42
+ if @container_logs.all?(&:empty?)
43
+ @logger.warn("No logs found for #{@parent_id}")
44
+ return
45
+ end
46
+
47
+ @container_logs.each do |cl|
48
+ if cl.empty?
49
+ @logger.warn("No logs found for #{@parent_id} container '#{cl.container_name}'")
50
+ else
51
+ @logger.info("Logs from #{@parent_id} container '#{cl.container_name}':")
52
+ cl.print_all
53
+ @logger.blank_line
54
+ end
55
+ end
56
+
57
+ @already_displayed = true
58
+ end
59
+ end
60
+ end
@@ -0,0 +1,118 @@
1
+ # frozen_string_literal: true
2
+ require 'tempfile'
3
+
4
+ require 'krane/common'
5
+ require 'krane/renderer'
6
+ require 'krane/template_sets'
7
+
8
+ module Krane
9
+ # Render templates
10
+ class RenderTask
11
+ # Initializes the render task
12
+ #
13
+ # @param logger [Object] Logger object (defaults to an instance of Krane::FormattedLogger)
14
+ # @param current_sha [String] The SHA of the commit
15
+ # @param filenames [Array<String>] An array of filenames and/or directories containing templates (*required*)
16
+ # @param bindings [Hash] Bindings parsed by Krane::BindingsParser
17
+ def initialize(logger: nil, current_sha:, filenames: [], bindings:)
18
+ @logger = logger || Krane::FormattedLogger.build
19
+ @filenames = filenames.map { |path| File.expand_path(path) }
20
+ @bindings = bindings
21
+ @current_sha = current_sha
22
+ end
23
+
24
+ # Runs the task, returning a boolean representing success or failure
25
+ #
26
+ # @return [Boolean]
27
+ def run(*args)
28
+ run!(*args)
29
+ true
30
+ rescue Krane::FatalDeploymentError
31
+ false
32
+ end
33
+
34
+ # Runs the task, raising exceptions in case of issues
35
+ #
36
+ # @param stream [IO] Place to stream the output to
37
+ #
38
+ # @return [nil]
39
+ def run!(stream:)
40
+ @logger.reset
41
+ @logger.phase_heading("Initializing render task")
42
+
43
+ ts = TemplateSets.from_dirs_and_files(paths: @filenames, logger: @logger)
44
+
45
+ validate_configuration(ts)
46
+ count = render_templates(stream, ts)
47
+
48
+ @logger.summary.add_action("Successfully rendered #{count} template(s)")
49
+ @logger.print_summary(:success)
50
+ rescue Krane::FatalDeploymentError
51
+ @logger.print_summary(:failure)
52
+ raise
53
+ end
54
+
55
+ private
56
+
57
+ def render_templates(stream, template_sets)
58
+ @logger.phase_heading("Rendering template(s)")
59
+ count = 0
60
+ template_sets.with_resource_definitions_and_filename(current_sha: @current_sha,
61
+ bindings: @bindings, raw: true) do |rendered_content, filename|
62
+ write_to_stream(rendered_content, filename, stream)
63
+ count += 1
64
+ end
65
+
66
+ count
67
+ rescue Krane::InvalidTemplateError => exception
68
+ log_invalid_template(exception)
69
+ raise
70
+ end
71
+
72
+ def write_to_stream(rendered_content, filename, stream)
73
+ file_basename = File.basename(filename)
74
+ @logger.info("Rendering #{file_basename}...")
75
+ implicit = []
76
+ YAML.parse_stream(rendered_content, "<rendered> #{filename}") { |d| implicit << d.implicit }
77
+ if rendered_content.present?
78
+ stream.puts "---\n" if implicit.first
79
+ stream.puts rendered_content
80
+ @logger.info("Rendered #{file_basename}")
81
+ else
82
+ @logger.warn("Rendered #{file_basename} successfully, but the result was blank")
83
+ end
84
+ rescue Psych::SyntaxError => exception
85
+ raise InvalidTemplateError.new("Template is not valid YAML. #{exception.message}", filename: filename)
86
+ end
87
+
88
+ def validate_configuration(template_sets)
89
+ @logger.info("Validating configuration")
90
+ errors = []
91
+ if @filenames.blank?
92
+ errors << "filenames must be set"
93
+ end
94
+
95
+ if !@current_sha.nil? && @current_sha.empty?
96
+ errors << "current-sha is optional but can not be blank"
97
+ end
98
+ errors += template_sets.validate
99
+
100
+ unless errors.empty?
101
+ @logger.summary.add_action("Configuration invalid")
102
+ @logger.summary.add_paragraph(errors.map { |err| "- #{err}" }.join("\n"))
103
+ raise Krane::TaskConfigurationError, "Configuration invalid: #{errors.join(', ')}"
104
+ end
105
+ end
106
+
107
+ def log_invalid_template(exception)
108
+ @logger.error("Failed to render #{exception.filename}")
109
+
110
+ debug_msg = ColorizedString.new("Invalid template: #{exception.filename}\n").red
111
+ debug_msg += "> Error message:\n#{FormattedLogger.indent_four(exception.to_s)}"
112
+ if exception.content
113
+ debug_msg += "\n> Template content:\n#{FormattedLogger.indent_four(exception.content)}"
114
+ end
115
+ @logger.summary.add_paragraph(debug_msg)
116
+ end
117
+ end
118
+ end
@@ -0,0 +1,118 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'erb'
4
+ require 'securerandom'
5
+ require 'yaml'
6
+ require 'json'
7
+
8
+ module Krane
9
+ class Renderer
10
+ class InvalidPartialError < InvalidTemplateError
11
+ attr_accessor :parents, :content, :filename
12
+ def initialize(msg, parents: [], content: nil, filename:)
13
+ @parents = parents
14
+ super(msg, content: content, filename: filename)
15
+ end
16
+ end
17
+ class PartialNotFound < InvalidTemplateError; end
18
+
19
+ def initialize(current_sha:, template_dir:, logger:, bindings: {})
20
+ @current_sha = current_sha
21
+ @template_dir = template_dir
22
+ @partials_dirs =
23
+ %w(partials ../partials).map { |d| File.expand_path(File.join(@template_dir, d)) }
24
+ @logger = logger
25
+ @bindings = bindings
26
+ # Max length of podname is only 63chars so try to save some room by truncating sha to 8 chars
27
+ @id = if ENV["TASK_ID"]
28
+ ENV["TASK_ID"]
29
+ elsif current_sha
30
+ current_sha[0...8] + "-#{SecureRandom.hex(4)}"
31
+ end
32
+ end
33
+
34
+ def render_template(filename, raw_template)
35
+ return raw_template unless File.extname(filename) == ".erb"
36
+
37
+ erb_binding = TemplateContext.new(self).template_binding
38
+ bind_template_variables(erb_binding, template_variables)
39
+
40
+ ERB.new(raw_template, nil, '-').result(erb_binding)
41
+ rescue InvalidPartialError => err
42
+ err.parents = err.parents.dup.unshift(filename)
43
+ err.filename = "#{err.filename} (partial included from: #{err.parents.join(' -> ')})"
44
+ raise err
45
+ rescue StandardError => err
46
+ raise InvalidTemplateError.new(err.message, filename: filename, content: raw_template)
47
+ end
48
+
49
+ def render_partial(partial, locals)
50
+ variables = template_variables.merge(locals)
51
+ erb_binding = TemplateContext.new(self).template_binding
52
+ bind_template_variables(erb_binding, variables)
53
+ erb_binding.local_variable_set("locals", locals)
54
+
55
+ partial_path = find_partial(partial)
56
+ template = File.read(partial_path)
57
+ expanded_template = ERB.new(template, nil, '-').result(erb_binding)
58
+
59
+ docs = Psych.parse_stream(expanded_template, partial_path)
60
+ # If the partial contains multiple documents or has an explicit document header,
61
+ # we know it cannot validly be indented in the parent, so return it immediately.
62
+ return expanded_template unless docs.children.one? && docs.children.first.implicit
63
+ # Make sure indentation isn't a problem by producing a single line of parseable YAML.
64
+ # Note that JSON is a subset of YAML.
65
+ JSON.generate(docs.children.first.to_ruby)
66
+ rescue PartialNotFound => err
67
+ # get the filename from the first parent, not the missing partial itself
68
+ raise err if err.filename == partial
69
+ raise InvalidPartialError.new(err.message, filename: partial, content: expanded_template || template)
70
+ rescue InvalidPartialError => err
71
+ err.parents = err.parents.dup.unshift(File.basename(partial_path))
72
+ raise err
73
+ rescue StandardError => err
74
+ raise InvalidPartialError.new(err.message, filename: partial_path, content: expanded_template || template)
75
+ end
76
+
77
+ private
78
+
79
+ def template_variables
80
+ {
81
+ 'current_sha' => @current_sha,
82
+ 'deployment_id' => @id,
83
+ }.merge(@bindings)
84
+ end
85
+
86
+ def bind_template_variables(erb_binding, variables)
87
+ variables.each do |var_name, value|
88
+ erb_binding.local_variable_set(var_name, value)
89
+ end
90
+ end
91
+
92
+ def find_partial(name)
93
+ partial_names = [name + '.yaml.erb', name + '.yml.erb']
94
+ @partials_dirs.each do |dir|
95
+ partial_names.each do |partial_name|
96
+ partial_path = File.join(dir, partial_name)
97
+ return partial_path if File.exist?(partial_path)
98
+ end
99
+ end
100
+ raise PartialNotFound.new("Could not find partial '#{name}' in any of #{@partials_dirs.join(':')}",
101
+ filename: name)
102
+ end
103
+
104
+ class TemplateContext
105
+ def initialize(renderer)
106
+ @_renderer = renderer
107
+ end
108
+
109
+ def template_binding
110
+ binding
111
+ end
112
+
113
+ def partial(partial, locals = {})
114
+ @_renderer.render_partial(partial, locals)
115
+ end
116
+ end
117
+ end
118
+ end
@@ -0,0 +1,68 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'concurrent/hash'
4
+
5
+ module Krane
6
+ class ResourceCache
7
+ delegate :namespace, :context, :logger, to: :@task_config
8
+
9
+ def initialize(task_config)
10
+ @task_config = task_config
11
+
12
+ @kind_fetcher_locks = Concurrent::Hash.new { |hash, key| hash[key] = Mutex.new }
13
+ @data = Concurrent::Hash.new
14
+ @kubectl = Kubectl.new(task_config: @task_config, log_failure_by_default: false)
15
+ end
16
+
17
+ def get_instance(kind, resource_name, raise_if_not_found: false)
18
+ instance = use_or_populate_cache(kind).fetch(resource_name, {})
19
+ if instance.blank? && raise_if_not_found
20
+ raise Krane::Kubectl::ResourceNotFoundError, "Resource does not exist (used cache for kind #{kind})"
21
+ end
22
+ instance
23
+ rescue KubectlError
24
+ {}
25
+ end
26
+
27
+ def get_all(kind, selector = nil)
28
+ instances = use_or_populate_cache(kind).values
29
+ return instances unless selector
30
+
31
+ instances.select do |r|
32
+ labels = r.dig("metadata", "labels") || {}
33
+ labels >= selector
34
+ end
35
+ rescue KubectlError
36
+ []
37
+ end
38
+
39
+ private
40
+
41
+ def statsd_tags
42
+ { namespace: namespace, context: context }
43
+ end
44
+
45
+ def use_or_populate_cache(kind)
46
+ @kind_fetcher_locks[kind].synchronize do
47
+ return @data[kind] if @data.key?(kind)
48
+ @data[kind] = fetch_by_kind(kind)
49
+ end
50
+ end
51
+
52
+ def fetch_by_kind(kind)
53
+ resource_class = KubernetesResource.class_for_kind(kind)
54
+ global_kind = @task_config.global_kinds.map(&:downcase).include?(kind.downcase)
55
+ output_is_sensitive = resource_class.nil? ? false : resource_class::SENSITIVE_TEMPLATE_CONTENT
56
+ raw_json, _, st = @kubectl.run("get", kind, "--chunk-size=0", attempts: 5, output: "json",
57
+ output_is_sensitive: output_is_sensitive, use_namespace: !global_kind)
58
+ raise KubectlError unless st.success?
59
+
60
+ instances = {}
61
+ JSON.parse(raw_json)["items"].each do |resource|
62
+ resource_name = resource.dig("metadata", "name")
63
+ instances[resource_name] = resource
64
+ end
65
+ instances
66
+ end
67
+ end
68
+ end