kubernetes-deploy 0.29.0 → 1.0.0.pre.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.buildkite/pipeline.nightly.yml +7 -0
- data/.rubocop.yml +0 -12
- data/.shopify-build/{kubernetes-deploy.yml → krane.yml} +8 -2
- data/1.0-Upgrade.md +109 -0
- data/CHANGELOG.md +60 -0
- data/CONTRIBUTING.md +2 -2
- data/Gemfile +1 -0
- data/README.md +86 -2
- data/dev.yml +3 -1
- data/dev/flamegraph-from-tests +1 -1
- data/exe/kubernetes-deploy +12 -9
- data/exe/kubernetes-render +9 -7
- data/exe/kubernetes-restart +3 -3
- data/exe/kubernetes-run +1 -1
- data/kubernetes-deploy.gemspec +5 -5
- data/lib/krane.rb +5 -3
- data/lib/{kubernetes-deploy → krane}/bindings_parser.rb +1 -1
- data/lib/krane/cli/deploy_command.rb +25 -13
- data/lib/krane/cli/global_deploy_command.rb +55 -0
- data/lib/krane/cli/krane.rb +12 -3
- data/lib/krane/cli/render_command.rb +19 -9
- data/lib/krane/cli/restart_command.rb +4 -4
- data/lib/krane/cli/run_command.rb +4 -4
- data/lib/krane/cli/version_command.rb +1 -1
- data/lib/krane/cluster_resource_discovery.rb +113 -0
- data/lib/{kubernetes-deploy → krane}/common.rb +8 -9
- data/lib/krane/concerns/template_reporting.rb +29 -0
- data/lib/{kubernetes-deploy → krane}/concurrency.rb +1 -1
- data/lib/{kubernetes-deploy → krane}/container_logs.rb +3 -2
- data/lib/{kubernetes-deploy → krane}/deferred_summary_logging.rb +2 -2
- data/lib/{kubernetes-deploy → krane}/delayed_exceptions.rb +0 -0
- data/lib/krane/deploy_task.rb +16 -0
- data/lib/krane/deploy_task_config_validator.rb +29 -0
- data/lib/krane/deprecated_deploy_task.rb +404 -0
- data/lib/{kubernetes-deploy → krane}/duration_parser.rb +1 -3
- data/lib/{kubernetes-deploy → krane}/ejson_secret_provisioner.rb +10 -13
- data/lib/krane/errors.rb +28 -0
- data/lib/{kubernetes-deploy → krane}/formatted_logger.rb +2 -2
- data/lib/krane/global_deploy_task.rb +210 -0
- data/lib/krane/global_deploy_task_config_validator.rb +12 -0
- data/lib/{kubernetes-deploy → krane}/kubeclient_builder.rb +13 -5
- data/lib/{kubernetes-deploy → krane}/kubectl.rb +14 -16
- data/lib/{kubernetes-deploy → krane}/kubernetes_resource.rb +110 -27
- data/lib/{kubernetes-deploy → krane}/kubernetes_resource/cloudsql.rb +1 -1
- data/lib/{kubernetes-deploy → krane}/kubernetes_resource/config_map.rb +1 -1
- data/lib/{kubernetes-deploy → krane}/kubernetes_resource/cron_job.rb +1 -1
- data/lib/{kubernetes-deploy → krane}/kubernetes_resource/custom_resource.rb +2 -2
- data/lib/{kubernetes-deploy → krane}/kubernetes_resource/custom_resource_definition.rb +1 -5
- data/lib/krane/kubernetes_resource/daemon_set.rb +90 -0
- data/lib/{kubernetes-deploy → krane}/kubernetes_resource/deployment.rb +2 -2
- data/lib/{kubernetes-deploy → krane}/kubernetes_resource/horizontal_pod_autoscaler.rb +1 -1
- data/lib/{kubernetes-deploy → krane}/kubernetes_resource/ingress.rb +1 -1
- data/lib/{kubernetes-deploy → krane}/kubernetes_resource/job.rb +1 -1
- data/lib/{kubernetes-deploy → krane}/kubernetes_resource/network_policy.rb +1 -1
- data/lib/{kubernetes-deploy → krane}/kubernetes_resource/persistent_volume_claim.rb +1 -1
- data/lib/{kubernetes-deploy → krane}/kubernetes_resource/pod.rb +6 -2
- data/lib/{kubernetes-deploy → krane}/kubernetes_resource/pod_disruption_budget.rb +2 -2
- data/lib/{kubernetes-deploy → krane}/kubernetes_resource/pod_set_base.rb +3 -3
- data/lib/{kubernetes-deploy → krane}/kubernetes_resource/pod_template.rb +1 -1
- data/lib/{kubernetes-deploy → krane}/kubernetes_resource/replica_set.rb +2 -2
- data/lib/{kubernetes-deploy → krane}/kubernetes_resource/resource_quota.rb +1 -1
- data/lib/{kubernetes-deploy → krane}/kubernetes_resource/role.rb +1 -1
- data/lib/{kubernetes-deploy → krane}/kubernetes_resource/role_binding.rb +1 -1
- data/lib/{kubernetes-deploy → krane}/kubernetes_resource/secret.rb +1 -1
- data/lib/{kubernetes-deploy → krane}/kubernetes_resource/service.rb +2 -2
- data/lib/{kubernetes-deploy → krane}/kubernetes_resource/service_account.rb +1 -1
- data/lib/{kubernetes-deploy → krane}/kubernetes_resource/stateful_set.rb +2 -2
- data/lib/{kubernetes-deploy → krane}/label_selector.rb +1 -1
- data/lib/{kubernetes-deploy → krane}/oj.rb +0 -0
- data/lib/{kubernetes-deploy → krane}/options_helper.rb +2 -2
- data/lib/{kubernetes-deploy → krane}/remote_logs.rb +2 -2
- data/lib/krane/render_task.rb +149 -0
- data/lib/{kubernetes-deploy → krane}/renderer.rb +1 -1
- data/lib/{kubernetes-deploy → krane}/resource_cache.rb +10 -9
- data/lib/krane/resource_deployer.rb +265 -0
- data/lib/{kubernetes-deploy → krane}/resource_watcher.rb +24 -25
- data/lib/krane/restart_task.rb +228 -0
- data/lib/{kubernetes-deploy → krane}/rollout_conditions.rb +1 -1
- data/lib/krane/runner_task.rb +212 -0
- data/lib/{kubernetes-deploy → krane}/runner_task_config_validator.rb +1 -1
- data/lib/{kubernetes-deploy → krane}/statsd.rb +13 -27
- data/lib/krane/task_config.rb +22 -0
- data/lib/{kubernetes-deploy → krane}/task_config_validator.rb +1 -1
- data/lib/{kubernetes-deploy → krane}/template_sets.rb +5 -5
- data/lib/krane/version.rb +4 -0
- data/lib/kubernetes-deploy/deploy_task.rb +6 -608
- data/lib/kubernetes-deploy/errors.rb +1 -26
- data/lib/kubernetes-deploy/render_task.rb +5 -122
- data/lib/kubernetes-deploy/rescue_krane_exceptions.rb +18 -0
- data/lib/kubernetes-deploy/restart_task.rb +6 -198
- data/lib/kubernetes-deploy/runner_task.rb +6 -184
- metadata +96 -70
- data/lib/kubernetes-deploy/cluster_resource_discovery.rb +0 -34
- data/lib/kubernetes-deploy/kubernetes_resource/daemon_set.rb +0 -54
- data/lib/kubernetes-deploy/task_config.rb +0 -16
- data/lib/kubernetes-deploy/version.rb +0 -4
@@ -1,5 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
module
|
2
|
+
module Krane
|
3
3
|
class Pod < KubernetesResource
|
4
4
|
TIMEOUT = 10.minutes
|
5
5
|
|
@@ -101,6 +101,10 @@ module KubernetesDeploy
|
|
101
101
|
exists? && !@stream_logs # don't print them a second time
|
102
102
|
end
|
103
103
|
|
104
|
+
def node_name
|
105
|
+
@instance_data.dig('spec', 'nodeName')
|
106
|
+
end
|
107
|
+
|
104
108
|
private
|
105
109
|
|
106
110
|
def failed_schedule_reason
|
@@ -137,7 +141,7 @@ module KubernetesDeploy
|
|
137
141
|
end
|
138
142
|
|
139
143
|
def logs
|
140
|
-
@logs ||=
|
144
|
+
@logs ||= Krane::RemoteLogs.new(
|
141
145
|
logger: @logger,
|
142
146
|
parent_id: id,
|
143
147
|
container_names: @containers.map(&:name),
|
@@ -1,5 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
module
|
2
|
+
module Krane
|
3
3
|
class PodDisruptionBudget < KubernetesResource
|
4
4
|
TIMEOUT = 10.seconds
|
5
5
|
|
@@ -13,7 +13,7 @@ module KubernetesDeploy
|
|
13
13
|
|
14
14
|
def deploy_method
|
15
15
|
# Required until https://github.com/kubernetes/kubernetes/issues/45398 changes
|
16
|
-
:replace_force
|
16
|
+
uses_generate_name? ? :create : :replace_force
|
17
17
|
end
|
18
18
|
|
19
19
|
def timeout_message
|
@@ -1,7 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
require '
|
2
|
+
require 'krane/kubernetes_resource/pod'
|
3
3
|
|
4
|
-
module
|
4
|
+
module Krane
|
5
5
|
class PodSetBase < KubernetesResource
|
6
6
|
def failure_message
|
7
7
|
pods.map(&:failure_message).compact.uniq.join("\n")
|
@@ -19,7 +19,7 @@ module KubernetesDeploy
|
|
19
19
|
end
|
20
20
|
|
21
21
|
def fetch_debug_logs
|
22
|
-
logs =
|
22
|
+
logs = Krane::RemoteLogs.new(
|
23
23
|
logger: @logger,
|
24
24
|
parent_id: id,
|
25
25
|
container_names: container_names,
|
File without changes
|
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
module
|
3
|
+
module Krane
|
4
4
|
module OptionsHelper
|
5
5
|
class OptionsError < StandardError; end
|
6
6
|
|
@@ -19,7 +19,7 @@ module KubernetesDeploy
|
|
19
19
|
end
|
20
20
|
|
21
21
|
if template_paths.include?("-")
|
22
|
-
Dir.mktmpdir("
|
22
|
+
Dir.mktmpdir("krane") do |dir|
|
23
23
|
template_dir_from_stdin(temp_dir: dir)
|
24
24
|
validated_paths << dir
|
25
25
|
yield validated_paths
|
@@ -0,0 +1,149 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'tempfile'
|
3
|
+
|
4
|
+
require 'krane/common'
|
5
|
+
require 'krane/renderer'
|
6
|
+
require 'krane/template_sets'
|
7
|
+
|
8
|
+
module Krane
|
9
|
+
# Render templates
|
10
|
+
class RenderTask
|
11
|
+
# Initializes the render task
|
12
|
+
#
|
13
|
+
# @param logger [Object] Logger object (defaults to an instance of Krane::FormattedLogger)
|
14
|
+
# @param current_sha [String] The SHA of the commit
|
15
|
+
# @param template_dir [String] Path to a directory with templates to render (deprecated)
|
16
|
+
# @param template_paths [Array<String>] An array of template paths to render
|
17
|
+
# @param bindings [Hash] Bindings parsed by Krane::BindingsParser
|
18
|
+
def initialize(logger: nil, current_sha:, template_dir: nil, template_paths: [], bindings:)
|
19
|
+
@logger = logger || Krane::FormattedLogger.build
|
20
|
+
@template_dir = template_dir
|
21
|
+
@template_paths = template_paths.map { |path| File.expand_path(path) }
|
22
|
+
@bindings = bindings
|
23
|
+
@current_sha = current_sha
|
24
|
+
end
|
25
|
+
|
26
|
+
# Runs the task, returning a boolean representing success or failure
|
27
|
+
#
|
28
|
+
# @return [Boolean]
|
29
|
+
def run(*args)
|
30
|
+
run!(*args)
|
31
|
+
true
|
32
|
+
rescue Krane::FatalDeploymentError
|
33
|
+
false
|
34
|
+
end
|
35
|
+
|
36
|
+
# Runs the task, raising exceptions in case of issues
|
37
|
+
#
|
38
|
+
# @param stream [IO] Place to stream the output to
|
39
|
+
# @param only_filenames [Array<String>] List of filenames to render
|
40
|
+
#
|
41
|
+
# @return [nil]
|
42
|
+
def run!(stream, only_filenames = [])
|
43
|
+
@logger.reset
|
44
|
+
@logger.phase_heading("Initializing render task")
|
45
|
+
|
46
|
+
ts = TemplateSets.from_dirs_and_files(paths: template_sets_paths(only_filenames), logger: @logger)
|
47
|
+
|
48
|
+
validate_configuration(ts, only_filenames)
|
49
|
+
count = render_templates(stream, ts)
|
50
|
+
|
51
|
+
@logger.summary.add_action("Successfully rendered #{count} template(s)")
|
52
|
+
@logger.print_summary(:success)
|
53
|
+
rescue Krane::FatalDeploymentError
|
54
|
+
@logger.print_summary(:failure)
|
55
|
+
raise
|
56
|
+
end
|
57
|
+
|
58
|
+
private
|
59
|
+
|
60
|
+
def template_sets_paths(only_filenames)
|
61
|
+
if @template_paths.present?
|
62
|
+
# Validation will catch @template_paths & @template_dir being present
|
63
|
+
@template_paths
|
64
|
+
elsif only_filenames.blank?
|
65
|
+
[File.expand_path(@template_dir || '')]
|
66
|
+
else
|
67
|
+
absolute_template_dir = File.expand_path(@template_dir || '')
|
68
|
+
only_filenames.map do |name|
|
69
|
+
File.join(absolute_template_dir, name)
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
def render_templates(stream, template_sets)
|
75
|
+
@logger.phase_heading("Rendering template(s)")
|
76
|
+
count = 0
|
77
|
+
template_sets.with_resource_definitions_and_filename(render_erb: true,
|
78
|
+
current_sha: @current_sha, bindings: @bindings, raw: true) do |rendered_content, filename|
|
79
|
+
write_to_stream(rendered_content, filename, stream)
|
80
|
+
count += 1
|
81
|
+
end
|
82
|
+
|
83
|
+
count
|
84
|
+
rescue Krane::InvalidTemplateError => exception
|
85
|
+
log_invalid_template(exception)
|
86
|
+
raise
|
87
|
+
end
|
88
|
+
|
89
|
+
def write_to_stream(rendered_content, filename, stream)
|
90
|
+
file_basename = File.basename(filename)
|
91
|
+
@logger.info("Rendering #{file_basename}...")
|
92
|
+
implicit = []
|
93
|
+
YAML.parse_stream(rendered_content, "<rendered> #{filename}") { |d| implicit << d.implicit }
|
94
|
+
if rendered_content.present?
|
95
|
+
stream.puts "---\n" if implicit.first
|
96
|
+
stream.puts rendered_content
|
97
|
+
@logger.info("Rendered #{file_basename}")
|
98
|
+
else
|
99
|
+
@logger.warn("Rendered #{file_basename} successfully, but the result was blank")
|
100
|
+
end
|
101
|
+
rescue Psych::SyntaxError => exception
|
102
|
+
raise InvalidTemplateError.new("Template is not valid YAML. #{exception.message}", filename: filename)
|
103
|
+
end
|
104
|
+
|
105
|
+
def validate_configuration(template_sets, filenames)
|
106
|
+
@logger.info("Validating configuration")
|
107
|
+
errors = []
|
108
|
+
if @template_dir.present? && @template_paths.present?
|
109
|
+
errors << "template_dir and template_paths can not be combined"
|
110
|
+
elsif @template_dir.blank? && @template_paths.blank?
|
111
|
+
errors << "template_dir or template_paths must be set"
|
112
|
+
end
|
113
|
+
|
114
|
+
if filenames.present?
|
115
|
+
if @template_dir.nil?
|
116
|
+
errors << "template_dir must be set to use filenames"
|
117
|
+
else
|
118
|
+
absolute_template_dir = File.expand_path(@template_dir)
|
119
|
+
filenames.each do |filename|
|
120
|
+
absolute_file = File.expand_path(File.join(@template_dir, filename))
|
121
|
+
unless absolute_file.start_with?(absolute_template_dir)
|
122
|
+
errors << "Filename \"#{absolute_file}\" is outside the template directory," \
|
123
|
+
" which was resolved as #{absolute_template_dir}"
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
129
|
+
errors += template_sets.validate
|
130
|
+
|
131
|
+
unless errors.empty?
|
132
|
+
@logger.summary.add_action("Configuration invalid")
|
133
|
+
@logger.summary.add_paragraph(errors.map { |err| "- #{err}" }.join("\n"))
|
134
|
+
raise Krane::TaskConfigurationError, "Configuration invalid: #{errors.join(', ')}"
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
def log_invalid_template(exception)
|
139
|
+
@logger.error("Failed to render #{exception.filename}")
|
140
|
+
|
141
|
+
debug_msg = ColorizedString.new("Invalid template: #{exception.filename}\n").red
|
142
|
+
debug_msg += "> Error message:\n#{FormattedLogger.indent_four(exception.to_s)}"
|
143
|
+
if exception.content
|
144
|
+
debug_msg += "\n> Template content:\n#{FormattedLogger.indent_four(exception.content)}"
|
145
|
+
end
|
146
|
+
@logger.summary.add_paragraph(debug_msg)
|
147
|
+
end
|
148
|
+
end
|
149
|
+
end
|
@@ -2,22 +2,22 @@
|
|
2
2
|
|
3
3
|
require 'concurrent/hash'
|
4
4
|
|
5
|
-
module
|
5
|
+
module Krane
|
6
6
|
class ResourceCache
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
@
|
7
|
+
delegate :namespace, :context, :logger, to: :@task_config
|
8
|
+
|
9
|
+
def initialize(task_config)
|
10
|
+
@task_config = task_config
|
11
11
|
|
12
12
|
@kind_fetcher_locks = Concurrent::Hash.new { |hash, key| hash[key] = Mutex.new }
|
13
13
|
@data = Concurrent::Hash.new
|
14
|
-
@kubectl = Kubectl.new(
|
14
|
+
@kubectl = Kubectl.new(task_config: @task_config, log_failure_by_default: false)
|
15
15
|
end
|
16
16
|
|
17
17
|
def get_instance(kind, resource_name, raise_if_not_found: false)
|
18
18
|
instance = use_or_populate_cache(kind).fetch(resource_name, {})
|
19
19
|
if instance.blank? && raise_if_not_found
|
20
|
-
raise
|
20
|
+
raise Krane::Kubectl::ResourceNotFoundError, "Resource does not exist (used cache for kind #{kind})"
|
21
21
|
end
|
22
22
|
instance
|
23
23
|
rescue KubectlError
|
@@ -39,7 +39,7 @@ module KubernetesDeploy
|
|
39
39
|
private
|
40
40
|
|
41
41
|
def statsd_tags
|
42
|
-
{ namespace:
|
42
|
+
{ namespace: namespace, context: context }
|
43
43
|
end
|
44
44
|
|
45
45
|
def use_or_populate_cache(kind)
|
@@ -51,9 +51,10 @@ module KubernetesDeploy
|
|
51
51
|
|
52
52
|
def fetch_by_kind(kind)
|
53
53
|
resource_class = KubernetesResource.class_for_kind(kind)
|
54
|
+
global_kind = @task_config.global_kinds.map(&:downcase).include?(kind.downcase)
|
54
55
|
output_is_sensitive = resource_class.nil? ? false : resource_class::SENSITIVE_TEMPLATE_CONTENT
|
55
56
|
raw_json, _, st = @kubectl.run("get", kind, "--chunk-size=0", attempts: 5, output: "json",
|
56
|
-
output_is_sensitive: output_is_sensitive)
|
57
|
+
output_is_sensitive: output_is_sensitive, use_namespace: !global_kind)
|
57
58
|
raise KubectlError unless st.success?
|
58
59
|
|
59
60
|
instances = {}
|
@@ -0,0 +1,265 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'krane/resource_watcher'
|
4
|
+
require 'krane/concerns/template_reporting'
|
5
|
+
|
6
|
+
module Krane
|
7
|
+
class ResourceDeployer
|
8
|
+
extend Krane::StatsD::MeasureMethods
|
9
|
+
include Krane::TemplateReporting
|
10
|
+
|
11
|
+
delegate :logger, to: :@task_config
|
12
|
+
attr_reader :statsd_tags
|
13
|
+
|
14
|
+
def initialize(task_config:, prune_whitelist:, max_watch_seconds:, current_sha: nil, selector:, statsd_tags:)
|
15
|
+
@task_config = task_config
|
16
|
+
@prune_whitelist = prune_whitelist
|
17
|
+
@max_watch_seconds = max_watch_seconds
|
18
|
+
@current_sha = current_sha
|
19
|
+
@selector = selector
|
20
|
+
@statsd_tags = statsd_tags
|
21
|
+
end
|
22
|
+
|
23
|
+
def deploy!(resources, verify_result, prune)
|
24
|
+
if verify_result
|
25
|
+
deploy_all_resources(resources, prune: prune, verify: true)
|
26
|
+
failed_resources = resources.reject(&:deploy_succeeded?)
|
27
|
+
success = failed_resources.empty?
|
28
|
+
if !success && failed_resources.all?(&:deploy_timed_out?)
|
29
|
+
raise DeploymentTimeoutError
|
30
|
+
end
|
31
|
+
raise FatalDeploymentError unless success
|
32
|
+
else
|
33
|
+
deploy_all_resources(resources, prune: prune, verify: false)
|
34
|
+
logger.summary.add_action("deployed #{resources.length} #{'resource'.pluralize(resources.length)}")
|
35
|
+
warning = <<~MSG
|
36
|
+
Deploy result verification is disabled for this deploy.
|
37
|
+
This means the desired changes were communicated to Kubernetes, but the deploy did not make sure they actually succeeded.
|
38
|
+
MSG
|
39
|
+
logger.summary.add_paragraph(ColorizedString.new(warning).yellow)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def predeploy_priority_resources(resource_list, predeploy_sequence)
|
44
|
+
bare_pods = resource_list.select { |resource| resource.is_a?(Pod) }
|
45
|
+
if bare_pods.count == 1
|
46
|
+
bare_pods.first.stream_logs = true
|
47
|
+
end
|
48
|
+
|
49
|
+
predeploy_sequence.each do |resource_type|
|
50
|
+
matching_resources = resource_list.select { |r| r.type == resource_type }
|
51
|
+
next if matching_resources.empty?
|
52
|
+
deploy_resources(matching_resources, verify: true, record_summary: false)
|
53
|
+
|
54
|
+
failed_resources = matching_resources.reject(&:deploy_succeeded?)
|
55
|
+
fail_count = failed_resources.length
|
56
|
+
if fail_count > 0
|
57
|
+
Krane::Concurrency.split_across_threads(failed_resources) do |r|
|
58
|
+
r.sync_debug_info(kubectl)
|
59
|
+
end
|
60
|
+
failed_resources.each { |r| logger.summary.add_paragraph(r.debug_message) }
|
61
|
+
raise FatalDeploymentError, "Failed to deploy #{fail_count} priority #{'resource'.pluralize(fail_count)}"
|
62
|
+
end
|
63
|
+
logger.blank_line
|
64
|
+
end
|
65
|
+
end
|
66
|
+
measure_method(:predeploy_priority_resources, 'priority_resources.duration')
|
67
|
+
|
68
|
+
private
|
69
|
+
|
70
|
+
def deploy_all_resources(resources, prune: false, verify:, record_summary: true)
|
71
|
+
deploy_resources(resources, prune: prune, verify: verify, record_summary: record_summary)
|
72
|
+
end
|
73
|
+
measure_method(:deploy_all_resources, 'normal_resources.duration')
|
74
|
+
|
75
|
+
def deploy_resources(resources, prune: false, verify:, record_summary: true)
|
76
|
+
return if resources.empty?
|
77
|
+
deploy_started_at = Time.now.utc
|
78
|
+
|
79
|
+
if resources.length > 1
|
80
|
+
logger.info("Deploying resources:")
|
81
|
+
resources.each do |r|
|
82
|
+
logger.info("- #{r.id} (#{r.pretty_timeout_type})")
|
83
|
+
end
|
84
|
+
else
|
85
|
+
resource = resources.first
|
86
|
+
logger.info("Deploying #{resource.id} (#{resource.pretty_timeout_type})")
|
87
|
+
end
|
88
|
+
|
89
|
+
# Apply can be done in one large batch, the rest have to be done individually
|
90
|
+
applyables, individuals = resources.partition { |r| r.deploy_method == :apply }
|
91
|
+
# Prunable resources should also applied so that they can be pruned
|
92
|
+
pruneable_types = @prune_whitelist.map { |t| t.split("/").last }
|
93
|
+
applyables += individuals.select { |r| pruneable_types.include?(r.type) }
|
94
|
+
|
95
|
+
individuals.each do |individual_resource|
|
96
|
+
individual_resource.deploy_started_at = Time.now.utc
|
97
|
+
|
98
|
+
case individual_resource.deploy_method
|
99
|
+
when :create
|
100
|
+
err, status = create_resource(individual_resource)
|
101
|
+
when :replace
|
102
|
+
err, status = replace_or_create_resource(individual_resource)
|
103
|
+
when :replace_force
|
104
|
+
err, status = replace_or_create_resource(individual_resource, force: true)
|
105
|
+
else
|
106
|
+
# Fail Fast! This is a programmer mistake.
|
107
|
+
raise ArgumentError, "Unexpected deploy method! (#{individual_resource.deploy_method.inspect})"
|
108
|
+
end
|
109
|
+
|
110
|
+
next if status.success?
|
111
|
+
|
112
|
+
raise FatalDeploymentError, <<~MSG
|
113
|
+
Failed to replace or create resource: #{individual_resource.id}
|
114
|
+
#{individual_resource.sensitive_template_content? ? '<suppressed sensitive output>' : err}
|
115
|
+
MSG
|
116
|
+
end
|
117
|
+
|
118
|
+
apply_all(applyables, prune)
|
119
|
+
|
120
|
+
if verify
|
121
|
+
watcher = Krane::ResourceWatcher.new(resources: resources, deploy_started_at: deploy_started_at,
|
122
|
+
timeout: @max_watch_seconds, task_config: @task_config, sha: @current_sha)
|
123
|
+
watcher.run(record_summary: record_summary)
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
def apply_all(resources, prune)
|
128
|
+
return unless resources.present?
|
129
|
+
command = %w(apply)
|
130
|
+
|
131
|
+
Dir.mktmpdir do |tmp_dir|
|
132
|
+
resources.each do |r|
|
133
|
+
FileUtils.symlink(r.file_path, tmp_dir)
|
134
|
+
r.deploy_started_at = Time.now.utc
|
135
|
+
end
|
136
|
+
command.push("-f", tmp_dir)
|
137
|
+
|
138
|
+
if prune && @prune_whitelist.present?
|
139
|
+
command.push("--prune")
|
140
|
+
if @selector
|
141
|
+
command.push("--selector", @selector.to_s)
|
142
|
+
else
|
143
|
+
command.push("--all")
|
144
|
+
end
|
145
|
+
@prune_whitelist.each { |type| command.push("--prune-whitelist=#{type}") }
|
146
|
+
end
|
147
|
+
|
148
|
+
output_is_sensitive = resources.any?(&:sensitive_template_content?)
|
149
|
+
global_mode = resources.all?(&:global?)
|
150
|
+
out, err, st = kubectl.run(*command, log_failure: false, output_is_sensitive: output_is_sensitive,
|
151
|
+
use_namespace: !global_mode)
|
152
|
+
|
153
|
+
if st.success?
|
154
|
+
log_pruning(out) if prune
|
155
|
+
else
|
156
|
+
record_apply_failure(err, resources: resources)
|
157
|
+
raise FatalDeploymentError, "Command failed: #{Shellwords.join(command)}"
|
158
|
+
end
|
159
|
+
end
|
160
|
+
end
|
161
|
+
measure_method(:apply_all)
|
162
|
+
|
163
|
+
def log_pruning(kubectl_output)
|
164
|
+
pruned = kubectl_output.scan(/^(.*) pruned$/)
|
165
|
+
return unless pruned.present?
|
166
|
+
|
167
|
+
logger.info("The following resources were pruned: #{pruned.join(', ')}")
|
168
|
+
logger.summary.add_action("pruned #{pruned.length} #{'resource'.pluralize(pruned.length)}")
|
169
|
+
end
|
170
|
+
|
171
|
+
def record_apply_failure(err, resources: [])
|
172
|
+
warn_msg = "WARNING: Any resources not mentioned in the error(s) below were likely created/updated. " \
|
173
|
+
"You may wish to roll back this deploy."
|
174
|
+
logger.summary.add_paragraph(ColorizedString.new(warn_msg).yellow)
|
175
|
+
|
176
|
+
unidentified_errors = []
|
177
|
+
filenames_with_sensitive_content = resources
|
178
|
+
.select(&:sensitive_template_content?)
|
179
|
+
.map { |r| File.basename(r.file_path) }
|
180
|
+
|
181
|
+
server_dry_run_validated_resource = resources
|
182
|
+
.select(&:server_dry_run_validated?)
|
183
|
+
.map { |r| File.basename(r.file_path) }
|
184
|
+
|
185
|
+
err.each_line do |line|
|
186
|
+
bad_files = find_bad_files_from_kubectl_output(line)
|
187
|
+
unless bad_files.present?
|
188
|
+
unidentified_errors << line
|
189
|
+
next
|
190
|
+
end
|
191
|
+
|
192
|
+
bad_files.each do |f|
|
193
|
+
err_msg = f[:err]
|
194
|
+
if filenames_with_sensitive_content.include?(f[:filename])
|
195
|
+
# Hide the error and template contents in case it has sensitive information
|
196
|
+
# we display full error messages as we assume there's no sensitive info leak after server-dry-run
|
197
|
+
err_msg = "SUPPRESSED FOR SECURITY" unless server_dry_run_validated_resource.include?(f[:filename])
|
198
|
+
record_invalid_template(logger: logger, err: err_msg, filename: f[:filename], content: nil)
|
199
|
+
else
|
200
|
+
record_invalid_template(logger: logger, err: err_msg, filename: f[:filename], content: f[:content])
|
201
|
+
end
|
202
|
+
end
|
203
|
+
end
|
204
|
+
return unless unidentified_errors.any?
|
205
|
+
|
206
|
+
if (filenames_with_sensitive_content - server_dry_run_validated_resource).present?
|
207
|
+
warn_msg = "WARNING: There was an error applying some or all resources. The raw output may be sensitive and " \
|
208
|
+
"so cannot be displayed."
|
209
|
+
logger.summary.add_paragraph(ColorizedString.new(warn_msg).yellow)
|
210
|
+
else
|
211
|
+
heading = ColorizedString.new('Unidentified error(s):').red
|
212
|
+
msg = FormattedLogger.indent_four(unidentified_errors.join)
|
213
|
+
logger.summary.add_paragraph("#{heading}\n#{msg}")
|
214
|
+
end
|
215
|
+
end
|
216
|
+
|
217
|
+
def replace_or_create_resource(resource, force: false)
|
218
|
+
args = if force
|
219
|
+
["replace", "--force", "--cascade", "-f", resource.file_path]
|
220
|
+
else
|
221
|
+
["replace", "-f", resource.file_path]
|
222
|
+
end
|
223
|
+
|
224
|
+
_, err, status = kubectl.run(*args, log_failure: false, output_is_sensitive: resource.sensitive_template_content?,
|
225
|
+
raise_if_not_found: true, use_namespace: !resource.global?)
|
226
|
+
|
227
|
+
[err, status]
|
228
|
+
rescue Krane::Kubectl::ResourceNotFoundError
|
229
|
+
# it doesn't exist so we can't replace it, we try to create it
|
230
|
+
create_resource(resource)
|
231
|
+
end
|
232
|
+
|
233
|
+
def create_resource(resource)
|
234
|
+
out, err, status = kubectl.run("create", "-f", resource.file_path, log_failure: false,
|
235
|
+
output: 'json', output_is_sensitive: resource.sensitive_template_content?,
|
236
|
+
use_namespace: !resource.global?)
|
237
|
+
|
238
|
+
# For resources that rely on a generateName attribute, we get the `name` from the result of the call to `create`
|
239
|
+
# We must explicitly set this name value so that the `apply` step for pruning can run successfully
|
240
|
+
if status.success? && resource.uses_generate_name?
|
241
|
+
resource.use_generated_name(JSON.parse(out))
|
242
|
+
end
|
243
|
+
|
244
|
+
[err, status]
|
245
|
+
end
|
246
|
+
|
247
|
+
# Inspect the file referenced in the kubectl stderr
|
248
|
+
# to make it easier for developer to understand what's going on
|
249
|
+
def find_bad_files_from_kubectl_output(line)
|
250
|
+
# stderr often contains one or more lines like the following, from which we can extract the file path(s):
|
251
|
+
# Error from server (TypeOfError): error when creating "/path/to/service-gqq5oh.yml": Service "web" is invalid:
|
252
|
+
|
253
|
+
line.scan(%r{"(/\S+\.ya?ml\S*)"}).each_with_object([]) do |matches, bad_files|
|
254
|
+
matches.each do |path|
|
255
|
+
content = File.read(path) if File.file?(path)
|
256
|
+
bad_files << { filename: File.basename(path), err: line, content: content }
|
257
|
+
end
|
258
|
+
end
|
259
|
+
end
|
260
|
+
|
261
|
+
def kubectl
|
262
|
+
@kubectl ||= Kubectl.new(task_config: @task_config, log_failure_by_default: true)
|
263
|
+
end
|
264
|
+
end
|
265
|
+
end
|