kubernetes-deploy 0.22.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.rubocop.yml +8 -0
- data/CHANGELOG.md +16 -0
- data/README.md +32 -0
- data/exe/kubernetes-deploy +2 -15
- data/exe/kubernetes-render +32 -0
- data/kubernetes-deploy.gemspec +5 -3
- data/lib/kubernetes-deploy.rb +5 -3
- data/lib/kubernetes-deploy/cluster_resource_discovery.rb +34 -0
- data/lib/kubernetes-deploy/container_logs.rb +25 -13
- data/lib/kubernetes-deploy/deploy_task.rb +68 -50
- data/lib/kubernetes-deploy/errors.rb +1 -0
- data/lib/kubernetes-deploy/formatted_logger.rb +16 -2
- data/lib/kubernetes-deploy/kubeclient_builder/google_friendly_config.rb +4 -6
- data/lib/kubernetes-deploy/kubectl.rb +20 -9
- data/lib/kubernetes-deploy/kubernetes_resource.rb +5 -6
- data/lib/kubernetes-deploy/kubernetes_resource/cloudsql.rb +3 -4
- data/lib/kubernetes-deploy/kubernetes_resource/daemon_set.rb +4 -5
- data/lib/kubernetes-deploy/kubernetes_resource/deployment.rb +7 -8
- data/lib/kubernetes-deploy/kubernetes_resource/memcached.rb +4 -5
- data/lib/kubernetes-deploy/kubernetes_resource/pod.rb +7 -5
- data/lib/kubernetes-deploy/kubernetes_resource/pod_set_base.rb +12 -6
- data/lib/kubernetes-deploy/kubernetes_resource/redis.rb +5 -6
- data/lib/kubernetes-deploy/kubernetes_resource/replica_set.rb +23 -5
- data/lib/kubernetes-deploy/kubernetes_resource/role.rb +22 -0
- data/lib/kubernetes-deploy/kubernetes_resource/service.rb +8 -4
- data/lib/kubernetes-deploy/kubernetes_resource/stateful_set.rb +2 -3
- data/lib/kubernetes-deploy/oj.rb +4 -0
- data/lib/kubernetes-deploy/options_helper.rb +27 -0
- data/lib/kubernetes-deploy/remote_logs.rb +10 -4
- data/lib/kubernetes-deploy/render_task.rb +119 -0
- data/lib/kubernetes-deploy/renderer.rb +1 -1
- data/lib/kubernetes-deploy/resource_cache.rb +64 -0
- data/lib/kubernetes-deploy/resource_watcher.rb +27 -6
- data/lib/kubernetes-deploy/restart_task.rb +5 -6
- data/lib/kubernetes-deploy/runner_task.rb +6 -10
- data/lib/kubernetes-deploy/statsd.rb +60 -7
- data/lib/kubernetes-deploy/template_discovery.rb +15 -0
- data/lib/kubernetes-deploy/version.rb +1 -1
- data/pull_request_template.md +8 -0
- metadata +47 -5
- data/lib/kubernetes-deploy/resource_discovery.rb +0 -19
- data/lib/kubernetes-deploy/sync_mediator.rb +0 -80
@@ -3,6 +3,7 @@ module KubernetesDeploy
|
|
3
3
|
class FatalDeploymentError < StandardError; end
|
4
4
|
class FatalKubeAPIError < FatalDeploymentError; end
|
5
5
|
class KubectlError < StandardError; end
|
6
|
+
class TaskConfigurationError < FatalDeploymentError; end
|
6
7
|
|
7
8
|
class InvalidTemplateError < FatalDeploymentError
|
8
9
|
attr_reader :content
|
@@ -6,12 +6,26 @@ module KubernetesDeploy
|
|
6
6
|
class FormattedLogger < Logger
|
7
7
|
include DeferredSummaryLogging
|
8
8
|
|
9
|
-
def self.
|
9
|
+
def self.indent_four(str)
|
10
|
+
" " + str.to_s.gsub("\n", "\n ")
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.build(namespace = nil, context = nil, stream = $stderr, verbose_prefix: false)
|
10
14
|
l = new(stream)
|
11
15
|
l.level = level_from_env
|
12
16
|
|
17
|
+
middle = if verbose_prefix
|
18
|
+
if namespace.blank?
|
19
|
+
raise ArgumentError, 'Must pass a namespace if logging verbosely'
|
20
|
+
end
|
21
|
+
if context.blank?
|
22
|
+
raise ArgumentError, 'Must pass a context if logging verbosely'
|
23
|
+
end
|
24
|
+
|
25
|
+
"[#{context}][#{namespace}]"
|
26
|
+
end
|
27
|
+
|
13
28
|
l.formatter = proc do |severity, datetime, _progname, msg|
|
14
|
-
middle = verbose_prefix ? "[#{context}][#{namespace}]" : ""
|
15
29
|
colorized_line = ColorizedString.new("[#{severity}][#{datetime}]#{middle}\t#{msg}\n")
|
16
30
|
|
17
31
|
case severity
|
@@ -32,12 +32,10 @@ module KubernetesDeploy
|
|
32
32
|
private
|
33
33
|
|
34
34
|
def json_error_message(body)
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
end
|
40
|
-
json_error_msg['message']
|
35
|
+
err = JSON.parse(body || '') || {}
|
36
|
+
err['message']
|
37
|
+
rescue JSON::ParserError
|
38
|
+
nil
|
41
39
|
end
|
42
40
|
end
|
43
41
|
end
|
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
module KubernetesDeploy
|
4
4
|
class Kubectl
|
5
|
-
DEFAULT_TIMEOUT =
|
5
|
+
DEFAULT_TIMEOUT = 15
|
6
6
|
NOT_FOUND_ERROR_TEXT = 'NotFound'
|
7
7
|
|
8
8
|
class ResourceNotFoundError < StandardError; end
|
@@ -20,32 +20,43 @@ module KubernetesDeploy
|
|
20
20
|
raise ArgumentError, "context is required" if context.blank?
|
21
21
|
end
|
22
22
|
|
23
|
-
def run(*args, log_failure: nil, use_context: true, use_namespace: true, raise_if_not_found: false)
|
23
|
+
def run(*args, log_failure: nil, use_context: true, use_namespace: true, raise_if_not_found: false, attempts: 1)
|
24
24
|
log_failure = @log_failure_by_default if log_failure.nil?
|
25
25
|
|
26
26
|
args = args.unshift("kubectl")
|
27
27
|
args.push("--namespace=#{@namespace}") if use_namespace
|
28
28
|
args.push("--context=#{@context}") if use_context
|
29
29
|
args.push("--request-timeout=#{@default_timeout}") if @default_timeout
|
30
|
+
out, err, st = nil
|
30
31
|
|
31
|
-
|
32
|
-
|
33
|
-
|
32
|
+
(1..attempts).to_a.each do |attempt|
|
33
|
+
@logger.debug "Running command (attempt #{attempt}): #{args.join(' ')}"
|
34
|
+
out, err, st = Open3.capture3(*args)
|
35
|
+
@logger.debug("Kubectl out: " + out.gsub(/\s+/, ' ')) unless output_is_sensitive?
|
36
|
+
|
37
|
+
break if st.success?
|
34
38
|
|
35
|
-
unless st.success?
|
36
39
|
if log_failure
|
37
|
-
@logger.warn("The following command failed: #{Shellwords.join(args)}")
|
40
|
+
@logger.warn("The following command failed (attempt #{attempt}/#{attempts}): #{Shellwords.join(args)}")
|
38
41
|
@logger.warn(err) unless output_is_sensitive?
|
39
42
|
end
|
40
43
|
|
41
|
-
if
|
42
|
-
raise
|
44
|
+
if err.match(NOT_FOUND_ERROR_TEXT)
|
45
|
+
raise(ResourceNotFoundError, err) if raise_if_not_found
|
46
|
+
else
|
47
|
+
@logger.debug("Kubectl err: #{err}") unless output_is_sensitive?
|
48
|
+
StatsD.increment('kubectl.error', 1, tags: { context: @context, namespace: @namespace, cmd: args[1] })
|
43
49
|
end
|
50
|
+
sleep retry_delay(attempt) unless attempt == attempts
|
44
51
|
end
|
45
52
|
|
46
53
|
[out.chomp, err.chomp, st]
|
47
54
|
end
|
48
55
|
|
56
|
+
def retry_delay(attempt)
|
57
|
+
attempt
|
58
|
+
end
|
59
|
+
|
49
60
|
def version_info
|
50
61
|
@version_info ||=
|
51
62
|
begin
|
@@ -121,8 +121,8 @@ module KubernetesDeploy
|
|
121
121
|
file.path
|
122
122
|
end
|
123
123
|
|
124
|
-
def sync(
|
125
|
-
@instance_data =
|
124
|
+
def sync(cache)
|
125
|
+
@instance_data = cache.get_instance(kubectl_resource_type, name, raise_if_not_found: true)
|
126
126
|
rescue KubernetesDeploy::Kubectl::ResourceNotFoundError
|
127
127
|
@disappeared = true if deploy_started?
|
128
128
|
@instance_data = {}
|
@@ -195,7 +195,7 @@ module KubernetesDeploy
|
|
195
195
|
|
196
196
|
def sync_debug_info(kubectl)
|
197
197
|
@debug_events = fetch_events(kubectl) unless ENV[DISABLE_FETCHING_EVENT_INFO]
|
198
|
-
@debug_logs = fetch_debug_logs
|
198
|
+
@debug_logs = fetch_debug_logs if print_debug_logs? && !ENV[DISABLE_FETCHING_LOG_INFO]
|
199
199
|
end
|
200
200
|
|
201
201
|
def debug_message(cause = nil, info_hash = {})
|
@@ -293,7 +293,7 @@ module KubernetesDeploy
|
|
293
293
|
|
294
294
|
def report_status_to_statsd(watch_time)
|
295
295
|
unless @statsd_report_done
|
296
|
-
|
296
|
+
StatsD.distribution('resource.duration', watch_time, tags: statsd_tags)
|
297
297
|
@statsd_report_done = true
|
298
298
|
end
|
299
299
|
end
|
@@ -407,8 +407,7 @@ module KubernetesDeploy
|
|
407
407
|
else
|
408
408
|
"unknown"
|
409
409
|
end
|
410
|
-
tags = %W(context:#{context} namespace:#{namespace}
|
411
|
-
type:#{type} sha:#{ENV['REVISION']} status:#{status})
|
410
|
+
tags = %W(context:#{context} namespace:#{namespace} type:#{type} status:#{status})
|
412
411
|
tags | @optional_statsd_tags
|
413
412
|
end
|
414
413
|
end
|
@@ -3,11 +3,10 @@ module KubernetesDeploy
|
|
3
3
|
class Cloudsql < KubernetesResource
|
4
4
|
TIMEOUT = 10.minutes
|
5
5
|
|
6
|
-
|
7
|
-
def sync(mediator)
|
6
|
+
def sync(cache)
|
8
7
|
super
|
9
|
-
@proxy_deployment =
|
10
|
-
@proxy_service =
|
8
|
+
@proxy_deployment = cache.get_instance(Deployment.kind, "cloudsql-#{cloudsql_resource_uuid}")
|
9
|
+
@proxy_service = cache.get_instance(Service.kind, "cloudsql-#{@name}")
|
11
10
|
end
|
12
11
|
|
13
12
|
def status
|
@@ -5,10 +5,9 @@ module KubernetesDeploy
|
|
5
5
|
TIMEOUT = 5.minutes
|
6
6
|
attr_reader :pods
|
7
7
|
|
8
|
-
|
9
|
-
def sync(mediator)
|
8
|
+
def sync(cache)
|
10
9
|
super
|
11
|
-
@pods = exists? ? find_pods(
|
10
|
+
@pods = exists? ? find_pods(cache) : []
|
12
11
|
end
|
13
12
|
|
14
13
|
def status
|
@@ -28,9 +27,9 @@ module KubernetesDeploy
|
|
28
27
|
observed_generation == current_generation
|
29
28
|
end
|
30
29
|
|
31
|
-
def fetch_debug_logs
|
30
|
+
def fetch_debug_logs
|
32
31
|
most_useful_pod = pods.find(&:deploy_failed?) || pods.find(&:deploy_timed_out?) || pods.first
|
33
|
-
most_useful_pod.fetch_debug_logs
|
32
|
+
most_useful_pod.fetch_debug_logs
|
34
33
|
end
|
35
34
|
|
36
35
|
def print_debug_logs?
|
@@ -6,10 +6,9 @@ module KubernetesDeploy
|
|
6
6
|
REQUIRED_ROLLOUT_TYPES = %w(maxUnavailable full none).freeze
|
7
7
|
DEFAULT_REQUIRED_ROLLOUT = 'full'
|
8
8
|
|
9
|
-
|
10
|
-
def sync(mediator)
|
9
|
+
def sync(cache)
|
11
10
|
super
|
12
|
-
@latest_rs = exists? ? find_latest_rs(
|
11
|
+
@latest_rs = exists? ? find_latest_rs(cache) : nil
|
13
12
|
end
|
14
13
|
|
15
14
|
def status
|
@@ -27,8 +26,8 @@ module KubernetesDeploy
|
|
27
26
|
@latest_rs.present?
|
28
27
|
end
|
29
28
|
|
30
|
-
def fetch_debug_logs
|
31
|
-
@latest_rs.fetch_debug_logs
|
29
|
+
def fetch_debug_logs
|
30
|
+
@latest_rs.fetch_debug_logs
|
32
31
|
end
|
33
32
|
|
34
33
|
def deploy_succeeded?
|
@@ -151,8 +150,8 @@ module KubernetesDeploy
|
|
151
150
|
progress_condition["status"] == 'False'
|
152
151
|
end
|
153
152
|
|
154
|
-
def find_latest_rs(
|
155
|
-
all_rs_data =
|
153
|
+
def find_latest_rs(cache)
|
154
|
+
all_rs_data = cache.get_all(ReplicaSet.kind, @instance_data["spec"]["selector"]["matchLabels"])
|
156
155
|
current_revision = @instance_data["metadata"]["annotations"]["deployment.kubernetes.io/revision"]
|
157
156
|
|
158
157
|
latest_rs_data = all_rs_data.find do |rs|
|
@@ -170,7 +169,7 @@ module KubernetesDeploy
|
|
170
169
|
parent: "#{@name.capitalize} deployment",
|
171
170
|
deploy_started_at: @deploy_started_at
|
172
171
|
)
|
173
|
-
rs.sync(
|
172
|
+
rs.sync(cache)
|
174
173
|
rs
|
175
174
|
end
|
176
175
|
|
@@ -4,12 +4,11 @@ module KubernetesDeploy
|
|
4
4
|
TIMEOUT = 5.minutes
|
5
5
|
CONFIGMAP_NAME = "memcached-url"
|
6
6
|
|
7
|
-
|
8
|
-
def sync(mediator)
|
7
|
+
def sync(cache)
|
9
8
|
super
|
10
|
-
@deployment =
|
11
|
-
@service =
|
12
|
-
@configmap =
|
9
|
+
@deployment = cache.get_instance(Deployment.kind, "memcached-#{@name}")
|
10
|
+
@service = cache.get_instance(Service.kind, "memcached-#{@name}")
|
11
|
+
@configmap = cache.get_instance(ConfigMap.kind, CONFIGMAP_NAME)
|
13
12
|
end
|
14
13
|
|
15
14
|
def status
|
@@ -25,12 +25,12 @@ module KubernetesDeploy
|
|
25
25
|
logger: logger, statsd_tags: statsd_tags)
|
26
26
|
end
|
27
27
|
|
28
|
-
def sync(
|
28
|
+
def sync(_cache)
|
29
29
|
super
|
30
30
|
raise_predates_deploy_error if exists? && unmanaged? && !deploy_started?
|
31
31
|
|
32
32
|
if exists?
|
33
|
-
logs.sync
|
33
|
+
logs.sync if unmanaged?
|
34
34
|
update_container_statuses(@instance_data["status"])
|
35
35
|
else # reset
|
36
36
|
@containers.each(&:reset_status)
|
@@ -85,8 +85,8 @@ module KubernetesDeploy
|
|
85
85
|
"#{phase_failure_message} #{container_problems}".strip.presence
|
86
86
|
end
|
87
87
|
|
88
|
-
def fetch_debug_logs
|
89
|
-
logs.sync
|
88
|
+
def fetch_debug_logs
|
89
|
+
logs.sync
|
90
90
|
logs
|
91
91
|
end
|
92
92
|
|
@@ -123,7 +123,9 @@ module KubernetesDeploy
|
|
123
123
|
@logs ||= KubernetesDeploy::RemoteLogs.new(
|
124
124
|
logger: @logger,
|
125
125
|
parent_id: id,
|
126
|
-
container_names: @containers.map(&:name)
|
126
|
+
container_names: @containers.map(&:name),
|
127
|
+
namespace: @namespace,
|
128
|
+
context: @context
|
127
129
|
)
|
128
130
|
end
|
129
131
|
|
@@ -16,9 +16,15 @@ module KubernetesDeploy
|
|
16
16
|
own_events.merge(most_useful_pod.fetch_events(kubectl))
|
17
17
|
end
|
18
18
|
|
19
|
-
def fetch_debug_logs
|
20
|
-
logs = KubernetesDeploy::RemoteLogs.new(
|
21
|
-
|
19
|
+
def fetch_debug_logs
|
20
|
+
logs = KubernetesDeploy::RemoteLogs.new(
|
21
|
+
logger: @logger,
|
22
|
+
parent_id: id,
|
23
|
+
container_names: container_names,
|
24
|
+
namespace: @namespace,
|
25
|
+
context: @context
|
26
|
+
)
|
27
|
+
logs.sync
|
22
28
|
logs
|
23
29
|
end
|
24
30
|
|
@@ -42,8 +48,8 @@ module KubernetesDeploy
|
|
42
48
|
regular_containers + init_containers
|
43
49
|
end
|
44
50
|
|
45
|
-
def find_pods(
|
46
|
-
all_pods =
|
51
|
+
def find_pods(cache)
|
52
|
+
all_pods = cache.get_all(Pod.kind, @instance_data["spec"]["selector"]["matchLabels"])
|
47
53
|
|
48
54
|
all_pods.each_with_object([]) do |pod_data, relevant_pods|
|
49
55
|
next unless parent_of_pod?(pod_data)
|
@@ -55,7 +61,7 @@ module KubernetesDeploy
|
|
55
61
|
parent: "#{name.capitalize} #{type}",
|
56
62
|
deploy_started_at: @deploy_started_at
|
57
63
|
)
|
58
|
-
pod.sync(
|
64
|
+
pod.sync(cache)
|
59
65
|
relevant_pods << pod
|
60
66
|
end
|
61
67
|
end
|
@@ -4,15 +4,14 @@ module KubernetesDeploy
|
|
4
4
|
TIMEOUT = 5.minutes
|
5
5
|
UUID_ANNOTATION = "redis.stable.shopify.io/owner_uid"
|
6
6
|
|
7
|
-
|
8
|
-
def sync(mediator)
|
7
|
+
def sync(cache)
|
9
8
|
super
|
10
9
|
|
11
|
-
@deployment =
|
12
|
-
@deployment =
|
10
|
+
@deployment = cache.get_instance(Deployment.kind, name)
|
11
|
+
@deployment = cache.get_instance(Deployment.kind, deprecated_name) if @deployment.empty?
|
13
12
|
|
14
|
-
@service =
|
15
|
-
@service =
|
13
|
+
@service = cache.get_instance(Service.kind, name)
|
14
|
+
@service = cache.get_instance(Service.kind, deprecated_name) if @service.empty?
|
16
15
|
end
|
17
16
|
|
18
17
|
def status
|
@@ -14,10 +14,9 @@ module KubernetesDeploy
|
|
14
14
|
logger: logger, statsd_tags: statsd_tags)
|
15
15
|
end
|
16
16
|
|
17
|
-
|
18
|
-
def sync(mediator)
|
17
|
+
def sync(cache)
|
19
18
|
super
|
20
|
-
@pods =
|
19
|
+
@pods = fetch_pods_if_needed(cache) || []
|
21
20
|
end
|
22
21
|
|
23
22
|
def status
|
@@ -26,7 +25,7 @@ module KubernetesDeploy
|
|
26
25
|
end
|
27
26
|
|
28
27
|
def deploy_succeeded?
|
29
|
-
|
28
|
+
return false if stale_status?
|
30
29
|
desired_replicas == rollout_data["availableReplicas"].to_i &&
|
31
30
|
desired_replicas == rollout_data["readyReplicas"].to_i
|
32
31
|
end
|
@@ -34,7 +33,7 @@ module KubernetesDeploy
|
|
34
33
|
def deploy_failed?
|
35
34
|
pods.present? &&
|
36
35
|
pods.all?(&:deploy_failed?) &&
|
37
|
-
|
36
|
+
!stale_status?
|
38
37
|
end
|
39
38
|
|
40
39
|
def desired_replicas
|
@@ -54,6 +53,25 @@ module KubernetesDeploy
|
|
54
53
|
|
55
54
|
private
|
56
55
|
|
56
|
+
def stale_status?
|
57
|
+
observed_generation != current_generation
|
58
|
+
end
|
59
|
+
|
60
|
+
def fetch_pods_if_needed(cache)
|
61
|
+
# If the ReplicaSet doesn't exist, its pods won't either
|
62
|
+
return unless exists?
|
63
|
+
# If the status hasn't been updated yet, we're not going to make a determination anyway
|
64
|
+
return if stale_status?
|
65
|
+
# If we don't want any pods at all, we don't need to look for them
|
66
|
+
return if desired_replicas == 0
|
67
|
+
# We only need to fetch pods so that deploy_failed? can check that they aren't ALL bad.
|
68
|
+
# If we can already tell some pods are ok from the RS data, don't bother fetching them (which can be expensive)
|
69
|
+
# Lower numbers here make us more susceptible to being fooled by replicas without probes briefly appearing ready
|
70
|
+
return if ready_replicas > 1
|
71
|
+
|
72
|
+
find_pods(cache)
|
73
|
+
end
|
74
|
+
|
57
75
|
def rollout_data
|
58
76
|
return { "replicas" => 0 } unless exists?
|
59
77
|
{ "replicas" => 0 }.merge(
|
@@ -0,0 +1,22 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
module KubernetesDeploy
|
3
|
+
class Role < KubernetesResource
|
4
|
+
TIMEOUT = 30.seconds
|
5
|
+
|
6
|
+
def status
|
7
|
+
exists? ? "Created" : "Unknown"
|
8
|
+
end
|
9
|
+
|
10
|
+
def deploy_succeeded?
|
11
|
+
exists?
|
12
|
+
end
|
13
|
+
|
14
|
+
def deploy_failed?
|
15
|
+
false
|
16
|
+
end
|
17
|
+
|
18
|
+
def timeout_message
|
19
|
+
UNUSUAL_FAILURE_MESSAGE
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -3,11 +3,15 @@ module KubernetesDeploy
|
|
3
3
|
class Service < KubernetesResource
|
4
4
|
TIMEOUT = 7.minutes
|
5
5
|
|
6
|
-
|
7
|
-
def sync(mediator)
|
6
|
+
def sync(cache)
|
8
7
|
super
|
9
|
-
|
10
|
-
|
8
|
+
if exists? && selector.present?
|
9
|
+
@related_deployments = cache.get_all(Deployment.kind, selector)
|
10
|
+
@related_pods = cache.get_all(Pod.kind, selector)
|
11
|
+
else
|
12
|
+
@related_deployments = []
|
13
|
+
@related_pods = []
|
14
|
+
end
|
11
15
|
end
|
12
16
|
|
13
17
|
def status
|