kubernetes-deploy 0.25.0 → 0.26.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.buildkite/pipeline.nightly.yml +0 -9
- data/.buildkite/pipeline.yml +0 -9
- data/CHANGELOG.md +23 -0
- data/CONTRIBUTING.md +164 -0
- data/README.md +29 -104
- data/dev.yml +3 -3
- data/exe/kubernetes-deploy +32 -21
- data/exe/kubernetes-render +20 -12
- data/exe/kubernetes-restart +5 -1
- data/lib/kubernetes-deploy.rb +1 -0
- data/lib/kubernetes-deploy/bindings_parser.rb +20 -8
- data/lib/kubernetes-deploy/cluster_resource_discovery.rb +1 -1
- data/lib/kubernetes-deploy/container_logs.rb +6 -0
- data/lib/kubernetes-deploy/deploy_task.rb +85 -44
- data/lib/kubernetes-deploy/ejson_secret_provisioner.rb +38 -84
- data/lib/kubernetes-deploy/errors.rb +8 -0
- data/lib/kubernetes-deploy/kubeclient_builder.rb +52 -20
- data/lib/kubernetes-deploy/kubeclient_builder/kube_config.rb +1 -1
- data/lib/kubernetes-deploy/kubectl.rb +11 -10
- data/lib/kubernetes-deploy/kubernetes_resource.rb +47 -9
- data/lib/kubernetes-deploy/kubernetes_resource/custom_resource.rb +1 -1
- data/lib/kubernetes-deploy/kubernetes_resource/custom_resource_definition.rb +1 -1
- data/lib/kubernetes-deploy/kubernetes_resource/deployment.rb +1 -1
- data/lib/kubernetes-deploy/kubernetes_resource/horizontal_pod_autoscaler.rb +12 -4
- data/lib/kubernetes-deploy/kubernetes_resource/network_policy.rb +22 -0
- data/lib/kubernetes-deploy/kubernetes_resource/pod.rb +2 -0
- data/lib/kubernetes-deploy/kubernetes_resource/secret.rb +23 -0
- data/lib/kubernetes-deploy/label_selector.rb +42 -0
- data/lib/kubernetes-deploy/options_helper.rb +31 -15
- data/lib/kubernetes-deploy/remote_logs.rb +6 -1
- data/lib/kubernetes-deploy/renderer.rb +5 -1
- data/lib/kubernetes-deploy/resource_cache.rb +4 -1
- data/lib/kubernetes-deploy/restart_task.rb +22 -10
- data/lib/kubernetes-deploy/runner_task.rb +5 -3
- data/lib/kubernetes-deploy/version.rb +1 -1
- metadata +6 -2
@@ -7,85 +7,64 @@ require 'kubernetes-deploy/kubectl'
|
|
7
7
|
module KubernetesDeploy
|
8
8
|
class EjsonSecretError < FatalDeploymentError
|
9
9
|
def initialize(msg)
|
10
|
-
super("
|
10
|
+
super("Generation of Kubernetes secrets from ejson failed: #{msg}")
|
11
11
|
end
|
12
12
|
end
|
13
13
|
|
14
14
|
class EjsonSecretProvisioner
|
15
|
-
|
16
|
-
|
15
|
+
EJSON_SECRET_ANNOTATION = "kubernetes-deploy.shopify.io/ejson-secret"
|
16
|
+
EJSON_SECRET_KEY = "kubernetes_secrets"
|
17
17
|
EJSON_SECRETS_FILE = "secrets.ejson"
|
18
18
|
EJSON_KEYS_SECRET = "ejson-keys"
|
19
19
|
|
20
|
-
def initialize(namespace:, context:, template_dir:, logger:,
|
20
|
+
def initialize(namespace:, context:, template_dir:, logger:, statsd_tags:, selector: nil)
|
21
21
|
@namespace = namespace
|
22
22
|
@context = context
|
23
23
|
@ejson_file = "#{template_dir}/#{EJSON_SECRETS_FILE}"
|
24
24
|
@logger = logger
|
25
|
-
@
|
25
|
+
@statsd_tags = statsd_tags
|
26
|
+
@selector = selector
|
26
27
|
@kubectl = Kubectl.new(
|
27
28
|
namespace: @namespace,
|
28
29
|
context: @context,
|
29
30
|
logger: @logger,
|
30
31
|
log_failure_by_default: false,
|
31
|
-
|
32
|
+
output_is_sensitive_default: true # output may contain ejson secrets
|
32
33
|
)
|
33
34
|
end
|
34
35
|
|
35
|
-
def
|
36
|
-
|
36
|
+
def resources
|
37
|
+
@resources ||= build_secrets
|
37
38
|
end
|
38
39
|
|
39
|
-
def
|
40
|
-
|
41
|
-
|
40
|
+
def ejson_keys_secret
|
41
|
+
@ejson_keys_secret ||= begin
|
42
|
+
out, err, st = @kubectl.run("get", "secret", EJSON_KEYS_SECRET, output: "json",
|
43
|
+
raise_if_not_found: true, attempts: 3, output_is_sensitive: true, log_failure: true)
|
44
|
+
unless st.success?
|
45
|
+
raise EjsonSecretError, "Error retrieving Secret/#{EJSON_KEYS_SECRET}: #{err}"
|
46
|
+
end
|
47
|
+
JSON.parse(out)
|
48
|
+
end
|
42
49
|
end
|
43
50
|
|
44
51
|
private
|
45
52
|
|
46
|
-
def
|
53
|
+
def build_secrets
|
54
|
+
return [] unless File.exist?(@ejson_file)
|
47
55
|
with_decrypted_ejson do |decrypted|
|
48
|
-
secrets = decrypted[
|
56
|
+
secrets = decrypted[EJSON_SECRET_KEY]
|
49
57
|
unless secrets.present?
|
50
|
-
@logger.warn("#{EJSON_SECRETS_FILE} does not have key #{
|
58
|
+
@logger.warn("#{EJSON_SECRETS_FILE} does not have key #{EJSON_SECRET_KEY}."\
|
51
59
|
"No secrets will be created.")
|
52
|
-
return
|
60
|
+
return []
|
53
61
|
end
|
54
62
|
|
55
|
-
secrets.
|
63
|
+
secrets.map do |secret_name, secret_spec|
|
56
64
|
validate_secret_spec(secret_name, secret_spec)
|
57
|
-
|
65
|
+
generate_secret_resource(secret_name, secret_spec["_type"], secret_spec["data"])
|
58
66
|
end
|
59
|
-
@logger.summary.add_action("created/updated #{secrets.length} #{'secret'.pluralize(secrets.length)}")
|
60
|
-
end
|
61
|
-
end
|
62
|
-
|
63
|
-
def prune_managed_secrets
|
64
|
-
ejson_secret_names = encrypted_ejson.fetch(MANAGED_SECRET_EJSON_KEY, {}).keys
|
65
|
-
live_secrets = run_kubectl_json("get", "secrets")
|
66
|
-
|
67
|
-
prune_count = 0
|
68
|
-
live_secrets.each do |secret|
|
69
|
-
secret_name = secret["metadata"]["name"]
|
70
|
-
next unless secret_managed?(secret)
|
71
|
-
next if ejson_secret_names.include?(secret_name)
|
72
|
-
|
73
|
-
@logger.info("Pruning secret #{secret_name}")
|
74
|
-
prune_count += 1
|
75
|
-
out, err, st = @kubectl.run("delete", "secret", secret_name)
|
76
|
-
@logger.debug(out)
|
77
|
-
raise EjsonSecretError, "Failed to prune secrets" unless st.success?
|
78
67
|
end
|
79
|
-
@logger.summary.add_action("pruned #{prune_count} #{'secret'.pluralize(prune_count)}") if prune_count > 0
|
80
|
-
end
|
81
|
-
|
82
|
-
def managed_secrets_exist?
|
83
|
-
all_secrets = run_kubectl_json("get", "secrets")
|
84
|
-
all_secrets.any? { |secret| secret_managed?(secret) }
|
85
|
-
end
|
86
|
-
|
87
|
-
def secret_managed?(secret)
|
88
|
-
secret["metadata"].fetch("annotations", {}).key?(MANAGEMENT_ANNOTATION)
|
89
68
|
end
|
90
69
|
|
91
70
|
def encrypted_ejson
|
@@ -110,23 +89,7 @@ module KubernetesDeploy
|
|
110
89
|
end
|
111
90
|
end
|
112
91
|
|
113
|
-
def
|
114
|
-
msg = secret_exists?(secret_name) ? "Updating secret #{secret_name}" : "Creating secret #{secret_name}"
|
115
|
-
@logger.info(msg)
|
116
|
-
|
117
|
-
secret_yaml = generate_secret_yaml(secret_name, secret_type, data)
|
118
|
-
file = Tempfile.new(secret_name)
|
119
|
-
file.write(secret_yaml)
|
120
|
-
file.close
|
121
|
-
|
122
|
-
out, err, st = @kubectl.run("apply", "--filename=#{file.path}")
|
123
|
-
@logger.debug(out)
|
124
|
-
raise EjsonSecretError, "Failed to create or update secrets" unless st.success?
|
125
|
-
ensure
|
126
|
-
file&.unlink
|
127
|
-
end
|
128
|
-
|
129
|
-
def generate_secret_yaml(secret_name, secret_type, data)
|
92
|
+
def generate_secret_resource(secret_name, secret_type, data)
|
130
93
|
unless data.is_a?(Hash) && data.values.all? { |v| v.is_a?(String) } # Secret data is map[string]string
|
131
94
|
raise EjsonSecretError, "Data for secret #{secret_name} was invalid. Only key-value pairs are permitted."
|
132
95
|
end
|
@@ -137,24 +100,25 @@ module KubernetesDeploy
|
|
137
100
|
encoded[secret_key] = Base64.strict_encode64(value)
|
138
101
|
end
|
139
102
|
|
103
|
+
labels = { "name" => secret_name }
|
104
|
+
labels.reverse_merge!(@selector) if @selector
|
105
|
+
|
140
106
|
secret = {
|
141
107
|
'kind' => 'Secret',
|
142
108
|
'apiVersion' => 'v1',
|
143
109
|
'type' => secret_type,
|
144
110
|
'metadata' => {
|
145
111
|
"name" => secret_name,
|
146
|
-
"labels" =>
|
112
|
+
"labels" => labels,
|
147
113
|
"namespace" => @namespace,
|
148
|
-
"annotations" => {
|
114
|
+
"annotations" => { EJSON_SECRET_ANNOTATION => "true" },
|
149
115
|
},
|
150
116
|
"data" => encoded_data,
|
151
117
|
}
|
152
|
-
secret.to_yaml
|
153
|
-
end
|
154
118
|
|
155
|
-
|
156
|
-
|
157
|
-
|
119
|
+
KubernetesDeploy::Secret.build(
|
120
|
+
namespace: @namespace, context: @context, logger: @logger, definition: secret, statsd_tags: @statsd_tags,
|
121
|
+
)
|
158
122
|
end
|
159
123
|
|
160
124
|
def load_ejson_from_file
|
@@ -175,33 +139,23 @@ module KubernetesDeploy
|
|
175
139
|
end
|
176
140
|
|
177
141
|
def decrypt_ejson(key_dir)
|
178
|
-
@logger.info("Decrypting #{EJSON_SECRETS_FILE}")
|
179
142
|
# ejson seems to dump both errors and output to STDOUT
|
180
143
|
out_err, st = Open3.capture2e("EJSON_KEYDIR=#{key_dir} ejson decrypt #{@ejson_file}")
|
181
144
|
raise EjsonSecretError, out_err unless st.success?
|
182
145
|
JSON.parse(out_err)
|
183
|
-
rescue JSON::ParserError
|
146
|
+
rescue JSON::ParserError
|
184
147
|
raise EjsonSecretError, "Failed to parse decrypted ejson"
|
185
148
|
end
|
186
149
|
|
187
150
|
def fetch_private_key_from_secret
|
188
|
-
|
189
|
-
|
190
|
-
secret = run_kubectl_json("get", "secret", EJSON_KEYS_SECRET)
|
191
|
-
encoded_private_key = secret["data"][public_key]
|
151
|
+
encoded_private_key = ejson_keys_secret["data"][public_key]
|
192
152
|
unless encoded_private_key
|
193
153
|
raise EjsonSecretError, "Private key for #{public_key} not found in #{EJSON_KEYS_SECRET} secret"
|
194
154
|
end
|
195
155
|
|
196
156
|
Base64.decode64(encoded_private_key)
|
197
|
-
|
198
|
-
|
199
|
-
def run_kubectl_json(*args)
|
200
|
-
args += ["--output=json"]
|
201
|
-
out, err, st = @kubectl.run(*args)
|
202
|
-
raise EjsonSecretError, err unless st.success?
|
203
|
-
result = JSON.parse(out)
|
204
|
-
result.fetch('items', result)
|
157
|
+
rescue Kubectl::ResourceNotFoundError
|
158
|
+
raise EjsonSecretError, "Secret/#{EJSON_KEYS_SECRET} is required to decrypt EJSON and could not be found"
|
205
159
|
end
|
206
160
|
end
|
207
161
|
end
|
@@ -23,6 +23,14 @@ module KubernetesDeploy
|
|
23
23
|
|
24
24
|
class DeploymentTimeoutError < FatalDeploymentError; end
|
25
25
|
|
26
|
+
class EjsonPrunableError < FatalDeploymentError
|
27
|
+
def initialize
|
28
|
+
super("Found #{KubernetesResource::LAST_APPLIED_ANNOTATION} annotation on " \
|
29
|
+
"#{EjsonSecretProvisioner::EJSON_KEYS_SECRET} secret. " \
|
30
|
+
"kubernetes-deploy will not continue since it is extremely unlikely that this secret should be pruned.")
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
26
34
|
module Errors
|
27
35
|
extend self
|
28
36
|
def server_version_warning(server_version)
|
@@ -3,25 +3,33 @@ require 'kubeclient'
|
|
3
3
|
require 'kubernetes-deploy/kubeclient_builder/kube_config'
|
4
4
|
|
5
5
|
module KubernetesDeploy
|
6
|
-
|
6
|
+
class KubeclientBuilder
|
7
7
|
class ContextMissingError < FatalDeploymentError
|
8
|
-
def initialize(context_name)
|
8
|
+
def initialize(context_name, kubeconfig)
|
9
9
|
super("`#{context_name}` context must be configured in your " \
|
10
|
-
"KUBECONFIG file(s) (#{
|
10
|
+
"KUBECONFIG file(s) (#{kubeconfig}).")
|
11
11
|
end
|
12
12
|
end
|
13
13
|
|
14
|
-
|
14
|
+
def self.kubeconfig
|
15
|
+
new.kubeconfig
|
16
|
+
end
|
17
|
+
|
18
|
+
attr_reader :kubeconfig
|
19
|
+
|
20
|
+
def initialize(kubeconfig: ENV["KUBECONFIG"])
|
21
|
+
@kubeconfig = kubeconfig || "#{Dir.home}/.kube/config"
|
22
|
+
end
|
15
23
|
|
16
24
|
def build_v1_kubeclient(context)
|
17
|
-
|
25
|
+
build_kubeclient(
|
18
26
|
api_version: "v1",
|
19
27
|
context: context
|
20
28
|
)
|
21
29
|
end
|
22
30
|
|
23
31
|
def build_v1beta1_kubeclient(context)
|
24
|
-
|
32
|
+
build_kubeclient(
|
25
33
|
api_version: "v1beta1",
|
26
34
|
context: context,
|
27
35
|
endpoint_path: "/apis/extensions/"
|
@@ -29,7 +37,7 @@ module KubernetesDeploy
|
|
29
37
|
end
|
30
38
|
|
31
39
|
def build_batch_v1beta1_kubeclient(context)
|
32
|
-
|
40
|
+
build_kubeclient(
|
33
41
|
api_version: "v1beta1",
|
34
42
|
context: context,
|
35
43
|
endpoint_path: "/apis/batch/"
|
@@ -37,7 +45,7 @@ module KubernetesDeploy
|
|
37
45
|
end
|
38
46
|
|
39
47
|
def build_batch_v1_kubeclient(context)
|
40
|
-
|
48
|
+
build_kubeclient(
|
41
49
|
api_version: "v1",
|
42
50
|
context: context,
|
43
51
|
endpoint_path: "/apis/batch/"
|
@@ -45,7 +53,7 @@ module KubernetesDeploy
|
|
45
53
|
end
|
46
54
|
|
47
55
|
def build_policy_v1beta1_kubeclient(context)
|
48
|
-
|
56
|
+
build_kubeclient(
|
49
57
|
api_version: "v1beta1",
|
50
58
|
context: context,
|
51
59
|
endpoint_path: "/apis/policy/"
|
@@ -53,7 +61,7 @@ module KubernetesDeploy
|
|
53
61
|
end
|
54
62
|
|
55
63
|
def build_apps_v1beta1_kubeclient(context)
|
56
|
-
|
64
|
+
build_kubeclient(
|
57
65
|
api_version: "v1beta1",
|
58
66
|
context: context,
|
59
67
|
endpoint_path: "/apis/apps"
|
@@ -61,7 +69,7 @@ module KubernetesDeploy
|
|
61
69
|
end
|
62
70
|
|
63
71
|
def build_apiextensions_v1beta1_kubeclient(context)
|
64
|
-
|
72
|
+
build_kubeclient(
|
65
73
|
api_version: "v1beta1",
|
66
74
|
context: context,
|
67
75
|
endpoint_path: "/apis/apiextensions.k8s.io"
|
@@ -69,7 +77,7 @@ module KubernetesDeploy
|
|
69
77
|
end
|
70
78
|
|
71
79
|
def build_autoscaling_v1_kubeclient(context)
|
72
|
-
|
80
|
+
build_kubeclient(
|
73
81
|
api_version: "v2beta1",
|
74
82
|
context: context,
|
75
83
|
endpoint_path: "/apis/autoscaling"
|
@@ -77,19 +85,48 @@ module KubernetesDeploy
|
|
77
85
|
end
|
78
86
|
|
79
87
|
def build_rbac_v1_kubeclient(context)
|
80
|
-
|
88
|
+
build_kubeclient(
|
81
89
|
api_version: "v1",
|
82
90
|
context: context,
|
83
91
|
endpoint_path: "/apis/rbac.authorization.k8s.io"
|
84
92
|
)
|
85
93
|
end
|
86
94
|
|
87
|
-
def
|
95
|
+
def build_networking_v1_kubeclient(context)
|
96
|
+
build_kubeclient(
|
97
|
+
api_version: "v1",
|
98
|
+
context: context,
|
99
|
+
endpoint_path: "/apis/networking.k8s.io"
|
100
|
+
)
|
101
|
+
end
|
102
|
+
|
103
|
+
def config_files
|
104
|
+
# Split the list by colon for Linux and Mac, and semicolon for Windows.
|
105
|
+
kubeconfig.split(/[:;]/).map!(&:strip).reject(&:empty?)
|
106
|
+
end
|
107
|
+
|
108
|
+
def validate_config_files
|
109
|
+
errors = []
|
110
|
+
if config_files.empty?
|
111
|
+
errors << "Kube config file name(s) not set in $KUBECONFIG"
|
112
|
+
else
|
113
|
+
config_files.each do |f|
|
114
|
+
unless File.file?(f)
|
115
|
+
errors << "Kube config not found at #{f}"
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
119
|
+
errors
|
120
|
+
end
|
121
|
+
|
122
|
+
private
|
123
|
+
|
124
|
+
def build_kubeclient(api_version:, context:, endpoint_path: nil)
|
88
125
|
# Find a context defined in kube conf files that matches the input context by name
|
89
126
|
configs = config_files.map { |f| KubeConfig.read(f) }
|
90
127
|
config = configs.find { |c| c.contexts.include?(context) }
|
91
128
|
|
92
|
-
raise ContextMissingError,
|
129
|
+
raise ContextMissingError.new(context, kubeconfig) unless config
|
93
130
|
|
94
131
|
kube_context = config.context(context)
|
95
132
|
client = Kubeclient::Client.new(
|
@@ -105,10 +142,5 @@ module KubernetesDeploy
|
|
105
142
|
client.discover
|
106
143
|
client
|
107
144
|
end
|
108
|
-
|
109
|
-
def config_files
|
110
|
-
# Split the list by colon for Linux and Mac, and semicolon for Windows.
|
111
|
-
ENV.fetch("KUBECONFIG").split(/[:;]/).map!(&:strip).reject(&:empty?)
|
112
|
-
end
|
113
145
|
end
|
114
146
|
end
|
@@ -8,43 +8,48 @@ module KubernetesDeploy
|
|
8
8
|
class ResourceNotFoundError < StandardError; end
|
9
9
|
|
10
10
|
def initialize(namespace:, context:, logger:, log_failure_by_default:, default_timeout: DEFAULT_TIMEOUT,
|
11
|
-
|
11
|
+
output_is_sensitive_default: false)
|
12
|
+
@kubeconfig = KubeclientBuilder.kubeconfig
|
12
13
|
@namespace = namespace
|
13
14
|
@context = context
|
14
15
|
@logger = logger
|
15
16
|
@log_failure_by_default = log_failure_by_default
|
16
17
|
@default_timeout = default_timeout
|
17
|
-
@
|
18
|
+
@output_is_sensitive_default = output_is_sensitive_default
|
18
19
|
|
19
20
|
raise ArgumentError, "namespace is required" if namespace.blank?
|
20
21
|
raise ArgumentError, "context is required" if context.blank?
|
21
22
|
end
|
22
23
|
|
23
|
-
def run(*args, log_failure: nil, use_context: true, use_namespace: true,
|
24
|
+
def run(*args, log_failure: nil, use_context: true, use_namespace: true, output: nil,
|
25
|
+
raise_if_not_found: false, attempts: 1, output_is_sensitive: nil)
|
24
26
|
log_failure = @log_failure_by_default if log_failure.nil?
|
27
|
+
output_is_sensitive = @output_is_sensitive_default if output_is_sensitive.nil?
|
25
28
|
|
26
29
|
args = args.unshift("kubectl")
|
30
|
+
args.push("--kubeconfig=#{@kubeconfig}")
|
27
31
|
args.push("--namespace=#{@namespace}") if use_namespace
|
28
32
|
args.push("--context=#{@context}") if use_context
|
33
|
+
args.push("--output=#{output}") if output
|
29
34
|
args.push("--request-timeout=#{@default_timeout}") if @default_timeout
|
30
35
|
out, err, st = nil
|
31
36
|
|
32
37
|
(1..attempts).to_a.each do |attempt|
|
33
38
|
@logger.debug("Running command (attempt #{attempt}): #{args.join(' ')}")
|
34
39
|
out, err, st = Open3.capture3(*args)
|
35
|
-
@logger.debug("Kubectl out: " + out.gsub(/\s+/, ' ')) unless output_is_sensitive
|
40
|
+
@logger.debug("Kubectl out: " + out.gsub(/\s+/, ' ')) unless output_is_sensitive
|
36
41
|
|
37
42
|
break if st.success?
|
38
43
|
|
39
44
|
if log_failure
|
40
45
|
@logger.warn("The following command failed (attempt #{attempt}/#{attempts}): #{Shellwords.join(args)}")
|
41
|
-
@logger.warn(err) unless output_is_sensitive
|
46
|
+
@logger.warn(err) unless output_is_sensitive
|
42
47
|
end
|
43
48
|
|
44
49
|
if err.match(NOT_FOUND_ERROR_TEXT)
|
45
50
|
raise(ResourceNotFoundError, err) if raise_if_not_found
|
46
51
|
else
|
47
|
-
@logger.debug("Kubectl err: #{err}") unless output_is_sensitive
|
52
|
+
@logger.debug("Kubectl err: #{err}") unless output_is_sensitive
|
48
53
|
StatsD.increment('kubectl.error', 1, tags: { context: @context, namespace: @namespace, cmd: args[1] })
|
49
54
|
end
|
50
55
|
sleep(retry_delay(attempt)) unless attempt == attempts
|
@@ -76,10 +81,6 @@ module KubernetesDeploy
|
|
76
81
|
|
77
82
|
private
|
78
83
|
|
79
|
-
def output_is_sensitive?
|
80
|
-
@output_is_sensitive
|
81
|
-
end
|
82
|
-
|
83
84
|
def extract_version_info_from_kubectl_response(response)
|
84
85
|
info = {}
|
85
86
|
response.each_line do |l|
|