krane 1.1.1 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. checksums.yaml +4 -4
  2. data/.github/CODEOWNERS +1 -0
  3. data/{ISSUE_TEMPLATE.md → .github/ISSUE_TEMPLATE.md} +0 -0
  4. data/{pull_request_template.md → .github/pull_request_template.md} +0 -0
  5. data/.rubocop-http---shopify-github-io-ruby-style-guide-rubocop-yml +27 -33
  6. data/.rubocop.yml +0 -12
  7. data/.shopify-build/krane.yml +20 -6
  8. data/1.0-Upgrade.md +1 -1
  9. data/CHANGELOG.md +57 -0
  10. data/CONTRIBUTING.md +2 -2
  11. data/README.md +17 -14
  12. data/bin/ci +1 -1
  13. data/bin/test +2 -2
  14. data/dev.yml +3 -2
  15. data/krane.gemspec +6 -4
  16. data/lib/krane/annotation.rb +11 -0
  17. data/lib/krane/cli/deploy_command.rb +2 -3
  18. data/lib/krane/cli/global_deploy_command.rb +3 -3
  19. data/lib/krane/cli/render_command.rb +3 -3
  20. data/lib/krane/cluster_resource_discovery.rb +10 -6
  21. data/lib/krane/concerns/template_reporting.rb +0 -6
  22. data/lib/krane/container_logs.rb +1 -1
  23. data/lib/krane/container_overrides.rb +33 -0
  24. data/lib/krane/deploy_task.rb +21 -18
  25. data/lib/krane/ejson_secret_provisioner.rb +1 -2
  26. data/lib/krane/global_deploy_task.rb +8 -14
  27. data/lib/krane/kubeclient_builder.rb +4 -2
  28. data/lib/krane/kubectl.rb +18 -5
  29. data/lib/krane/kubernetes_resource.rb +32 -42
  30. data/lib/krane/kubernetes_resource/custom_resource.rb +2 -2
  31. data/lib/krane/kubernetes_resource/custom_resource_definition.rb +13 -10
  32. data/lib/krane/kubernetes_resource/deployment.rb +5 -7
  33. data/lib/krane/kubernetes_resource/persistent_volume_claim.rb +1 -0
  34. data/lib/krane/kubernetes_resource/pod.rb +12 -8
  35. data/lib/krane/psych_k8s_compatibility.rb +36 -0
  36. data/lib/krane/render_task.rb +2 -2
  37. data/lib/krane/renderer.rb +2 -0
  38. data/lib/krane/resource_deployer.rb +9 -5
  39. data/lib/krane/resource_watcher.rb +1 -1
  40. data/lib/krane/restart_task.rb +8 -8
  41. data/lib/krane/runner_task.rb +21 -24
  42. data/lib/krane/statsd.rb +2 -2
  43. data/lib/krane/task_config.rb +7 -2
  44. data/lib/krane/task_config_validator.rb +3 -3
  45. data/lib/krane/template_sets.rb +1 -1
  46. data/lib/krane/version.rb +1 -1
  47. metadata +17 -13
  48. data/lib/krane/kubernetes_resource/cloudsql.rb +0 -44
@@ -28,6 +28,8 @@ module Krane
28
28
  ENV["TASK_ID"]
29
29
  elsif current_sha
30
30
  current_sha[0...8] + "-#{SecureRandom.hex(4)}"
31
+ else
32
+ SecureRandom.hex(8)
31
33
  end
32
34
  end
33
35
 
@@ -46,8 +46,13 @@ module Krane
46
46
  bare_pods.first.stream_logs = true
47
47
  end
48
48
 
49
- predeploy_sequence.each do |resource_type|
50
- matching_resources = resource_list.select { |r| r.type == resource_type }
49
+ predeploy_sequence.each do |resource_type, attributes|
50
+ matching_resources = resource_list.select do |r|
51
+ r.type == resource_type &&
52
+ (!attributes[:group] || r.group == attributes[:group])
53
+ end
54
+ StatsD.client.gauge('priority_resources.count', matching_resources.size, tags: statsd_tags)
55
+
51
56
  next if matching_resources.empty?
52
57
  deploy_resources(matching_resources, verify: true, record_summary: false)
53
58
 
@@ -90,11 +95,10 @@ module Krane
90
95
  applyables, individuals = resources.partition { |r| r.deploy_method == :apply }
91
96
  # Prunable resources should also applied so that they can be pruned
92
97
  pruneable_types = @prune_whitelist.map { |t| t.split("/").last }
93
- applyables += individuals.select { |r| pruneable_types.include?(r.type) }
98
+ applyables += individuals.select { |r| pruneable_types.include?(r.type) && !r.deploy_method_override }
94
99
 
95
100
  individuals.each do |individual_resource|
96
101
  individual_resource.deploy_started_at = Time.now.utc
97
-
98
102
  case individual_resource.deploy_method
99
103
  when :create
100
104
  err, status = create_resource(individual_resource)
@@ -148,7 +152,7 @@ module Krane
148
152
  output_is_sensitive = resources.any?(&:sensitive_template_content?)
149
153
  global_mode = resources.all?(&:global?)
150
154
  out, err, st = kubectl.run(*command, log_failure: false, output_is_sensitive: output_is_sensitive,
151
- use_namespace: !global_mode)
155
+ attempts: 2, use_namespace: !global_mode)
152
156
 
153
157
  if st.success?
154
158
  log_pruning(out) if prune
@@ -68,7 +68,7 @@ module Krane
68
68
  end
69
69
 
70
70
  def global_timeout?(started_at)
71
- @timeout && (Time.now.utc - started_at > @timeout)
71
+ @timeout && (Time.now.utc - started_at >= @timeout)
72
72
  end
73
73
 
74
74
  def sleep_until_next_sync(min_interval)
@@ -22,15 +22,19 @@ module Krane
22
22
  HTTP_OK_RANGE = 200..299
23
23
  ANNOTATION = "shipit.shopify.io/restart"
24
24
 
25
+ attr_reader :task_config
26
+
27
+ delegate :kubeclient_builder, to: :task_config
28
+
25
29
  # Initializes the restart task
26
30
  #
27
31
  # @param context [String] Kubernetes context / cluster (*required*)
28
32
  # @param namespace [String] Kubernetes namespace (*required*)
29
33
  # @param logger [Object] Logger object (defaults to an instance of Krane::FormattedLogger)
30
34
  # @param global_timeout [Integer] Timeout in seconds
31
- def initialize(context:, namespace:, logger: nil, global_timeout: nil)
35
+ def initialize(context:, namespace:, logger: nil, global_timeout: nil, kubeconfig: nil)
32
36
  @logger = logger || Krane::FormattedLogger.build(namespace, context)
33
- @task_config = Krane::TaskConfig.new(context, namespace, @logger)
37
+ @task_config = Krane::TaskConfig.new(context, namespace, @logger, kubeconfig)
34
38
  @context = context
35
39
  @namespace = namespace
36
40
  @global_timeout = global_timeout
@@ -39,8 +43,8 @@ module Krane
39
43
  # Runs the task, returning a boolean representing success or failure
40
44
  #
41
45
  # @return [Boolean]
42
- def run(*args)
43
- perform!(*args)
46
+ def run(**args)
47
+ perform!(**args)
44
48
  true
45
49
  rescue FatalDeploymentError
46
50
  false
@@ -220,9 +224,5 @@ module Krane
220
224
  def v1beta1_kubeclient
221
225
  @v1beta1_kubeclient ||= kubeclient_builder.build_v1beta1_kubeclient(@context)
222
226
  end
223
-
224
- def kubeclient_builder
225
- @kubeclient_builder ||= KubeclientBuilder.new
226
- end
227
227
  end
228
228
  end
@@ -9,13 +9,16 @@ require 'krane/resource_watcher'
9
9
  require 'krane/kubernetes_resource'
10
10
  require 'krane/kubernetes_resource/pod'
11
11
  require 'krane/runner_task_config_validator'
12
+ require 'krane/container_overrides'
12
13
 
13
14
  module Krane
14
15
  # Run a pod that exits upon completing a task
15
16
  class RunnerTask
16
17
  class TaskTemplateMissingError < TaskConfigurationError; end
17
18
 
18
- attr_reader :pod_name
19
+ attr_reader :pod_name, :task_config
20
+
21
+ delegate :kubeclient_builder, to: :task_config
19
22
 
20
23
  # Initializes the runner task
21
24
  #
@@ -23,9 +26,9 @@ module Krane
23
26
  # @param context [String] Kubernetes context / cluster (*required*)
24
27
  # @param logger [Object] Logger object (defaults to an instance of Krane::FormattedLogger)
25
28
  # @param global_timeout [Integer] Timeout in seconds
26
- def initialize(namespace:, context:, logger: nil, global_timeout: nil)
29
+ def initialize(namespace:, context:, logger: nil, global_timeout: nil, kubeconfig: nil)
27
30
  @logger = logger || Krane::FormattedLogger.build(namespace, context)
28
- @task_config = Krane::TaskConfig.new(context, namespace, @logger)
31
+ @task_config = Krane::TaskConfig.new(context, namespace, @logger, kubeconfig)
29
32
  @namespace = namespace
30
33
  @context = context
31
34
  @global_timeout = global_timeout
@@ -34,8 +37,8 @@ module Krane
34
37
  # Runs the task, returning a boolean representing success or failure
35
38
  #
36
39
  # @return [Boolean]
37
- def run(*args)
38
- run!(*args)
40
+ def run(**args)
41
+ run!(**args)
39
42
  true
40
43
  rescue DeploymentTimeoutError, FatalDeploymentError
41
44
  false
@@ -50,7 +53,7 @@ module Krane
50
53
  # @param verify_result [Boolean] Wait for completion and verify pod success
51
54
  #
52
55
  # @return [nil]
53
- def run!(template:, command:, arguments:, env_vars: [], verify_result: true)
56
+ def run!(template:, command:, arguments:, env_vars: [], image_tag: nil, verify_result: true)
54
57
  start = Time.now.utc
55
58
  @logger.reset
56
59
 
@@ -59,8 +62,13 @@ module Krane
59
62
  @logger.info("Validating configuration")
60
63
  verify_config!(template)
61
64
  @logger.info("Using namespace '#{@namespace}' in context '#{@context}'")
62
-
63
- pod = build_pod(template, command, arguments, env_vars, verify_result)
65
+ container_overrides = ContainerOverrides.new(
66
+ command: command,
67
+ arguments: arguments,
68
+ env_vars: env_vars,
69
+ image_tag: image_tag
70
+ )
71
+ pod = build_pod(template, container_overrides, verify_result)
64
72
  validate_pod(pod)
65
73
 
66
74
  @logger.phase_heading("Running pod")
@@ -98,11 +106,12 @@ module Krane
98
106
  raise FatalDeploymentError, msg
99
107
  end
100
108
 
101
- def build_pod(template_name, command, args, env_vars, verify_result)
109
+ def build_pod(template_name, container_overrides, verify_result)
102
110
  task_template = get_template(template_name)
103
111
  @logger.info("Using template '#{template_name}'")
104
112
  pod_template = build_pod_definition(task_template)
105
- set_container_overrides!(pod_template, command, args, env_vars)
113
+ container = extract_task_runner_container(pod_template)
114
+ container_overrides.apply!(container)
106
115
  ensure_valid_restart_policy!(pod_template, verify_result)
107
116
  Pod.new(namespace: @namespace, context: @context, logger: @logger, stream_logs: true,
108
117
  definition: pod_template.to_hash.deep_stringify_keys, statsd_tags: [])
@@ -165,7 +174,7 @@ module Krane
165
174
  pod_definition
166
175
  end
167
176
 
168
- def set_container_overrides!(pod_definition, command, args, env_vars)
177
+ def extract_task_runner_container(pod_definition)
169
178
  container = pod_definition.spec.containers.find { |cont| cont.name == 'task-runner' }
170
179
  if container.nil?
171
180
  message = "Pod spec does not contain a template container called 'task-runner'"
@@ -173,15 +182,7 @@ module Krane
173
182
  raise TaskConfigurationError, message
174
183
  end
175
184
 
176
- container.command = command if command
177
- container.args = args if args
178
-
179
- env_args = env_vars.map do |env|
180
- key, value = env.split('=', 2)
181
- { name: key, value: value }
182
- end
183
- container.env ||= []
184
- container.env = container.env.map(&:to_h) + env_args
185
+ container
185
186
  end
186
187
 
187
188
  def ensure_valid_restart_policy!(template, verify)
@@ -201,10 +202,6 @@ module Krane
201
202
  @kubeclient ||= kubeclient_builder.build_v1_kubeclient(@context)
202
203
  end
203
204
 
204
- def kubeclient_builder
205
- @kubeclient_builder ||= KubeclientBuilder.new
206
- end
207
-
208
205
  def statsd_tags(status)
209
206
  %W(namespace:#{@namespace} context:#{@context} status:#{status})
210
207
  end
@@ -35,10 +35,10 @@ module Krane
35
35
  end
36
36
 
37
37
  metric ||= "#{method_name}.duration"
38
- self::InstrumentationProxy.send(:define_method, method_name) do |*args, &block|
38
+ self::InstrumentationProxy.send(:define_method, method_name) do |*args, **kwargs, &block|
39
39
  begin
40
40
  start_time = Time.now.utc
41
- super(*args, &block)
41
+ super(*args, **kwargs, &block)
42
42
  rescue
43
43
  error = true
44
44
  raise
@@ -4,12 +4,13 @@ require 'krane/cluster_resource_discovery'
4
4
 
5
5
  module Krane
6
6
  class TaskConfig
7
- attr_reader :context, :namespace, :logger
7
+ attr_reader :context, :namespace, :logger, :kubeconfig
8
8
 
9
- def initialize(context, namespace, logger = nil)
9
+ def initialize(context, namespace, logger = nil, kubeconfig = nil)
10
10
  @context = context
11
11
  @namespace = namespace
12
12
  @logger = logger || FormattedLogger.build(@namespace, @context)
13
+ @kubeconfig = kubeconfig || ENV['KUBECONFIG']
13
14
  end
14
15
 
15
16
  def global_kinds
@@ -18,5 +19,9 @@ module Krane
18
19
  cluster_resource_discoverer.fetch_resources(namespaced: false).map { |g| g["kind"] }
19
20
  end
20
21
  end
22
+
23
+ def kubeclient_builder
24
+ @kubeclient_builder ||= KubeclientBuilder.new(kubeconfig: kubeconfig)
25
+ end
21
26
  end
22
27
  end
@@ -45,7 +45,7 @@ module Krane
45
45
  end
46
46
 
47
47
  _, err, st = @kubectl.run("config", "get-contexts", context, "-o", "name",
48
- use_namespace: false, use_context: false, log_failure: false)
48
+ use_namespace: false, use_context: false, log_failure: false, attempts: 2)
49
49
 
50
50
  unless st.success?
51
51
  @errors << if err.match("error: context #{context} not found")
@@ -58,7 +58,7 @@ module Krane
58
58
 
59
59
  def validate_context_reachable
60
60
  _, err, st = @kubectl.run("get", "namespaces", "-o", "name",
61
- use_namespace: false, log_failure: false)
61
+ use_namespace: false, log_failure: false, attempts: 2)
62
62
 
63
63
  unless st.success?
64
64
  @errors << "Something went wrong connecting to #{context}. #{err} "
@@ -71,7 +71,7 @@ module Krane
71
71
  end
72
72
 
73
73
  _, err, st = @kubectl.run("get", "namespace", "-o", "name", namespace,
74
- use_namespace: false, log_failure: false)
74
+ use_namespace: false, log_failure: false, attempts: 3)
75
75
 
76
76
  unless st.success?
77
77
  @errors << if err.match("Error from server [(]NotFound[)]: namespace")
@@ -150,7 +150,7 @@ module Krane
150
150
 
151
151
  if rendering_erb_disabled? && deploying_with_erb_files?
152
152
  errors << "ERB template discovered with rendering disabled. If you were trying to render ERB and " \
153
- "deploy the result, try piping the output of `krane render` to `krane-deploy` with the --stdin flag"
153
+ "deploy the result, try piping the output of `krane render` to `krane-deploy -f -`"
154
154
  end
155
155
 
156
156
  errors
@@ -1,4 +1,4 @@
1
1
  # frozen_string_literal: true
2
2
  module Krane
3
- VERSION = "1.1.1"
3
+ VERSION = "2.1.0"
4
4
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: krane
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.1.1
4
+ version: 2.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Katrina Verey
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: exe
12
12
  cert_chain: []
13
- date: 2020-01-22 00:00:00.000000000 Z
13
+ date: 2020-10-06 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: activesupport
@@ -150,7 +150,7 @@ dependencies:
150
150
  requirements:
151
151
  - - ">="
152
152
  - !ruby/object:Gem::Version
153
- version: 0.20.3
153
+ version: '1.0'
154
154
  - - "<"
155
155
  - !ruby/object:Gem::Version
156
156
  version: '2.0'
@@ -160,7 +160,7 @@ dependencies:
160
160
  requirements:
161
161
  - - ">="
162
162
  - !ruby/object:Gem::Version
163
- version: 0.20.3
163
+ version: '1.0'
164
164
  - - "<"
165
165
  - !ruby/object:Gem::Version
166
166
  version: '2.0'
@@ -212,14 +212,14 @@ dependencies:
212
212
  requirements:
213
213
  - - "~>"
214
214
  - !ruby/object:Gem::Version
215
- version: '5.0'
215
+ version: '5.12'
216
216
  type: :development
217
217
  prerelease: false
218
218
  version_requirements: !ruby/object:Gem::Requirement
219
219
  requirements:
220
220
  - - "~>"
221
221
  - !ruby/object:Gem::Version
222
- version: '5.0'
222
+ version: '5.12'
223
223
  - !ruby/object:Gem::Dependency
224
224
  name: minitest-stub-const
225
225
  requirement: !ruby/object:Gem::Requirement
@@ -366,14 +366,14 @@ dependencies:
366
366
  requirements:
367
367
  - - "~>"
368
368
  - !ruby/object:Gem::Version
369
- version: 0.76.0
369
+ version: 0.89.1
370
370
  type: :development
371
371
  prerelease: false
372
372
  version_requirements: !ruby/object:Gem::Requirement
373
373
  requirements:
374
374
  - - "~>"
375
375
  - !ruby/object:Gem::Version
376
- version: 0.76.0
376
+ version: 0.89.1
377
377
  - !ruby/object:Gem::Dependency
378
378
  name: codecov
379
379
  requirement: !ruby/object:Gem::Requirement
@@ -398,7 +398,10 @@ extensions: []
398
398
  extra_rdoc_files: []
399
399
  files:
400
400
  - ".buildkite/pipeline.nightly.yml"
401
+ - ".github/CODEOWNERS"
402
+ - ".github/ISSUE_TEMPLATE.md"
401
403
  - ".github/probots.yml"
404
+ - ".github/pull_request_template.md"
402
405
  - ".gitignore"
403
406
  - ".rubocop-http---shopify-github-io-ruby-style-guide-rubocop-yml"
404
407
  - ".rubocop.yml"
@@ -409,7 +412,6 @@ files:
409
412
  - CODE_OF_CONDUCT.md
410
413
  - CONTRIBUTING.md
411
414
  - Gemfile
412
- - ISSUE_TEMPLATE.md
413
415
  - LICENSE.txt
414
416
  - README.md
415
417
  - Rakefile
@@ -421,6 +423,7 @@ files:
421
423
  - exe/krane
422
424
  - krane.gemspec
423
425
  - lib/krane.rb
426
+ - lib/krane/annotation.rb
424
427
  - lib/krane/bindings_parser.rb
425
428
  - lib/krane/cli/deploy_command.rb
426
429
  - lib/krane/cli/global_deploy_command.rb
@@ -434,6 +437,7 @@ files:
434
437
  - lib/krane/concerns/template_reporting.rb
435
438
  - lib/krane/concurrency.rb
436
439
  - lib/krane/container_logs.rb
440
+ - lib/krane/container_overrides.rb
437
441
  - lib/krane/deferred_summary_logging.rb
438
442
  - lib/krane/delayed_exceptions.rb
439
443
  - lib/krane/deploy_task.rb
@@ -447,7 +451,6 @@ files:
447
451
  - lib/krane/kubeclient_builder.rb
448
452
  - lib/krane/kubectl.rb
449
453
  - lib/krane/kubernetes_resource.rb
450
- - lib/krane/kubernetes_resource/cloudsql.rb
451
454
  - lib/krane/kubernetes_resource/config_map.rb
452
455
  - lib/krane/kubernetes_resource/cron_job.rb
453
456
  - lib/krane/kubernetes_resource/custom_resource.rb
@@ -474,6 +477,7 @@ files:
474
477
  - lib/krane/label_selector.rb
475
478
  - lib/krane/oj.rb
476
479
  - lib/krane/options_helper.rb
480
+ - lib/krane/psych_k8s_compatibility.rb
477
481
  - lib/krane/remote_logs.rb
478
482
  - lib/krane/render_task.rb
479
483
  - lib/krane/renderer.rb
@@ -489,7 +493,6 @@ files:
489
493
  - lib/krane/task_config_validator.rb
490
494
  - lib/krane/template_sets.rb
491
495
  - lib/krane/version.rb
492
- - pull_request_template.md
493
496
  - screenshots/deploy-demo.gif
494
497
  - screenshots/migrate-logs.png
495
498
  - screenshots/missing-secret-fail.png
@@ -498,7 +501,8 @@ files:
498
501
  homepage: https://github.com/Shopify/krane
499
502
  licenses:
500
503
  - MIT
501
- metadata: {}
504
+ metadata:
505
+ allowed_push_host: https://rubygems.org
502
506
  post_install_message:
503
507
  rdoc_options: []
504
508
  require_paths:
@@ -507,7 +511,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
507
511
  requirements:
508
512
  - - ">="
509
513
  - !ruby/object:Gem::Version
510
- version: 2.4.0
514
+ version: 2.5.0
511
515
  required_rubygems_version: !ruby/object:Gem::Requirement
512
516
  requirements:
513
517
  - - ">="
@@ -1,44 +0,0 @@
1
- # frozen_string_literal: true
2
- module Krane
3
- class Cloudsql < KubernetesResource
4
- TIMEOUT = 10.minutes
5
- SYNC_DEPENDENCIES = %w(Deployment Service)
6
-
7
- def sync(cache)
8
- super
9
- @proxy_deployment = cache.get_instance(Deployment.kind, "cloudsql-#{cloudsql_resource_uuid}")
10
- @proxy_service = cache.get_instance(Service.kind, "cloudsql-#{@name}")
11
- end
12
-
13
- def status
14
- deploy_succeeded? ? "Provisioned" : "Unknown"
15
- end
16
-
17
- def deploy_succeeded?
18
- proxy_deployment_ready? && proxy_service_ready?
19
- end
20
-
21
- def deploy_failed?
22
- false
23
- end
24
-
25
- private
26
-
27
- def proxy_deployment_ready?
28
- return false unless (status = @proxy_deployment["status"])
29
- # all cloudsql-proxy pods are running
30
- status.fetch("availableReplicas", -1) == status.fetch("replicas", 0)
31
- end
32
-
33
- def proxy_service_ready?
34
- return false unless @proxy_service.present?
35
- # the service has an assigned cluster IP and is therefore functioning
36
- @proxy_service.dig("spec", "clusterIP").present?
37
- end
38
-
39
- def cloudsql_resource_uuid
40
- return unless @instance_data
41
- @instance_data.dig("metadata", "uid")
42
- end
43
- end
44
- end