kube_deploy_tools 3.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE.txt +177 -0
  3. data/README.md +142 -0
  4. data/bin/deploy +60 -0
  5. data/bin/generate +28 -0
  6. data/bin/kdt +17 -0
  7. data/bin/make_configmap +20 -0
  8. data/bin/publish +28 -0
  9. data/bin/push +57 -0
  10. data/bin/render_deploys_hook +18 -0
  11. data/bin/templater +34 -0
  12. data/bin/upgrade +23 -0
  13. data/lib/kube_deploy_tools.rb +17 -0
  14. data/lib/kube_deploy_tools/artifact_registry.rb +30 -0
  15. data/lib/kube_deploy_tools/artifact_registry/driver.rb +13 -0
  16. data/lib/kube_deploy_tools/artifact_registry/driver_artifactory.rb +155 -0
  17. data/lib/kube_deploy_tools/artifact_registry/driver_base.rb +37 -0
  18. data/lib/kube_deploy_tools/artifact_registry/driver_gcs.rb +120 -0
  19. data/lib/kube_deploy_tools/built_artifacts_file.rb +28 -0
  20. data/lib/kube_deploy_tools/concurrency.rb +18 -0
  21. data/lib/kube_deploy_tools/deferred_summary_logging.rb +69 -0
  22. data/lib/kube_deploy_tools/deploy.rb +215 -0
  23. data/lib/kube_deploy_tools/deploy/options.rb +114 -0
  24. data/lib/kube_deploy_tools/deploy_config_file.rb +286 -0
  25. data/lib/kube_deploy_tools/deploy_config_file/deep_merge.rb +38 -0
  26. data/lib/kube_deploy_tools/deploy_config_file/util.rb +39 -0
  27. data/lib/kube_deploy_tools/errors.rb +5 -0
  28. data/lib/kube_deploy_tools/file_filter.rb +43 -0
  29. data/lib/kube_deploy_tools/formatted_logger.rb +59 -0
  30. data/lib/kube_deploy_tools/generate.rb +145 -0
  31. data/lib/kube_deploy_tools/generate/options.rb +66 -0
  32. data/lib/kube_deploy_tools/image_registry.rb +30 -0
  33. data/lib/kube_deploy_tools/image_registry/driver.rb +18 -0
  34. data/lib/kube_deploy_tools/image_registry/driver/aws.rb +121 -0
  35. data/lib/kube_deploy_tools/image_registry/driver/base.rb +50 -0
  36. data/lib/kube_deploy_tools/image_registry/driver/gcp.rb +71 -0
  37. data/lib/kube_deploy_tools/image_registry/driver/login.rb +26 -0
  38. data/lib/kube_deploy_tools/image_registry/driver/noop.rb +15 -0
  39. data/lib/kube_deploy_tools/image_registry/image.rb +17 -0
  40. data/lib/kube_deploy_tools/kdt.rb +52 -0
  41. data/lib/kube_deploy_tools/kubectl.rb +25 -0
  42. data/lib/kube_deploy_tools/kubernetes_resource.rb +57 -0
  43. data/lib/kube_deploy_tools/kubernetes_resource/deployment.rb +56 -0
  44. data/lib/kube_deploy_tools/make_configmap.rb +51 -0
  45. data/lib/kube_deploy_tools/make_configmap/options.rb +39 -0
  46. data/lib/kube_deploy_tools/object.rb +11 -0
  47. data/lib/kube_deploy_tools/publish.rb +40 -0
  48. data/lib/kube_deploy_tools/publish/options.rb +34 -0
  49. data/lib/kube_deploy_tools/push.rb +129 -0
  50. data/lib/kube_deploy_tools/push/options.rb +46 -0
  51. data/lib/kube_deploy_tools/render_deploys_hook.rb +95 -0
  52. data/lib/kube_deploy_tools/shellrunner.rb +46 -0
  53. data/lib/kube_deploy_tools/tag.rb +33 -0
  54. data/lib/kube_deploy_tools/templater.rb +63 -0
  55. data/lib/kube_deploy_tools/templater/options.rb +74 -0
  56. data/lib/kube_deploy_tools/version.rb +3 -0
  57. metadata +191 -0
@@ -0,0 +1,38 @@
1
+ # Downloaded from
2
+ # https://raw.githubusercontent.com/rails/rails/v6.0.2.2/activesupport/lib/active_support/core_ext/hash/deep_merge.rb
3
+ # to avoid a dependency on activesupport.
4
+
5
+ # frozen_string_literal: true
6
+
7
+ class Hash
8
+ # Returns a new hash with +self+ and +other_hash+ merged recursively.
9
+ #
10
+ # h1 = { a: true, b: { c: [1, 2, 3] } }
11
+ # h2 = { a: false, b: { x: [3, 4, 5] } }
12
+ #
13
+ # h1.deep_merge(h2) # => { a: false, b: { c: [1, 2, 3], x: [3, 4, 5] } }
14
+ #
15
+ # Like with Hash#merge in the standard library, a block can be provided
16
+ # to merge values:
17
+ #
18
+ # h1 = { a: 100, b: 200, c: { c1: 100 } }
19
+ # h2 = { b: 250, c: { c1: 200 } }
20
+ # h1.deep_merge(h2) { |key, this_val, other_val| this_val + other_val }
21
+ # # => { a: 100, b: 450, c: { c1: 300 } }
22
+ def deep_merge(other_hash, &block)
23
+ dup.deep_merge!(other_hash, &block)
24
+ end
25
+
26
+ # Same as +deep_merge+, but modifies +self+.
27
+ def deep_merge!(other_hash, &block)
28
+ merge!(other_hash) do |key, this_val, other_val|
29
+ if this_val.is_a?(Hash) && other_val.is_a?(Hash)
30
+ this_val.deep_merge(other_val, &block)
31
+ elsif block_given?
32
+ block.call(key, this_val, other_val)
33
+ else
34
+ other_val
35
+ end
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,39 @@
1
+
2
+ module KubeDeployTools
3
+ module DeployConfigFileUtil
4
+ def check_and_err(condition, error)
5
+ if ! condition
6
+ Logger.error("Error in configuration #{@filename}")
7
+ raise ArgumentError, error
8
+ end
9
+ end
10
+
11
+ def check_and_warn(condition, warning)
12
+ if ! condition
13
+ Logger.warn("Warning in configuration #{@filename}")
14
+ Logger.warn(warning)
15
+ end
16
+ end
17
+
18
+ def load_library(lib)
19
+ # All paths must be valid accessible gcs paths for the current user.
20
+ # To modify gcloud identity being used by this process, set
21
+ # GOOGLE_APPLICATION_CREDENTIALS or sign in with `gcloud auth login`
22
+ lib_config = nil
23
+ if lib.start_with?('gs://')
24
+ Tempfile.open(['gs-kdt-library', '.yaml']) do |t|
25
+ out = Shellrunner.check_call('gsutil', 'cat', lib)
26
+ t.write(out)
27
+ t.flush
28
+ lib_config = DeployConfigFile.new(t.path)
29
+ end
30
+ elsif File.exist?(lib)
31
+ lib_config = DeployConfigFile.new(lib)
32
+ else
33
+ raise "Unsupported or non-existent library: #{lib}"
34
+ end
35
+
36
+ lib_config
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,5 @@
1
+
2
+ module KubeDeployTools
3
+ class FatalDeploymentError < StandardError; end
4
+ end
5
+
@@ -0,0 +1,43 @@
1
+ require 'kube_deploy_tools/formatted_logger'
2
+
3
+ # frozen_string_literal: true
4
+ module KubeDeployTools
5
+ module FileFilter
6
+ VALID_FILTER_NAMES = %w(include_dir exclude_dir)
7
+
8
+ def self.filter_files(filters:, files_path:)
9
+ all_files = Dir[File.join(files_path, '**', '*')].to_set
10
+ filtered_files = if filters.any? { |k, _| k =~ /include_/ }
11
+ Set.new
12
+ else
13
+ Set.new(all_files)
14
+ end
15
+
16
+ filters.each do |action, globpath|
17
+ if action =~ /_dir$/
18
+ # ensure globpath correctly selects an entire dir recursively
19
+ # always starts with files_path
20
+ # always ends with /**/* (to recusively select entire dir)
21
+ globpath = File.join(files_path, globpath.gsub(/^[\/*]+|[\/*]+$/, ""), '**', '*')
22
+ end
23
+
24
+ case action
25
+ when /^include_(files|dir)$/
26
+ filtered_files.merge( all_files.select{ |f| File.fnmatch?(globpath, f, File::FNM_PATHNAME) } )
27
+ when /^exclude_(files|dir)$/
28
+ filtered_files.reject!{ |f| File.fnmatch?(globpath, f, File::FNM_PATHNAME) }
29
+ else
30
+ raise "you want to #{action}: wat do"
31
+ end
32
+ end
33
+ Logger.debug("\nYour filter generates following paths: \n#{filtered_files.to_a.join("\n")}")
34
+ filtered_files
35
+ end
36
+
37
+ def self.filters_from_hash(h)
38
+ VALID_FILTER_NAMES.flat_map do |filter_name|
39
+ Array(h[filter_name]).map { |dir_path| [filter_name, dir_path] }
40
+ end
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,59 @@
1
+ require 'forwardable'
2
+ require 'logger'
3
+
4
+ require 'colorized_string'
5
+
6
+ require 'kube_deploy_tools/deferred_summary_logging'
7
+
8
+ module KubeDeployTools
9
+ class FormattedLogger < ::Logger
10
+ include DeferredSummaryLogging
11
+
12
+ def self.build(context: nil, namespace: nil, stream: $stderr)
13
+ l = new(stream)
14
+ l.level = level_from_env
15
+
16
+ l.formatter = proc do |severity, datetime, _progname, msg|
17
+ middle = context ? "[#{context}]" : ""
18
+ if ! namespace.nil? && namespace != 'default'
19
+ middle += "[#{namespace}]"
20
+ end
21
+
22
+ dt = datetime.strftime('%F %T')
23
+ colorized_line = ColorizedString.new("[#{severity}][#{dt}]#{middle} #{msg}\n")
24
+
25
+ case severity
26
+ when "FATAL"
27
+ ColorizedString.new("[#{severity}][#{dt}]#{middle} ").red + "#{msg}\n"
28
+ when "ERROR"
29
+ colorized_line.red
30
+ when "WARN"
31
+ colorized_line.yellow
32
+ else
33
+ colorized_line
34
+ end
35
+ end
36
+ l
37
+ end
38
+
39
+ def self.level_from_env
40
+ return ::Logger::DEBUG if ENV["DEBUG"]
41
+
42
+ if ENV["LEVEL"]
43
+ ::Logger.const_get(ENV["LEVEL"].upcase)
44
+ else
45
+ ::Logger::INFO
46
+ end
47
+ end
48
+ private_class_method :level_from_env
49
+ end
50
+
51
+ class Logger
52
+ class << self
53
+ extend Forwardable
54
+
55
+ attr_accessor :logger
56
+ def_delegators :@logger, *(::Logger.public_instance_methods(false) + DeferredSummaryLogging.public_instance_methods(false))
57
+ end
58
+ end
59
+ end
@@ -0,0 +1,145 @@
1
+ require 'erb'
2
+ require 'fileutils'
3
+ require 'tempfile'
4
+ require 'time'
5
+ require 'yaml'
6
+
7
+ require 'kube_deploy_tools/render_deploys_hook'
8
+ require 'kube_deploy_tools/deploy_config_file'
9
+ require 'kube_deploy_tools/file_filter'
10
+ require 'kube_deploy_tools/shellrunner'
11
+ require 'kube_deploy_tools/tag'
12
+
13
+ DEFAULT_HOOK_SCRIPT = 'render_deploys_hook'
14
+ DEFAULT_HOOK_SCRIPT_LABEL = 'default'
15
+
16
+ module KubeDeployTools
17
+ DEFAULT_FLAGS = {
18
+ 'image_tag' => tag_from_local_env,
19
+ 'tag' => tag_from_local_env,
20
+ }.freeze
21
+ class Generate
22
+ def initialize(manifest, input_dir, output_dir, file_filters: [], print_flags_only: false, literals: {})
23
+ @project = KubeDeployTools::PROJECT
24
+ @build_number = KubeDeployTools::BUILD_NUMBER
25
+
26
+ @config = DeployConfigFile.new(manifest)
27
+ @literals = literals
28
+
29
+ @input_dir = input_dir
30
+ @output_dir = output_dir
31
+ FileUtils.mkdir_p @output_dir
32
+
33
+ @file_filters = file_filters
34
+
35
+ @print_flags_only = print_flags_only
36
+ end
37
+
38
+ def git_commit
39
+ commit_sha = Shellrunner.check_call(*%w(git rev-parse HEAD))
40
+ commit_sha ? commit_sha.strip : ''
41
+ end
42
+
43
+ def git_project
44
+ project_url = Shellrunner.check_call(*%w(git config --get remote.origin.url))
45
+ project_url ? project_url.strip : ''
46
+ end
47
+
48
+ def generate
49
+ permutations = {}
50
+ @config.artifacts.each do |c|
51
+ artifact = c.fetch('name')
52
+
53
+ # Get metadata for this target/environment pair from manifest
54
+ cluster_flags = DEFAULT_FLAGS.dup
55
+
56
+ cluster_flags['image_registry'] = c.fetch('image_registry')
57
+
58
+ # Merge in configured default flags
59
+ cluster_flags.merge!(@config.default_flags)
60
+
61
+ # Update and merge deploy flags for rendering
62
+ cluster_flags.merge!(generate_erb_flags(c.fetch('flags', {})))
63
+
64
+ cluster_flags['image_registry'] = @config.valid_image_registries[cluster_flags['image_registry']]
65
+ # Allow deploy.yaml to gate certain flavors to certain targets.
66
+ cluster_flavors = @config.flavors.select { |key, value| c['flavors'].nil? || c['flavors'].include?(key) }
67
+ cluster_flavors.each do |flavor, flavor_flags|
68
+ full_flags = cluster_flags.clone
69
+ full_flags.merge!(generate_erb_flags(flavor_flags)) if flavor_flags
70
+ # Project information used to identify source of various manifests
71
+ full_flags.merge!({
72
+ 'git_commit' => git_commit,
73
+ 'git_project' => git_project
74
+ })
75
+
76
+ full_flags.merge!(@literals)
77
+
78
+ # Print all flags for each artifact-flavor
79
+ puts "artifact '#{artifact}', flavor '#{flavor}'"
80
+ full_flags.
81
+ sort_by { |k, v| k }.
82
+ each do |k, v|
83
+ puts "config['#{k}'] = #{v}"
84
+ end
85
+ puts ""
86
+ # Skip rendering ERB templates and generating artifacts
87
+ # if printing flags only
88
+ next if @print_flags_only
89
+
90
+ file_filters = FileFilter.filters_from_hash(c) + @file_filters
91
+
92
+ # Call individual templating hook with the rendered configuration
93
+ # and a prefix to place all the files. Run many hooks in the
94
+ # background.
95
+ flavor_dir = File.join(@output_dir, "#{artifact}_#{flavor}")
96
+ FileUtils.rm_rf flavor_dir
97
+ FileUtils.mkdir_p flavor_dir
98
+ pid = fork do
99
+ # Save rendered release configuration to a temp file.
100
+ rendered = Tempfile.new('deploy_config')
101
+ rendered << YAML.dump(full_flags)
102
+ rendered.flush
103
+
104
+ # Run every hook sequentially. 'default' hook is special.
105
+ @config.hooks.each do |hook|
106
+ if hook == DEFAULT_HOOK_SCRIPT_LABEL
107
+ # TODO(joshk): render_deploys method should take a hash for testability
108
+ KubeDeployTools::RenderDeploysHook.render_deploys(rendered.path, @input_dir, flavor_dir, file_filters)
109
+ else
110
+ Shellrunner.check_call(hook, rendered.path, @input_dir, flavor_dir)
111
+ end
112
+ end
113
+ end
114
+
115
+ permutations[pid] = "#{artifact}_#{flavor}"
116
+ end
117
+ end
118
+
119
+ failure = false
120
+ Process.waitall.each do |pid, status|
121
+ if status.exitstatus != 0
122
+ Logger.error "Rendering #{permutations[pid]} failed: exit status #{status.exitstatus}"
123
+ failure = true
124
+ end
125
+ end
126
+
127
+ raise 'rendering deploy configurations failed' if failure
128
+ end
129
+
130
+ def generate_erb_flags(flags)
131
+ result = Hash.new
132
+
133
+ flags.each do |key, template|
134
+ if template.is_a?(String)
135
+ templater = ERB.new(template)
136
+ result[key] = templater.result
137
+ else
138
+ result[key] = template
139
+ end
140
+ end
141
+
142
+ result
143
+ end
144
+ end
145
+ end
@@ -0,0 +1,66 @@
1
+ require 'optparse'
2
+
3
+ module KubeDeployTools
4
+ class Generate::Optparser
5
+ class Options
6
+ attr_accessor :manifest_file, :input_path, :output_path, :file_filters, :print_flags_only, :literals
7
+
8
+ def initialize
9
+ self.input_path = File.join('kubernetes/')
10
+ self.output_path = File.join('build', 'kubernetes')
11
+ self.file_filters = []
12
+ self.literals = {}
13
+ end
14
+
15
+ def define_options(parser)
16
+ parser.on('-mMANIFEST', '--manifest MANIFEST', 'The configuration MANIFEST to render deploys with.') do |f|
17
+ self.manifest_file = f
18
+ end
19
+
20
+ parser.on('-iPATH', '--input-path PATH', 'Path where Kubernetes manifests and manifest templates (.erb) are located.') do |p|
21
+ self.input_path = p
22
+ end
23
+
24
+ parser.on('-oPATH', '--output-path PATH', 'Path where rendered manifests should be written.') do |p|
25
+ self.output_path = p
26
+ end
27
+
28
+ parser.on('-p', '--print', 'Print all available ERB config values only.') do |p|
29
+ self.print_flags_only = p
30
+ end
31
+
32
+ parser.on('--from-literal KEY=VALUE', "Specify a key and literal value in the ERB context e.g. mykey=myvalue") do |p|
33
+ parts = p.split('=')
34
+ raise ArgumentError, "Expected --from-literal to be in the format key=value, but got '#{p}'" if parts.length != 2
35
+ key, value = parts
36
+ self.literals[key] = value
37
+ end
38
+
39
+ parser.on('--include INCLUDE', "Include glob pattern. Example: --include=**/* will include every file. Default is ''.") do |p|
40
+ self.file_filters.push(["include_files", p])
41
+ end
42
+
43
+ parser.on('--exclude EXCLUDE', "Exclude glob pattern. Example: --exclude=**/gazette/* will exclude every file in gazette folder. Default is ''.") do |p|
44
+ self.file_filters.push(["exclude_files", p])
45
+ end
46
+
47
+ parser.on('--include-dir INCLUDE', "Recursively include all files in a directory and its subdirectories. Example: --include-dir=gazette/ (equivalent of --include=**/gazette/**/*)") do |p|
48
+ self.file_filters.push(["include_dir", p])
49
+ end
50
+
51
+ parser.on('--exclude-dir EXCLUDE', "Recursively exclude all files in a directory and its subdirectories. Example: --exclude-dir=gazette/ (equivalent of --exclude=**/gazette/**/*)") do |p|
52
+ self.file_filters.push(["exclude_dir", p])
53
+ end
54
+ end
55
+ end
56
+
57
+ def parse(args)
58
+ @options = Options.new
59
+ OptionParser.new do |parser|
60
+ @options.define_options(parser)
61
+ parser.parse(args)
62
+ end
63
+ @options
64
+ end
65
+ end
66
+ end
@@ -0,0 +1,30 @@
1
+ module KubeDeployTools
2
+ # Read-only model for image_registries[] array element in KDT deploy.yaml
3
+ # configuration file.
4
+ class ImageRegistry
5
+ attr_accessor :name, :driver, :prefix, :config
6
+
7
+ def initialize(h)
8
+ @name = h['name']
9
+ @driver = h['driver']
10
+ @prefix = h['prefix']
11
+ @config = h['config']
12
+ end
13
+
14
+ def ==(o)
15
+ @name == o.name
16
+ @driver == o.driver
17
+ @prefix == o.prefix
18
+ @config == o.config
19
+ end
20
+
21
+ def to_h
22
+ {
23
+ 'name' => @name,
24
+ 'driver' => @driver,
25
+ 'prefix' => @prefix,
26
+ 'config' => @config,
27
+ }
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,18 @@
1
+ require_relative 'driver/base'
2
+ require_relative 'driver/aws'
3
+ require_relative 'driver/gcp'
4
+ require_relative 'driver/login'
5
+ require_relative 'driver/noop'
6
+
7
+ module KubeDeployTools
8
+ class ImageRegistry
9
+ module Driver
10
+ MAPPINGS = {
11
+ 'aws' => Aws,
12
+ 'gcp' => Gcp,
13
+ 'login' => Login,
14
+ 'noop' => Noop
15
+ }
16
+ end
17
+ end
18
+ end