kube_deploy_tools 3.0.5

data/lib/kube_deploy_tools/deploy_config_file/deep_merge.rb

@@ -0,0 +1,38 @@
+# Downloaded from
+# https://raw.githubusercontent.com/rails/rails/v6.0.2.2/activesupport/lib/active_support/core_ext/hash/deep_merge.rb
+# to avoid a dependency on activesupport.
+
+# frozen_string_literal: true
+
+class Hash
+  # Returns a new hash with +self+ and +other_hash+ merged recursively.
+  #
+  #   h1 = { a: true, b: { c: [1, 2, 3] } }
+  #   h2 = { a: false, b: { x: [3, 4, 5] } }
+  #
+  #   h1.deep_merge(h2) # => { a: false, b: { c: [1, 2, 3], x: [3, 4, 5] } }
+  #
+  # Like with Hash#merge in the standard library, a block can be provided
+  # to merge values:
+  #
+  #   h1 = { a: 100, b: 200, c: { c1: 100 } }
+  #   h2 = { b: 250, c: { c1: 200 } }
+  #   h1.deep_merge(h2) { |key, this_val, other_val| this_val + other_val }
+  #   # => { a: 100, b: 450, c: { c1: 300 } }
+  def deep_merge(other_hash, &block)
+    dup.deep_merge!(other_hash, &block)
+  end
+
+  # Same as +deep_merge+, but modifies +self+.
+  def deep_merge!(other_hash, &block)
+    merge!(other_hash) do |key, this_val, other_val|
+      if this_val.is_a?(Hash) && other_val.is_a?(Hash)
+        this_val.deep_merge(other_val, &block)
+      elsif block_given?
+        block.call(key, this_val, other_val)
+      else
+        other_val
+      end
+    end
+  end
+end

data/lib/kube_deploy_tools/deploy_config_file/util.rb

@@ -0,0 +1,39 @@
+
+module KubeDeployTools
+  module DeployConfigFileUtil
+    def check_and_err(condition, error)
+      if ! condition
+        Logger.error("Error in configuration #{@filename}")
+        raise ArgumentError, error
+      end
+    end
+
+    def check_and_warn(condition, warning)
+      if ! condition
+        Logger.warn("Warning in configuration #{@filename}")
+        Logger.warn(warning)
+      end
+    end
+
+    def load_library(lib)
+      # All paths must be valid accessible gcs paths for the current user.
+      # To modify gcloud identity being used by this process, set
+      # GOOGLE_APPLICATION_CREDENTIALS or sign in with `gcloud auth login`
+      lib_config = nil
+      if lib.start_with?('gs://')
+        Tempfile.open(['gs-kdt-library', '.yaml']) do |t|
+          out = Shellrunner.check_call('gsutil', 'cat', lib)
+          t.write(out)
+          t.flush
+          lib_config = DeployConfigFile.new(t.path)
+        end
+      elsif File.exist?(lib)
+        lib_config = DeployConfigFile.new(lib)
+      else
+        raise "Unsupported or non-existent library: #{lib}"
+      end
+
+      lib_config
+    end
+  end
+end

data/lib/kube_deploy_tools/errors.rb

@@ -0,0 +1,5 @@
+
+module KubeDeployTools
+  class FatalDeploymentError < StandardError; end
+end
+

data/lib/kube_deploy_tools/file_filter.rb

@@ -0,0 +1,43 @@
+require 'kube_deploy_tools/formatted_logger'
+
+# frozen_string_literal: true
+module KubeDeployTools
+  module FileFilter
+    VALID_FILTER_NAMES = %w(include_dir exclude_dir)
+
+    def self.filter_files(filters:, files_path:)
+      all_files = Dir[File.join(files_path, '**', '*')].to_set
+      filtered_files = if filters.any? { |k, _| k =~ /include_/ }
+        Set.new
+      else
+        Set.new(all_files)
+      end
+
+      filters.each do |action, globpath|
+        if action =~ /_dir$/
+          # ensure globpath correctly selects an entire dir recursively
+          # always starts with files_path
+          # always ends with /**/* (to recusively select entire dir)
+          globpath = File.join(files_path, globpath.gsub(/^[\/*]+|[\/*]+$/, ""), '**', '*')
+        end
+
+        case action
+        when /^include_(files|dir)$/
+          filtered_files.merge( all_files.select{ |f| File.fnmatch?(globpath, f, File::FNM_PATHNAME) } )
+        when /^exclude_(files|dir)$/
+          filtered_files.reject!{ |f| File.fnmatch?(globpath, f, File::FNM_PATHNAME) }
+        else
+          raise "you want to #{action}: wat do"
+        end
+      end
+      Logger.debug("\nYour filter generates following paths: \n#{filtered_files.to_a.join("\n")}")
+      filtered_files
+    end
+
+    def self.filters_from_hash(h)
+      VALID_FILTER_NAMES.flat_map do |filter_name|
+        Array(h[filter_name]).map { |dir_path| [filter_name, dir_path] }
+      end
+    end
+  end
+end

data/lib/kube_deploy_tools/formatted_logger.rb

@@ -0,0 +1,59 @@
+require 'forwardable'
+require 'logger'
+
+require 'colorized_string'
+
+require 'kube_deploy_tools/deferred_summary_logging'
+
+module KubeDeployTools
+  class FormattedLogger < ::Logger
+    include DeferredSummaryLogging
+
+    def self.build(context: nil, namespace: nil, stream: $stderr)
+      l = new(stream)
+      l.level = level_from_env
+
+      l.formatter = proc do |severity, datetime, _progname, msg|
+        middle = context ? "[#{context}]" : ""
+        if ! namespace.nil? && namespace != 'default'
+          middle += "[#{namespace}]"
+        end
+
+        dt = datetime.strftime('%F %T')
+        colorized_line = ColorizedString.new("[#{severity}][#{dt}]#{middle} #{msg}\n")
+
+        case severity
+        when "FATAL"
+          ColorizedString.new("[#{severity}][#{dt}]#{middle} ").red + "#{msg}\n"
+        when "ERROR"
+          colorized_line.red
+        when "WARN"
+          colorized_line.yellow
+        else
+          colorized_line
+        end
+      end
+      l
+    end
+
+    def self.level_from_env
+      return ::Logger::DEBUG if ENV["DEBUG"]
+
+      if ENV["LEVEL"]
+        ::Logger.const_get(ENV["LEVEL"].upcase)
+      else
+        ::Logger::INFO
+      end
+    end
+    private_class_method :level_from_env
+  end
+
+  class Logger
+    class << self
+      extend Forwardable
+
+      attr_accessor :logger
+      def_delegators :@logger, *(::Logger.public_instance_methods(false) + DeferredSummaryLogging.public_instance_methods(false))
+    end
+  end
+end

data/lib/kube_deploy_tools/generate.rb

@@ -0,0 +1,145 @@
+require 'erb'
+require 'fileutils'
+require 'tempfile'
+require 'time'
+require 'yaml'
+
+require 'kube_deploy_tools/render_deploys_hook'
+require 'kube_deploy_tools/deploy_config_file'
+require 'kube_deploy_tools/file_filter'
+require 'kube_deploy_tools/shellrunner'
+require 'kube_deploy_tools/tag'
+
+DEFAULT_HOOK_SCRIPT = 'render_deploys_hook'
+DEFAULT_HOOK_SCRIPT_LABEL = 'default'
+
+module KubeDeployTools
+  DEFAULT_FLAGS = {
+    'image_tag' => tag_from_local_env,
+    'tag' => tag_from_local_env,
+  }.freeze
+  class Generate
+    def initialize(manifest, input_dir, output_dir, file_filters: [], print_flags_only: false, literals: {})
+      @project = KubeDeployTools::PROJECT
+      @build_number = KubeDeployTools::BUILD_NUMBER
+
+      @config = DeployConfigFile.new(manifest)
+      @literals = literals
+
+      @input_dir = input_dir
+      @output_dir = output_dir
+      FileUtils.mkdir_p @output_dir
+
+      @file_filters = file_filters
+
+      @print_flags_only = print_flags_only
+    end
+
+    def git_commit
+      commit_sha = Shellrunner.check_call(*%w(git rev-parse HEAD))
+      commit_sha ? commit_sha.strip : ''
+    end
+
+    def git_project
+      project_url = Shellrunner.check_call(*%w(git config --get remote.origin.url))
+      project_url ? project_url.strip : ''
+    end
+
+    def generate
+      permutations = {}
+      @config.artifacts.each do |c|
+        artifact = c.fetch('name')
+
+        # Get metadata for this target/environment pair from manifest
+        cluster_flags = DEFAULT_FLAGS.dup
+
+        cluster_flags['image_registry'] = c.fetch('image_registry')
+
+        # Merge in configured default flags
+        cluster_flags.merge!(@config.default_flags)
+
+        # Update and merge deploy flags for rendering
+        cluster_flags.merge!(generate_erb_flags(c.fetch('flags', {})))
+
+        cluster_flags['image_registry'] = @config.valid_image_registries[cluster_flags['image_registry']]
+        # Allow deploy.yaml to gate certain flavors to certain targets.
+        cluster_flavors = @config.flavors.select { |key, value| c['flavors'].nil? || c['flavors'].include?(key) }
+        cluster_flavors.each do |flavor, flavor_flags|
+          full_flags = cluster_flags.clone
+          full_flags.merge!(generate_erb_flags(flavor_flags)) if flavor_flags
+          # Project information used to identify source of various manifests
+          full_flags.merge!({
+            'git_commit' => git_commit,
+            'git_project' => git_project
+          })
+
+          full_flags.merge!(@literals)
+
+          # Print all flags for each artifact-flavor
+          puts "artifact '#{artifact}', flavor '#{flavor}'"
+          full_flags.
+            sort_by { |k, v| k }.
+            each do |k, v|
+              puts "config['#{k}'] = #{v}"
+            end
+          puts ""
+          # Skip rendering ERB templates and generating artifacts
+          # if printing flags only
+          next if @print_flags_only
+
+          file_filters = FileFilter.filters_from_hash(c) + @file_filters
+
+          # Call individual templating hook with the rendered configuration
+          # and a prefix to place all the files. Run many hooks in the
+          # background.
+          flavor_dir = File.join(@output_dir, "#{artifact}_#{flavor}")
+          FileUtils.rm_rf flavor_dir
+          FileUtils.mkdir_p flavor_dir
+          pid = fork do
+            # Save rendered release configuration to a temp file.
+            rendered = Tempfile.new('deploy_config')
+            rendered << YAML.dump(full_flags)
+            rendered.flush
+
+            # Run every hook sequentially. 'default' hook is special.
+            @config.hooks.each do |hook|
+              if hook == DEFAULT_HOOK_SCRIPT_LABEL
+                # TODO(joshk): render_deploys method should take a hash for testability
+                KubeDeployTools::RenderDeploysHook.render_deploys(rendered.path, @input_dir, flavor_dir, file_filters)
+              else
+                Shellrunner.check_call(hook, rendered.path, @input_dir, flavor_dir)
+              end
+            end
+          end
+
+          permutations[pid] = "#{artifact}_#{flavor}"
+        end
+      end
+
+      failure = false
+      Process.waitall.each do |pid, status|
+        if status.exitstatus != 0
+          Logger.error "Rendering #{permutations[pid]} failed: exit status #{status.exitstatus}"
+          failure = true
+        end
+      end
+
+      raise 'rendering deploy configurations failed' if failure
+    end
+
+    def generate_erb_flags(flags)
+      result = Hash.new
+
+      flags.each do |key, template|
+        if template.is_a?(String)
+          templater = ERB.new(template)
+          result[key] = templater.result
+        else
+          result[key] = template
+        end
+      end
+
+      result
+    end
+  end
+end

data/lib/kube_deploy_tools/generate/options.rb

@@ -0,0 +1,66 @@
+require 'optparse'
+
+module KubeDeployTools
+  class Generate::Optparser
+    class Options
+      attr_accessor :manifest_file, :input_path, :output_path, :file_filters, :print_flags_only, :literals
+
+      def initialize
+        self.input_path = File.join('kubernetes/')
+        self.output_path = File.join('build', 'kubernetes')
+        self.file_filters = []
+        self.literals = {}
+      end
+
+      def define_options(parser)
+        parser.on('-mMANIFEST', '--manifest MANIFEST', 'The configuration MANIFEST to render deploys with.') do |f|
+          self.manifest_file = f
+        end
+
+        parser.on('-iPATH', '--input-path PATH', 'Path where Kubernetes manifests and manifest templates (.erb) are located.') do |p|
+          self.input_path = p
+        end
+
+        parser.on('-oPATH', '--output-path PATH', 'Path where rendered manifests should be written.') do |p|
+          self.output_path = p
+        end
+
+        parser.on('-p', '--print', 'Print all available ERB config values only.') do |p|
+          self.print_flags_only = p
+        end
+
+        parser.on('--from-literal KEY=VALUE', "Specify a key and literal value in the ERB context e.g. mykey=myvalue") do |p|
+          parts = p.split('=')
+          raise ArgumentError, "Expected --from-literal to be in the format key=value, but got '#{p}'" if parts.length != 2
+          key, value = parts
+          self.literals[key] = value
+        end
+
+        parser.on('--include INCLUDE', "Include glob pattern. Example: --include=**/* will include every file. Default is ''.") do |p|
+          self.file_filters.push(["include_files", p])
+        end
+
+        parser.on('--exclude EXCLUDE', "Exclude glob pattern. Example: --exclude=**/gazette/* will exclude every file in gazette folder. Default is ''.") do |p|
+          self.file_filters.push(["exclude_files", p])
+        end
+
+        parser.on('--include-dir INCLUDE', "Recursively include all files in a directory and its subdirectories. Example: --include-dir=gazette/ (equivalent of --include=**/gazette/**/*)") do |p|
+          self.file_filters.push(["include_dir", p])
+        end
+
+        parser.on('--exclude-dir EXCLUDE', "Recursively exclude all files in a directory and its subdirectories. Example: --exclude-dir=gazette/ (equivalent of --exclude=**/gazette/**/*)") do |p|
+          self.file_filters.push(["exclude_dir", p])
+        end
+      end
+    end
+
+    def parse(args)
+      @options = Options.new
+      OptionParser.new do |parser|
+        @options.define_options(parser)
+        parser.parse(args)
+      end
+      @options
+    end
+  end
+end

data/lib/kube_deploy_tools/image_registry.rb

@@ -0,0 +1,30 @@
+module KubeDeployTools
+  # Read-only model for image_registries[] array element in KDT deploy.yaml
+  # configuration file.
+  class ImageRegistry
+    attr_accessor :name, :driver, :prefix, :config
+
+    def initialize(h)
+      @name = h['name']
+      @driver = h['driver']
+      @prefix = h['prefix']
+      @config = h['config']
+    end
+
+    def ==(o)
+      @name == o.name
+      @driver == o.driver
+      @prefix == o.prefix
+      @config == o.config
+    end
+
+    def to_h
+      {
+        'name' => @name,
+        'driver' => @driver,
+        'prefix' => @prefix,
+        'config' => @config,
+      }
+    end
+  end
+end

data/lib/kube_deploy_tools/image_registry/driver.rb

@@ -0,0 +1,18 @@
+require_relative 'driver/base'
+require_relative 'driver/aws'
+require_relative 'driver/gcp'
+require_relative 'driver/login'
+require_relative 'driver/noop'
+
+module KubeDeployTools
+  class ImageRegistry
+    module Driver
+      MAPPINGS = {
+        'aws' => Aws,
+        'gcp' => Gcp,
+        'login' => Login,
+        'noop' => Noop
+      }
+    end
+  end
+end