krane 1.0.0

data/lib/krane/label_selector.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Krane
+  class LabelSelector
+    def self.parse(string)
+      selector = {}
+
+      string.split(',').each do |kvp|
+        key, value = kvp.split('=', 2)
+
+        if key.blank?
+          raise ArgumentError, "key is blank"
+        end
+
+        if key.end_with?("!")
+          raise ArgumentError, "!= selectors are not supported"
+        end
+
+        if value&.start_with?("=")
+          raise ArgumentError, "== selectors are not supported"
+        end
+
+        selector[key] = value
+      end
+
+      new(selector)
+    end
+
+    def initialize(hash)
+      @selector = hash
+    end
+
+    def to_h
+      @selector
+    end
+
+    def to_s
+      return "" if @selector.nil?
+      @selector.map { |k, v| "#{k}=#{v}" }.join(",")
+    end
+  end
+end

data/lib/krane/oj.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+require 'oj'
+
+Oj.mimic_JSON

data/lib/krane/options_helper.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Krane
+  module OptionsHelper
+    class OptionsError < StandardError; end
+
+    STDIN_TEMP_FILE = "from_stdin.yml"
+    class << self
+      def with_processed_template_paths(template_paths, render_erb: false)
+        validated_paths = []
+        template_paths.uniq!
+        template_paths.each do |template_path|
+          next if template_path == '-'
+          validated_paths << template_path
+        end
+
+        if template_paths.include?("-")
+          Dir.mktmpdir("krane") do |dir|
+            template_dir_from_stdin(temp_dir: dir, render_erb: render_erb)
+            validated_paths << dir
+            yield validated_paths
+          end
+        else
+          yield validated_paths
+        end
+      end
+
+      private
+
+      def template_dir_from_stdin(temp_dir:, render_erb:)
+        tempfile = STDIN_TEMP_FILE
+        tempfile += ".erb" if render_erb
+        File.open(File.join(temp_dir, tempfile), 'w+') { |f| f.print($stdin.read) }
+      rescue IOError, Errno::ENOENT => e
+        raise OptionsError, e.message
+      end
+    end
+  end
+end

data/lib/krane/remote_logs.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+require 'krane/container_logs'
+
+module Krane
+  class RemoteLogs
+    attr_reader :container_logs
+
+    def initialize(logger:, parent_id:, container_names:, namespace:, context:)
+      @logger = logger
+      @parent_id = parent_id
+      @container_logs = container_names.map do |n|
+        ContainerLogs.new(
+          logger: logger,
+          container_name: n,
+          parent_id: parent_id,
+          namespace: namespace,
+          context: context
+        )
+      end
+    end
+
+    def empty?
+      @container_logs.all?(&:empty?)
+    end
+
+    def sync
+      @container_logs.each(&:sync)
+    end
+
+    def print_latest
+      @container_logs.each do |cl|
+        unless cl.printing_started?
+          @logger.info("Streaming logs from #{@parent_id} container '#{cl.container_name}':")
+        end
+        cl.print_latest(prefix: @container_logs.length > 1)
+      end
+    end
+
+    def print_all(prevent_duplicate: true)
+      return if @already_displayed && prevent_duplicate
+
+      if @container_logs.all?(&:empty?)
+        @logger.warn("No logs found for #{@parent_id}")
+        return
+      end
+
+      @container_logs.each do |cl|
+        if cl.empty?
+          @logger.warn("No logs found for #{@parent_id} container '#{cl.container_name}'")
+        else
+          @logger.info("Logs from #{@parent_id} container '#{cl.container_name}':")
+          cl.print_all
+          @logger.blank_line
+        end
+      end
+
+      @already_displayed = true
+    end
+  end
+end

data/lib/krane/render_task.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+require 'tempfile'
+
+require 'krane/common'
+require 'krane/renderer'
+require 'krane/template_sets'
+
+module Krane
+  # Render templates
+  class RenderTask
+    # Initializes the render task
+    #
+    # @param logger [Object] Logger object (defaults to an instance of Krane::FormattedLogger)
+    # @param current_sha [String] The SHA of the commit
+    # @param filenames [Array<String>] An array of filenames and/or directories containing templates (*required*)
+    # @param bindings [Hash] Bindings parsed by Krane::BindingsParser
+    def initialize(logger: nil, current_sha:, filenames: [], bindings:)
+      @logger = logger || Krane::FormattedLogger.build
+      @filenames = filenames.map { |path| File.expand_path(path) }
+      @bindings = bindings
+      @current_sha = current_sha
+    end
+
+    # Runs the task, returning a boolean representing success or failure
+    #
+    # @return [Boolean]
+    def run(*args)
+      run!(*args)
+      true
+    rescue Krane::FatalDeploymentError
+      false
+    end
+
+    # Runs the task, raising exceptions in case of issues
+    #
+    # @param stream [IO] Place to stream the output to
+    #
+    # @return [nil]
+    def run!(stream:)
+      @logger.reset
+      @logger.phase_heading("Initializing render task")
+
+      ts = TemplateSets.from_dirs_and_files(paths: @filenames, logger: @logger)
+
+      validate_configuration(ts)
+      count = render_templates(stream, ts)
+
+      @logger.summary.add_action("Successfully rendered #{count} template(s)")
+      @logger.print_summary(:success)
+    rescue Krane::FatalDeploymentError
+      @logger.print_summary(:failure)
+      raise
+    end
+
+    private
+
+    def render_templates(stream, template_sets)
+      @logger.phase_heading("Rendering template(s)")
+      count = 0
+      template_sets.with_resource_definitions_and_filename(current_sha: @current_sha,
+        bindings: @bindings, raw: true) do |rendered_content, filename|
+        write_to_stream(rendered_content, filename, stream)
+        count += 1
+      end
+
+      count
+    rescue Krane::InvalidTemplateError => exception
+      log_invalid_template(exception)
+      raise
+    end
+
+    def write_to_stream(rendered_content, filename, stream)
+      file_basename = File.basename(filename)
+      @logger.info("Rendering #{file_basename}...")
+      implicit = []
+      YAML.parse_stream(rendered_content, "<rendered> #{filename}") { |d| implicit << d.implicit }
+      if rendered_content.present?
+        stream.puts "---\n" if implicit.first
+        stream.puts rendered_content
+        @logger.info("Rendered #{file_basename}")
+      else
+        @logger.warn("Rendered #{file_basename} successfully, but the result was blank")
+      end
+    rescue Psych::SyntaxError => exception
+      raise InvalidTemplateError.new("Template is not valid YAML. #{exception.message}", filename: filename)
+    end
+
+    def validate_configuration(template_sets)
+      @logger.info("Validating configuration")
+      errors = []
+      if @filenames.blank?
+        errors << "filenames must be set"
+      end
+
+      if !@current_sha.nil? && @current_sha.empty?
+        errors << "current-sha is optional but can not be blank"
+      end
+      errors += template_sets.validate
+
+      unless errors.empty?
+        @logger.summary.add_action("Configuration invalid")
+        @logger.summary.add_paragraph(errors.map { |err| "- #{err}" }.join("\n"))
+        raise Krane::TaskConfigurationError, "Configuration invalid: #{errors.join(', ')}"
+      end
+    end
+
+    def log_invalid_template(exception)
+      @logger.error("Failed to render #{exception.filename}")
+
+      debug_msg = ColorizedString.new("Invalid template: #{exception.filename}\n").red
+      debug_msg += "> Error message:\n#{FormattedLogger.indent_four(exception.to_s)}"
+      if exception.content
+        debug_msg += "\n> Template content:\n#{FormattedLogger.indent_four(exception.content)}"
+      end
+      @logger.summary.add_paragraph(debug_msg)
+    end
+  end
+end

data/lib/krane/renderer.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+require 'erb'
+require 'securerandom'
+require 'yaml'
+require 'json'
+
+module Krane
+  class Renderer
+    class InvalidPartialError < InvalidTemplateError
+      attr_accessor :parents, :content, :filename
+      def initialize(msg, parents: [], content: nil, filename:)
+        @parents = parents
+        super(msg, content: content, filename: filename)
+      end
+    end
+    class PartialNotFound < InvalidTemplateError; end
+
+    def initialize(current_sha:, template_dir:, logger:, bindings: {})
+      @current_sha = current_sha
+      @template_dir = template_dir
+      @partials_dirs =
+        %w(partials ../partials).map { |d| File.expand_path(File.join(@template_dir, d)) }
+      @logger = logger
+      @bindings = bindings
+      # Max length of podname is only 63chars so try to save some room by truncating sha to 8 chars
+      @id = if ENV["TASK_ID"]
+        ENV["TASK_ID"]
+      elsif current_sha
+        current_sha[0...8] + "-#{SecureRandom.hex(4)}"
+      end
+    end
+
+    def render_template(filename, raw_template)
+      return raw_template unless File.extname(filename) == ".erb"
+
+      erb_binding = TemplateContext.new(self).template_binding
+      bind_template_variables(erb_binding, template_variables)
+
+      ERB.new(raw_template, nil, '-').result(erb_binding)
+    rescue InvalidPartialError => err
+      err.parents = err.parents.dup.unshift(filename)
+      err.filename = "#{err.filename} (partial included from: #{err.parents.join(' -> ')})"
+      raise err
+    rescue StandardError => err
+      raise InvalidTemplateError.new(err.message, filename: filename, content: raw_template)
+    end
+
+    def render_partial(partial, locals)
+      variables = template_variables.merge(locals)
+      erb_binding = TemplateContext.new(self).template_binding
+      bind_template_variables(erb_binding, variables)
+      erb_binding.local_variable_set("locals", locals)
+
+      partial_path = find_partial(partial)
+      template = File.read(partial_path)
+      expanded_template = ERB.new(template, nil, '-').result(erb_binding)
+
+      docs = Psych.parse_stream(expanded_template, partial_path)
+      # If the partial contains multiple documents or has an explicit document header,
+      # we know it cannot validly be indented in the parent, so return it immediately.
+      return expanded_template unless docs.children.one? && docs.children.first.implicit
+      # Make sure indentation isn't a problem by producing a single line of parseable YAML.
+      # Note that JSON is a subset of YAML.
+      JSON.generate(docs.children.first.to_ruby)
+    rescue PartialNotFound => err
+      # get the filename from the first parent, not the missing partial itself
+      raise err if err.filename == partial
+      raise InvalidPartialError.new(err.message, filename: partial, content: expanded_template || template)
+    rescue InvalidPartialError => err
+      err.parents = err.parents.dup.unshift(File.basename(partial_path))
+      raise err
+    rescue StandardError => err
+      raise InvalidPartialError.new(err.message, filename: partial_path, content: expanded_template || template)
+    end
+
+    private
+
+    def template_variables
+      {
+        'current_sha' => @current_sha,
+        'deployment_id' => @id,
+      }.merge(@bindings)
+    end
+
+    def bind_template_variables(erb_binding, variables)
+      variables.each do |var_name, value|
+        erb_binding.local_variable_set(var_name, value)
+      end
+    end
+
+    def find_partial(name)
+      partial_names = [name + '.yaml.erb', name + '.yml.erb']
+      @partials_dirs.each do |dir|
+        partial_names.each do |partial_name|
+          partial_path = File.join(dir, partial_name)
+          return partial_path if File.exist?(partial_path)
+        end
+      end
+      raise PartialNotFound.new("Could not find partial '#{name}' in any of #{@partials_dirs.join(':')}",
+        filename: name)
+    end
+
+    class TemplateContext
+      def initialize(renderer)
+        @_renderer = renderer
+      end
+
+      def template_binding
+        binding
+      end
+
+      def partial(partial, locals = {})
+        @_renderer.render_partial(partial, locals)
+      end
+    end
+  end
+end

data/lib/krane/resource_cache.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require 'concurrent/hash'
+
+module Krane
+  class ResourceCache
+    delegate :namespace, :context, :logger, to: :@task_config
+
+    def initialize(task_config)
+      @task_config = task_config
+
+      @kind_fetcher_locks = Concurrent::Hash.new { |hash, key| hash[key] = Mutex.new }
+      @data = Concurrent::Hash.new
+      @kubectl = Kubectl.new(task_config: @task_config, log_failure_by_default: false)
+    end
+
+    def get_instance(kind, resource_name, raise_if_not_found: false)
+      instance = use_or_populate_cache(kind).fetch(resource_name, {})
+      if instance.blank? && raise_if_not_found
+        raise Krane::Kubectl::ResourceNotFoundError, "Resource does not exist (used cache for kind #{kind})"
+      end
+      instance
+    rescue KubectlError
+      {}
+    end
+
+    def get_all(kind, selector = nil)
+      instances = use_or_populate_cache(kind).values
+      return instances unless selector
+
+      instances.select do |r|
+        labels = r.dig("metadata", "labels") || {}
+        labels >= selector
+      end
+    rescue KubectlError
+      []
+    end
+
+    private
+
+    def statsd_tags
+      { namespace: namespace, context: context }
+    end
+
+    def use_or_populate_cache(kind)
+      @kind_fetcher_locks[kind].synchronize do
+        return @data[kind] if @data.key?(kind)
+        @data[kind] = fetch_by_kind(kind)
+      end
+    end
+
+    def fetch_by_kind(kind)
+      resource_class = KubernetesResource.class_for_kind(kind)
+      global_kind = @task_config.global_kinds.map(&:downcase).include?(kind.downcase)
+      output_is_sensitive = resource_class.nil? ? false : resource_class::SENSITIVE_TEMPLATE_CONTENT
+      raw_json, _, st = @kubectl.run("get", kind, "--chunk-size=0", attempts: 5, output: "json",
+         output_is_sensitive: output_is_sensitive, use_namespace: !global_kind)
+      raise KubectlError unless st.success?
+
+      instances = {}
+      JSON.parse(raw_json)["items"].each do |resource|
+        resource_name = resource.dig("metadata", "name")
+        instances[resource_name] = resource
+      end
+      instances
+    end
+  end
+end