kubernetes_template_rendering 0.1.0.pre.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: bd6a61ce9b0278d451c6e402ffab894a703846d04704061be05b1bdd30ddfef1
+  data.tar.gz: '079c1784c5357f79fa1e6d6e5a15ef00ea6f2bdf72f37b1ff3949283d3d328ef'
+SHA512:
+  metadata.gz: 2eee87d74893f113a7b999c312457001cc5a8df0fed38b3827b7e773b5f243f2f73625b69ceb7e330576a4438671aa7f7c6491bf80b1b3e82b2167ab31d272ed
+  data.tar.gz: 17698c156cc055c8d0e42c1194f88d1ba99b1152712c2ee212393222ac5a93c75ca1029f424cf01f86993f95113fec39183546f4aef04a3d46f3f02de8e6190c

data/.buildkite/pipeline.yml

@@ -0,0 +1,13 @@
+---
+common_config: &common_config
+  agents:
+    queue: "ruby-3-1"
+  timeout_in_minutes: 5
+steps:
+- label: ":ruby: Render Test Matrix"
+  timeout_in_minutes: 5
+  plugins:
+    - ssh://git@github.com/Invoca/invoca-ruby-test-matrix-buildkite-plugin.git#main:
+        min_ruby_version: '3.1'
+        slack_notification_channel: '#dev-team-release-backend-tools-octothorpe'
+        test_command: "bundle exec rspec"

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,6 @@
+# Inherit from the Invoca style guide
+inherit_from: https://raw.githubusercontent.com/Invoca/style-guide/master/ruby/.rubocop.yml
+
+AllCops:
+  TargetRubyVersion: 3.1
+Gemspec/RequireMFA: false

data/.ruby-version

@@ -0,0 +1 @@
+3.1.4

data/CHANGELOG.md

@@ -0,0 +1,9 @@
+# CHANGELOG for `kubernetes_template_rendering`
+
+Inspired by [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
+
+Note: this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [0.1.0] - Unreleased
+
+- Initial release

data/README.md

@@ -0,0 +1,48 @@
+# KubernetesTemplateRendering
+
+The `invoca-kubernetes_template` gem is a thin wrapper around `jsonnet` and `erb` to allow for the generation of
+Kubernetes manifests from a set of templates combined with a `definitions.yaml` file which stores environmental
+configuration for various deployment environments.
+
+## Installation
+
+Install the gem and add to the application's Gemfile by executing:
+```
+bundle add kubernetes_template_rendering
+```
+
+If bundler is not being used to manage dependencies, install the gem by executing:
+```
+gem install kubernetes_template_rendering
+```
+
+## Usage
+
+This gem is meant to be used as a command line tool for rendering templates that are either written in `jsonnet` or `erb`.
+To use this gem you can either install it, and use the `render_kubernetes_template` executable directly, or you can use
+`gem exec` to execute the command without first installing the gem.
+
+### Example Usage
+```bash
+gem exec -g kubernetes_template_rendering render_kubernetes_templates \
+    --jsonnet-library-path deployment/vendor \
+    --rendered_directory path/to/resources \
+    deployment/templates
+```
+
+### Options
+
+To see a full list of options and how to use them, run the following command:
+```bash
+gem exec -g kubernetes_template_rendering render_kubernetes_templates --help
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/invoca/kubernetes_template_rendering.

data/Rakefile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+require "rubocop/rake_task"
+
+RuboCop::RakeTask.new
+
+task default: %i[spec rubocop]

data/exe/render_templates

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "erb"
+require "fileutils"
+require "optparse"
+require "ostruct"
+require "pathname"
+require "yaml"
+require_relative "../lib/kubernetes_template_rendering/cli"
+
+KubernetesTemplateRendering::CLI.parse_and_render(ARGV)

data/lib/kubernetes_template_rendering/cli.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require_relative "template_directory_renderer"
+require_relative "cli_arguments"
+
+module KubernetesTemplateRendering
+  class CLI
+    DEFINITIONS_FILENAME = "definitions.yaml"
+
+    Arguments = CLIArguments
+
+    class << self
+      def parse_and_render(options)
+        renderer, args = parse(options)
+
+        renderer.render(args)
+      end
+
+      private
+
+      def parse(options)
+        args = Arguments.new
+
+        parser = OptionParser.new do |op|
+          op.banner = "Usage: #{$PROGRAM_NAME} --rendered-directory=<directory> <template directory>"
+
+          op.on("--rendered-directory=RENDERED_DIRECTORY",     "set the directory where rendered output is written")                  { args.rendered_directory = _1 }
+          op.on("--[no-]fork",                                 "disable/enable fork")                                                 { |fork| args.fork = fork }
+          op.on("--makeflags=MAKEFLAGS",                       "pass through makeflags so that we can infer fork preference from -j") { args.makeflags = _1 }
+          op.on("--jsonnet_library_path=JSONNET_LIBRARY_PATH", "set the jsonnet library path")                                        { args.jsonnet_library_path = _1 }
+          op.on("--cluster_type=CLUSTER_TYPE",                 "set the specific cluster type to render")                             { args.cluster_type = _1 }
+          op.on("--region=REGION",                             "set the specific region to render")                                   { args.region = _1 }
+          op.on("--color=COLOR",                               "set the specific color to render")                                    { args.color = _1 }
+
+          op.on("--variable-override=KEY:VALUE", "override a variable value set within definitions.yaml") do |override|
+            args.variable_overrides ||= {}
+            args.variable_overrides.merge!(Hash[[override.split(":", 2)]])
+          end
+
+          op.on("-h", "--help") do
+            puts op
+            exit
+          end
+        end
+
+        parser.parse!(options)
+        args.template_directory = options.first
+
+        unless args.valid?
+          STDERR.puts(parser)
+          exit(1)
+        end
+
+        [renderer_from_args(args), args]
+      end
+
+      def renderer_from_args(args)
+        directories = template_directories(args.template_directory, DEFINITIONS_FILENAME)
+
+        TemplateDirectoryRenderer.new(
+          directories: directories,
+          rendered_directory: args.rendered_directory,
+          cluster_type: args.cluster_type,
+          region: args.region,
+          color: args.color,
+          variable_overrides: args.variable_overrides
+        )
+      end
+
+      def template_directories(template_directory, definitions_file)
+        File.directory?(template_directory) or raise ArgumentError, "template directory not found--make sure to include templates/ prefix: #{template_directory}"
+        directories = Dir["#{template_directory}/**/#{definitions_file}"]
+        directories.map { |directory| File.dirname(directory) }
+      end
+    end
+  end
+end

data/lib/kubernetes_template_rendering/cli_arguments.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module KubernetesTemplateRendering
+  CLIArguments =
+    Struct.new(
+      :rendered_directory,
+      :template_directory,
+      :fork,
+      :makeflags,
+      :jsonnet_library_path,
+      :cluster_type,
+      :region,
+      :color,
+      :variable_overrides
+    ) do
+      def valid?
+        rendered_directory && template_directory
+      end
+
+      def fork?
+        if fork.nil?
+          makeflags&.include?('-j')
+        else
+          fork
+        end
+      end
+
+      def render_files?
+        true
+      end
+    end
+end

data/lib/kubernetes_template_rendering/color.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module KubernetesTemplateRendering
+  module Color
+    class << self
+      def black(str)
+        "\e[30m#{str}\e[0m"
+      end
+
+      def red(str)
+        "\e[31m#{str}\e[0m"
+      end
+
+      def green(str)
+        "\e[32m#{str}\e[0m"
+      end
+
+      def brown(str)
+        "\e[33m#{str}\e[0m"
+      end
+
+      def blue(str)
+        "\e[34m#{str}\e[0m"
+      end
+
+      def magenta(str)
+        "\e[35m#{str}\e[0m"
+      end
+    end
+  end
+end

data/lib/kubernetes_template_rendering/deploy_grouped_resource.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require_relative "resource"
+
+module KubernetesTemplateRendering
+  class DeployGroupedResource
+    DEFAULT_GROUP_VARIABLE_NAME = "deploy_group"
+
+    attr_reader :groups_to_render, :variables, :template_path, :output_directory, :template_path_exclusions, :group_variable_name
+
+    def initialize(template_path:, definitions_path:, variables:, output_directory:, groups_to_render:, template_path_exclusions:, group_variable_name: nil)
+      @template_path    = template_path
+      @definitions_path = definitions_path
+      @variables        = variables
+      @output_directory = output_directory
+      @groups_to_render = groups_to_render
+      @template_path_exclusions = template_path_exclusions || {}
+      @group_variable_name = group_variable_name || DEFAULT_GROUP_VARIABLE_NAME
+    end
+
+    def render(args)
+      @resources =
+        groups_to_render.map do |deploy_group|
+          if template_is_excluded?(deploy_group)
+            puts "Skipping #{Color.magenta(template_path_basename)} for #{deploy_group} deploy group due to it being " \
+              "excluded within the deploy group config\n\n"
+            nil
+          else
+            vars     = variables.merge(group_variable_name => deploy_group)
+            filename = filename_for_deploy_group(deploy_group)
+            Resource.new(template_path: template_path, definitions_path: @definitions_path, variables: vars, output_directory: output_directory, output_filename: filename).tap do |resource|
+              resource.render(args) if args.render_files?
+            end
+          end
+        end.compact
+    end
+
+    private
+
+    def template_path_basename
+      @template_path_basename ||= File.basename(template_path)
+    end
+
+    def template_is_excluded?(deploy_group)
+      @template_path_exclusions[deploy_group]&.include?(template_path_basename)
+    end
+
+    def filename_for_deploy_group(group)
+      case @template_path
+      when /.erb/
+        File.basename(template_path, ".erb").sub(/([^-.]+)\.yaml$/, "#{group}-\\1.yaml")
+      when /.jsonnet/
+        File.basename(template_path).sub(/([^-.]+)\.jsonnet$/, "#{group}-\\1.yaml")
+      else
+        raise "unexpected template_path #{template_path.inspect}"
+      end
+    end
+  end
+end

data/lib/kubernetes_template_rendering/erb_template.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+require_relative "template"
+
+module KubernetesTemplateRendering
+  class ErbTemplate < Template
+    module Snippet
+      def snippet(file)
+        snippet = "#{File.dirname(@template_path)}/#{file}"
+        content = File.read(snippet)
+        erb = ERB.new(content, trim_mode: "-")
+        erb.filename = snippet
+        erb.result(variables_object._binding)
+      end
+    end
+
+    include Snippet
+
+    class VariablesClass < BasicObject
+      include Snippet
+
+      def initialize(template_path, variables)
+        @template_path = template_path
+        @variables = variables
+      end
+
+      def _binding
+        ::Kernel.binding
+      end
+
+      def keys
+        @variables.keys
+      end
+
+      def method_missing(sym, *args, &block)
+        @variables.fetch(sym.to_s) # will raise KeyError if not in @variables hash
+      end
+
+      private
+
+      def variables_object
+        self
+      end
+    end
+
+    def render(erb_binding: nil, jsonnet_library_path: nil)
+      rendered_erb = render_erb(erb_binding)
+      if template_path.end_with?("yaml.erb")
+        with_auto_generated_yaml_comment(sort_yaml(rendered_erb))
+      else
+        rendered_erb
+      end
+    end
+
+    private
+
+    def variables_object
+      VariablesClass.new(template_path, variables)
+    end
+
+    def render_erb(erb_binding)
+      content = File.read(template_path)
+      erb = ERB.new(content, trim_mode: "-")
+      erb.filename = template_path
+      erb_binding.nil? ? erb.result(variables_object._binding) : erb.result(erb_binding) # here is where we eval the template
+    end
+
+    def sort_yaml(erb_yaml)
+      if (yaml_docs = YAML.load_stream(erb_yaml)).any?
+        yaml_docs.map { |yaml_doc| sort_keys(yaml_doc).to_yaml }.join("\n")
+      else
+        erb_yaml
+      end
+    end
+
+    def sort_keys(json_doc)
+      case json_doc
+      when Array
+        json_doc.map { |v| sort_keys(v) }
+      when Hash
+        with_sorted_values = json_doc.transform_values { |v| sort_keys(v) }
+        with_sorted_values.sort.to_h
+      else
+        json_doc
+      end
+    end
+  end
+end

data/lib/kubernetes_template_rendering/jsonnet_template.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require "jsonnet"
+require_relative "template"
+
+module KubernetesTemplateRendering
+  class JsonnetTemplate < Template
+    MULTI_FILE_RENDER_KEY = "MULTI_FILE_RENDER"
+    MULTI_FILE_RENDER_FILE_NAME_KEY = "MULTI_FILE_RENDER_NAME"
+
+    class MultiFileJsonnetRenderError < StandardError; end
+
+    def render(erb_binding: nil, jsonnet_library_path: nil)
+      json_doc = render_json_doc_from_template(jsonnet_library_path)
+      if multi_file_jsonnet_doc?(json_doc)
+        render_multi_file_jsonnet!(json_doc)
+      else
+        with_auto_generated_yaml_comment(json_doc.to_yaml)
+      end
+    end
+
+    private
+
+    def render_json_doc_from_template(jsonnet_library_path)
+      vm = Jsonnet::VM.new
+      vm.tla_code("vars", variables.to_json)
+      if jsonnet_library_path.nil?
+        vm.jpath_add(File.expand_path('../../../vendor-jb', __dir__))
+      else
+        vm.jpath_add(jsonnet_library_path)
+      end
+      JSON.parse(vm.evaluate_file(template_path))
+    end
+
+    def multi_file_jsonnet_doc?(json_doc)
+      json_doc[MULTI_FILE_RENDER_KEY]
+    end
+
+    # Multi-File JSONNET Template Structuring:
+    # Top Level - Multi File Hash
+    # Keys map to Either Hash or Array
+    # If Array, the values of the array must be Hashs.
+    # Hash can be either a normal Hash or another Multi-File Hash
+
+    def render_multi_file_jsonnet!(json_doc, file_name_to_yaml_hash = {})
+      json_doc.delete(MULTI_FILE_RENDER_KEY)
+      json_doc.each do |key, value|
+        case value
+        when Hash
+          render_json_doc(value, key, file_name_to_yaml_hash)
+        when NilClass
+          next
+        else
+          raise ArgumentError, "must be a Hash or NilClass, was #{value.inspect}"
+        end
+      end
+      file_name_to_yaml_hash
+    end
+
+    def render_json_doc(json_doc, default_file_name, file_name_to_yaml_hash)
+      if multi_file_jsonnet_doc?(json_doc)
+        render_multi_file_jsonnet!(json_doc, file_name_to_yaml_hash)
+      else
+        file_name = file_name_from_object(json_doc, default: default_file_name)
+        file_name_to_yaml_hash["#{file_name}.yaml"] = with_auto_generated_yaml_comment(json_doc.to_yaml)
+      end
+    end
+
+    def validate_json_doc!(json_doc, context)
+      json_doc.is_a?(Hash) or raise MultiFileJsonnetRenderError, context
+    end
+
+    def file_name_from_object(object, default: nil)
+      if (base_name = object.delete(MULTI_FILE_RENDER_FILE_NAME_KEY))
+        base_name
+      else
+        default
+      end
+    end
+  end
+end

data/lib/kubernetes_template_rendering/resource.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require "pathname"
+require_relative "color"
+require_relative "erb_template"
+require_relative "jsonnet_template"
+
+module KubernetesTemplateRendering
+  class Resource
+    class UnexpectedFileTypeError < StandardError; end
+
+    attr_reader :variables, :template_path, :output_directory
+
+    def initialize(template_path:, definitions_path:, variables:, output_directory:, output_filename: nil)
+      @template_path    = template_path
+      @definitions_path = definitions_path
+      @variables        = variables
+      @output_directory = output_directory
+      @output_filename  = output_filename || template_filename(template_path)
+    end
+
+    def render(args)
+      write_template(args)
+    end
+
+    private
+
+    def rendered_template(args)
+      @rendered_template ||= template_klass.render(@template_path, variables, jsonnet_library_path: args.jsonnet_library_path)
+    end
+
+    def write_template(args)
+      print_status
+
+      # If a Hash is returned, that means this is a multi-file template, meaning we need to iterate over the hash.
+      # Else a String is returned and we can write it directly to the output file.
+      rt = rendered_template(args)
+
+      if args.render_files?
+        if rt.is_a?(Hash)
+          rt.each do |filename, contents|
+            File.write(output_path(filename), contents)
+          end
+        else
+          File.write(output_path(@output_filename), rt)
+        end
+      end
+    end
+
+    def template_klass
+      case @template_path
+      when /\.erb\z/
+        ErbTemplate
+      when /\.jsonnet\z/
+        JsonnetTemplate
+      else
+        raise UnexpectedFileTypeError, "Unexpected file type #{@template_path}"
+      end
+    end
+
+    def print_status
+      variable_output = variables.map { |k, v| "#{Color.magenta(k)}=#{Color.blue(v)}" }.join(', ')
+      puts "Writing #{Color.magenta(File.basename(output_path(@output_filename)))} with variables #{variable_output}\n\n"
+    end
+
+    def output_path(filename)
+      File.join(@output_directory, filename)
+    end
+
+    def template_filename(template_path)
+      if template_path.match(/\.erb\z/)
+        File.basename(template_path, '.erb')
+
+      elsif template_path.match(/\.jsonnet\z/)
+        Pathname.new(template_path).basename.sub_ext('.yaml')
+
+      else
+        raise "Unexpected template_path format: #{template_path}"
+      end
+    end
+  end
+end

data/lib/kubernetes_template_rendering/resource_set.rb

@@ -0,0 +1,229 @@
+# frozen_string_literal: true
+
+require_relative "resource"
+require_relative "deploy_grouped_resource"
+
+# This class points to the resources in a given template_directory for a given kubernetes_cluster_type like 'ops' or 'prod' or 'ci'.
+# `config` contains the definitions found in `definitions_path`.
+#   The most important config is "directory" which is a pattern like "%{region}/%{type}/%{color}/staging-ops".
+#   The config "regions", "colors" are the sets of regions and colors to render for.
+# It renders into `rendered_directory`.
+module KubernetesTemplateRendering
+  class ResourceSet
+    attr_reader :variables, :output_directory, :deploy_group_config, :omitted_resources,
+                :template_directory, :target_output_directory, :regions, :colors,
+                :definitions_path, :kubernetes_cluster_type
+
+    def initialize(config:, template_directory:, rendered_directory:, definitions_path:, kubernetes_cluster_type:)
+      @variables               = config["variables"] || {}
+      @deploy_group_config     = config["deploy_groups"]
+      @omitted_resources       = config["omitted_resources"]
+      @template_directory      = template_directory
+      @target_output_directory = config["directory"] or raise ArgumentError, "missing 'directory:' in #{config.inspect}"
+      @regions                 = config["regions"] || []
+      @colors                  = config["colors"] || []
+      @rendered_directory      = rendered_directory
+      @definitions_path        = definitions_path
+      @kubernetes_cluster_type = kubernetes_cluster_type
+      @resources               = {}
+
+      if @kubernetes_cluster_type != "kube-platform"
+        @target_output_directory.include?("%{plain_region}") or raise "#{@template_directory}: target_output_directory #{@target_output_directory} needs %{plain_region}"
+      end
+    end
+
+    def normal_render(args)
+      dynamic_output_directory? and raise "Directory must not be dynamic: #{target_output_directory}"
+
+      variables["kubernetes_cluster_type"] = @kubernetes_cluster_type
+
+      if (plain_region = variables["plain_region"])
+        default_region_vars(plain_region)
+      end
+
+      output_directory = File.join(@rendered_directory, target_output_directory)
+      render_create_directory(args, output_directory)
+    end
+
+    def render(args)
+      @regions.any? or raise "#{template_directory}: must have at least one region"
+      @colors.any? or raise "#{template_directory}: must have at least one color"
+
+      variables["kubernetes_cluster_type"] = @kubernetes_cluster_type
+
+      @regions.each do |plain_region|
+        default_region_vars(plain_region)
+
+        @colors.each do |c|
+          variables["color"] = c
+          output_directory = File.join(@rendered_directory, format(@target_output_directory, plain_region: plain_region, color: c, type: @kubernetes_cluster_type))
+          render_create_directory(args, output_directory)
+        end
+      end
+    end
+
+    def render_create_directory(args, output_directory)
+      create_directory(output_directory)
+      puts "Rendering templates to: #{Color.magenta(output_directory)}"
+      puts "Variable assignments:"
+      variables.each { |k, v| puts "\t#{Color.magenta(k)}=#{Color.blue(v)}" }
+      puts
+      if omitted_resources
+        puts "Omitted resources:"
+        omitted_resources.each { |ot| puts "\t#{ot}" }
+      end
+      puts
+      resources(output_directory).each do |resource|
+        resource.render(args)
+      end
+      puts
+    end
+
+    private
+
+    CLOUD_REGION_TO_PROVIDER_AND_DATACENTER = {
+      # Note: The names below should match RegionDiscovery from process_settings-production.
+      # https://github.com/Invoca/process_settings-production/blob/main/settings/region_discovery/production_regions.yml
+      "us-east-1"    => ['aws', "AWS-us-east-1"],
+      "us-east-2"    => ['aws', "AWS-us-east-2"],
+      "us-central1"  => ['gcp', "GCE-us-central1"],
+      "us-west2"     => ['gcp', "GCE-us-west2"],
+      "eu-central-1" => ['aws', "AWS-eu-central-1"],
+      "eu-west-1"    => ['aws', "AWS-eu-west-1"],
+      "europe-west4" => ['gcp', "GCE-europe-west4"],
+
+      # other regions
+      "us-east1"    => ['gcp', "GCE-us-east1"],
+      "us-west1"    => ['gcp', "GCE-us-west1"],
+      "local"       => ['',    "local"]
+    }.freeze
+
+    # The zone to use for failure-domain.beta.kubernetes.io/zone or topology.kubernetes.io/zone by DeployGroup
+    AVAILABILITY_ZONE_FOR_DEPLOY_GROUP = {
+      "us-east-1" => {
+        "primary" => "us-east-1c",
+        "secondary" => "us-east-1d",
+        "tertiary" => "us-east-1b"
+      },
+      "us-east-2" => {
+        "primary" => "us-east-2a",
+        "secondary" => "us-east-2b",
+        "tertiary" => "us-east-2c"
+      },
+      "us-central1" => {
+        "primary" => "us-central1-a",
+        "secondary" => "us-central1-b",
+        "tertiary" => "us-central1-c"
+      },
+      "us-west2" => {
+        "primary" => "us-west2-a",
+        "secondary" => "us-west2-b",
+        "tertiary" => "us-west2-c"
+      },
+      "eu-central-1" => {
+        "primary" => "eu-central-1a",
+        "secondary" => "eu-central-1b",
+        "tertiary" => "eu-central-1c"
+      },
+      "eu-west-1" => {
+        "primary" => "eu-west-1a",
+        "secondary" => "eu-west-1b",
+        "tertiary" => "eu-west-1c"
+      },
+      "europe-west4" => {
+        "primary" => "europe-west4-a",
+        "secondary" => "europe-west4-b",
+        "tertiary" => "europe-west4-c"
+      },
+      "local" => {},
+      "us-west1" => {},
+      "us-east1" => {}
+    }
+
+    def default_region_vars(plain_region)
+      variables.has_key?("region") and raise "replace region with plain_region"
+      variables["plain_region"] = plain_region
+      cloud_datacenter_and_provider = CLOUD_REGION_TO_PROVIDER_AND_DATACENTER[plain_region] or raise "no CLOUD_REGION_TO_PROVIDER_AND_DATACENTER entry found for #{plain_region.inspect}"
+      variables["cloud_provider"], variables["cloud_datacenter"] = cloud_datacenter_and_provider
+      variables["cloud_region"] = variables["cloud_datacenter"] # for compatibility with old resource files; cloud_datacenter is preferred now
+      if plain_region == "local"
+        variables["data_silo"] ||= "local"
+      elsif plain_region.start_with?("us-")
+        variables["data_silo"] ||= "us"
+      end
+
+      # cannot do ||= because we want to reset when plain_region changes
+      variables["availability_zone_for_deploy_group"] = AVAILABILITY_ZONE_FOR_DEPLOY_GROUP[plain_region] or raise "no AVAILABILITY_ZONE_FOR_DEPLOY_GROUP mapping found for #{plain_region.inspect}"
+    end
+
+    def create_directory(directory)
+      unless File.exist?(directory)
+        puts <<~MESSAGE
+
+          Directory #{Color.magenta(directory)} doesn't exist, #{Color.green('creating it')}
+
+        MESSAGE
+        FileUtils.mkdir_p(directory)
+      end
+    end
+
+    def resources(output_directory)
+      @resources[output_directory] ||= standard_resources(output_directory) + grouped_resources(output_directory)
+    end
+
+    def standard_resources(output_directory)
+      standard_template_paths.map do |path|
+        Resource.new(template_path: path, definitions_path: @definitions_path, variables: variables, output_directory: output_directory)
+      end
+    end
+
+    def grouped_resources(output_directory)
+      deploy_grouped_template_paths.map do |path|
+        DeployGroupedResource.new(
+          template_path: path,
+          definitions_path: @definitions_path,
+          variables: variables,
+          output_directory: output_directory,
+          groups_to_render: deploy_groups_to_render,
+          template_path_exclusions: deploy_group_config["exclude_files"],
+          group_variable_name: deploy_group_config["variable_name"]
+        )
+      end
+    end
+
+    def standard_template_paths
+      @standard_template_paths ||=
+        (Dir[File.join(template_directory, "*.yaml.erb")] +
+          Dir[File.join(template_directory, '*.jsonnet')]) -
+        deploy_grouped_template_paths - omitted_resource_paths
+    end
+
+    def deploy_grouped_template_paths
+      @deploy_grouped_template_paths ||=
+        if deploy_group_config
+          deploy_group_config["files"]&.map { |file| File.join(template_directory, file) } ||
+            (Dir[File.join(template_directory, "*-deploy.yaml.erb")] + Dir[File.join(template_directory, "*-deploy.jsonnet")]) - omitted_resource_paths
+        else
+          []
+        end
+    end
+
+    def omitted_resource_paths
+      @omitted_resource_paths ||= omitted_resources&.map { |file| File.join(template_directory, file) } || []
+    end
+
+    def deploy_groups_to_render
+      group_names = deploy_group_config["group_names"]
+      if array_of_arrays?(group_names)
+        first, *rest = group_names
+        first.product(*rest).map { |group| group.join("-") }
+      else
+        group_names
+      end
+    end
+
+    def array_of_arrays?(array)
+      array.all? { |item| item.is_a?(Array) }
+    end
+  end
+end

data/lib/kubernetes_template_rendering/template.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require "ostruct"
+require "open3"
+require "shellwords"
+require "yaml"
+
+# This is a base class for all Templates. Derived classes must implement the render method.
+module KubernetesTemplateRendering
+  class Template < OpenStruct
+    attr_reader :template_path, :variables
+    def initialize(template_path, variables)
+      @template_path = template_path
+      @variables = variables
+    end
+
+    def render(args)
+      raise "must be defined by subclass"
+    end
+
+    private
+
+    def with_auto_generated_yaml_comment(yaml_string)
+      comment = <<~EOS
+      # WARNING: DO NO EDIT THIS FILE!
+      # Any changes made here will be lost.
+      # This file is autogenerated from #{template_path}
+      EOS
+      comment + yaml_string
+    end
+
+    class << self
+      # @param [String] template_path: file path to template file that needs to be rendered.
+      # @param [Hash]: variables that will be used in the template file to generate distict files.
+      #
+      # @return [String] generated YAML file
+      # @return [Hash] Hash of file names as keys with the values being corresponding generated YAML.
+      #
+      # @raise [UnexpectedFileTypeError] if file type doesn't match [yaml.erb, erb, or jsonnet]
+      # @raise [MultiFileJsonnetRenderError] if Jsonnet template files don't follow proper MultiFileJsonnet templating.
+
+      # TODO
+      # The ErbTemplate and JsonnetTemplate classes both inherit from the Template class and implement a render method.
+      # However, the erb_binding parameter is used just in ErbTemplate, while the jsonnet_library_path parameter is used just in JsonnetTemplate.
+      # This is a little awkward. Potentially this could be refactored.
+      def render(template_path, variables, erb_binding: nil, jsonnet_library_path: nil)
+        new(template_path, variables).render(erb_binding: erb_binding, jsonnet_library_path: jsonnet_library_path)
+      end
+    end
+  end
+end

data/lib/kubernetes_template_rendering/template_directory_renderer.rb

@@ -0,0 +1,174 @@
+# frozen_string_literal: true
+
+require 'active_support'
+require 'active_support/core_ext' # for deep_merge
+require 'invoca/utils'
+require 'yaml'
+require_relative 'resource_set'
+require 'set'
+
+# This class points to a collection of template directories to render, and a rendered_directory to render into.
+# Optionally, some of the template directories may be omitted by including them in `omitted_names`.
+module KubernetesTemplateRendering
+  class TemplateDirectoryRenderer
+    DEFINITIONS_FILENAME = "definitions.yaml"
+
+    attr_reader :directories, :omitted_names, :rendered_directory, :cluster_type, :region, :color, :variable_overrides
+
+    def initialize(directories:, rendered_directory:, omitted_names: [], cluster_type: nil, region: nil, color: nil, variable_overrides: nil)
+      @directories        = directories_with_definitions(Array(directories))
+      @omitted_names      = Array(omitted_names)
+      @rendered_directory = rendered_directory
+      @cluster_type       = cluster_type
+      @region             = region
+      @color              = color
+      @variable_overrides = variable_overrides || {}
+    end
+
+    def render(args)
+      child_pids = []
+
+      resource_sets.each do |name, resource_sets|
+        puts "Rendering templates for definition #{Color.red(name)}..."
+        resource_sets.each do |resource_set|
+          if args.fork?
+            if (pid = Process.fork)
+              # this is the parent
+              child_pids << pid
+              wait_if_max_forked(child_pids)
+            else
+              # this is the child
+              render_set(args, resource_set)
+              Kernel.exit! # skip at_exit handlers since parent will run those
+            end
+          else
+            render_set(args, resource_set)
+          end
+        end
+      end
+
+      if args.fork?
+        Process.waitall
+      end
+    end
+
+    private
+
+    def read_definitions(path)
+      File.read(path)
+    end
+
+    MAX_FORKED_PROCESSES = 9
+
+    def wait_if_max_forked(child_pids)
+      while child_pids.size >= MAX_FORKED_PROCESSES
+        begin
+          Process.waitpid # this is a race condition because 1 or more processes could exit before we get here
+        rescue SystemCallError # this will happen if they all exited before we called waitpid
+        end
+        child_pids.delete_if do |pid|
+          Process.waitpid(pid, Process::WNOHANG)
+        rescue Errno::ECHILD # No child processes
+          true
+        end
+      end
+    end
+
+    def render_set(args, resource_set)
+      resource_set.render(args)
+    rescue => ex
+      raise "error rendering ResourceSet from #{resource_set.definitions_path}\n#{ex.class}: #{ex.message}"
+    end
+
+    def directories_with_definitions(directories)
+      directories.select do |dir|
+        definitions_path = definitions_path_for_dir(dir)
+        File.exist?(definitions_path)
+      end
+    end
+
+    def definitions_path_for_dir(dir)
+      File.join(dir, DEFINITIONS_FILENAME)
+    end
+
+    def resource_sets
+      @resource_sets ||= @directories.each_with_object({}) do |dir, hash|
+        definitions_path = definitions_path_for_dir(dir)
+        config = load_config(definitions_path)
+
+        config.map do |name, config|
+          next if omitted_names.include?(name)
+
+          kubernetes_cluster_type = name.sub('SPP-PLACEHOLDER', 'staging').sub(/\..*/, '') # prod.gcp => prod
+
+          hash[name] ||= []
+          hash[name] << ResourceSet.new(config: config, template_directory: dir, rendered_directory: @rendered_directory, kubernetes_cluster_type: kubernetes_cluster_type, definitions_path: definitions_path)
+        end
+      end
+    end
+
+    def build_libsonnet(dir, config)
+      fname = File.join(dir, 'definitions.libsonnet')
+      existing = File.exists?(fname) ? File.read(fname) : ""
+      proposed = build_json(config)
+
+      if existing != proposed
+        puts("Generating updated #{Color.magenta(File.basename(fname))}")
+        File.write(fname, proposed)
+      end
+    end
+
+    def build_json(config)
+      hash = transform_for_jsonnet(config)
+      JSON.pretty_generate(hash)
+    end
+
+    # This method ensures that OpenStructs are
+    # converted to hashes to support to_json operation
+    # It also converts any embedded variable place holders
+    # into a Jsonnet friendly format:
+    #
+    # %{variable} is converted to %(variable)s which can
+    # then be used with the Jsonnet function std.format
+    def transform_for_jsonnet(hash)
+      hash.transform_values do |value|
+        case value
+        when OpenStruct
+          transform_for_jsonnet(value.to_h)
+        when Hash
+          transform_for_jsonnet(value)
+        else
+          value
+        end
+      end
+    end
+
+    def load_config(definitions_path)
+      begin
+        config = YAML.safe_load(read_definitions(definitions_path), aliases: true)
+      rescue => ex
+        raise "error loading YAML from #{definitions_path}:\n#{ex.class}: #{ex.message}"
+      end
+
+      expand_config(config).each_with_object({}) do |(name, data), hash|
+        if !cluster_type || cluster_type == name.sub('SPP-PLACEHOLDER', 'staging').sub(/\..*/, '') # prod.gcp => prod
+          cluster_type_config = OpenStruct.new(data)
+
+          cluster_type_config.regions   = cluster_type_config.regions & [region] if region
+          cluster_type_config.colors    = cluster_type_config.colors & [color]   if color
+          cluster_type_config.variables = (cluster_type_config.variables || {}).merge(variable_overrides)
+
+          hash[name] = cluster_type_config if (region.nil? && color.nil?) || (cluster_type_config.regions.any? && cluster_type_config.colors.any?)
+        end
+      end
+    end
+
+    # returns a copy of the given config hash with the COMMON: k-v removed and deep merged into the other config values
+    # (explicit config values take precedence over COMMON: ones)
+    def expand_config(config)
+      common = config.delete('COMMON') || {}
+
+      Hash[config.map { |k, v| [k, common.deep_merge(v)] }]
+    end
+  end
+end

data/lib/kubernetes_template_rendering/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module KubernetesTemplateRendering
+  VERSION = "0.1.0.pre.1"
+end

data/lib/kubernetes_template_rendering.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require_relative "kubernetes_template_rendering/version"
+
+module KubernetesTemplateRendering
+  class Error < StandardError; end
+  # Your code goes here...
+end

data/sig/invoca/kubernetes_templates.rbs

@@ -0,0 +1,6 @@
+module Invoca
+  module KubernetesTemplateRendering
+    VERSION: String
+    # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+  end
+end

metadata

@@ -0,0 +1,110 @@
+--- !ruby/object:Gem::Specification
+name: kubernetes_template_rendering
+version: !ruby/object:Gem::Version
+  version: 0.1.0.pre.1
+platform: ruby
+authors:
+- Octothorpe
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2024-04-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: jsonnet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: invoca-utils
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Tool for rendering ERB and Jsonnet templates
+email:
+- octothorpe@invoca.com
+executables:
+- render_templates
+extensions: []
+extra_rdoc_files: []
+files:
+- ".buildkite/pipeline.yml"
+- ".rspec"
+- ".rubocop.yml"
+- ".ruby-version"
+- CHANGELOG.md
+- README.md
+- Rakefile
+- exe/render_templates
+- lib/kubernetes_template_rendering.rb
+- lib/kubernetes_template_rendering/cli.rb
+- lib/kubernetes_template_rendering/cli_arguments.rb
+- lib/kubernetes_template_rendering/color.rb
+- lib/kubernetes_template_rendering/deploy_grouped_resource.rb
+- lib/kubernetes_template_rendering/erb_template.rb
+- lib/kubernetes_template_rendering/jsonnet_template.rb
+- lib/kubernetes_template_rendering/resource.rb
+- lib/kubernetes_template_rendering/resource_set.rb
+- lib/kubernetes_template_rendering/template.rb
+- lib/kubernetes_template_rendering/template_directory_renderer.rb
+- lib/kubernetes_template_rendering/version.rb
+- sig/invoca/kubernetes_templates.rbs
+homepage: https://github.com/Invoca/kubernetes_template_rendering
+licenses: []
+metadata:
+  allowed_push_host: https://rubygems.org
+  homepage_uri: https://github.com/Invoca/kubernetes_template_rendering
+  source_code_uri: https://github.com/Invoca/kubernetes_template_rendering
+  changelog_uri: https://github.com/Invoca/kubernetes_template_rendering/blob/main/CHANGELOG.md
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.1.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">"
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubygems_version: 3.4.21
+signing_key:
+specification_version: 4
+summary: Tool for rendering ERB and Jsonnet templates
+test_files: []