invoca-kubernetes_templates 0.1.0.pre.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 56823d5fc55ba306d3ab46f15c1f5228fa3a57f1edcb930461c66be155291b6c
+  data.tar.gz: 99c85338f9c8f68433bb0443677204121a58947b87621a34c419cd3f81072b89
+SHA512:
+  metadata.gz: 7d7c3c7d72c82e8d6360217bf552f4bee43b019072707e34832500a1ab56874b0072a8fc448d4856b0465a0a85071aabfd266732c8ca2642d712a38cb477d965
+  data.tar.gz: aa8dc2d7092e56247436689f1ef6572e4401a7fb07eabced192475f1c0858ef719e0483f855d071337e0ecbc3cc7d26a9b512c69c184e074d11af65c5f9f8c37

data/.buildkite/pipeline.yml

@@ -0,0 +1,13 @@
+---
+common_config: &common_config
+  agents:
+    queue: "ruby-3-1"
+  timeout_in_minutes: 5
+steps:
+- label: ":ruby: Render Test Matrix"
+  timeout_in_minutes: 5
+  plugins:
+    - ssh://git@github.com/Invoca/invoca-ruby-test-matrix-buildkite-plugin.git#main:
+        min_ruby_version: '3.1'
+        slack_notification_channel: '#dev-team-release-backend-tools-octothorpe'
+        test_command: "bundle exec rspec"

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,6 @@
+# Inherit from the Invoca style guide
+inherit_from: https://raw.githubusercontent.com/Invoca/style-guide/master/ruby/.rubocop.yml
+
+AllCops:
+  TargetRubyVersion: 3.1
+Gemspec/RequireMFA: false

data/.ruby-version

@@ -0,0 +1 @@
+3.1.4

data/CHANGELOG.md

@@ -0,0 +1,9 @@
+# CHANGELOG for `invoca-kubernetes_templates`
+
+Inspired by [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
+
+Note: this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [0.1.0] - Unreleased
+
+- Initial release

data/README.md

@@ -0,0 +1,48 @@
+# Invoca::KubernetesTemplates
+
+The `invoca-kubernetes_template` gem is a thin wrapper around `jsonnet` and `erb` to allow for the generation of
+Kubernetes manifests from a set of templates combined with a `definitions.yaml` file which stores environmental
+configuration for various deployment environments.
+
+## Installation
+
+Install the gem and add to the application's Gemfile by executing:
+```
+bundle add invoca-kubernetes_templates
+```
+
+If bundler is not being used to manage dependencies, install the gem by executing:
+```
+gem install invoca-kubernetes_templates
+```
+
+## Usage
+
+This gem is meant to be used as a command line tool for rendering templates that are either written in `jsonnet` or `erb`.
+To use this gem you can either install it, and use the `render_kubernetes_template` executable directly, or you can use
+`gem exec` to execute the command without first installing the gem.
+
+### Example Usage
+```bash
+gem exec -g invoca-kubernetes_templates render_kubernetes_templates -- \
+    --jsonnet-library-path deployment/vendor \
+    --rendered_directory path/to/resources \
+    deployment/templates
+```
+
+### Options
+
+To see a full list of options and how to use them, run the following command:
+```bash
+gem exec -g invoca-kubernetes_templates render_kubernetes_templates -- --help
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/invoca/invoca-kubernetes_templates.

data/Rakefile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+require "rubocop/rake_task"
+
+RuboCop::RakeTask.new
+
+task default: %i[spec rubocop]

data/exe/render_kubernetes_templates

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "erb"
+require "fileutils"
+require "optparse"
+require "ostruct"
+require "pathname"
+require "yaml"
+require_relative "../lib/invoca/kubernetes_templates/cli"
+
+Invoca::KubernetesTemplates::CLI.parse_and_render(ARGV)

data/lib/invoca/kubernetes_templates/cli.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require_relative "template_directory_renderer"
+require_relative "cli_arguments"
+
+module Invoca
+  module KubernetesTemplates
+    class CLI
+      DEFINITIONS_FILENAME = "definitions.yaml"
+
+      Arguments = CLIArguments
+
+      class << self
+        def parse_and_render(options)
+          renderer, args = parse(options)
+
+          renderer.render(args)
+        end
+
+        private
+
+        def parse(options)
+          args = Arguments.new
+
+          parser = OptionParser.new do |op|
+            op.banner = "Usage: #{$PROGRAM_NAME} --rendered-directory=<directory> <template directory>"
+
+            op.on("--rendered-directory=RENDERED_DIRECTORY",     "set the directory where rendered output is written")                  { args.rendered_directory = _1 }
+            op.on("--[no-]fork",                                 "disable/enable fork")                                                 { |fork| args.fork = fork }
+            op.on("--makeflags=MAKEFLAGS",                       "pass through makeflags so that we can infer fork preference from -j") { args.makeflags = _1 }
+            op.on("--jsonnet_library_path=JSONNET_LIBRARY_PATH", "set the jsonnet library path")                                        { args.jsonnet_library_path = _1 }
+            op.on("--cluster_type=CLUSTER_TYPE",                 "set the specific cluster type to render")                             { args.cluster_type = _1 }
+            op.on("--region=REGION",                             "set the specific region to render")                                   { args.region = _1 }
+            op.on("--color=COLOR",                               "set the specific color to render")                                    { args.color = _1 }
+
+            op.on("--variable-override=KEY:VALUE", "override a variable value set within definitions.yaml") do |override|
+              args.variable_overrides ||= {}
+              args.variable_overrides.merge!(Hash[[override.split(":", 2)]])
+            end
+
+            op.on("-h", "--help") do
+              puts op
+              exit
+            end
+          end
+
+          parser.parse!(options)
+          args.template_directory = options.first
+
+          unless args.valid?
+            STDERR.puts(parser)
+            exit(1)
+          end
+
+          [renderer_from_args(args), args]
+        end
+
+        def renderer_from_args(args)
+          directories = template_directories(args.template_directory, DEFINITIONS_FILENAME)
+
+          TemplateDirectoryRenderer.new(
+            directories: directories,
+            rendered_directory: args.rendered_directory,
+            cluster_type: args.cluster_type,
+            region: args.region,
+            color: args.color,
+            variable_overrides: args.variable_overrides
+          )
+        end
+
+        def template_directories(template_directory, definitions_file)
+          File.directory?(template_directory) or raise ArgumentError, "template directory not found--make sure to include templates/ prefix: #{template_directory}"
+          directories = Dir["#{template_directory}/**/#{definitions_file}"]
+          directories.map { |directory| File.dirname(directory) }
+        end
+      end
+    end
+  end
+end

data/lib/invoca/kubernetes_templates/cli_arguments.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Invoca
+  module KubernetesTemplates
+    CLIArguments =
+      Struct.new(
+        :rendered_directory,
+        :template_directory,
+        :fork,
+        :makeflags,
+        :jsonnet_library_path,
+        :cluster_type,
+        :region,
+        :color,
+        :variable_overrides
+      ) do
+        def valid?
+          rendered_directory && template_directory
+        end
+
+        def fork?
+          if fork.nil?
+            makeflags&.include?('-j')
+          else
+            fork
+          end
+        end
+
+        def render_files?
+          true
+        end
+      end
+  end
+end

data/lib/invoca/kubernetes_templates/color.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Invoca
+  module KubernetesTemplates
+    module Color
+      class << self
+        def black(str)
+          "\e[30m#{str}\e[0m"
+        end
+
+        def red(str)
+          "\e[31m#{str}\e[0m"
+        end
+
+        def green(str)
+          "\e[32m#{str}\e[0m"
+        end
+
+        def brown(str)
+          "\e[33m#{str}\e[0m"
+        end
+
+        def blue(str)
+          "\e[34m#{str}\e[0m"
+        end
+
+        def magenta(str)
+          "\e[35m#{str}\e[0m"
+        end
+      end
+    end
+  end
+end

data/lib/invoca/kubernetes_templates/deploy_grouped_resource.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require_relative "resource"
+
+module Invoca
+  module KubernetesTemplates
+    class DeployGroupedResource
+      DEFAULT_GROUP_VARIABLE_NAME = "deploy_group"
+
+      attr_reader :groups_to_render, :variables, :template_path, :output_directory, :template_path_exclusions, :group_variable_name
+
+      def initialize(template_path:, definitions_path:, variables:, output_directory:, groups_to_render:, template_path_exclusions:, group_variable_name: nil)
+        @template_path    = template_path
+        @definitions_path = definitions_path
+        @variables        = variables
+        @output_directory = output_directory
+        @groups_to_render = groups_to_render
+        @template_path_exclusions = template_path_exclusions || {}
+        @group_variable_name = group_variable_name || DEFAULT_GROUP_VARIABLE_NAME
+      end
+
+      def render(args)
+        @resources =
+          groups_to_render.map do |deploy_group|
+            if template_is_excluded?(deploy_group)
+              puts "Skipping #{Color.magenta(template_path_basename)} for #{deploy_group} deploy group due to it being " \
+                "excluded within the deploy group config\n\n"
+              nil
+            else
+              vars     = variables.merge(group_variable_name => deploy_group)
+              filename = filename_for_deploy_group(deploy_group)
+              Resource.new(template_path: template_path, definitions_path: @definitions_path, variables: vars, output_directory: output_directory, output_filename: filename).tap do |resource|
+                resource.render(args) if args.render_files?
+              end
+            end
+          end.compact
+      end
+
+      private
+
+      def template_path_basename
+        @template_path_basename ||= File.basename(template_path)
+      end
+
+      def template_is_excluded?(deploy_group)
+        @template_path_exclusions[deploy_group]&.include?(template_path_basename)
+      end
+
+      def filename_for_deploy_group(group)
+        case @template_path
+        when /.erb/
+          File.basename(template_path, ".erb").sub(/([^-.]+)\.yaml$/, "#{group}-\\1.yaml")
+        when /.jsonnet/
+          File.basename(template_path).sub(/([^-.]+)\.jsonnet$/, "#{group}-\\1.yaml")
+        else
+          raise "unexpected template_path #{template_path.inspect}"
+        end
+      end
+    end
+  end
+end

data/lib/invoca/kubernetes_templates/erb_template.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require_relative "template"
+
+module Invoca
+  module KubernetesTemplates
+    class ErbTemplate < Template
+      module Snippet
+        def snippet(file)
+          snippet = "#{File.dirname(@template_path)}/#{file}"
+          content = File.read(snippet)
+          erb = ERB.new(content, trim_mode: "-")
+          erb.filename = snippet
+          erb.result(variables_object._binding)
+        end
+      end
+
+      include Snippet
+
+      class VariablesClass < BasicObject
+        include Snippet
+
+        def initialize(template_path, variables)
+          @template_path = template_path
+          @variables = variables
+        end
+
+        def _binding
+          ::Kernel.binding
+        end
+
+        def keys
+          @variables.keys
+        end
+
+        def method_missing(sym, *args, &block)
+          @variables.fetch(sym.to_s) # will raise KeyError if not in @variables hash
+        end
+
+        private
+
+        def variables_object
+          self
+        end
+      end
+
+      def render(erb_binding: nil, jsonnet_library_path: nil)
+        rendered_erb = render_erb(erb_binding)
+        if template_path.end_with?("yaml.erb")
+          with_auto_generated_yaml_comment(sort_yaml(rendered_erb))
+        else
+          rendered_erb
+        end
+      end
+
+      private
+
+      def variables_object
+        VariablesClass.new(template_path, variables)
+      end
+
+      def render_erb(erb_binding)
+        content = File.read(template_path)
+        erb = ERB.new(content, trim_mode: "-")
+        erb.filename = template_path
+        erb_binding.nil? ? erb.result(variables_object._binding) : erb.result(erb_binding) # here is where we eval the template
+      end
+
+      def sort_yaml(erb_yaml)
+        if (yaml_docs = YAML.load_stream(erb_yaml)).any?
+          yaml_docs.map { |yaml_doc| sort_keys(yaml_doc).to_yaml }.join("\n")
+        else
+          erb_yaml
+        end
+      end
+
+      def sort_keys(json_doc)
+        case json_doc
+        when Array
+          json_doc.map { |v| sort_keys(v) }
+        when Hash
+          with_sorted_values = json_doc.transform_values { |v| sort_keys(v) }
+          with_sorted_values.sort.to_h
+        else
+          json_doc
+        end
+      end
+    end
+  end
+end

data/lib/invoca/kubernetes_templates/jsonnet_template.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require "jsonnet"
+require_relative "template"
+
+module Invoca
+  module KubernetesTemplates
+    class JsonnetTemplate < Template
+      MULTI_FILE_RENDER_KEY = "MULTI_FILE_RENDER"
+      MULTI_FILE_RENDER_FILE_NAME_KEY = "MULTI_FILE_RENDER_NAME"
+
+      class MultiFileJsonnetRenderError < StandardError; end
+
+      def render(erb_binding: nil, jsonnet_library_path: nil)
+        json_doc = render_json_doc_from_template(jsonnet_library_path)
+        if multi_file_jsonnet_doc?(json_doc)
+          render_multi_file_jsonnet!(json_doc)
+        else
+          with_auto_generated_yaml_comment(json_doc.to_yaml)
+        end
+      end
+
+      private
+
+      def render_json_doc_from_template(jsonnet_library_path)
+        vm = Jsonnet::VM.new
+        vm.tla_code("vars", variables.to_json)
+        if jsonnet_library_path.nil?
+          vm.jpath_add(File.expand_path('../../../../../vendor-jb', __dir__))
+        else
+          vm.jpath_add(jsonnet_library_path)
+        end
+        JSON.parse(vm.evaluate_file(template_path))
+      end
+
+      def multi_file_jsonnet_doc?(json_doc)
+        json_doc[MULTI_FILE_RENDER_KEY]
+      end
+
+      # Multi-File JSONNET Template Structuring:
+      # Top Level - Multi File Hash
+      # Keys map to Either Hash or Array
+      # If Array, the values of the array must be Hashs.
+      # Hash can be either a normal Hash or another Multi-File Hash
+
+      def render_multi_file_jsonnet!(json_doc, file_name_to_yaml_hash = {})
+        json_doc.delete(MULTI_FILE_RENDER_KEY)
+        json_doc.each do |key, value|
+          case value
+          when Hash
+            render_json_doc(value, key, file_name_to_yaml_hash)
+          when NilClass
+            next
+          else
+            raise ArgumentError, "must be a Hash or NilClass, was #{value.inspect}"
+          end
+        end
+        file_name_to_yaml_hash
+      end
+
+      def render_json_doc(json_doc, default_file_name, file_name_to_yaml_hash)
+        if multi_file_jsonnet_doc?(json_doc)
+          render_multi_file_jsonnet!(json_doc, file_name_to_yaml_hash)
+        else
+          file_name = file_name_from_object(json_doc, default: default_file_name)
+          file_name_to_yaml_hash["#{file_name}.yaml"] = with_auto_generated_yaml_comment(json_doc.to_yaml)
+        end
+      end
+
+      def validate_json_doc!(json_doc, context)
+        json_doc.is_a?(Hash) or raise MultiFileJsonnetRenderError, context
+      end
+
+      def file_name_from_object(object, default: nil)
+        if (base_name = object.delete(MULTI_FILE_RENDER_FILE_NAME_KEY))
+          base_name
+        else
+          default
+        end
+      end
+    end
+  end
+end

data/lib/invoca/kubernetes_templates/resource.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require "pathname"
+require_relative "color"
+require_relative "erb_template"
+require_relative "jsonnet_template"
+
+module Invoca
+  module KubernetesTemplates
+    class Resource
+      class UnexpectedFileTypeError < StandardError; end
+
+      attr_reader :variables, :template_path, :output_directory
+
+      def initialize(template_path:, definitions_path:, variables:, output_directory:, output_filename: nil)
+        @template_path    = template_path
+        @definitions_path = definitions_path
+        @variables        = variables
+        @output_directory = output_directory
+        @output_filename  = output_filename || template_filename(template_path)
+      end
+
+      def render(args)
+        write_template(args)
+      end
+
+      private
+
+      def rendered_template(args)
+        @rendered_template ||= template_klass.render(@template_path, variables, jsonnet_library_path: args.jsonnet_library_path)
+      end
+
+      def write_template(args)
+        print_status
+
+        # If a Hash is returned, that means this is a multi-file template, meaning we need to iterate over the hash.
+        # Else a String is returned and we can write it directly to the output file.
+        rt = rendered_template(args)
+
+        if args.render_files?
+          if rt.is_a?(Hash)
+            rt.each do |filename, contents|
+              File.write(output_path(filename), contents)
+            end
+          else
+            File.write(output_path(@output_filename), rt)
+          end
+        end
+      end
+
+      def template_klass
+        case @template_path
+        when /\.erb\z/
+          ErbTemplate
+        when /\.jsonnet\z/
+          JsonnetTemplate
+        else
+          raise UnexpectedFileTypeError, "Unexpected file type #{@template_path}"
+        end
+      end
+
+      def print_status
+        variable_output = variables.map { |k, v| "#{Color.magenta(k)}=#{Color.blue(v)}" }.join(', ')
+        puts "Writing #{Color.magenta(File.basename(output_path(@output_filename)))} with variables #{variable_output}\n\n"
+      end
+
+      def output_path(filename)
+        File.join(@output_directory, filename)
+      end
+
+      def template_filename(template_path)
+        if template_path.match(/\.erb\z/)
+          File.basename(template_path, '.erb')
+
+        elsif template_path.match(/\.jsonnet\z/)
+          Pathname.new(template_path).basename.sub_ext('.yaml')
+
+        else
+          raise "Unexpected template_path format: #{template_path}"
+        end
+      end
+    end
+  end
+end

data/lib/invoca/kubernetes_templates/resource_set.rb

@@ -0,0 +1,231 @@
+# frozen_string_literal: true
+
+require_relative "resource"
+require_relative "deploy_grouped_resource"
+
+# This class points to the resources in a given template_directory for a given kubernetes_cluster_type like 'ops' or 'prod' or 'ci'.
+# `config` contains the definitions found in `definitions_path`.
+#   The most important config is "directory" which is a pattern like "%{region}/%{type}/%{color}/staging-ops".
+#   The config "regions", "colors" are the sets of regions and colors to render for.
+# It renders into `rendered_directory`.
+module Invoca
+  module KubernetesTemplates
+    class ResourceSet
+      attr_reader :variables, :output_directory, :deploy_group_config, :omitted_resources,
+                  :template_directory, :target_output_directory, :regions, :colors,
+                  :definitions_path, :kubernetes_cluster_type
+
+      def initialize(config:, template_directory:, rendered_directory:, definitions_path:, kubernetes_cluster_type:)
+        @variables               = config["variables"] || {}
+        @deploy_group_config     = config["deploy_groups"]
+        @omitted_resources       = config["omitted_resources"]
+        @template_directory      = template_directory
+        @target_output_directory = config["directory"] or raise ArgumentError, "missing 'directory:' in #{config.inspect}"
+        @regions                 = config["regions"] || []
+        @colors                  = config["colors"] || []
+        @rendered_directory      = rendered_directory
+        @definitions_path        = definitions_path
+        @kubernetes_cluster_type = kubernetes_cluster_type
+        @resources               = {}
+
+        if @kubernetes_cluster_type != "kube-platform"
+          @target_output_directory.include?("%{plain_region}") or raise "#{@template_directory}: target_output_directory #{@target_output_directory} needs %{plain_region}"
+        end
+      end
+
+      def normal_render(args)
+        dynamic_output_directory? and raise "Directory must not be dynamic: #{target_output_directory}"
+
+        variables["kubernetes_cluster_type"] = @kubernetes_cluster_type
+
+        if (plain_region = variables["plain_region"])
+          default_region_vars(plain_region)
+        end
+
+        output_directory = File.join(@rendered_directory, target_output_directory)
+        render_create_directory(args, output_directory)
+      end
+
+      def render(args)
+        @regions.any? or raise "#{template_directory}: must have at least one region"
+        @colors.any? or raise "#{template_directory}: must have at least one color"
+
+        variables["kubernetes_cluster_type"] = @kubernetes_cluster_type
+
+        @regions.each do |plain_region|
+          default_region_vars(plain_region)
+
+          @colors.each do |c|
+            variables["color"] = c
+            output_directory = File.join(@rendered_directory, format(@target_output_directory, plain_region: plain_region, color: c, type: @kubernetes_cluster_type))
+            render_create_directory(args, output_directory)
+          end
+        end
+      end
+
+      def render_create_directory(args, output_directory)
+        create_directory(output_directory)
+        puts "Rendering templates to: #{Color.magenta(output_directory)}"
+        puts "Variable assignments:"
+        variables.each { |k, v| puts "\t#{Color.magenta(k)}=#{Color.blue(v)}" }
+        puts
+        if omitted_resources
+          puts "Omitted resources:"
+          omitted_resources.each { |ot| puts "\t#{ot}" }
+        end
+        puts
+        resources(output_directory).each do |resource|
+          resource.render(args)
+        end
+        puts
+      end
+
+      private
+
+      CLOUD_REGION_TO_PROVIDER_AND_DATACENTER = {
+        # Note: The names below should match RegionDiscovery from process_settings-production.
+        # https://github.com/Invoca/process_settings-production/blob/main/settings/region_discovery/production_regions.yml
+        "us-east-1"    => ['aws', "AWS-us-east-1"],
+        "us-east-2"    => ['aws', "AWS-us-east-2"],
+        "us-central1"  => ['gcp', "GCE-us-central1"],
+        "us-west2"     => ['gcp', "GCE-us-west2"],
+        "eu-central-1" => ['aws', "AWS-eu-central-1"],
+        "eu-west-1"    => ['aws', "AWS-eu-west-1"],
+        "europe-west4" => ['gcp', "GCE-europe-west4"],
+
+        # other regions
+        "us-east1"    => ['gcp', "GCE-us-east1"],
+        "us-west1"    => ['gcp', "GCE-us-west1"],
+        "local"       => ['',    "local"]
+      }.freeze
+
+      # The zone to use for failure-domain.beta.kubernetes.io/zone or topology.kubernetes.io/zone by DeployGroup
+      AVAILABILITY_ZONE_FOR_DEPLOY_GROUP = {
+        "us-east-1" => {
+          "primary" => "us-east-1c",
+          "secondary" => "us-east-1d",
+          "tertiary" => "us-east-1b"
+        },
+        "us-east-2" => {
+          "primary" => "us-east-2a",
+          "secondary" => "us-east-2b",
+          "tertiary" => "us-east-2c"
+        },
+        "us-central1" => {
+          "primary" => "us-central1-a",
+          "secondary" => "us-central1-b",
+          "tertiary" => "us-central1-c"
+        },
+        "us-west2" => {
+          "primary" => "us-west2-a",
+          "secondary" => "us-west2-b",
+          "tertiary" => "us-west2-c"
+        },
+        "eu-central-1" => {
+          "primary" => "eu-central-1a",
+          "secondary" => "eu-central-1b",
+          "tertiary" => "eu-central-1c"
+        },
+        "eu-west-1" => {
+          "primary" => "eu-west-1a",
+          "secondary" => "eu-west-1b",
+          "tertiary" => "eu-west-1c"
+        },
+        "europe-west4" => {
+          "primary" => "europe-west4-a",
+          "secondary" => "europe-west4-b",
+          "tertiary" => "europe-west4-c"
+        },
+        "local" => {},
+        "us-west1" => {},
+        "us-east1" => {}
+      }
+
+      def default_region_vars(plain_region)
+        variables.has_key?("region") and raise "replace region with plain_region"
+        variables["plain_region"] = plain_region
+        cloud_datacenter_and_provider = CLOUD_REGION_TO_PROVIDER_AND_DATACENTER[plain_region] or raise "no CLOUD_REGION_TO_PROVIDER_AND_DATACENTER entry found for #{plain_region.inspect}"
+        variables["cloud_provider"], variables["cloud_datacenter"] = cloud_datacenter_and_provider
+        variables["cloud_region"] = variables["cloud_datacenter"] # for compatibility with old resource files; cloud_datacenter is preferred now
+        if plain_region == "local"
+          variables["data_silo"] ||= "local"
+        elsif plain_region.start_with?("us-")
+          variables["data_silo"] ||= "us"
+        end
+
+        # cannot do ||= because we want to reset when plain_region changes
+        variables["availability_zone_for_deploy_group"] = AVAILABILITY_ZONE_FOR_DEPLOY_GROUP[plain_region] or raise "no AVAILABILITY_ZONE_FOR_DEPLOY_GROUP mapping found for #{plain_region.inspect}"
+      end
+
+      def create_directory(directory)
+        unless File.exist?(directory)
+          puts <<~MESSAGE
+
+            Directory #{Color.magenta(directory)} doesn't exist, #{Color.green('creating it')}
+
+          MESSAGE
+          FileUtils.mkdir_p(directory)
+        end
+      end
+
+      def resources(output_directory)
+        @resources[output_directory] ||= standard_resources(output_directory) + grouped_resources(output_directory)
+      end
+
+      def standard_resources(output_directory)
+        standard_template_paths.map do |path|
+          Resource.new(template_path: path, definitions_path: @definitions_path, variables: variables, output_directory: output_directory)
+        end
+      end
+
+      def grouped_resources(output_directory)
+        deploy_grouped_template_paths.map do |path|
+          DeployGroupedResource.new(
+            template_path: path,
+            definitions_path: @definitions_path,
+            variables: variables,
+            output_directory: output_directory,
+            groups_to_render: deploy_groups_to_render,
+            template_path_exclusions: deploy_group_config["exclude_files"],
+            group_variable_name: deploy_group_config["variable_name"]
+          )
+        end
+      end
+
+      def standard_template_paths
+        @standard_template_paths ||=
+          (Dir[File.join(template_directory, "*.yaml.erb")] +
+           Dir[File.join(template_directory, '*.jsonnet')]) -
+          deploy_grouped_template_paths - omitted_resource_paths
+      end
+
+      def deploy_grouped_template_paths
+        @deploy_grouped_template_paths ||=
+          if deploy_group_config
+            deploy_group_config["files"]&.map { |file| File.join(template_directory, file) } ||
+              (Dir[File.join(template_directory, "*-deploy.yaml.erb")] + Dir[File.join(template_directory, "*-deploy.jsonnet")]) - omitted_resource_paths
+          else
+            []
+          end
+      end
+
+      def omitted_resource_paths
+        @omitted_resource_paths ||= omitted_resources&.map { |file| File.join(template_directory, file) } || []
+      end
+
+      def deploy_groups_to_render
+        group_names = deploy_group_config["group_names"]
+        if array_of_arrays?(group_names)
+          first, *rest = group_names
+          first.product(*rest).map { |group| group.join("-") }
+        else
+          group_names
+        end
+      end
+
+      def array_of_arrays?(array)
+        array.all? { |item| item.is_a?(Array) }
+      end
+    end
+  end
+end

data/lib/invoca/kubernetes_templates/template.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require "ostruct"
+require "open3"
+require "shellwords"
+require "yaml"
+
+# This is a base class for all Templates. Derived classes must implement the render method.
+module Invoca
+  module KubernetesTemplates
+    class Template < OpenStruct
+      attr_reader :template_path, :variables
+      def initialize(template_path, variables)
+        @template_path = template_path
+        @variables = variables
+      end
+
+      def render(args)
+        raise "must be defined by subclass"
+      end
+
+      private
+
+      def with_auto_generated_yaml_comment(yaml_string)
+        comment = <<~EOS
+        # WARNING: DO NO EDIT THIS FILE!
+        # Any changes made here will be lost.
+        # This file is autogenerated from #{template_path}
+        EOS
+        comment + yaml_string
+      end
+
+      class << self
+        # @param [String] template_path: file path to template file that needs to be rendered.
+        # @param [Hash]: variables that will be used in the template file to generate distict files.
+        #
+        # @return [String] generated YAML file
+        # @return [Hash] Hash of file names as keys with the values being corresponding generated YAML.
+        #
+        # @raise [UnexpectedFileTypeError] if file type doesn't match [yaml.erb, erb, or jsonnet]
+        # @raise [MultiFileJsonnetRenderError] if Jsonnet template files don't follow proper MultiFileJsonnet templating.
+
+        # TODO
+        # The ErbTemplate and JsonnetTemplate classes both inherit from the Template class and implement a render method.
+        # However, the erb_binding parameter is used just in ErbTemplate, while the jsonnet_library_path parameter is used just in JsonnetTemplate.
+        # This is a little awkward. Potentially this could be refactored.
+        def render(template_path, variables, erb_binding: nil, jsonnet_library_path: nil)
+          new(template_path, variables).render(erb_binding: erb_binding, jsonnet_library_path: jsonnet_library_path)
+        end
+      end
+    end
+  end
+end

data/lib/invoca/kubernetes_templates/template_directory_renderer.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+require 'active_support'
+require 'active_support/core_ext' # for deep_merge
+require 'invoca/utils'
+require 'yaml'
+require_relative 'resource_set'
+require 'set'
+
+# This class points to a collection of template directories to render, and a rendered_directory to render into.
+# Optionally, some of the template directories may be omitted by including them in `omitted_names`.
+module Invoca
+  module KubernetesTemplates
+    class TemplateDirectoryRenderer
+      DEFINITIONS_FILENAME = "definitions.yaml"
+
+      attr_reader :directories, :omitted_names, :rendered_directory, :cluster_type, :region, :color, :variable_overrides
+
+      def initialize(directories:, rendered_directory:, omitted_names: [], cluster_type: nil, region: nil, color: nil, variable_overrides: nil)
+        @directories        = directories_with_definitions(Array(directories))
+        @omitted_names      = Array(omitted_names)
+        @rendered_directory = rendered_directory
+        @cluster_type       = cluster_type
+        @region             = region
+        @color              = color
+        @variable_overrides = variable_overrides || {}
+      end
+
+      def render(args)
+        child_pids = []
+
+        resource_sets.each do |name, resource_sets|
+          puts "Rendering templates for definition #{Color.red(name)}..."
+          resource_sets.each do |resource_set|
+            if args.fork?
+              if (pid = Process.fork)
+                # this is the parent
+                child_pids << pid
+                wait_if_max_forked(child_pids)
+              else
+                # this is the child
+                render_set(args, resource_set)
+                Kernel.exit! # skip at_exit handlers since parent will run those
+              end
+            else
+              render_set(args, resource_set)
+            end
+          end
+        end
+
+        if args.fork?
+          Process.waitall
+        end
+      end
+
+      private
+
+      def read_definitions(path)
+        File.read(path)
+      end
+
+      MAX_FORKED_PROCESSES = 9
+
+      def wait_if_max_forked(child_pids)
+        while child_pids.size >= MAX_FORKED_PROCESSES
+          begin
+            Process.waitpid # this is a race condition because 1 or more processes could exit before we get here
+          rescue SystemCallError # this will happen if they all exited before we called waitpid
+          end
+          child_pids.delete_if do |pid|
+            Process.waitpid(pid, Process::WNOHANG)
+          rescue Errno::ECHILD # No child processes
+            true
+          end
+        end
+      end
+
+      def render_set(args, resource_set)
+        resource_set.render(args)
+      rescue => ex
+        raise "error rendering ResourceSet from #{resource_set.definitions_path}\n#{ex.class}: #{ex.message}"
+      end
+
+      def directories_with_definitions(directories)
+        directories.select do |dir|
+          definitions_path = definitions_path_for_dir(dir)
+          File.exist?(definitions_path)
+        end
+      end
+
+      def definitions_path_for_dir(dir)
+        File.join(dir, DEFINITIONS_FILENAME)
+      end
+
+      def resource_sets
+        @resource_sets ||= @directories.each_with_object({}) do |dir, hash|
+          definitions_path = definitions_path_for_dir(dir)
+          config = load_config(definitions_path)
+
+          config.map do |name, config|
+            next if omitted_names.include?(name)
+
+            kubernetes_cluster_type = name.sub('SPP-PLACEHOLDER', 'staging').sub(/\..*/, '') # prod.gcp => prod
+
+            hash[name] ||= []
+            hash[name] << ResourceSet.new(config: config, template_directory: dir, rendered_directory: @rendered_directory, kubernetes_cluster_type: kubernetes_cluster_type, definitions_path: definitions_path)
+          end
+        end
+      end
+
+      def build_libsonnet(dir, config)
+        fname = File.join(dir, 'definitions.libsonnet')
+        existing = File.exists?(fname) ? File.read(fname) : ""
+        proposed = build_json(config)
+
+        if existing != proposed
+          puts("Generating updated #{Color.magenta(File.basename(fname))}")
+          File.write(fname, proposed)
+        end
+      end
+
+      def build_json(config)
+        hash = transform_for_jsonnet(config)
+        JSON.pretty_generate(hash)
+      end
+
+      # This method ensures that OpenStructs are
+      # converted to hashes to support to_json operation
+      # It also converts any embedded variable place holders
+      # into a Jsonnet friendly format:
+      #
+      # %{variable} is converted to %(variable)s which can
+      # then be used with the Jsonnet function std.format
+      def transform_for_jsonnet(hash)
+        hash.transform_values do |value|
+          case value
+          when OpenStruct
+            transform_for_jsonnet(value.to_h)
+          when Hash
+            transform_for_jsonnet(value)
+          else
+            value
+          end
+        end
+      end
+
+      def load_config(definitions_path)
+        begin
+          config = YAML.safe_load(read_definitions(definitions_path), aliases: true)
+        rescue => ex
+          raise "error loading YAML from #{definitions_path}:\n#{ex.class}: #{ex.message}"
+        end
+
+        expand_config(config).each_with_object({}) do |(name, data), hash|
+          if !cluster_type || cluster_type == name.sub('SPP-PLACEHOLDER', 'staging').sub(/\..*/, '') # prod.gcp => prod
+            cluster_type_config = OpenStruct.new(data)
+
+            cluster_type_config.regions   = cluster_type_config.regions & [region] if region
+            cluster_type_config.colors    = cluster_type_config.colors & [color]   if color
+            cluster_type_config.variables = cluster_type_config.variables.merge(variable_overrides)
+
+            hash[name] = cluster_type_config if (region.nil? && color.nil?) || (cluster_type_config.regions.any? && cluster_type_config.colors.any?)
+          end
+        end
+      end
+
+      # returns a copy of the given config hash with the COMMON: k-v removed and deep merged into the other config values
+      # (explicit config values take precedence over COMMON: ones)
+      def expand_config(config)
+        common = config.delete('COMMON') || {}
+
+        Hash[config.map { |k, v| [k, common.deep_merge(v)] }]
+      end
+    end
+  end
+end

data/lib/invoca/kubernetes_templates/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Invoca
+  module KubernetesTemplates
+    VERSION = "0.1.0.pre.1"
+  end
+end

data/lib/invoca/kubernetes_templates.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require_relative "kubernetes_templates/version"
+
+module Invoca
+  module KubernetesTemplates
+    class Error < StandardError; end
+    # Your code goes here...
+  end
+end

data/sig/invoca/kubernetes_templates.rbs

@@ -0,0 +1,6 @@
+module Invoca
+  module KubernetesTemplates
+    VERSION: String
+    # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+  end
+end

metadata

@@ -0,0 +1,110 @@
+--- !ruby/object:Gem::Specification
+name: invoca-kubernetes_templates
+version: !ruby/object:Gem::Version
+  version: 0.1.0.pre.1
+platform: ruby
+authors:
+- Octothorpe
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2024-04-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: jsonnet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: invoca-utils
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Tool for rendering ERB and Jsonnet templates
+email:
+- octothorpe@invoca.com
+executables:
+- render_kubernetes_templates
+extensions: []
+extra_rdoc_files: []
+files:
+- ".buildkite/pipeline.yml"
+- ".rspec"
+- ".rubocop.yml"
+- ".ruby-version"
+- CHANGELOG.md
+- README.md
+- Rakefile
+- exe/render_kubernetes_templates
+- lib/invoca/kubernetes_templates.rb
+- lib/invoca/kubernetes_templates/cli.rb
+- lib/invoca/kubernetes_templates/cli_arguments.rb
+- lib/invoca/kubernetes_templates/color.rb
+- lib/invoca/kubernetes_templates/deploy_grouped_resource.rb
+- lib/invoca/kubernetes_templates/erb_template.rb
+- lib/invoca/kubernetes_templates/jsonnet_template.rb
+- lib/invoca/kubernetes_templates/resource.rb
+- lib/invoca/kubernetes_templates/resource_set.rb
+- lib/invoca/kubernetes_templates/template.rb
+- lib/invoca/kubernetes_templates/template_directory_renderer.rb
+- lib/invoca/kubernetes_templates/version.rb
+- sig/invoca/kubernetes_templates.rbs
+homepage: https://github.com/Invoca/inovca-kubernetes_templates
+licenses: []
+metadata:
+  allowed_push_host: https://rubygems.org
+  homepage_uri: https://github.com/Invoca/inovca-kubernetes_templates
+  source_code_uri: https://github.com/Invoca/inovca-kubernetes_templates
+  changelog_uri: https://github.com/Invoca/inovca-kubernetes_templates/blob/main/CHANGELOG.md
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.1.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">"
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubygems_version: 3.4.21
+signing_key:
+specification_version: 4
+summary: Tool for rendering ERB and Jsonnet templates
+test_files: []