matsuri 0.1.0 → 0.3.0

data/lib/matsuri/presenter.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require 'rlet'
+
+module Matsuri
+  # A presenter is one manifest-shaped step in a delivery pipeline:
+  # Manifest -> Manifest, zero-config and context-free. Construction takes
+  # nothing; parameterization happens before #call ever runs — definition
+  # lets filling a concern's declared holes, or declaration-time values
+  # closed over in a leaf Callable.
+  #
+  # #call is framework-owned: it makes the one boundary copy and delegates
+  # to #transform, which authors implement against a carrier that is already
+  # theirs to mutate. The base #transform is the identity, so the bare
+  # Presenter is the identity element. A fresh instance serves each call, so
+  # let memoization scopes to a single manifest pass. The Kubernetes dialect
+  # (Transform::Kubernetes, carrying Algebra + Env) is bare vocabulary
+  # inside #transform.
+  class Presenter
+    include Let
+    include Matsuri::Transform::Kubernetes
+
+    class << self
+      def call(manifest)
+        new.call(manifest)
+      end
+
+      def load_path
+        Matsuri::Config.presenters_path
+      end
+
+      def definition_module_name
+        'Presenters'
+      end
+    end
+
+    # Marshal is the copy because the carrier is data — Map.new alone shares
+    # nested structure with the input.
+    def call(manifest)
+      transform(Map.new(Marshal.load(Marshal.dump(manifest))))
+    end
+
+    def transform(manifest)
+      manifest
+    end
+  end
+end

data/lib/matsuri/presenters/helpers.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Matsuri
+  module Presenters
+    # Declaration-surface constructors: each closes a seed concern's hole
+    # over its argument as an anonymous Presenter subclass — the named
+    # definition form minus the registry entry, with the boundary copy
+    # riding along.
+    module Helpers
+      def seed_replicas(count)
+        Class.new(Matsuri::Presenter) do
+          include Matsuri::Presenters::SeedReplicas
+          let(:replicas) { count }
+        end
+      end
+
+      def seed_image(image_value)
+        Class.new(Matsuri::Presenter) do
+          include Matsuri::Presenters::SeedImage
+          let(:image) { image_value }
+        end
+      end
+
+      def seed_resources(resources_spec)
+        Class.new(Matsuri::Presenter) do
+          include Matsuri::Presenters::SeedResources
+          let(:resources) { resources_spec }
+        end
+      end
+
+      def prune_empty(except: [])
+        Class.new(Matsuri::Presenter) do
+          include Matsuri::Presenters::PruneEmpty
+          let(:exceptions) { except }
+        end
+      end
+    end
+  end
+end

data/lib/matsuri/presenters/prune_empty.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Matsuri
+  module Presenters
+    # Prunes empty values — nil, {}, [] — from the payload, bottom-up,
+    # so emptiness never rides a declaration. Under server-side apply an
+    # empty collection is an ownership claim over a field the definition
+    # does not manage, and inside an atomic struct (.spec.selector) it
+    # diverges from the server's persisted normal form, turning the
+    # seeding protocol's equal-values co-ownership into a conflict. A
+    # pruned payload declares in persistence normal form: ceding a field
+    # and saying nothing are the same act.
+    #
+    # The merge is not uniformly monoidal — for some fields an empty
+    # value is a genuine assertion (an atomic field held empty against
+    # other writers). let(:exceptions) names those paths, each kept even
+    # when empty. It ships empty; fill the let in a named definition or
+    # subclass. Paths address hash keys only — they do not descend into
+    # arrays.
+    #
+    # Empty strings are values, not emptiness ("" is meaningful to K8s —
+    # storageClassName among others), and array elements are never
+    # dropped — removing elements changes list semantics; only their
+    # interiors prune.
+    module PruneEmpty
+      extend Concern
+
+      included do
+        let(:exceptions)      { [] }
+        let(:exception_paths) { exceptions.map { |e| Array(e).map(&:to_s) } }
+      end
+
+      def transform(manifest)
+        prune(manifest, [])
+      end
+
+      private
+
+      def prune(value, path)
+        case value
+        when Hash
+          value.each_with_object(value.class.new) do |(k, v), acc|
+            child = prune(v, path + [k.to_s])
+            acc[k] = child unless prunable?(child, path + [k.to_s])
+          end
+        when Array
+          value.map { |v| prune(v, path) }
+        else
+          value
+        end
+      end
+
+      def prunable?(value, path)
+        return false if exception_paths.include?(path)
+        value.nil? || (!value.is_a?(String) && value.respond_to?(:empty?) && value.empty?)
+      end
+    end
+  end
+end

data/lib/matsuri/presenters/seed_image.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Matsuri
+  module Presenters
+    # Completes a create payload with the boot image on every container
+    # that does not carry one. Additive only: a container that declares
+    # its image has opted out structurally, and the seed yields. The
+    # including definition owns what the value is: let(:image).
+    #
+    # One seed concern per presenter; compose seeds in the pipeline, not
+    # by stacking includes.
+    module SeedImage
+      extend Concern
+
+      included do
+        let(:image) { fail Matsuri::UndefinedLet, 'Must define let(:image)' }
+      end
+
+      def transform(manifest)
+        containers = manifest.get(*pod_spec_path(manifest), :containers) || []
+        containers.each do |container|
+          container.set(:image, image) if container[:image].nil?
+        end
+        manifest
+      end
+    end
+  end
+end

data/lib/matsuri/presenters/seed_replicas.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Matsuri
+  module Presenters
+    # Completes a create payload with the boot replica count. Additive
+    # only: a manifest that declares spec.replicas has opted out
+    # structurally, and the seed yields. The including definition owns
+    # what the value is: let(:replicas).
+    #
+    # One seed concern per presenter; compose seeds in the pipeline, not
+    # by stacking includes.
+    module SeedReplicas
+      extend Concern
+
+      included do
+        let(:replicas) { fail Matsuri::UndefinedLet, 'Must define let(:replicas)' }
+      end
+
+      def transform(manifest)
+        manifest.set([:spec, :replicas], replicas) if manifest.get(:spec, :replicas).nil?
+        manifest
+      end
+    end
+  end
+end

data/lib/matsuri/presenters/seed_resources.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Matsuri
+  module Presenters
+    # Completes a create payload with boot resource settings on every
+    # container that does not carry its own. Additive only: a container
+    # that declares resources has opted out structurally, and the seed
+    # yields. The including definition owns what the value is:
+    # let(:resources).
+    #
+    # One seed concern per presenter; compose seeds in the pipeline, not
+    # by stacking includes.
+    module SeedResources
+      extend Concern
+
+      included do
+        let(:resources) { fail Matsuri::UndefinedLet, 'Must define let(:resources)' }
+      end
+
+      def transform(manifest)
+        containers = manifest.get(*pod_spec_path(manifest), :containers) || []
+        containers.each do |container|
+          container.set(:resources, resources) if container[:resources].nil?
+        end
+        manifest
+      end
+    end
+  end
+end

data/lib/matsuri/registry.rb

@@ -0,0 +1,267 @@
+# frozen_string_literal: true
+
+require 'singleton'
+require 'map'
+require 'matsuri/core_ext'
+
+module Matsuri
+  class Registry
+    include Singleton
+
+    def data
+      @data ||= Map.new
+    end
+
+    def k8s_class_registry
+      @k8s_class_registry ||= Map.new
+    end
+
+    def k8s_class_aliases
+      @k8s_class_aliases ||= Map.new
+    end
+
+    class << self
+      # Registers a K8S class, which can then be used for definitions
+      def register_class(name, opt = {})
+        fail ArgumentError, 'need to pass class: parameter' unless opt[:class]
+        instance.k8s_class_registry[name] = opt[:class]
+        Array(opt[:aliases]).each do |class_alias|
+          instance.k8s_class_aliases[class_alias] = name
+        end
+      end
+
+      # Helper to load class definitions
+      def load_definition(type, name)
+        def_path = definition_path(type, name)
+
+        # [Hosh] Somehow, TOPLEVEL_BINDING got infected with version and str
+        # local variables. When thinking about it, we probably want to require
+        # instead of evaluating this with the TOPLEVEL_BINDING anyways.
+
+        #eval(File.read(def_path), TOPLEVEL_BINDING, def_path)
+        require def_path
+
+        _klass = instance.data.get(type, name)
+        return _klass if _klass
+
+        Matsuri.log :fatal, "Unable to find #{type}/#{name} defined in #{def_path}"
+      end
+
+      def definition_path(type, name)
+        load_path    = load_path_for(type)
+        def_file     = "#{name}.rb"
+        env_def_path = File.join(load_path, Matsuri.environment, def_file)
+        return env_def_path if File.file?(env_def_path)
+        Matsuri.log :debug, "Unable to find #{env_def_path} ... trying global definition"
+
+        def_path = File.join(load_path, def_file)
+        return def_path if File.file?(def_path)
+        Matsuri.log :fatal, "Unable to find #{env_def_path} or #{def_path}"
+      end
+
+      # Helper to generate Kubernetes artifact definitions
+      def define(type, name, inherits: nil, &blk)
+        type = normalize_and_validate_type(type)
+        parent_klass = parent_class(type, inherits)
+        klass = Class.new(parent_klass)
+        module_for(type).const_set(class_name_for(name), klass)
+        klass.class_eval do
+          let(:name) { name }
+        end
+
+        klass.class_eval(&blk) if blk
+
+        # TODO: consider whether to use the class or to go ahead
+        # and instantiate it
+        instance.data.set(type, name, klass)
+        return klass
+      end
+
+      def register(type, name, obj)
+        _type = normalize_and_validate_type(type)
+        instance.data.set(_type, name, obj)
+      end
+
+      def fetch_or_load(type, name)
+        _type = normalize_and_validate_type(type)
+
+        if name == :not_specified
+          print_options_for_type(_type)
+        end
+
+        resource = instance.data.get(_type, name)
+        return resource if resource
+
+        load_definition(_type, name)
+      end
+
+      def print_options_for_type(type)
+        options = options_for_type(type)
+        if options.empty?
+          global_load_path = load_path_for(type)
+          env_def_path     = File.join(global_load_path, Matsuri.environment)
+          Matsuri.log :fatal, "Unable to find any definition files in #{env_def_path} or #{global_load_path}"
+        end
+
+        Matsuri.log :error, "Which #{type}?\n\n"
+
+        options.each do |def_name|
+          Matsuri.log :error, "   #{def_name}"
+        end
+
+        Matsuri.log :fatal, "\nPlease specify a #{type} from the above list."
+      end
+
+      def options_for_type(type)
+        global_load_path = load_path_for(type)
+        env_def_path     = File.join(global_load_path, Matsuri.environment)
+        (Dir.glob(File.join(env_def_path, "*.rb")) + Dir.glob(File.join(global_load_path, "*.rb")))
+          .map  {|file| File.basename(file, File.extname(file))}
+          .uniq
+          .sort
+      end
+
+      def imported_manifest(name)
+        fetch_or_load :imported_manifest, name
+      end
+
+      def pod(name)
+        fetch_or_load :pod, name
+      end
+
+      def service(name)
+        fetch_or_load :service, name
+      end
+
+      def ingress(name)
+        fetch_or_load :ingress, name
+      end
+
+      def replica_set(name)
+        fetch_or_load :replica_set, name
+      end
+
+      def stateful_set(name)
+        fetch_or_load :stateful_set, name
+      end
+
+      def daemon_set(name)
+        fetch_or_load :daemon_set, name
+      end
+
+      def deployment(name)
+        fetch_or_load :deployment, name
+      end
+
+      def cron_job(name)
+        fetch_or_load :cron_job, name
+      end
+
+      def persistent_volume(name)
+        fetch_or_load :persistent_volume, name
+      end
+
+      def pv(name)
+        persistent_volume(name)
+      end
+
+      def persistent_volume_claim(name)
+        fetch_or_load :persistent_volume_claim, name
+      end
+
+      def pvc(name)
+        persistent_volume_claim(name)
+      end
+
+      def storage_class(name)
+        fetch_or_load :storage_class, name
+      end
+
+      def endpoints(name)
+        fetch_or_load :endpoints, name
+      end
+
+      def secret(name)
+        fetch_or_load :secret, name
+      end
+
+      def config_map(name)
+        fetch_or_load :config_map, name
+      end
+
+      def horizontal_pod_autoscaler(name)
+        fetch_or_load :horizontal_pod_autoscaler, name
+      end
+
+      def hpa(name)
+        horizontal_pod_autoscaler(name)
+      end
+
+      def stack(name)
+        fetch_or_load :stack, name
+      end
+
+      def acceptor(name)
+        fetch_or_load :acceptor, name
+      end
+
+      def presenter(name)
+        fetch_or_load :presenter, name
+      end
+
+      private
+
+      def parent_class(type, parent_name = nil)
+        return fetch_or_load(type, parent_name) if parent_name
+        instance.k8s_class_registry[normalize_and_validate_type(type)]
+      end
+
+      def load_path_for(type)
+        parent_class(type).load_path
+      end
+
+      def class_name_for(name)
+        name.to_s.gsub(/-/, '_').camelize
+      end
+
+      def module_for(type)
+        maybe_define_module(parent_class(type).definition_module_name)
+      end
+
+      def maybe_define_module(mod)
+        return Object.const_get(mod) if Object.const_defined?(mod)
+        Object.const_set(mod, Module.new)
+      end
+
+      def normalize_and_validate_type(type)
+        _type = type.to_s.freeze
+        _type = instance.k8s_class_aliases[_type] if instance.k8s_class_aliases.key?(_type)
+        return _type if instance.k8s_class_registry.key?(_type)
+
+        fail "Registery type #{type.inspect} invalid. Use one of #{instance.k8s_class_registry.keys}"
+      end
+    end
+  end
+end
+
+# Register the definition classes
+# TODO: Consider making this dynamically and lazy loaded
+Matsuri::Registry.register_class 'stack',                     class: Matsuri::Stack
+Matsuri::Registry.register_class 'acceptor',                  class: Matsuri::Acceptor
+Matsuri::Registry.register_class 'presenter',                 class: Matsuri::Presenter
+Matsuri::Registry.register_class 'imported_manifest',         class: Matsuri::Kubernetes::ImportedManifest
+Matsuri::Registry.register_class 'pod',                       class: Matsuri::Kubernetes::Pod
+Matsuri::Registry.register_class 'service',                   class: Matsuri::Kubernetes::Service
+Matsuri::Registry.register_class 'ingress',                   class: Matsuri::Kubernetes::Ingress
+Matsuri::Registry.register_class 'endpoints',                 class: Matsuri::Kubernetes::Endpoints
+Matsuri::Registry.register_class 'secret',                    class: Matsuri::Kubernetes::Secret
+Matsuri::Registry.register_class 'config_map',                class: Matsuri::Kubernetes::ConfigMap
+Matsuri::Registry.register_class 'replica_set',               class: Matsuri::Kubernetes::ReplicaSet,            aliases: %w[rs]
+Matsuri::Registry.register_class 'stateful_set',              class: Matsuri::Kubernetes::StatefulSet,           aliases: %w[sts]
+Matsuri::Registry.register_class 'daemon_set',                class: Matsuri::Kubernetes::DaemonSet,             aliases: %w[ds]
+Matsuri::Registry.register_class 'deployment',                class: Matsuri::Kubernetes::Deployment
+Matsuri::Registry.register_class 'cron_job',                  class: Matsuri::Kubernetes::CronJob
+Matsuri::Registry.register_class 'persistent_volume',         class: Matsuri::Kubernetes::PersistentVolume, aliases: %w[pv]
+Matsuri::Registry.register_class 'persistent_volume_claim',   class: Matsuri::Kubernetes::PersistentVolumeClaim, aliases: %w[pvc]
+Matsuri::Registry.register_class 'storage_class',             class: Matsuri::Kubernetes::StorageClass
+Matsuri::Registry.register_class 'horizontal_pod_autoscaler', class: Matsuri::Kubernetes::HorizontalPodAutoscaler, aliases: %w[hpa]

data/lib/matsuri/shell_out.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+require 'mixlib/shellout'
+require 'rainbow/ext/string'
+
+module Matsuri
+  module ShellOut
+    module_function
+
+    # Override
+    def verbose
+      Matsuri::Config.verbose
+    end
+
+    def debug
+      Matsuri::Config.debug
+    end
+
+    def shellout_defaults
+      Matsuri::Config.shellout_defaults
+    end
+
+    # Overridable
+    # By default, the environment name is used. However, if you are using
+    # something like GCP, it is easier to set this in config/platform.rb
+    # and pull it from there. To override this, use let() or just define
+    # a method. For example, put this in a Staging mixin:
+    # let(:kube_context) { platform.staging.kube_context }
+    def kube_context
+      if Matsuri::Config.map_env_to_kube_context
+        Matsuri::Platform.send(Matsuri::Config.environment).kube_context
+      else
+        Matsuri::Config.environment
+      end
+    end
+
+    # Here for class method to get default namespace
+    # This is overriden by Matsuri::Kubernetes::Base when used in K8S resources
+    def namespace
+      Matsuri::Platform.send(Matsuri::Config.environment).namespace || 'default'
+    end
+
+    def shell_out(_cmd, options = {})
+      echo_level = options.delete(:echo_level) || :info
+      no_stdout = options.delete(:no_stdout)
+
+      Matsuri.log echo_level, "$ #{_cmd}".color(:green)
+      options = shellout_defaults.merge(timeout: 3600).merge(options)
+      cmd = Mixlib::ShellOut.new(_cmd, options)
+      cmd.live_stream = STDOUT unless no_stdout && !debug
+      cmd.run_command
+      cmd
+    end
+
+    def shell_out!(_cmd, options = {})
+      cmd = shell_out(_cmd, options)
+      return cmd if cmd.status.success?
+      $stderr.print "ERROR: #{cmd.exitstatus}\nSTDOUT:\n#{cmd.stdout}\n\nSTDERR:\n#{cmd.stderr}\n".color(:red).bright
+      exit 1
+    end
+
+    # Log and exec a command. Useful for things such as
+    # attaching to a remote shell or console on the K8S cluster
+    def log_and_exec(_cmd, options = {})
+      echo_level = options.delete(:echo_level) || :info
+      Matsuri.log echo_level, "$ #{_cmd}".color(:green)
+      exec _cmd
+    end
+
+    # This is so that it is easier to write stack commands
+    def kubectl_cmd(_cmd, options = {})
+      kube_options = { context: kube_context, namespace: namespace }.merge(options).compact
+      kube_cli_options = kube_options.map { |k,v| "--#{k}=#{v}" }.join(' ')
+      "kubectl #{kube_cli_options} #{_cmd}"
+    end
+
+    def kubectl(_cmd, options = {})
+      kube_options = options.delete(:kube_options) || {}
+      shell_out(kubectl_cmd(_cmd, kube_options), options)
+    end
+
+    def kubectl!(_cmd, options = {})
+      kube_options = options.delete(:kube_options) || {}
+      shell_out!(kubectl_cmd(_cmd, kube_options), options)
+    end
+
+    def docker(_cmd, options = {})
+      shell_out("docker #{_cmd}", options)
+    end
+
+    def docker!(_cmd, options = {})
+      shell_out!("docker #{_cmd}", options)
+    end
+
+    def docker_run(_cmd, options = {})
+      docker("run #{_cmd}", options)
+    end
+
+    def docker_run!(_cmd, options = {})
+      docker!("run #{_cmd}", options)
+    end
+
+    def bundle_exec(cmd, options = {})
+      shell_out "bundle exec #{cmd}", options
+    end
+
+    def bundle_exec!(cmd, options = {})
+      shell_out! "bundle exec #{cmd}", options
+    end
+
+    def sudo(cmd, options = {})
+      shell_out "sudo #{cmd}", options
+    end
+
+    def sudo!(cmd, options = {})
+      shell_out! "sudo #{cmd}", options
+    end
+  end
+end