matsuri 0.1.0 → 0.3.0

data/lib/matsuri/dsl/cluster/service_account.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require 'rlet'
+
+module Matsuri
+  module DSL
+    module Cluster
+      # Cluster resources here in Matsuri are a bit different as the are meant to
+      # be driven by the cluster dsl
+      class ServiceAccount
+        include Let
+        include Matsuri::Concerns::ExportManifest
+        include Matsuri::DSL::Concerns::Metadata
+        include Matsuri::DSL::Concerns::ManifestSet
+
+        attr_accessor :manifest_config
+
+        let(:api_version) { 'v1' }
+        let(:kind)        { 'ServiceAccount' }
+
+        ### Manifest Set
+        let(:cluster_manifests) { manifest }
+
+        ### Manifest
+
+        let(:manifest) do
+          {
+            apiVersion:  api_version,
+            kind:        kind,
+            metadata:    final_metadata,
+
+            automountServiceAccountToken: manifest_config[:automount_token],
+            imagePullSecrets:             manifest_config[:image_pull_secrets].presence,
+            secrets:                      manifest_config[:secrets].presence
+          }.compact
+        end
+
+        def initialize(options = {}, &block)
+          initialize_metadata_dsl(options)
+          self.manifest_config = {
+            automount_token:    nil,
+            image_pull_secrets: [],
+            secrets:            []
+          }
+
+          configure(&block) if block
+        end
+
+        def configure(&block)
+          instance_eval(&block)
+        end
+
+        ### DSL Helpers
+        def automount_token(flag = true)
+          manifest_config[:automount_token] = flag
+        end
+
+        def image_pull_secret(name)
+          manifest_config[:image_pull_secrets] << { name: name }
+        end
+
+        # Usually the name is sufficient.
+        def secret(name, options = {})
+          manifest_config[:secrets] << {
+            name: name,
+            namespace:       options[:namespace],
+            apiVersion:      options[:api_version],
+            kind:            options[:kind],
+            resourceVersion: options[:version],
+            uid:             options[:uid]
+          }.compact
+        end
+      end
+    end
+  end
+end

data/lib/matsuri/dsl/concerns/default_binding.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require 'rlet/concern'
+
+module Matsuri
+  module DSL
+    module Concerns
+      module DefaultBinding
+        extend Concern
+
+        included do
+          let(:default_binding) { Matsuri::DSL::Cluster::Binding.new(default_binding_options) }
+
+          let(:default_binding_name)      { name }      # Use the name of the role or cluster_role
+          let(:default_binding_namespace) { namespace } # Use the namespace of the role
+          let(:default_binding_type)      { kind }
+
+          let(:default_binding_options) do
+            {
+              name:           default_binding_name,
+              namespace:      default_binding_namespace,
+              type:           default_binding_type,
+              scoped_options: scoped_options
+            }
+          end
+        end
+
+        def bind_to(name, kind:, namespace: nil)
+          default_binding.subject(name, kind: kind, namespace: namespace || self.namespace)
+        end
+      end
+    end
+  end
+end

data/lib/matsuri/dsl/concerns/manifest_set.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require 'rlet/concern'
+
+module Matsuri
+  module DSL
+    module Concerns
+      # Defines the callbacks for being able to generate a ManifestSet
+      module ManifestSet
+        extend Concern
+
+        included do
+          # By default, return nil. This will exclude the manifest from being generated
+          # To include a manifest, override it with something like let(:rbac_manifest) { manifest }
+          let(:rbac_manifests)    { nil }
+          let(:cluster_manifests) { nil }
+        end
+      end
+    end
+  end
+end

data/lib/matsuri/dsl/concerns/metadata.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require 'rlet/concern'
+
+module Matsuri
+  module DSL
+    module Concerns
+      module Metadata
+        extend Concern
+
+        included do
+          attr_accessor :labels, :annotations, :options
+
+          let(:dsl_version)    { Matsuri::Config.cluster_dsl_version }
+
+          let(:name)           { options[:name] }
+          let(:namespace)      { (options[:namespace] || parent_scope[:namespace])&.to_s }
+          let(:source_file)    { parent_scope[:source_file] }
+          let(:skip?)          { options[:skip] || parent_scope[:skip] }
+
+          let(:parent_scope)   { options[:scoped_options] || {} }
+
+          let(:scoped_options) do
+            {
+              namespace:   namespace,
+              source_file: source_file,
+              skip:        skip?
+            }
+          end
+
+          let(:final_metadata)      { default_metadata.merge(metadata) }
+          let(:default_metadata)    { { name: name, namespace: namespace, labels: final_labels, annotations: final_annotations }.compact }
+          let(:metadata)            { { } }
+          let(:default_labels)      { { 'matsuri/dsl' => dsl_version } }
+          let(:default_annotations) { { 'matsuri/source_file' => source_file }.compact } # Added for easier debugging
+          let(:final_labels)        { default_labels.merge(labels.to_h) }
+          let(:final_annotations)   { default_annotations.merge(annotations.to_h) }
+        end
+
+        def initialize_metadata_dsl(options = {})
+          self.options     = options
+          self.labels      = []
+          self.annotations = []
+        end
+
+        def label(key, value)
+          labels << [key, value]
+        end
+
+        def annotate(key, value)
+          annotations << [key, value]
+        end
+      end
+    end
+  end
+end

data/lib/matsuri/dsl/concerns/rbac_rules.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require 'rlet/concern'
+
+module Matsuri
+  module DSL
+    module Concerns
+      module RbacRules
+        extend Concern
+
+        included do
+          attr_accessor :rules
+        end
+
+        def initialize_rbac_rules_dsl
+          self.rules = []
+        end
+
+        def rule(api_groups: '', urls: nil, resources: nil, names: nil, verbs: nil)
+          rules << {
+            'apiGroups'       => api_groups,
+            'nonResourceURLs' => urls,
+            'resourceNames'   => names,
+            'resources'       => resources,
+            'verbs'           => verbs
+          }.compact.map(&method(:normalize_string_array)).to_h
+        end
+
+        def resources(resources, names: nil, verbs: nil, api_groups: '')
+          rule api_groups: api_groups, resources: resources, names: names, verbs: verbs
+        end
+
+        def normalize_string_array(k, v)
+          [k, Array(v).map(&:to_s)]
+        end
+      end
+    end
+  end
+end

data/lib/matsuri/initializer.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require 'thor'
+require 'json'
+
+module Matsuri
+  module Initializer
+    def self.start(args)
+      # Flush stdout to keep CI harnesses updated
+      STDOUT.sync = true
+
+      # Look for the matsuri repo by walking up the parent directories for
+      # config/matsuri.rb. ENV['PWD'] preserves the path as the shell
+      # presented it; non-shell execs may not set it, hence the fallback.
+      matsuri_base_path = find_matsuri_base(ENV['PWD'] || Dir.pwd)
+      Matsuri.log :fatal, 'Unable to find the Matsuri repo. Looking for a config/matsuri.rb file.' unless matsuri_base_path
+      Matsuri::Config.base_path = matsuri_base_path
+
+      # The repo contributes code through matsuri, not through its launcher;
+      # every repo-authored file from here on may assume lib/ is loadable.
+      # See docs/design/boot.md
+      $LOAD_PATH.unshift(Matsuri::Config.lib_path) if File.directory?(Matsuri::Config.lib_path)
+
+      # The environment has exactly one source: the first positional
+      # argument (see docs/design/boot.md). Set before loading configuration
+      # so config files and the custom initializer can read it.
+      Matsuri::Config.environment = args.first
+      Matsuri::Config.load_configuration(Matsuri::Config.repo_defaults_path)
+      Matsuri::Platform.load_configuration(Matsuri::Config.platform_load_paths)
+
+      require Matsuri::Config.initializer_path if File.file?(Matsuri::Config.initializer_path)
+      Matsuri::Cmds::Cli.start(args.drop(1))
+    end
+
+    # Wish we have tail recursion here ...
+    def self.find_matsuri_base(cwd)
+      return cwd if File.file?(File.join(cwd, 'config/matsuri.rb'))
+      find_matsuri_base(File.expand_path('..', cwd)) unless cwd == '/'
+    end
+  end
+end

data/lib/matsuri/kubernetes/base.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require 'rlet'
+
+require 'matsuri/core_ext'
+
+require 'rainbow/ext/string'
+require 'rlet/lazy_options'
+
+module Matsuri
+  module Kubernetes
+    class Base
+      include Let
+      include RLet::LazyOptions
+      include Matsuri::Concerns::ExportManifest
+      include Matsuri::Concerns::RegistryHelpers
+      include Matsuri::Concerns::Diffable
+
+      class_attribute :presenters
+
+      # Declaration-surface vocabulary, as on stacks.
+      extend Matsuri::Transform::Combinators::Helpers
+      extend Matsuri::Presenters::Helpers
+
+      # Appends transform members to a scenario's pipeline, as on stacks:
+      # a scenario accumulates, declaration order is member order. Default
+      # is none. A resource's own pipeline completes its self-knowledge
+      # and runs before any stack-declared members — the stack's steps
+      # admit the finished manifest.
+      def self.present(scenario, *steps)
+        key = scenario.to_sym
+        current = presenters || {}
+        self.presenters = current.merge(key => Array(current[key]) + steps)
+      end
+
+      # Namespace resolution
+      let(:namespace)             { namespace_from_option || default_namespace }
+
+      # Override default namespace instead of namespace in order to override namespace
+      # from the command line
+      let(:default_namespace)     { namespace_from_config || 'default' }
+      let(:namespace_from_option) { options[:namespace] }
+      let(:namespace_from_config) { env_config.namespace }
+      let(:env_config)            { Matsuri::Platform.send(Matsuri.environment) }
+
+      # Kubernetes manifest
+      let(:manifest) do
+        {
+          apiVersion:  api_version,
+          kind:        kind,
+          metadata:    final_metadata,
+          spec:        spec
+        }
+      end
+
+      let(:final_metadata)      { default_metadata.merge(metadata) }
+      let(:default_metadata)    { { name: name, namespace: namespace, labels: final_labels, annotations: final_annotations } }
+      # Needed for autodetecting 'current' for rolling-updates. However, this is obsolete with Deployments
+      let(:default_labels)      { { 'matsuri-name' => name, 'matsuri-env' => matsuri_env, 'namespace' => namespace } }
+      let(:default_annotations) { { } }
+      let(:final_labels)        { default_labels.merge(labels) }
+      let(:final_annotations)   { default_annotations.merge(annotations) }
+      let(:resource_type)       { kind.to_s.downcase }
+      let(:labels)              { { } }
+      let(:annotations)         { { } }
+      let(:matsuri_env)         { Matsuri.environment }
+
+      # Optional parameters
+      # The acceptor instance the owning stack bound for this converge; nil
+      # when the definition is invoked standalone.
+      let(:stack_config) { options[:stack_config] }
+
+      # Delivery policy, read by the Tasks::KubectlOps boundary. Resources
+      # declare what delivery should do; the mechanism lives in the tasks.
+      # save_config resolves most-specific-first: a definition's own let,
+      # the owning stack's declaration (arriving as an option), then the
+      # repo-wide Matsuri::Platform.save_config (default true). It governs
+      # the normal apply path's create/recreate only. field_manager names
+      # the identity server-side delivery writes under; repo-wide
+      # Matsuri::Platform.field_manager (default 'matsuri'), overridable
+      # per definition.
+      let(:converge_via)  { :apply }
+      let(:apply_via)     { :apply }
+      let(:save_config)   { options.key?(:save_config) ? options[:save_config] : Matsuri::Platform.save_config }
+      let(:field_manager) { Matsuri::Platform.field_manager }
+
+      # Diff policy, read by Tasks::HashDiff. The path to the primary
+      # container within the manifest; resources with a pod template
+      # declare it so the focused diff (--primary-container) can project
+      # both manifests to the same container.
+      let(:primary_container_path) { nil }
+
+      # Overridables
+      let(:spec)        { fail Matsuri::UndefinedLet, 'Must define let(:spec)' }
+
+      # Helper functions
+      # Conditionally returns block or nil
+      # Useful for conditionally adding evironmental variables or list of volumes
+      # Example:
+      #   let(:use_keyfile?) { false }
+      #   let(:volumes)      { [maybe(use_keyfile?) { mongo-keyfile-volume }].compact }
+      def maybe(cond)
+        yield if cond
+      end
+    end
+  end
+end

data/lib/matsuri/kubernetes/config_map.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require 'base64'
+
+module Matsuri
+  module Kubernetes
+    class ConfigMap < Matsuri::Kubernetes::Base
+      let(:api_version) { 'v1' }    # K8S 1.10
+      let(:kind)        { 'ConfigMap' }
+
+      # Overridables
+      let(:manifest) do
+        {
+          apiVersion: api_version,
+          kind:       kind,
+          metadata:   final_metadata,
+          data:       data,
+          binaryData: binary_data
+        }
+      end
+
+      let(:data)         { nil }
+      let(:binary_data)  { nil }
+
+      # Helpers
+
+      # Binary Files must be Base64 Encoded. This load the secret from
+      # disk (default: config/secrets/)  base64 encode it.
+      def binary_from(path)
+        binary_from_path(config_overlays_path(path))
+      end
+
+      # Load secret file and base64 encode it from any file location
+      def binary_from_path(path)
+        base64(File.read(path))
+      end
+
+      def text_from(path)
+        text_from_path(config_overlays_path(path))
+      end
+
+      def text_from_path(path)
+        File.read(path)
+      end
+
+      def all_text_from(dir)
+        all_from(dir, &method(:text_from_path))
+      end
+
+      def all_bin_from(dir)
+        all_from(dir, &method(:bin_from_path))
+      end
+
+      def all_from(dir, &block)
+        all_from_dir(config_overlays_path(dir), &block)
+      end
+
+      # This will only look at the first-level and will not recursively descend
+      def all_from_dir(dir, &block)
+        Dir.glob(File.join(dir, '*')).
+          map { |path| [File.basename(path), yield(path)] }.
+          to_h
+      end
+
+      # Helper to load secrets file from default location
+      def config_overlays_path(path)
+        File.join(Matsuri::Config.config_overlays_path, path)
+      end
+
+      # Helper for base64 encoder. This allows us to pull
+      # secrets from elsewhere, such as from Hashicorp Vault
+      # or from an encrypted AWS S3 bucket item
+      def base64(file)
+        Base64.encode64(file).split("\n").join # Strip all newlines
+      end
+
+      # Helper to transform the values of a hash to base64
+      def base64_hash(hash)
+        Hash[hash.map { |k,v| [k, base64(v) ] }]
+      end
+
+      ### Delivery policy
+
+      # Like secrets, config maps are set up by the cluster administrator:
+      # if one is present on the server, we can skip it. Better to use
+      # recreate than apply.
+      let(:converge_via) { :recreate }
+
+      class << self
+        def load_path
+          Matsuri::Config.config_maps_path
+        end
+
+        def definition_module_name
+          'Configmaps'
+        end
+      end
+    end
+  end
+end

data/lib/matsuri/kubernetes/cron_job.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module Matsuri
+  module Kubernetes
+    class CronJob < Matsuri::Kubernetes::Base
+      include Matsuri::Concerns::PodTemplate
+
+      let(:api_version) { 'batch/v1' } # k8s v1.21
+      let(:kind)        { 'CronJob' }  # https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/
+
+
+      # Parameters passed from command line
+      let(:image_tag)            { options[:image_tag] }
+
+      # https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#cronjobspec-v1-batch
+      let(:spec) do
+        {
+          schedule:                   schedule,
+          timeZone:                   time_zone,
+          suspend:                    suspend,
+          concurrencyPolicy:          concurrency_policy,
+          successfulJobsHistoryLimit: successful_jobs_history_limit,
+          failedJobsHistoryLimit:     failed_jobs_history_limit,
+          startingDeadlineSeconds:    starting_deadline_seconds,
+
+          jobTemplate: {
+            metadata: job_template_metadata,
+            spec: job_template_spec
+          }.compact,
+        }.compact
+      end
+
+      let(:concurrency_policy)            { nil } # 'Allow' | 'Forbid' | 'Replace', default: 'Allow'
+      let(:failed_jobs_history_limit)     { nil } # non-negative integer, default: 1
+      let(:schedule)                      { fail 'Must define let(:schedule)' } # string, cron schedule format
+      let(:starting_deadline_seconds)     { nil } # seconds, integer, optional
+      let(:successful_jobs_history_limit) { nil } # non-negative integer, default: 3
+      let(:suspend)                       { nil } # boolean, default: false
+
+      # k8s v1.27
+      # See: https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#time-zones
+      # See: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
+      let(:time_zone)                     { nil } # string, TZ info, optional
+
+      let(:job_template_metadata)         { { labels: pod_labels, annotations: pod_annotations } }
+
+
+      # https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#jobspec-v1-batch
+      let(:job_template_spec) do
+        {
+          selector:                selector,
+          completions:             completions,
+          parallelism:             parallelism,
+          activeDeadlineSeconds:   active_deadline_seconds,
+          ttlSecondsAfterFinished: ttl_seconds_after_finished,
+          backoffLimit:            backoff_limit,
+          managedBy:               managed_by,
+          manualSelector:          manual_selector,
+          podReplacementPolicy:    pod_replacement_policy,
+          suspend:                 job_template_suspend,
+
+          template:                template, # defined in Matsuri::Concerns::PodTemplate
+
+          # Indexed mode (for partitioning jobs), beta as of 1.31
+          completionMode:          completion_mode,
+          backoffLimitPerIndex:    backoff_limit_per_index,
+          maxFailedIndexes:        max_failed_indexes,
+          podFailurePolicy:        pod_failure_policy,
+          successPolicy:           success_policy,
+        }.compact
+      end
+
+      let(:active_deadline_seconds)    { nil } # integer
+      let(:backoff_limit)              { nil } # integer, default: 6
+      let(:completions)                { nil } # null | integer
+      let(:managed_by)                 { nil } # null | string
+      let(:manual_selector)            { nil } # null | boolean
+      let(:parallelism)                { nil } # null | integer
+      let(:ttl_seconds_after_finished) { nil } # null | integer
+
+      let(:selector)                   { nil } # This will be autopopulated by cronjob
+      #let(:selector)                   { { matchLabels: match_labels, matchExpressions: match_expressions } }
+      let(:match_labels)               { fail Matsuri::UndefinedLet, 'Must define let(:match_labels)' }
+      let(:match_expressions)          { [] }
+
+      let(:job_template_suspend)       { nil } # boolean, default: false
+
+
+      let(:pod_replacement_policy)     { nil } # 'TerminatingOrFailed' | 'Failed'
+
+      let(:completion_mode)            { nil } # 'NonIndexed' | 'Indexed', default: 'Indexed'
+      let(:backoff_limit_per_index)    { nil } # integer
+      let(:max_failed_indexes)         { nil } # null | integer
+
+      # See: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#podfailurepolicy-v1-batch
+      let(:pod_failure_policy)         { nil } # null | PodFailurePolicySpec
+
+      # See: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#successpolicy-v1-batch
+      let(:success_policy)             { nil } # null | SuccessPolicySpec
+
+      let(:primary_container_path)    { ['spec', 'jobTemplate', 'spec', 'template', 'spec', 'containers', 0] }
+
+
+      class << self
+        def load_path
+          Matsuri::Config.cron_jobs_path
+        end
+
+        def definition_module_name
+          'CronJobs'
+        end
+      end
+    end
+  end
+end

data/lib/matsuri/kubernetes/daemon_set.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Matsuri
+  module Kubernetes
+    class DaemonSet < Matsuri::Kubernetes::Base
+      include Matsuri::Concerns::PodTemplate
+
+      let(:api_version) { 'apps/v1' }   # K8S 1.10
+      let(:kind)        { 'DaemonSet' } # http://kubernetes.io/docs/user-guide/deployments/
+
+      # Overridables
+      let(:spec) do
+        {
+          selector:             selector,
+          template:             template,
+          minReadySeconds:      min_ready_seconds,
+          revisionHistoryLimit: revision_history_limit,
+          updateStrategy:       update_strategy
+        }.compact
+      end
+
+      let(:selector) { { matchLabels: match_labels, matchExpressions: match_expressions } }
+
+      # Parameters
+      let(:image_tag) { options[:image_tag] }
+
+      # Explicitly define replicas
+      let(:match_labels)      { fail Matsuri::UndefinedLet, 'Must define let(:match_labels)' }
+      let(:match_expressions) { [] }
+
+      # Update strategy
+      # See: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#rollingupdatedaemonset-v1-app
+      let(:update_strategy) { { type: 'RollingUpdate', rollingUpdate: rolling_update } }
+      let(:rolling_update)  { { maxUnavailable: max_unavailable } }
+      let(:max_unavailable) { 1 }
+
+      # Minimum number of seconds for which a newly created pod should be ready without
+      # any of its container crashing, for it to be considered available.
+      # Defaults to 0 (pod will be considered available as soon as it is ready)
+      let(:min_ready_seconds) { 0 }
+
+      # The number of old ReplicaSets to retain to allow rollback. This is a pointer to
+      # distinguish between explicit zero and not specified.
+      let(:revision_history_limit) { nil }
+
+      let(:primary_container_path) { ['spec', 'template', 'spec', 'containers', 0] }
+
+      ### Helpers
+
+      ### Matsuri Registry
+      class << self
+        def load_path
+          Matsuri::Config.daemon_sets_path
+        end
+
+        def definition_module_name
+          'DaemonSets'
+        end
+      end
+    end
+  end
+end