matsuri 0.1.0 → 0.3.0

data/lib/matsuri/kubernetes/pod.rb

@@ -0,0 +1,355 @@
+# frozen_string_literal: true
+
+require 'matsuri/core_ext'
+require 'json'
+
+# rubocop:disable Lint/MissingCopEnableDirective
+# rubocop:disable Style/Alias
+module Matsuri
+  module Kubernetes
+    class Pod < Matsuri::Kubernetes::Base
+      include Matsuri::Transform::Env
+
+      let(:api_version) { 'v1' }
+      let(:kind)        { 'Pod' }
+
+      let(:primary_container_path) { ['spec', 'containers', 0] }
+
+      # Overridables
+      let(:spec) do
+        {
+          containers:     containers,
+          initContainers: init_containers,
+          volumes:        volumes,
+
+          imagePullSecrets: image_pull_secrets,
+
+          # Networking and Host
+          hostname:    hostname,
+          subdomain:   subdomain,
+          dnsConfig:   dns_config,
+          dnsPolicy:   dns_policy,
+          hostAliases: host_aliases,
+          hostIPC:     host_ipc,
+          hostNetwork: host_network,
+
+          # Scheduling
+          nodeName:                  node_name,
+          nodeSelector:              node_selector,
+          affinity:                  affinity,
+          tolerations:               tolerations,
+          priority:                  priority,
+          priorityClassName:         priority_class_name,
+          schedulerName:             scheduler_name,
+          topologySpreadConstraints: topology_spread_constraints,
+
+          # Lifecycle
+          readinessGates:                readiness_gates,
+          restartPolicy:                 restart_policy,
+          activeDeadlineSeconds:         active_deadline_seconds,
+          terminationGracePeriodSeconds: termination_grace_period_seconds,
+
+          # Security
+          hostPID:                       host_pid,
+          shareProcessNamespace:         share_process_namespace,
+          securityContext:               pod_security_context,
+          serviceAccountName:            service_account_name,
+          automountServiceAccountToken:  automount_service_account_token
+        }.compact
+      end
+
+      # Get image tag from the command line
+      let(:image_tag)          { options[:image_tag] }
+
+      let(:containers)         { [container] }
+      let(:init_containers)    { nil }
+      let(:volumes)            { [volume] }
+      let(:image_pull_secrets) { [] }
+
+      # Scheduling
+      let(:node_name)                   { nil }
+      let(:node_selector)               { { } }
+      let(:tolerations)                 { [toleration].compact }
+      let(:toleration)                  { nil }
+      let(:affinity)                    { { nodeAffinity: node_affinity, podAffinity: pod_affinity, podAntiAffinity: pod_anti_affinity }.compact }
+      let(:node_affinity)               { nil } # https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#nodeaffinity-v1-core
+      let(:pod_affinity)                { nil } # https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#podaffinity-v1-core
+      let(:pod_anti_affinity)           { nil } # https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#podantiaffinity-v1-core
+      let(:priority)                    { nil }
+      let(:priority_class_name)         { nil }
+      let(:scheduler_name)              { nil }
+      let(:topology_spread_constraints) { nil } # https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/
+
+      # Networking and Host
+      let(:hostname)           { nil } # https://kubernetes.io/docs/admin/dns/
+      let(:subdomain)          { nil }
+      let(:dns_config)         { nil }
+      let(:dns_policy)         { nil }
+      let(:host_aliases)       { nil }
+      let(:host_ipc)           { nil }
+      let(:host_network)       { nil }
+
+      # Lifecycle
+      let(:readiness_gates)                  { nil }
+      let(:restart_policy)                   { 'Always' }
+      let(:active_deadline_seconds)          { nil }
+      let(:termination_grace_period_seconds) { nil }
+
+      # Security and Execution Context
+      let(:host_pid)                         { nil }
+      let(:share_process_namespace)          { nil }
+      let(:service_account_name)             { nil }
+      let(:automount_service_account_token)  { nil }
+
+      # https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#podsecuritycontext-v1-core
+      let(:pod_security_context) do
+        {
+          runAsUser:          pod_run_as_user,
+          runAsGroup:         pod_run_as_group,
+          runAsNonRoot:       pod_run_as_non_root,
+          seLinuxOptions:     pod_se_linux_options,
+          fsGroup:            fs_group,
+          supplementalGroups: supplemental_groups,
+          sysctls:            sysctls
+        }.compact
+      end
+
+      let(:pod_run_as_user)      { nil }
+      let(:pod_run_as_group)     { nil }
+      let(:pod_run_as_non_root)  { nil }
+      let(:pod_se_linux_options) { nil }
+      let(:fs_group)             { nil }
+      let(:supplemental_groups)  { nil }
+      let(:sysctls)              { nil }
+
+      # Default container
+      let(:container)   { fail Matsuri::UndefinedLet, 'Must define let(:container)'}
+      let(:volume)      { fail Matsuri::UndefinedLet, 'Must define let(:volume)' }
+
+      let(:primary_container) { containers.first[:name] }
+      let(:primary_image)     { fail Matsuri::UndefinedLet, 'Must define let(:primary_image) for deployment or replication controller' }
+
+      # Resources for default container
+      let(:resources)          { { requests: resource_requests, limits: resource_limits } }
+      let(:resource_requests)  { { cpu: cpu_request, memory: mem_request }.compact }
+      let(:resource_limits)    { { cpu: cpu_limit,   memory: mem_limit  }.compact }
+
+      # We want to make sure all limits are defined
+      let(:cpu_limit)   { fail Matsuri::UndefinedLet, 'Must define let(:cpu_limit)' }
+      let(:mem_limit)   { fail Matsuri::UndefinedLet, 'Must define let(:mem_limit)' }
+      let(:cpu_request) { cpu_limit }
+      let(:mem_request) { mem_limit }
+
+      # Default container security context
+      let(:security_context) do
+        {
+          runAsUser:                 run_as_user,
+          runAsGroup:                run_as_group,
+          runAsNonRoot:              run_as_non_root,
+          privileged:                privileged,
+          allowPrivilegeEscalation:  allow_privilege_escalation,
+          seLinuxOptions:            se_linux_options,
+          capabilities:              capabilities
+        }.compact
+      end
+
+      let(:run_as_user)                 { nil }
+      let(:run_as_group)                { nil }
+      let(:run_as_non_root)             { nil }
+      let(:privileged)                  { nil }
+      let(:allow_privilege_escalation)  { nil }
+      let(:se_linux_options)            { nil }
+      let(:capabilities)                { nil }
+
+      ### Helpers
+
+      # Pods are practically useless with using apply. When we converge, we want
+      # to recreate instead. Pods unmanaged by a workload controller are more
+      # useful in dev mode than anywhere else
+      let(:converge_via) { :recreate }
+
+      # Helper methods
+      def config_file(path)
+        File.join config.config_path, path
+      end
+
+      def cache_path(path)
+        File.join config.cache_path, path
+      end
+
+      def src_path(path)
+        File.join config.src_path, path
+      end
+
+      def base_path(path)
+        File.join config.base_path, path
+      end
+
+      def env_from_secret(name:, key:)
+        { secretKeyRef: { name: name, key: key } }
+      end
+
+      def env_from_config_map(name:, key:)
+        { configMapKeyRef: { name: name, key: key } }
+      end
+
+      def env_from_field_ref(path:)
+        { fieldRef: { fieldPath: path } }
+      end
+
+      def env_from_resource(resource:, container: nil, divisor: nil)
+        { resourceFieldRef: { resource: resource, containerName: container, divisor: divisor } }
+      end
+
+      def sorted_env(hash)
+        expand_env(hash).sort { |a,b| a[:name] <=> b[:name] }
+      end
+
+      def port(num, protocol: 'TCP', name: nil, host_ip: nil, host_port: nil)
+        {
+          name:          name.try(:to_s),
+          containerPort: num,
+          protocol:      protocol.to_s,
+          hostIP:        host_ip,
+          hostPort:      host_port
+        }.compact
+      end
+
+      def mount(name, path, options = {})
+        {
+          name:                name,
+          mountPath:           path,
+          readOnly:            options[:read_only] || false,
+          mountPropogation:    options[:mount_propogation],
+          recursivelyReadOnly: options[:recursively_read_only],
+          subPath:             options[:sub_path],
+          subPathExpr:         options[:sub_path_expr]
+        }.compact
+      end
+
+      def host_path_volume(name, host_path)
+        { name: name, hostPath: { path: host_path } }
+      end
+
+      def empty_dir_volume(name, medium: nil)
+        { name: name, emptyDir: { medium: medium }.compact }
+      end
+
+      def tmpfs_volume(name)
+        empty_dir_volume(name, medium: 'Memory')
+      end
+
+      def secret_volume(name, secret_name:, items: nil, default_mode: nil)
+        { name: name, secret: { secretName: secret_name, items: items, defaultMode: default_mode }.compact }
+      end
+
+      def secret_item(key, path, mode: nil)
+        { key: key, path: path, mode: mode }.compact
+      end
+
+      def config_map_volume(name, config_map_name: nil, items: nil, default_mode: nil)
+        config_map_name ||= name
+        { name: name, configMap: { name: config_map_name, items: items, defaultMode: default_mode }.compact }
+      end
+
+      def gce_volume(name, pd_name:, fs_type: 'ext4')
+        { name: name, gcePersistentDisk: { pdName: pd_name, fsType: fs_type } }
+      end
+
+      def label_selector(match_labels: nil, match_expressions: nil)
+        {
+          matchLabels:      match_labels,
+          matchExpressions: match_expressions.present? ? Array(match_expressions) : nil
+        }.compact
+      end
+
+      def match_labels_selector(matcher)
+        label_selector(match_labels: matcher)
+      end
+
+      def match_expressions_selector(matcher)
+        label_selector(match_expressions: matcher)
+      end
+
+      def match_expression(key:, operator:, values: nil)
+        { key: key, operator: operator, values: values }.compact
+      end
+
+      def tolerate(key: nil, value: nil, operator: 'Equal', effect: nil, toleration_seconds: nil)
+        {
+          key:               key,
+          value:             value,
+          effect:            effect,
+          operator:          operator,
+          tolerationSeconds: toleration_seconds
+        }.compact
+      end
+
+      # Helper to generate pod affinity term
+      # Defaults to node name for topology key, and empty namespaces
+      def pod_affinity_term(match_labels: nil, match_expressions: nil, namespaces: [], topology_key: 'kubernetes.io/hostname')
+        {
+          labelSelector: label_selector(match_labels: match_labels, match_expressions: match_expressions),
+          namespaces:    namespaces,
+          topologyKey:   topology_key
+        }.compact
+      end
+
+      alias_method :affinity_term, :pod_affinity_term
+
+      def weighted_pod_affinity_term(weight: 100, match_labels: nil, match_expressions: nil, namespaces: [], topology_key: 'kubernetes.io/hostname')
+        {
+          weight: weight,
+          podAffinityTerm: pod_affinity_term(match_labels: match_labels, match_expressions: match_expressions, namespaces: namespaces, topology_key: topology_key)
+        }
+      end
+
+      alias_method :weighted_affinity_term, :weighted_pod_affinity_term
+
+      def preferred_during_scheduling_ignored_during_execution(*weighted_affinity_terms)
+        { preferredDuringSchedulingIgnoredDuringExecution: weighted_affinity_terms }
+      end
+
+      alias_method :preferred_during_scheduling, :preferred_during_scheduling_ignored_during_execution
+
+      def required_during_scheduling_ignored_during_execution(*affinity_terms)
+        { requiredDuringSchedulingIgnoredDuringExecution: affinity_terms }
+      end
+
+      alias_method :required_during_scheduling, :required_during_scheduling_ignored_during_execution
+
+      def topology_spread_constraint(opts = {})
+        {
+          maxSkew:           opts[:max_skew],
+          topologyKey:       opts[:topology_key],
+          whenUnsatisfiable: opts[:when_unsatisfiable],
+          labelSelector:     opts[:label_selector]
+        }.compact
+      end
+
+      def spread_by_node(opts = {})
+        topology_spread_constraint({topology_key: 'kubernetes.io/hostname'}.merge(opts))
+      end
+
+      def spread_by_zone(opts = {})
+        topology_spread_constraint({topology_key: 'topology.kubernetes.io/zone'}.merge(opts))
+      end
+
+      def spread_by_region(opts = {})
+        topology_spread_constraint({topology_key: 'topology.kubernetes.io/region'}.merge(opts))
+      end
+
+      # Helpers
+
+      class << self
+        def load_path
+          Matsuri::Config.pods_path
+        end
+
+        def definition_module_name
+          'Pods'
+        end
+      end
+    end
+  end
+end

data/lib/matsuri/kubernetes/replica_set.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Matsuri
+  module Kubernetes
+    class ReplicaSet < Matsuri::Kubernetes::Base
+      include Matsuri::Concerns::PodTemplate
+
+      let(:api_version) { 'apps/v1' }    # K8S 1.10
+      let(:kind)        { 'ReplicaSet' } # http://kubernetes.io/docs/user-guide/replicasets/
+
+      # Overridables
+
+      let(:spec) do
+        {
+          replicas: replicas,
+          selector: selector,
+          template: template
+        }
+      end
+
+      let(:selector) { { matchLabels: match_labels, matchExpressions: match_expressions } }
+
+      # Supplied by the owning stack through dep_options, or by the CLI
+      # positional; pod templates read it.
+      let(:image_tag) { options[:image_tag] }
+
+      # Explicitly define replicas
+      let(:replicas)          { fail Matsuri::UndefinedLet, 'Must define let(:replicas)' }
+      let(:match_labels)      { fail Matsuri::UndefinedLet, 'Must define let(:match_labels)' }
+
+      let(:match_expressions) { [] }
+
+      let(:primary_container_path) { ['spec', 'template', 'spec', 'containers', 0] }
+
+      class << self
+        def load_path
+          Matsuri::Config.replica_sets_path
+        end
+
+        def definition_module_name
+          'ReplicaSets'
+        end
+      end
+    end
+  end
+end

data/lib/matsuri/kubernetes/secret.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require "base64"
+
+module Matsuri
+  module Kubernetes
+    class Secret < Matsuri::Kubernetes::Base
+      let(:api_version) { 'v1' }    # K8S 1.10
+      let(:kind)        { 'Secret' }
+
+      # Overridables
+      let(:manifest) do
+        {
+          apiVersion: api_version,
+          kind:       kind,
+          metadata:   final_metadata,
+          type:       secret_type,
+          data:       data
+        }
+      end
+
+      # Looks like Kubernetes 1.0.6 supports only Opaque secrets
+      let(:secret_type) { 'Opaque' }
+      let(:data) { { secret_key => secret_value } }
+
+      let(:secret_key)   { fail Matsuri::UndefinedLet, 'Must define let(:secret_key)' }
+      let(:secret_value) { secret_from_path(full_secret_path) }
+      let(:secret_path)  { fail Matsuri::UndefinedLet, 'Must define let(:secret_path)' }
+
+      let(:full_secret_path) { secret_file(secret_path) }
+
+      # Override let(:data) with yaml_data if the secret file is
+      # in yaml form
+      let(:yaml_data)   { base64_hash(yaml_secret) }
+      let(:yaml_secret) { YAML.load_file(secret_file(secret_path)) }
+
+      # Helpers
+
+      # Secrets must be Base64 Encoded. This load the secret from
+      # disk (default: config/secrets/)  base64 encode it.
+      def secret_from(path)
+        secret_from_path(secret_file(path))
+      end
+
+      # Load secret file and base64 encode it from any file location
+      def secret_from_path(path)
+        base64(File.read(path))
+      end
+
+      # Helper to load secrets file from default location
+      def secret_file(path)
+        File.join(Matsuri::Config.config_secrets_path, path)
+      end
+
+      # Helper for base64 encoder. This allows us to pull
+      # secrets from elsewhere, such as from Hashicorp Vault
+      # or from an encrypted AWS S3 bucket item
+      def base64(secret)
+        Base64.encode64(secret).split("\n").join # Strip all newlines
+      end
+
+      # Helper to transform the values of a hash to base64
+      def base64_hash(hash)
+        Hash[hash.map { |k,v| [k, base64(v) ] }]
+      end
+
+      ### Overrides
+
+      # Secrets are not generally something passed around in developer copies
+      # These are things set up by the cluster administrator, so we can assume
+      # if it is present on the server, we can skip it. As such, better to use
+      # recreate than apply for secrets.
+      let(:converge_via) { :recreate }
+
+      class << self
+        def load_path
+          Matsuri::Config.secrets_path
+        end
+
+        def definition_module_name
+          'Secrets'
+        end
+      end
+    end
+  end
+end

data/lib/matsuri/kubernetes/service.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Matsuri
+  module Kubernetes
+    class Service < Matsuri::Kubernetes::Base
+      let(:api_version) { 'v1' }    # K8S 1.10
+      let(:kind)        { 'Service' }
+
+      # Overridables
+      let(:spec) do
+        {
+          ports:     ports,
+          selector:  selector,
+          type:      service_type,
+          clusterIP: cluster_ip
+        }.compact
+      end
+
+      let(:ports)        { [port(port_num)] }
+      let(:port_num)     { fail Matsuri::UndefinedLet, 'Must define let(:port_num)' }
+      let(:selector)     { fail Matsuri::UndefinedLet, 'Must define let(:selector)' }
+      let(:service_type) { 'ClusterIP' }
+      let(:cluster_ip)   { nil }
+
+      # Helpers
+      # Optional:
+      #  target_port: the port exposed externally
+      #  protocol:    connection protocol (ex: tcp)
+      def port(num, target_port: nil, protocol: nil, name: nil, node_port: nil)
+        _port = { port: num }
+        _port[:targetPort] = target_port   if target_port
+        _port[:nodePort]   = node_port     if node_port
+        _port[:protocol]   = protocol.to_s if protocol
+        _port[:name]       = name.to_s     if name
+        return _port
+      end
+
+      class << self
+        def load_path
+          Matsuri::Config.services_path
+        end
+
+        def definition_module_name
+          'Services'
+        end
+      end
+    end
+  end
+end

data/lib/matsuri/kubernetes/stateful_set.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Matsuri
+  module Kubernetes
+    class StatefulSet < Matsuri::Kubernetes::Base
+      include Matsuri::Concerns::PodTemplate
+
+      let(:api_version) { 'apps/v1' }
+      let(:kind)        { 'StatefulSet' } # https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/
+
+      let(:spec) do
+        {
+          replicas:             replicas,
+          selector:             selector,
+          template:             template,
+          serviceName:          service_name,
+          podManagementPolicy:  pod_management_policy,
+          updateStrategy:       update_strategy,
+          revisionHistoryLimit: revision_history_limit,
+          volumeClaimTemplates: volume_claim_templates
+        }.compact
+      end
+
+      # Supplied by the owning stack through dep_options, or by the CLI
+      # positional; pod templates read it.
+      let(:image_tag) { options[:image_tag] }
+
+      let(:selector) { { matchLabels: match_labels, matchExpressions: match_expressions }.compact }
+      let(:service_name) { fail Matsuri::UndefinedLet, 'Must define let(:service_name). Will define "pod-specific-string.serviceName.default.svc.cluster.local"' }
+
+      # Explicitly define replicas
+      let(:replicas)          { fail Matsuri::UndefinedLet, 'Must define let(:replicas)' }
+      let(:match_labels)      { fail Matsuri::UndefinedLet, 'Must define let(:match_labels)' }
+      let(:match_expressions) { nil} # Default this to nil so it drops off unless defined
+
+      let(:revision_history_limit) { nil }
+
+      # Deployment Strategy. Defaults to Rolling Update. Recreate is the other one.
+      let(:pod_management_policy)   { fail Matsuri::UndefinedLet, 'Must define let(:pod_management_policy) as OrderedReady or Parallel' }
+      let(:update_strategy)         { fail Matsuri::UndefinedLet, 'Must define let(:update_strategy). See: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#update-strategies' }
+      let(:on_delete_strategy)      { { type: 'OnDelete' } }
+      let(:rolling_update_strategy) { { type: 'RollingUpdate', rollingUpdate: rolling_update } }
+      let(:rolling_update)          { { partition: partition } }
+      let(:partition)               { 0 }
+
+      let(:volume_claim_templates)  { nil } # Make sure this is nil, otherwise stateful set cannot be updated with apply
+
+      let(:primary_container_path)  { ['spec', 'template', 'spec', 'containers', 0] }
+
+      ### Helpers
+      def volume_claim_template(name:, storage_class: nil, access_modes: 'ReadWriteOnce', storage_request:)
+        {
+          metadata: { name: name },
+          spec: {
+            accessModes: Array(access_modes),
+            storageClassName: storage_class,
+            resources: {
+              requests: { storage: storage_request }
+            }.compact
+          }
+        }
+      end
+
+      class << self
+        def load_path
+          Matsuri::Config.stateful_sets_path
+        end
+
+        def definition_module_name
+          'StatefulSets'
+        end
+      end
+    end
+  end
+end

data/lib/matsuri/kubernetes/storage_class.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Matsuri
+  module Kubernetes
+    # Kubernetes Persistent Volume
+    # http://kubernetes.io/docs/user-guide/persistent-volumes/#storageclasses
+    # No official API definition found
+    class StorageClass < Matsuri::Kubernetes::Base
+      let(:kind)        { 'StorageClass' }
+      let(:api_version) { 'storage.k8s.io/v1' }
+      let(:provisioner) { fail Matsuri::UndefinedLet, 'Must define let(:provisioner). Example: let(:provisoner) { gce_pd }' }
+
+      let(:gce_pd)      { 'kubernetes.io/gce-pd' }
+      let(:aws_ebs)     { 'kubernetes.io/aws-ebs' }
+
+      let(:parameters)  { fail Matsuri::UndefinedLet, 'Must define let(:parameters)' }
+
+      ### K8S default is "Immediate", but "WaitForFirstConsumer" is the sensible default
+      let(:volume_binding_mode) { 'WaitForFirstConsumer' }
+      let(:allowed_topologies)  { nil }
+
+      let(:manifest) do
+        {
+          apiVersion:  api_version,
+          kind:        kind,
+          metadata:    final_metadata,
+          provisioner: provisioner,
+          parameters:  parameters,
+          volumeBindingMode: volume_binding_mode,
+          allowedTopologies: allowed_topologies
+        }
+      end
+
+      class << self
+        def load_path
+          Matsuri::Config.storage_classes_path
+        end
+
+        def definition_module_name
+          'StorageClasses'
+        end
+      end
+    end
+  end
+end

data/lib/matsuri/platform.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require 'mixlib/config'
+require 'pathname'
+
+module Matsuri
+  # This contains configs specific to the platform. Unlike Matsuri::Config, these
+  # is freeform. Think of it as a key-value store which can be loaded and layered
+  # from a set of files. You would define the list of platform config files to load,
+  # and Matsuri will load those values in here. Files loaded later will override values
+  # loaded from earlier files.
+  #
+  # Platform config is where you will put things like dev secrets, API test keys, to
+  # fill into the environment spec for Kubernetes pods.
+  module Platform
+    extend Mixlib::Config
+    config_strict_mode false # Allow users to extend the key space
+
+    # Repo-wide save-config policy for the normal apply path's create and
+    # recreate; stacks override per stack, resources per definition.
+    default :save_config, true
+
+    # Repo-wide field manager for server-side delivery. It declares the
+    # repo's ongoing-delivery identity: the conflict frontier forms
+    # exactly where manager names differ, and a shared name is a single
+    # manager — writes within it never conflict.
+    default :field_manager, 'matsuri'
+
+    # Loads a platform file if it exists. Each subsequent file loaded
+    # will override older ones.
+    def self.load_configuration(load_paths = [])
+      load_paths.each do |path|
+        next unless File.file?(path)
+        from_file(path)
+      end
+    end
+  end
+end