matsuri 0.1.0 → 0.3.0

data/lib/matsuri/stack.rb

@@ -0,0 +1,276 @@
+# frozen_string_literal: true
+
+require 'map'
+require 'matsuri/core_ext'
+require 'matsuri/acceptors/image_tag'
+require 'rainbow/ext/string'
+require 'rlet'
+require 'rlet/lazy_options'
+
+module Matsuri
+  class Stack
+    include Let
+    include RLet::LazyOptions
+    include Matsuri::Concerns::RegistryHelpers
+    include Matsuri::Concerns::Awaiting
+
+    class_attribute :build_order
+    class_attribute :acceptors
+    class_attribute :presenters
+
+    class_attribute :save_config_policy
+
+    # Declaration-surface vocabulary: match / match_kind / match_deployments
+    # wrap a present member so it applies selectively; seed_* constructors
+    # close a seed concern's hole over a declaration-time value.
+    extend Matsuri::Transform::Combinators::Helpers
+    extend Matsuri::Presenters::Helpers
+
+    # The collection name is the options key: each acceptor admits its own
+    # component of the invocation options, whole. The built-in :image_tag
+    # collection carries the converge positional and owns the 'latest'
+    # default; :config carries the stack-config artifact, already loaded —
+    # whoever constructs the stack supplies data, never a path.
+    let(:image_tag)    { accepted(:image_tag).image_tag }
+    let(:stack_config) { accepted(:config) }
+
+    # Defines a list of deps, used to  to idempotently converge Kubernetes
+    # resources. Generally, you will add a stack as a dep for resources you are
+    # sharing with other stacks, while directly referencing Kubernetes resources
+    # for things this stack is directly responsible for.
+    #
+    # Examples:
+    # needs :stack,   'postgresql-95'
+    # needs :stack,   'redis-28'
+    # needs :stack,      'load-balancer'
+    # needs :deployment, 'rails-app-worker'
+    # needs :deployment, 'rails-app'
+    # needs :service,    'rails-app'
+    #
+    # Hooks:
+    #   check_only: when set to true, do not create or apply resource. Only check for it's presence
+    #   on_failure: a symbol of a method to call when check fails.
+    #
+    # @TODO Consider making this lazy-loaded. We don't need to generate this elaborate data structure
+    # unless we are converging. (But it might not really make things that much slower).
+    class << self
+      def needs(type, name, on_failure: nil, check_only: false)
+        self.build_order ||= []
+        self.build_order << [type.to_sym, name.to_s, { on_failure: on_failure, check_only: check_only }]
+      end
+
+      def config_file(path)
+        File.join(Matsuri::Config.config_path, path)
+      end
+
+      # Registers an acceptor under a named collection. Last declaration
+      # per collection wins; a stack overrides a built-in by re-declaring it.
+      def accept(collection, acceptor)
+        self.acceptors = (acceptors || {}).merge(collection.to_sym => acceptor)
+      end
+
+      # Appends transform members to a scenario's pipeline. Unlike accept,
+      # where a collection names a slot and last declaration wins, a
+      # scenario accumulates: declaration order is member order. Members
+      # are presenter names, presenter classes, Callables, or anything
+      # #call-able; scenarios are intent regimes (:on_create brings into
+      # existence, :on_apply delivers to something existing) and each
+      # delivery verb assembles its own.
+      def present(scenario, *steps)
+        key = scenario.to_sym
+        current = presenters || {}
+        self.presenters = current.merge(key => Array(current[key]) + steps)
+      end
+
+      # Declares the save-config policy for every resource member this
+      # stack delivers (overriding the repo-wide Platform setting); a
+      # resource overrides again with its own let(:save_config). Normal
+      # apply path only.
+      def save_config(value)
+        self.save_config_policy = value
+      end
+    end
+
+    accept :image_tag, Matsuri::Acceptors::ImageTag
+    # Empty acceptor: the collection is total, its surface is empty until
+    # a platform re-declares it.
+    accept :config, Matsuri::Acceptor
+
+    # Override this. This commands defines how to display the status
+    # Maybe we can find some useful, standardized way of displaying
+    # status.
+    def status!
+      fail NoMethodError, 'Must implement #status!'
+    end
+
+    # Renders every member manifest as one kubectl-compatible YAML stream.
+    def show!(_opt = {})
+      members.each { |member| puts member.to_yaml }
+    end
+
+    def diff!(opt = {})
+      members.each do |member|
+        # --primary-container is a HashDiff-only projection, so it
+        # implies --hash-diff
+        if opt[:hash_diff] || opt[:primary_container]
+          Matsuri::Tasks::HashDiff.new(resource: member, primary_container: opt[:primary_container],
+                                       presenters: stack_presenters).run!
+        else
+          Matsuri::Tasks::KubectlOps.diff(member, presenters: stack_presenters)
+        end
+      end
+    end
+
+    # Override this. This command defines how all the stack depenencies
+    # are started, such as services, endpoints, replication controllers, etc.
+    def create!
+      converge! # Most of the time, we want to converge
+    end
+
+    # Override this. This command defines how all stack dependencies are stopped
+    def delete!
+      self.class.build_order.each do |(type, name)|
+        if type == :stack
+          puts "Skipping stack #{name}"
+          next
+        end
+
+        # Same options as converge, so a member whose identity derives from
+        # an accepted value addresses the same resource it created.
+        Matsuri::Tasks::KubectlOps.delete(dep(type, name).new(dep_options(type)))
+      end
+    end
+
+    def recreate!
+      delete!
+      create!
+    end
+
+    def rebuild!
+      build!
+      recreate!
+    end
+
+    def converge!(opts = {})
+      puts "Converging #{name}".color(:red).bright if config.verbose
+      self.class.build_order.each do |(type, name, options)|
+        case type
+        when :file
+          Matsuri.log :info, "Checking for required file #{name}"
+          next if File.file?(name)
+          Matsuri.log(:fatal, "Cannot find required file #{name}") unless options[:on_failure]
+          send(options[:on_failure], type, name) if options[:on_failure]
+        when :task
+          Matsuri.log :info, "Executing task #{name}"
+          send(name, opts)
+          next
+        end
+
+        resource = dep(type, name).new(dep_options(type))
+        # @TODO if-elsif-end ladder code smell, refactor
+        if options[:check_only]
+          next if Matsuri::Tasks::KubectlOps.exists?(resource)
+          send(options[:on_failure], type, name) || Matsuri.log(:fatal, "#{type}/#{name} not found on cluster")
+        elsif type == :stack && !opts[:reboot]
+          puts "Shallow converging stack #{name}".color(:red).bright if config.verbose
+          resource.converge!
+        elsif type == :stack
+          resource.converge!(opts)
+        else
+          Matsuri::Tasks::KubectlOps.converge(resource, opts, presenters: stack_presenters)
+        end
+      end
+    end
+
+    # Hooks
+    def reload!(version, opt={})
+      Matsuri.log :fatal, "I don't know how to reload #{name}. Define me in the stack definition."
+    end
+
+    def update!(version, opt={})
+      Matsuri.log :fatal, "I don't know how to update #{name}. Define me in the stack definition."
+    end
+
+    def sh!(opt, args)
+      Matsuri.log :fatal, "I don't know how to shell into #{name}. Define me in the stack definition."
+    end
+
+    def console!(opt, args)
+      Matsuri.log :fatal, "I don't know how to get to the console for #{name}. Define me in the stack definition."
+    end
+
+    def build!(opt={})
+      Matsuri.log :fatal, "I don't know how to build #{name}. Define me in the stack definition."
+    end
+
+    def push!(opt={})
+      Matsuri.log :fatal, "I don't know how to push #{name}. Define me in the stack definition."
+    end
+
+    # The docker mechanism lives in Tasks::Docker; the hook declares, the
+    # task delivers. Definition-supplied build!/push! hooks call this
+    # unchanged. Stacks themselves no longer shell out.
+    def docker!(cmd, options = {})
+      Matsuri::Tasks::Docker.new.run!(cmd, options)
+    end
+
+    # Helper functions
+    # Like stack-config, present declarations never cross a stack boundary:
+    # a nested stack assembles from its own class.
+    def stack_presenters
+      self.class.presenters || {}
+    end
+
+    # stack-config never crosses a stack boundary: instance identity is the
+    # (stack, stack-config) pair, and a nested stack is a different instance.
+    # The save-config policy rides to resource members only — a nested
+    # stack declares its own.
+    def dep_options(type)
+      return { image_tag: image_tag } if type == :stack
+
+      opts = { image_tag: image_tag, stack_config: stack_config }
+      opts[:save_config] = self.class.save_config_policy unless self.class.save_config_policy.nil?
+      opts
+    end
+
+    def members
+      self.class.build_order.filter_map do |(type, name, _)|
+        next if type == :file || type == :task
+
+        if type == :stack
+          puts "Skipping stack #{name}"
+          next
+        end
+
+        dep(type, name).new(dep_options(type))
+      end
+    end
+
+    def accepted(collection)
+      @accepted ||= {}
+      @accepted[collection] ||= acceptor_class_for(self.class.acceptors.fetch(collection)).new(options[collection])
+    end
+
+    def acceptor_class_for(name)
+      name.is_a?(Class) ? name : Matsuri::Registry.acceptor(name)
+    end
+
+    def dep(type, name)
+      Matsuri::Registry.fetch_or_load type, name
+    end
+
+    def stack(name, opt={})
+      Matsuri::Registry.stack(name).new(opt)
+    end
+
+    class << self
+      def load_path
+        Matsuri::Config.stacks_path
+      end
+
+      def definition_module_name
+        'Stacks'
+      end
+    end
+  end
+end

data/lib/matsuri/task.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require 'rlet'
+require 'rlet/lazy_options'
+
+module Matsuri
+  module Task
+    extend Concern
+
+    included do
+      include Let
+      include RLet::LazyOptions
+      include Matsuri::ShellOut
+
+      let(:verbose) { options[:verbose] || config.verbose }
+      let(:debug)   { options[:debug]   || config.debug }
+      let(:config)  { Matsuri::Config }
+    end
+  end
+end

data/lib/matsuri/tasks/cluster.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require 'yaml'
+require 'matsuri/core_ext'
+
+module Matsuri
+  module Tasks
+    # Converge cluster
+    class Cluster
+      include Matsuri::Task
+
+      let(:root_scope)             { Matsuri::DSL::Cluster::Scope.new(namespace: nil) }
+
+      let(:all_files)              { global_files + environment_files }
+      let(:global_files)           { Dir.glob(File.join(clusters_path, Matsuri.environment, '**', '*.rb')) }
+      let(:environment_files)      { Dir.glob(File.join(clusters_path, '*.rb')) }
+      let(:clusters_path)          { Matsuri::Config.clusters_path }
+
+      let(:all_rbac_manifests)             { load_all! && root_scope.rbac_manifests.flatten.compact }
+      let(:rbac_manifests_to_yaml)         { all_rbac_manifests.map(&:deep_stringify_keys).map(&:to_yaml) }
+      let(:rbac_manifests_to_json)         { all_rbac_manifests.map(&:to_json) }
+      let(:rbac_manifests_to_pretty_json)  { all_rbac_manifests.map(&JSON.method(:pretty_generate)) }
+
+      let(:all_cluster_manifests)            { load_all! && root_scope.cluster_manifests.flatten.compact }
+      let(:cluster_manifests_to_yaml)        { all_cluster_manifests.map(&:deep_stringify_keys).map(&:to_yaml) }
+      let(:cluster_manifests_to_json)        { all_cluster_manifests.map(&:to_json) }
+      let(:cluster_manifests_to_pretty_json) { all_cluster_manifests.map(&JSON.method(:pretty_generate)) }
+
+      # Memoized so it only evaluates once
+      let(:load_all!)              { all_files.each(&method(:load_definition!)) }
+
+      def load_definition!(filename)
+        root_scope.import(filename)
+      end
+
+      def show!(opt = {})
+        if opt[:json]
+          show_json!
+        else
+          show_yaml!
+        end
+      end
+
+      def show_json!
+        puts '//--- RBAC resources'
+        puts rbac_manifests_to_pretty_json
+        puts '', '//--- Other resources'
+        puts cluster_manifests_to_pretty_json
+      end
+
+      def show_yaml!
+        puts '##### RBAC resources'
+        puts rbac_manifests_to_yaml
+        puts '', '##### Other resources'
+        puts cluster_manifests_to_yaml
+      end
+
+      def reconcile!(opt = {})
+        puts 'Converging cluster-wide configuration'.color(:yellow) if config.verbose
+        reconcile_rbac!(opt)
+        apply_cluster!(opt)
+      end
+
+      def reconcile_rbac!(_opt)
+        Matsuri.log :info, 'Reconciling RBAC'
+
+        # kubectl_options = opt[:dry_run] ? '--dry-run' : ''
+
+        kubectl! 'auth reconcile -f -', input: rbac_manifests_to_json.join("\n"), kube_options: { namespace: nil }
+      end
+
+      def apply_cluster!(_opt)
+        Matsuri.log :info, 'Applying non-RBAC Cluster Resources'
+
+        kubectl! 'apply -f -', input: cluster_manifests_to_json.join("\n"), kube_options: { namespace: nil }
+      end
+    end
+  end
+end

data/lib/matsuri/tasks/docker.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Matsuri
+  module Tasks
+    class Docker
+      include Matsuri::Task
+
+      # The delivery surface stack hooks delegate to: the hook declares,
+      # this task shells out. (ShellOut's own docker! is private on
+      # includers — module_function.)
+      def run!(cmd, options = {})
+        docker!(cmd, options)
+      end
+
+      def kill_all!
+        docker! 'stop $(docker ps -a -q)'
+        docker! 'rm -f $(docker ps -a -q)'
+      end
+
+      def fix_pts!
+        puts 'sudo mount -o remount,ptmxmode=666 /dev/pts'
+      end
+    end
+  end
+end

data/lib/matsuri/tasks/hash_diff.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require 'rainbow/ext/string'
+
+module Matsuri
+  module Tasks
+    # Client-side diff: fetches the current manifest, asks the resource for
+    # the filtered deltas (Concerns::Diffable), and prints them. Sibling to
+    # KubectlOps rather than inside it — the comparison happens here, kubectl
+    # is only the fetch. The server-side counterpart (KubectlOps::Diff,
+    # wrapping kubectl diff) shows what the cluster will make of the
+    # manifest, admission injections included; this one shows drift relative
+    # to the declaration with the noise filtered. Both are kept.
+    class HashDiff
+      include Matsuri::Task
+
+      let(:resource)          { options.fetch(:resource) }
+      let(:primary_container) { options[:primary_container] }
+      let(:presenters)        { options[:presenters] || {} }
+
+      def run!
+        current = Matsuri::Tasks::KubectlOps.manifest(resource)
+        unless current
+          Matsuri.log :fatal, "Cannot fetch current manifest for #{Matsuri::Tasks::KubectlOps.description_for(resource)}"
+        end
+
+        deltas =
+          if primary_container
+            resource.primary_container_diff(current, desired: desired_manifest)
+          else
+            resource.diff(current, desired: desired_manifest)
+          end
+        print_diff(deltas)
+      end
+
+      # The declaration through the same :on_apply pipeline apply
+      # assembles, over the same basis: the applied manifest. Drift reads
+      # against what apply would send — never against a step a presenter
+      # owns, never against a field the resource restricts out of its
+      # applied_manifest. The JSON round trip is the comparison
+      # normalization diff always applied to the declaration.
+      def desired_manifest
+        ops = Matsuri::Tasks::KubectlOps
+        presentation = ops.presentation(resource.applied_manifest_for_export,
+                                        ops.presenters_for(resource, presenters), :on_apply)
+        JSON.parse(presentation.to_json)
+      end
+
+      def print_diff(deltas)
+        deltas.each do |line|
+          color = case line[0]
+                  when '-' then :red
+                  when '+' then :green
+                  when '~' then :yellow
+                  end
+          puts line.join(' ').color(color).bright
+        end
+      end
+    end
+  end
+end

data/lib/matsuri/tasks/kubectl.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Matsuri
+  module Tasks
+    class Kubectl
+      include Matsuri::Task
+
+      let(:environment)  { Matsuri.environment }
+      let(:env_config)   { Matsuri::Platform.send(environment) }
+      let(:namespace)    { env_config.namespace || 'default' }
+      let(:kube_user)    { env_config.user || kube_context }
+      let(:kube_cluster) { env_config.kube_cluster || kube_context }
+
+      # Set up aliases for kubectl. This allows you you to use
+      # `kubectl --context=staging` instead of the name given by GKE
+      def setup!
+        shell_out! "kubectl config set-context #{environment} --cluster=#{kube_cluster} --namespace=#{namespace} --user=#{kube_user}"
+      end
+    end
+  end
+end

data/lib/matsuri/tasks/kubectl_ops/apply.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Matsuri
+  module Tasks
+    module KubectlOps
+      # Client-side apply of a rendered manifest payload. The payload is
+      # whatever the caller finished rendering — single-doc or multi-doc
+      # YAML — so one task serves generated and imported manifests alike.
+      class Apply
+        include Matsuri::Task
+
+        let(:input)       { options.fetch(:input) }
+        let(:namespace)   { options.fetch(:namespace) }
+        let(:description) { options.fetch(:description) }
+
+        def run!
+          puts "Applying (create or update) #{description}".color(:yellow).bright if verbose
+          puts input if debug
+          kubectl! 'apply -f -', input: input
+        end
+      end
+    end
+  end
+end

data/lib/matsuri/tasks/kubectl_ops/create.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Matsuri
+  module Tasks
+    module KubectlOps
+      # Imperative create of a rendered manifest payload. --save-config
+      # seeds the last-applied-configuration annotation so a later
+      # client-side apply gets its three-way merge; the boundary pins it
+      # false (consults_last_applied?) when ongoing delivery never
+      # consults the annotation.
+      class Create
+        include Matsuri::Task
+
+        let(:input)       { options.fetch(:input) }
+        let(:namespace)   { options.fetch(:namespace) }
+        let(:description) { options.fetch(:description) }
+        let(:save_config) { options.fetch(:save_config, true) }
+
+        def run!
+          puts "Creating #{description}".color(:yellow).bright if verbose
+          puts input if debug
+          kubectl! "create --save-config=#{save_config} -f -", input: input
+        end
+      end
+    end
+  end
+end

data/lib/matsuri/tasks/kubectl_ops/delete.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Matsuri
+  module Tasks
+    module KubectlOps
+      # Deletes a live object addressed by type/name.
+      class Delete
+        include Matsuri::Task
+
+        let(:resource_type) { options.fetch(:resource_type) }
+        let(:name)          { options.fetch(:name) }
+        let(:namespace)     { options.fetch(:namespace) }
+
+        def run!
+          puts "Deleting #{resource_type}/#{name}".color(:yellow).bright if verbose
+          kubectl! "delete #{resource_type}/#{name}"
+        end
+      end
+    end
+  end
+end

data/lib/matsuri/tasks/kubectl_ops/diff.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require 'rainbow/ext/string'
+
+module Matsuri
+  module Tasks
+    module KubectlOps
+      # Server-side diff: the apiserver dry-runs the payload through
+      # defaulting and admission, and kubectl prints live-vs-admitted.
+      # Companion to Tasks::HashDiff, which shows drift relative to the
+      # declaration; this one shows what the cluster will actually do
+      # with the manifest, admission injections included.
+      #
+      # kubectl diff exit codes carry meaning: 0 = no differences,
+      # 1 = differences found, >1 = error. Only >1 is a failure, and
+      # silence on no-differences is kubectl's own convention.
+      class Diff
+        include Matsuri::Task
+
+        let(:input)         { options.fetch(:input) }
+        let(:namespace)     { options.fetch(:namespace) }
+        let(:description)   { options.fetch(:description) }
+        let(:server_side)   { options.fetch(:server_side, false) }
+        let(:field_manager) { options.fetch(:field_manager, nil) }
+
+        def run!
+          puts "Diffing (live vs admitted) #{description}".color(:yellow).bright if verbose
+          puts input if debug
+
+          cmd = kubectl "diff#{server_side_args} -f -", input: input
+          if cmd.exitstatus > 1
+            $stderr.print "ERROR: #{cmd.exitstatus}\nSTDOUT:\n#{cmd.stdout}\n\nSTDERR:\n#{cmd.stderr}\n".color(:red).bright
+            exit 1
+          end
+
+          puts "No changes for #{description}".color(:yellow) if verbose && cmd.exitstatus.zero?
+        end
+
+        # The manager must be the one apply delivers under, or the
+        # preview shows a different conflict frontier than delivery hits.
+        def server_side_args
+          server_side ? " --server-side --field-manager=#{field_manager}" : ''
+        end
+      end
+    end
+  end
+end

data/lib/matsuri/tasks/kubectl_ops/get.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Matsuri
+  module Tasks
+    module KubectlOps
+      # Read-side of the delivery mechanism: existence checks and manifest
+      # reads against the live cluster, addressed by type/name. Queries,
+      # not effects, so it exposes readers instead of run!. Under verbose,
+      # exists? narrates what it found.
+      class Get
+        include Matsuri::Task
+
+        let(:resource_type) { options.fetch(:resource_type) }
+        let(:name)          { options.fetch(:name) }
+        let(:namespace)     { options.fetch(:namespace) }
+
+        def exists?
+          cmd = kubectl "get #{resource_type}/#{name}", echo_level: :debug, no_stdout: true
+          if verbose
+            status = cmd.status.success? ? 'already started' : 'not started'
+            Matsuri.log :info, "#{resource_type}/#{name} #{status}".color(:yellow)
+          end
+          cmd.status.success?
+        end
+
+        # Plain parsed hash, string keys; nil when not on the cluster.
+        def manifest
+          cmd = kubectl "get #{resource_type}/#{name} -o json", echo_level: :debug, no_stdout: true
+          return nil unless cmd.status.success?
+
+          JSON.parse(cmd.stdout)
+        end
+      end
+    end
+  end
+end

data/lib/matsuri/tasks/kubectl_ops/recreate.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Matsuri
+  module Tasks
+    module KubectlOps
+      # Delete-then-create lifecycle: tears down the live object if
+      # present, waits until the deletion lands, then runs the creation
+      # task the boundary selected — plain create for client-side
+      # delivery, the seeding protocol for server-side. The payload must
+      # be rendered before this task runs — deletion severs anything the
+      # render would have read back.
+      class Recreate
+        include Matsuri::Task
+        include Matsuri::Concerns::Awaiting
+
+        let(:resource_type) { options.fetch(:resource_type) }
+        let(:name)          { options.fetch(:name) }
+        let(:namespace)     { options.fetch(:namespace) }
+        let(:description)   { options.fetch(:description, "#{resource_type}/#{name}") }
+        let(:create)        { options.fetch(:create) }
+
+        let(:identity) { { resource_type: resource_type, name: name, namespace: namespace } }
+
+        def run!
+          if Get.new(identity).exists?
+            Delete.new(identity).run!
+            awaiting!('resource deletion', interval: 0.5) { !Get.new(identity).exists? }
+          end
+
+          create.run!
+        end
+      end
+    end
+  end
+end