porkadot 0.21.0 → 0.23.0

data/lib/porkadot/assets/kubernetes/manifests/kube-scheduler.yaml.erb

@@ -1,6 +1,6 @@
 <% k8s = global_config.k8s -%>
 ---
-apiVersion: policy/v1beta1
+apiVersion: policy/v1
 kind: PodDisruptionBudget
 metadata:
   name: kube-scheduler
@@ -113,6 +113,11 @@ spec:
       annotations:
         scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
+      securityContext:
+        seccompProfile:
+          type: RuntimeDefault
+        runAsNonRoot: true
+        runAsUser: 65534
       affinity:
         podAntiAffinity:
           preferredDuringSchedulingIgnoredDuringExecution:
@@ -141,17 +146,26 @@ spec:
         - <%= k %><% if v ;%>=<%= v %><%; end %>
         <%- end -%>
         livenessProbe:
+          failureThreshold: 8
           httpGet:
             path: /healthz
-            port: 10251  # Note: Using default port. Update if --port option is set differently.
-          initialDelaySeconds: 15
+            port: 10259
+            scheme: HTTPS
+          initialDelaySeconds: 10
+          periodSeconds: 10
+          timeoutSeconds: 15
+        startupProbe:
+          failureThreshold: 24
+          httpGet:
+            path: /healthz
+            port: 10259
+            scheme: HTTPS
+          initialDelaySeconds: 10
+          periodSeconds: 10
           timeoutSeconds: 15
       priorityClassName: system-cluster-critical
       nodeSelector:
         k8s.unstable.cloud/master: ""
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 65534
       serviceAccountName: kube-scheduler
       tolerations:
       - key: CriticalAddonsOnly

data/lib/porkadot/assets/kubernetes/manifests/kubelet.yaml.erb

@@ -15,7 +15,6 @@ roleRef:
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: auto-approve-csrs-for-group
   name: porkadot:node-autoapprove-bootstrap
 subjects:
 - kind: Group

data/lib/porkadot/assets/kubernetes/manifests/kustomization.yaml.erb

@@ -0,0 +1,8 @@
+resources:
+- addons
+- kube-apiserver.yaml
+- kube-controller-manager.yaml
+- kube-proxy.yaml
+- kube-scheduler.yaml
+- kubelet.yaml
+- porkadot.yaml

data/lib/porkadot/assets/kubernetes.rb

@@ -17,34 +17,107 @@ module Porkadot; module Assets
 
     def render
       logger.info "--> Rendering kubernetes manifests"
-      unless File.directory?(config.manifests_path)
-        FileUtils.mkdir_p(config.manifests_path)
-      end
-      unless File.directory?(config.manifests_secrets_path)
-        FileUtils.mkdir_p(config.manifests_secrets_path)
-      end
-      lb = global_config.lb
-      cni = global_config.cni
       render_erb 'manifests/porkadot.yaml'
       render_erb 'manifests/kubelet.yaml'
-      render_erb "manifests/000-#{lb.type}.yaml"
-      render_erb "manifests/#{lb.type}.yaml"
-      render_erb "manifests/#{lb.type}.config.yaml"
-      render_secrets_erb "manifests/#{lb.type}.secrets.yaml"
-      render_erb "manifests/#{cni.type}.yaml"
-      render_erb "manifests/coredns.yaml"
-      render_erb "manifests/dns-horizontal-autoscaler.yaml"
       render_erb "manifests/kube-apiserver.yaml"
       render_secrets_erb "manifests/kube-apiserver.secrets.yaml"
       render_erb "manifests/kube-proxy.yaml"
       render_erb "manifests/kube-scheduler.yaml"
       render_erb "manifests/kube-controller-manager.yaml"
       render_secrets_erb "manifests/kube-controller-manager.secrets.yaml"
-      render_erb "manifests/kubelet-rubber-stamp.yaml"
-      render_erb "manifests/storage-version-migrator.yaml"
       render_secrets_erb "kubeconfig.yaml"
-      render_erb 'install.sh'
+      render_erb 'manifests/kustomization.yaml'
+      render_erb 'kustomization.yaml', force: false
+      render_erb 'install.sh', prune_allowlist: prune_allowlist
+      render_secrets_erb 'install.secrets.sh'
+
+      addons = Addons.new(global_config)
+      addons.render
+    end
+
+    def prune_allowlist
+      return %w[
+        apiextensions.k8s.io/v1/customresourcedefinition
+        apps/v1/daemonset
+        apps/v1/deployment
+        core/v1/configmap
+        core/v1/namespace
+        core/v1/service
+        core/v1/serviceaccount
+        policy/v1/poddisruptionbudget
+        policy/v1beta1/podsecuritypolicy
+        rbac.authorization.k8s.io/v1/clusterrole
+        rbac.authorization.k8s.io/v1/clusterrolebinding
+        rbac.authorization.k8s.io/v1/role
+        rbac.authorization.k8s.io/v1/rolebinding
+      ]
     end
+  end
+
+  class Addons
+    include Porkadot::Assets
+    TEMPLATE_DIR = File.join(File.dirname(__FILE__), "kubernetes", "manifests", "addons")
+    attr_reader :global_config
+    attr_reader :config
+    attr_reader :logger
+
+    def initialize global_config
+      @global_config = global_config
+      @config = global_config.addons
+      @logger = global_config.logger
+    end
+
+    def render
+      logger.info "--> Rendering kubernetes addons"
+      render_erb "kustomization.yaml"
+
+      self.config.enabled.each do |name|
+        manifests = @@manifests[name]
+        manifests.each do |m|
+          render_erb(m)
+        end
+        secrets = @@secrets_manifests[name]
+        secrets.each do |m|
+          render_secrets_erb(m)
+        end
+      end
+    end
+
+    def self.register_manifests name, manifests, secrets: []
+      @@manifests ||= {}
+      @@manifests[name] = manifests
+      @@secrets_manifests ||= {}
+      @@secrets_manifests[name] = secrets
+    end
+
+    register_manifests('flannel', [
+      'flannel/flannel.yaml',
+      'flannel/kustomization.yaml'
+    ])
+
+    register_manifests('coredns', [
+      'coredns/coredns.yaml',
+      'coredns/dns-horizontal-autoscaler.yaml',
+      'coredns/kustomization.yaml'
+    ])
+
+    register_manifests('metallb', [
+      'metallb/000-metallb.yaml',
+      'metallb/metallb.yaml',
+      'metallb/metallb.config.yaml',
+      'metallb/kustomization.yaml'
+    ])
+
+
+    register_manifests('kubelet-rubber-stamp', [
+      'kubelet-rubber-stamp/kubelet-rubber-stamp.yaml',
+      'kubelet-rubber-stamp/kustomization.yaml'
+    ])
+
+    register_manifests('storage-version-migrator', [
+      'storage-version-migrator/storage-version-migrator.yaml',
+      'storage-version-migrator/kustomization.yaml'
+    ])
 
   end
 end; end

data/lib/porkadot/assets.rb

@@ -15,7 +15,7 @@ module Porkadot::Assets
     end
   end
 
-  def render_erb file, opts={}
+  def render_erb file, **opts
     file = file.to_s
     opts[:config] = self.config
     opts[:global_config] = self.global_config
@@ -23,8 +23,15 @@ module Porkadot::Assets
     opts[:u] = ErbUtils.new
 
     logger.info "----> #{file}"
+    asset = config.asset_path(file)
+    if opts[:force] != nil && File.file?(asset)
+      logger.debug "------> Already exists: skipping #{file}"
+      return
+    end
+    asset_dir = File.dirname(asset)
+    FileUtils.mkdir_p(asset_dir) unless File.directory?(asset_dir)
     open(File.join(self.class::TEMPLATE_DIR, "#{file}.erb")) do |io|
-      open(config.asset_path(file), 'w') do |out|
+      open(asset, 'w') do |out|
         out.write ERB.new(io.read, trim_mode: '-').result_with_hash(opts)
       end
     end
@@ -38,8 +45,11 @@ module Porkadot::Assets
     opts[:u] = ErbUtils.new
 
     logger.info "----> #{file}"
+    secret = config.secrets_path(file)
+    secret_dir = File.dirname(secret)
+    FileUtils.mkdir_p(secret_dir) unless File.directory?(secret_dir)
     open(File.join(self.class::TEMPLATE_DIR, "#{file}.erb")) do |io|
-      open(config.secrets_path(file), 'w') do |out|
+      open(secret, 'w') do |out|
         out.write ERB.new(io.read, trim_mode: '-').result_with_hash(opts)
       end
     end

data/lib/porkadot/cmd/cli.rb

@@ -13,15 +13,22 @@ module Porkadot; module Cmd
     desc "install", "Install kubernetes"
     subcommand "install", Porkadot::Cmd::Install::Cli
 
+    desc "etcd", "Interact with etcd"
+    subcommand "etcd", Porkadot::Cmd::Etcd::Cli
+
     desc "setup-containerd", "Setup containerd"
     option :node, type: :string
     option :force, type: :boolean, default: false
+    option :bootstrap, type: :boolean, default: false
     def setup_containerd
       logger.info "Setup containerd"
       kubelets = Porkadot::Install::KubeletList.new(self.config)
       nodes = []
       if node = options[:node]
         nodes = kubelets[node]
+      elsif options[:bootstrap]
+        bootstrap = Porkadot::Install::Bootstrap.new(self.config)
+        nodes = bootstrap.host
       else
         nodes = kubelets.kubelets.values
       end
@@ -29,6 +36,26 @@ module Porkadot; module Cmd
       ""
     end
 
+    desc "setup-node", "Setup node default settings"
+    option :node, type: :string
+    option :force, type: :boolean, default: false
+    option :bootstrap, type: :boolean, default: false
+    def setup_node
+      logger.info "Setup node default"
+      kubelets = Porkadot::Install::KubeletList.new(self.config)
+      nodes = []
+      if node = options[:node]
+        nodes = kubelets[node]
+      elsif options[:bootstrap]
+        bootstrap = Porkadot::Install::Bootstrap.new(self.config)
+        nodes = bootstrap.host
+      else
+        nodes = kubelets.kubelets.values
+      end
+      kubelets.setup_default hosts: nodes, force: options[:force]
+      ""
+    end
+
     desc "set-config", "Set cluster to kubeconfig"
     def set_config
       name = config.k8s.cluster_name

data/lib/porkadot/cmd/etcd.rb

@@ -0,0 +1,68 @@
+
+module Porkadot; module Cmd; module Etcd
+  class Cli < Porkadot::SubCommandBase
+    include Porkadot::Utils
+
+    default_task :all
+    desc "all", "Interact with etcd"
+    def all
+      "Use restore or backup sub commands."
+    end
+
+    desc "backup", "Backup etcd data"
+    option :node, type: :string
+    option :path, type: :string, default: "./backup", desc: "Directory where etcd backup data will be stored."
+    def backup
+      require 'date'
+
+      filename = "etcd-#{DateTime.now.to_s}.db"
+      path = File.join(options[:path], filename)
+
+      logger.info "Backing up etcd data to #{path}"
+      kubelets = Porkadot::Install::KubeletList.new(self.config)
+      kubelets.backup_etcd host: options[:node], path: path
+      ""
+    end
+
+    desc "start", "Start etcd"
+    option :node, type: :string
+    def start
+      logger.info "Start etcd"
+      kubelets = Porkadot::Install::KubeletList.new(self.config)
+      kubelets.start_etcd hosts: options[:node]
+      ""
+    end
+
+    desc "stop", "Stop etcd"
+    option :node, type: :string
+    def stop
+      logger.info "Start etcd"
+      kubelets = Porkadot::Install::KubeletList.new(self.config)
+      kubelets.stop_etcd hosts: options[:node]
+      ""
+    end
+
+    desc "restore", "Restore etcd data"
+    option :path, type: :string, default: "./backup", desc: "Directory where etcd backup data is stored."
+    def restore
+      invoke :stop, [], options
+
+      path = Dir.glob(File.join(options[:path], "etcd-*.db")).sort.reverse[0]
+      unless path
+        return "No backup data found...: #{options[:path]}"
+      end
+
+      logger.info "Restore etcd from #{path}"
+      kubelets = Porkadot::Install::KubeletList.new(self.config)
+      kubelets.restore_etcd path: path
+
+      invoke :start, [], options
+      ""
+    end
+
+    def self.subcommand_prefix
+      'etcd'
+    end
+  end
+end; end; end
+

data/lib/porkadot/cmd/install.rb

@@ -26,6 +26,21 @@ module Porkadot; module Cmd; module Install
       ""
     end
 
+    desc "kubernetes", "Install kubernetes"
+    option :node, type: :string
+    def kubernetes
+      logger.info "Installing kubernetes"
+      kubelets = Porkadot::Install::KubeletList.new(self.config)
+      if node = options[:node]
+        nodes = kubelets[node]
+      else
+        nodes = Porkadot::Install::Bootstrap.new(self.config).host
+      end
+      k8s = Porkadot::Install::Kubernetes.new(self.config)
+      k8s.install(nodes)
+      ""
+    end
+
     desc "bootstrap", "Install bootstrap components"
     subcommand "bootstrap", Porkadot::Cmd::Install::Bootstrap::Cli
 

data/lib/porkadot/config.rb

@@ -31,16 +31,15 @@ module Porkadot
       self.raw.connection
     end
 
+    def addons
+      @addons ||= Porkadot::Configs::Addons.new(self)
+    end
+
     def lb
       @lb ||= Porkadot::Configs::Lb.new(self)
       return @lb
     end
 
-    def cni
-      @cni ||= Porkadot::Configs::Cni.new(self)
-      return @cni
-    end
-
     def bootstrap
       @bootstrap ||= Porkadot::Configs::Bootstrap.new(self)
       return @bootstrap
@@ -57,6 +56,11 @@ module Porkadot
       return @etcd
     end
 
+    def kubelet_default
+      @kubelet_default ||= Porkadot::Configs::KubeletDefault.new(self)
+      return @kubelet_default
+    end
+
     def nodes
       @nodes ||= {}.tap do |nodes|
         self.raw.nodes.each do |k, v|

data/lib/porkadot/configs/addons.rb

@@ -0,0 +1,21 @@
+
+module Porkadot; module Configs
+  class Addons
+    include Porkadot::ConfigUtils
+
+    def initialize config
+      @config = config
+      @raw = config.raw.addons
+    end
+
+    def target_path
+      File.join(self.config.assets_dir, 'kubernetes', 'manifests', 'addons')
+    end
+
+    def target_secrets_path
+      File.join(self.config.secrets_root_dir, 'kubernetes', 'manifests', 'addons')
+    end
+
+  end
+end; end
+

data/lib/porkadot/configs/certs.rb

@@ -9,6 +9,9 @@ module Porkadot; module Configs
     end
 
     def ipaddr?(addr)
+      if addr.nil?
+        return false
+      end
       IPAddr.new(addr)
       return true
     rescue IPAddr::InvalidAddressError

data/lib/porkadot/configs/etcd.rb

@@ -39,6 +39,33 @@ module Porkadot; module Configs
       return (self.raw.labels && self.raw.labels[Porkadot::ETCD_ADDRESS_LABEL]) || self.raw.hostname || self.name
     end
 
+    def listen_address label_key
+      listen_address = nil
+      if self.raw.labels
+        listen_address = self.raw.labels[label_key] || self.raw.labels[Porkadot::ETCD_LISTEN_ADDRESS_LABEL]
+      end
+
+      if !listen_adress
+        if self.ipaddr?(self.raw.hostname)
+          listen_address = self.raw.hostname
+        elsif self.ipaddr?(self.raw.name)
+          listen_address = self.raw.name
+        else
+          listen_address = '0.0.0.0'
+        end
+      end
+
+      return listen_address
+    end
+
+    def listen_client_address
+      return self.listen_address(Porkadot::ETCD_LISTEN_CLIENT_ADDRESS_LABEL)
+    end
+
+    def listen_peer_address
+      return self.listen_address(Porkadot::ETCD_LISTEN_PEER_ADDRESS_LABEL)
+    end
+
     def advertise_client_urls
      ["https://#{member_address}:2379"]
     end
@@ -48,11 +75,16 @@ module Porkadot; module Configs
     end
 
     def listen_client_urls
-      self.advertise_client_urls + ["https://127.0.0.1:2379"]
+      address = self.listen_client_address
+      if address != '0.0.0.0'
+        return  ["https://#{address}:2379", "https://127.0.0.1:2379"]
+      else
+        return  ["https://#{address}:2379"]
+      end
     end
 
     def listen_peer_urls
-      self.advertise_peer_urls
+      ["https://#{self.listen_client_address}:2380"]
     end
 
     def initial_cluster
@@ -72,6 +104,7 @@ module Porkadot; module Configs
           sans << "DNS:#{san}"
         end
       end
+      sans << "IP:127.0.0.1"
       return sans
     end
 

data/lib/porkadot/configs/kubelet.rb

@@ -1,4 +1,30 @@
 module Porkadot; module Configs
+  class KubeletDefault
+    include Porkadot::ConfigUtils
+
+    def initialize config
+      @config = config
+      @raw = ::Porkadot::Raw.new
+    end
+
+    def target_path
+      File.join(self.config.assets_dir, 'kubelet-default')
+    end
+
+    def target_secrets_path
+      File.join(self.config.secrets_root_dir, 'kubelet-default')
+    end
+
+    def addon_path
+      File.join(self.target_path, 'addons')
+    end
+
+    def addon_secrets_path
+      File.join(self.target_secrets_path, 'addons')
+    end
+
+  end
+
   class Kubelet
     include Porkadot::ConfigUtils
     attr_reader :name

data/lib/porkadot/configs/kubernetes.rb

@@ -1,4 +1,3 @@
-
 module Porkadot; module Configs
   class Kubernetes
     include Porkadot::ConfigUtils
@@ -35,10 +34,6 @@ module Porkadot; module Configs
       File.join(self.target_path, 'manifests')
     end
 
-    def manifests_secrets_path
-      File.join(self.target_secrets_path, 'manifests')
-    end
-
     def control_plane_endpoint_host_and_port
       endpoint = self.config.k8s.control_plane_endpoint
       raise "kubernetes.control_plane_endpoint should not be nil" unless endpoint
@@ -196,9 +191,9 @@ module Porkadot; module Configs
           --cluster-signing-key-file=/etc/kubernetes/pki/kubernetes/ca.key
           --controllers=*,bootstrapsigner,tokencleaner
           --leader-elect=true
-          --node-cidr-mask-size=24
           --root-ca-file=/etc/kubernetes/pki/kubernetes/ca.crt
           --service-account-private-key-file=/etc/kubernetes/pki/kubernetes/sa.key
+          --service-cluster-ip-range=#{config.k8s.networking.service_subnet}
           --use-service-account-credentials=true
           --v=#{self.log_level}
         ).map {|i| i.split('=', 2)}.to_h
@@ -249,13 +244,35 @@ module Porkadot; module Configs
       end
 
       def kubernetes_ip
-        cluster_ip_range = IPAddr.new(self.service_subnet)
-        cluster_ip_range.to_range.first(2)[1].to_s
+        cluster_ip_range = IPAddr.new(self.default_service_subnet)
+        cluster_ip_range.to_range.first(2)[1]
       end
 
       def dns_ip
-        cluster_ip_range = IPAddr.new(self.service_subnet)
-        cluster_ip_range.to_range.first(11)[10].to_s
+        cluster_ip_range = IPAddr.new(self.default_service_subnet)
+        cluster_ip_range.to_range.first(11)[10]
+      end
+
+      def default_service_subnet
+        self.service_subnet.split(',')[0]
+      end
+
+      def pod_v4subnet
+        if ip = self._pod_subnet.find{ |net| net.ipv4? }
+          return "#{ip.to_s}/#{ip.prefix}"
+        end
+      end
+      alias enable_ipv4 pod_v4subnet
+
+      def pod_v6subnet
+        if ip = self._pod_subnet.find{ |net| net.ipv6? }
+          return "#{ip.to_s}/#{ip.prefix}"
+        end
+      end
+      alias enable_ipv6 pod_v6subnet
+
+      def _pod_subnet
+        self.pod_subnet.split(",").map{|net| IPAddr.new(net)}
       end
     end
   end

data/lib/porkadot/const.rb

@@ -5,4 +5,7 @@ module Porkadot
   K8S_MASTER_LABEL = "k8s.unstable.cloud/master"
   ETCD_MEMBER_LABEL = "etcd.unstable.cloud/member"
   ETCD_ADDRESS_LABEL = "etcd.unstable.cloud/address"
+  ETCD_LISTEN_ADDRESS_LABEL = "etcd.unstable.cloud/listen-address"
+  ETCD_LISTEN_CLIENT_ADDRESS_LABEL = "etcd.unstable.cloud/listen-client-address"
+  ETCD_LISTEN_PEER_ADDRESS_LABEL = "etcd.unstable.cloud/listen-client-address"
 end

data/lib/porkadot/default.yaml

@@ -10,13 +10,25 @@ nodes: {}
 
 bootstrap: {}
 
-cni:
-  type: flannel
+addons:
+  enabled: [flannel, coredns, metallb, kubelet-rubber-stamp, storage-version-migrator]
+
   flannel:
     backend: vxlan
+    plugin_image_repository: rancher/mirrored-flannelcni-flannel-cni-plugin
+    plugin_image_tag: v1.0.1
+    daemon_image_repository: rancher/mirrored-flannelcni-flannel
+    daemon_image_tag: v0.17.0
+    resources:
+      requests:
+        cpu: "100m"
+        memory: "50Mi"
+      limits:
+        cpu: "100m"
+        memory: "50Mi"
+
+  coredns: {}
 
-lb:
-  type: metallb
   metallb:
     config: |
       address-pools:
@@ -25,20 +37,26 @@ lb:
         addresses:
         - 192.168.1.240-192.168.1.250
 
+  kubelet-rubber-stamp: {}
+
+  storage-version-migrator: {}
+
 etcd:
   image_repository: gcr.io/etcd-development/etcd
   image_tag: v3.4.13
   extra_env: []
 
 kubernetes:
-  kubernetes_version: v1.21.3
+  kubernetes_version: v1.23.5
+  crictl_version: v1.23.0
   image_repository: k8s.gcr.io
 
   networking:
-    cni_version: v0.8.2
+    cni_version: v1.0.1
     service_subnet: '10.254.0.0/24'
     pod_subnet: '10.244.0.0/16'
     dns_domain: 'cluster.local'
+    additional_domains: []
 
   apiserver:
     bind_port: 6443