porkadot 0.19.1 → 0.22.2

data/lib/porkadot/assets/kubernetes/manifests/kube-apiserver.secrets.yaml.erb

@@ -11,6 +11,7 @@ data:
   kubelet-client.crt: <%= certs.kubernetes.to_base64(:kubelet_client_cert) %>
   kubelet-client.key: <%= certs.kubernetes.to_base64(:kubelet_client_key) %>
   sa.pub: <%= certs.kubernetes.to_base64(:sa_public_key) %>
+  sa.key: <%= certs.kubernetes.to_base64(:sa_private_key) %>
 kind: Secret
 metadata:
   name: kube-apiserver

data/lib/porkadot/assets/kubernetes/manifests/kube-apiserver.yaml.erb

@@ -24,6 +24,9 @@ spec:
       annotations:
         checkpointer.alpha.coreos.com/checkpoint: "true"
     spec:
+      securityContext:
+        seccompProfile:
+          type: RuntimeDefault
       containers:
       - name: kube-apiserver
         resources:
@@ -35,6 +38,35 @@ spec:
         <%- k8s.apiserver.args.each do |k, v| -%>
         - <%= k %><% if v ;%>=<%= v %><%; end %>
         <%- end -%>
+        livenessProbe:
+          failureThreshold: 8
+          httpGet:
+            host: 127.0.0.1
+            path: /livez
+            port: 6443
+            scheme: HTTPS
+          initialDelaySeconds: 10
+          periodSeconds: 10
+          timeoutSeconds: 15
+        readinessProbe:
+          failureThreshold: 3
+          httpGet:
+            host: 127.0.0.1
+            path: /readyz
+            port: 6443
+            scheme: HTTPS
+          periodSeconds: 1
+          timeoutSeconds: 15
+        startupProbe:
+          failureThreshold: 24
+          httpGet:
+            host: 127.0.0.1
+            path: /livez
+            port: 6443
+            scheme: HTTPS
+          initialDelaySeconds: 10
+          periodSeconds: 10
+          timeoutSeconds: 15
         env:
         - name: POD_IP
           valueFrom:

data/lib/porkadot/assets/kubernetes/manifests/kube-controller-manager.yaml.erb

@@ -1,6 +1,6 @@
 <% k8s = global_config.k8s -%>
 ---
-apiVersion: policy/v1beta1
+apiVersion: policy/v1
 kind: PodDisruptionBudget
 metadata:
   name: kube-controller-manager
@@ -69,6 +69,11 @@ spec:
       annotations:
         scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
+      securityContext:
+        seccompProfile:
+          type: RuntimeDefault
+        runAsNonRoot: true
+        runAsUser: 65534
       affinity:
         podAntiAffinity:
           preferredDuringSchedulingIgnoredDuringExecution:
@@ -97,10 +102,22 @@ spec:
         - <%= k %><% if v ;%>=<%= v %><%; end %>
         <%- end -%>
         livenessProbe:
+          failureThreshold: 8
           httpGet:
             path: /healthz
-            port: 10252  # Note: Using default port. Update if --port option is set differently.
-          initialDelaySeconds: 15
+            port: 10257
+            scheme: HTTPS
+          initialDelaySeconds: 10
+          periodSeconds: 10
+          timeoutSeconds: 15
+        startupProbe:
+          failureThreshold: 24
+          httpGet:
+            path: /healthz
+            port: 10257
+            scheme: HTTPS
+          initialDelaySeconds: 10
+          periodSeconds: 10
           timeoutSeconds: 15
         volumeMounts:
         - name: var-run-kubernetes
@@ -122,9 +139,6 @@ spec:
       priorityClassName: system-cluster-critical
       nodeSelector:
         k8s.unstable.cloud/master: ""
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 65534
       serviceAccountName: kube-controller-manager
       tolerations:
       - key: CriticalAddonsOnly

data/lib/porkadot/assets/kubernetes/manifests/kube-scheduler.yaml.erb

@@ -1,6 +1,6 @@
 <% k8s = global_config.k8s -%>
 ---
-apiVersion: policy/v1beta1
+apiVersion: policy/v1
 kind: PodDisruptionBudget
 metadata:
   name: kube-scheduler
@@ -113,6 +113,11 @@ spec:
       annotations:
         scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
+      securityContext:
+        seccompProfile:
+          type: RuntimeDefault
+        runAsNonRoot: true
+        runAsUser: 65534
       affinity:
         podAntiAffinity:
           preferredDuringSchedulingIgnoredDuringExecution:
@@ -141,17 +146,26 @@ spec:
         - <%= k %><% if v ;%>=<%= v %><%; end %>
         <%- end -%>
         livenessProbe:
+          failureThreshold: 8
           httpGet:
             path: /healthz
-            port: 10251  # Note: Using default port. Update if --port option is set differently.
-          initialDelaySeconds: 15
+            port: 10259
+            scheme: HTTPS
+          initialDelaySeconds: 10
+          periodSeconds: 10
+          timeoutSeconds: 15
+        startupProbe:
+          failureThreshold: 24
+          httpGet:
+            path: /healthz
+            port: 10259
+            scheme: HTTPS
+          initialDelaySeconds: 10
+          periodSeconds: 10
           timeoutSeconds: 15
       priorityClassName: system-cluster-critical
       nodeSelector:
         k8s.unstable.cloud/master: ""
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 65534
       serviceAccountName: kube-scheduler
       tolerations:
       - key: CriticalAddonsOnly

data/lib/porkadot/assets/kubernetes/manifests/kubelet.yaml.erb

@@ -15,7 +15,6 @@ roleRef:
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: auto-approve-csrs-for-group
   name: porkadot:node-autoapprove-bootstrap
 subjects:
 - kind: Group

data/lib/porkadot/assets/kubernetes/manifests/kustomization.yaml.erb

@@ -0,0 +1,8 @@
+resources:
+- addons
+- kube-apiserver.yaml
+- kube-controller-manager.yaml
+- kube-proxy.yaml
+- kube-scheduler.yaml
+- kubelet.yaml
+- porkadot.yaml

data/lib/porkadot/assets/kubernetes.rb

@@ -17,31 +17,109 @@ module Porkadot; module Assets
 
     def render
       logger.info "--> Rendering kubernetes manifests"
-      unless File.directory?(config.manifests_path)
-        FileUtils.mkdir_p(config.manifests_path)
-      end
-      unless File.directory?(config.manifests_secrets_path)
-        FileUtils.mkdir_p(config.manifests_secrets_path)
-      end
-      lb = global_config.lb
-      cni = global_config.cni
       render_erb 'manifests/porkadot.yaml'
       render_erb 'manifests/kubelet.yaml'
-      render_erb "manifests/#{lb.type}.yaml"
-      render_secrets_erb "manifests/#{lb.type}.secrets.yaml"
-      render_erb "manifests/#{cni.type}.yaml"
-      render_erb "manifests/coredns.yaml"
-      render_erb "manifests/dns-horizontal-autoscaler.yaml"
       render_erb "manifests/kube-apiserver.yaml"
       render_secrets_erb "manifests/kube-apiserver.secrets.yaml"
       render_erb "manifests/kube-proxy.yaml"
       render_erb "manifests/kube-scheduler.yaml"
       render_erb "manifests/kube-controller-manager.yaml"
       render_secrets_erb "manifests/kube-controller-manager.secrets.yaml"
-      render_erb "manifests/kubelet-rubber-stamp.yaml"
-      render_erb "manifests/storage-version-migrator.yaml"
-      render_erb 'install.sh'
+      render_secrets_erb "kubeconfig.yaml"
+      render_erb 'manifests/kustomization.yaml'
+      render_erb 'kustomization.yaml', force: false
+      render_erb 'install.sh', prune_allowlist: prune_allowlist
+      render_secrets_erb 'install.secrets.sh'
+
+      addons = Addons.new(global_config)
+      addons.render
+    end
+
+    def prune_allowlist
+      return %w[
+        apiextensions.k8s.io/v1/customresourcedefinition
+        apps/v1/daemonset
+        apps/v1/deployment
+        core/v1/configmap
+        core/v1/namespace
+        core/v1/service
+        core/v1/serviceaccount
+        policy/v1/poddisruptionbudget
+        policy/v1beta1/podsecuritypolicy
+        rbac.authorization.k8s.io/v1/clusterrole
+        rbac.authorization.k8s.io/v1/clusterrolebinding
+        rbac.authorization.k8s.io/v1/role
+        rbac.authorization.k8s.io/v1/rolebinding
+      ]
     end
+  end
+
+  class Addons
+    include Porkadot::Assets
+    TEMPLATE_DIR = File.join(File.dirname(__FILE__), "kubernetes", "manifests", "addons")
+    attr_reader :global_config
+    attr_reader :config
+    attr_reader :logger
+
+    def initialize global_config
+      @global_config = global_config
+      @config = global_config.addons
+      @logger = global_config.logger
+    end
+
+    def render
+      logger.info "--> Rendering kubernetes addons"
+      render_erb "kustomization.yaml"
+
+      self.config.enabled.each do |name|
+        manifests = @@manifests[name]
+        manifests.each do |m|
+          render_erb(m)
+        end
+        secrets = @@secrets_manifests[name]
+        secrets.each do |m|
+          render_secrets_erb(m)
+        end
+      end
+    end
+
+    def self.register_manifests name, manifests, secrets: []
+      @@manifests ||= {}
+      @@manifests[name] = manifests
+      @@secrets_manifests ||= {}
+      @@secrets_manifests[name] = secrets
+    end
+
+    register_manifests('flannel', [
+      'flannel/flannel.yaml',
+      'flannel/kustomization.yaml'
+    ])
+
+    register_manifests('coredns', [
+      'coredns/coredns.yaml',
+      'coredns/dns-horizontal-autoscaler.yaml',
+      'coredns/kustomization.yaml'
+    ])
+
+    register_manifests('metallb', [
+      'metallb/000-metallb.yaml',
+      'metallb/metallb.yaml',
+      'metallb/metallb.config.yaml',
+      'metallb/kustomization.yaml'
+    ], secrets: [
+      'metallb/metallb.secrets.yaml'
+    ])
+
+
+    register_manifests('kubelet-rubber-stamp', [
+      'kubelet-rubber-stamp/kubelet-rubber-stamp.yaml',
+      'kubelet-rubber-stamp/kustomization.yaml'
+    ])
+
+    register_manifests('storage-version-migrator', [
+      'storage-version-migrator/storage-version-migrator.yaml',
+      'storage-version-migrator/kustomization.yaml'
+    ])
 
   end
 end; end

data/lib/porkadot/assets.rb

@@ -15,7 +15,7 @@ module Porkadot::Assets
     end
   end
 
-  def render_erb file, opts={}
+  def render_erb file, **opts
     file = file.to_s
     opts[:config] = self.config
     opts[:global_config] = self.global_config
@@ -23,8 +23,15 @@ module Porkadot::Assets
     opts[:u] = ErbUtils.new
 
     logger.info "----> #{file}"
+    asset = config.asset_path(file)
+    if opts[:force] != nil && File.file?(asset)
+      logger.debug "------> Already exists: skipping #{file}"
+      return
+    end
+    asset_dir = File.dirname(asset)
+    FileUtils.mkdir_p(asset_dir) unless File.directory?(asset_dir)
     open(File.join(self.class::TEMPLATE_DIR, "#{file}.erb")) do |io|
-      open(config.asset_path(file), 'w') do |out|
+      open(asset, 'w') do |out|
         out.write ERB.new(io.read, trim_mode: '-').result_with_hash(opts)
       end
     end
@@ -38,8 +45,11 @@ module Porkadot::Assets
     opts[:u] = ErbUtils.new
 
     logger.info "----> #{file}"
+    secret = config.secrets_path(file)
+    secret_dir = File.dirname(secret)
+    FileUtils.mkdir_p(secret_dir) unless File.directory?(secret_dir)
     open(File.join(self.class::TEMPLATE_DIR, "#{file}.erb")) do |io|
-      open(config.secrets_path(file), 'w') do |out|
+      open(secret, 'w') do |out|
         out.write ERB.new(io.read, trim_mode: '-').result_with_hash(opts)
       end
     end

data/lib/porkadot/cmd/cli.rb

@@ -29,6 +29,22 @@ module Porkadot; module Cmd
       ""
     end
 
+    desc "setup-node", "Setup node default settings"
+    option :node, type: :string
+    option :force, type: :boolean, default: false
+    def setup_node
+      logger.info "Setup node default"
+      kubelets = Porkadot::Install::KubeletList.new(self.config)
+      nodes = []
+      if node = options[:node]
+        nodes = kubelets[node]
+      else
+        nodes = kubelets.kubelets.values
+      end
+      kubelets.setup_default hosts: nodes, force: options[:force]
+      ""
+    end
+
     desc "set-config", "Set cluster to kubeconfig"
     def set_config
       name = config.k8s.cluster_name

data/lib/porkadot/cmd/install.rb

@@ -26,6 +26,21 @@ module Porkadot; module Cmd; module Install
       ""
     end
 
+    desc "kubernetes", "Install kubernetes"
+    option :node, type: :string
+    def kubernetes
+      logger.info "Installing kubernetes"
+      kubelets = Porkadot::Install::KubeletList.new(self.config)
+      if node = options[:node]
+        nodes = kubelets[node]
+      else
+        nodes = Porkadot::Install::Bootstrap.new(self.config).host
+      end
+      k8s = Porkadot::Install::Kubernetes.new(self.config)
+      k8s.install(nodes)
+      ""
+    end
+
     desc "bootstrap", "Install bootstrap components"
     subcommand "bootstrap", Porkadot::Cmd::Install::Bootstrap::Cli
 

data/lib/porkadot/config.rb

@@ -31,16 +31,15 @@ module Porkadot
       self.raw.connection
     end
 
+    def addons
+      @addons ||= Porkadot::Configs::Addons.new(self)
+    end
+
     def lb
       @lb ||= Porkadot::Configs::Lb.new(self)
       return @lb
     end
 
-    def cni
-      @cni ||= Porkadot::Configs::Cni.new(self)
-      return @cni
-    end
-
     def bootstrap
       @bootstrap ||= Porkadot::Configs::Bootstrap.new(self)
       return @bootstrap
@@ -57,6 +56,11 @@ module Porkadot
       return @etcd
     end
 
+    def kubelet_default
+      @kubelet_default ||= Porkadot::Configs::KubeletDefault.new(self)
+      return @kubelet_default
+    end
+
     def nodes
       @nodes ||= {}.tap do |nodes|
         self.raw.nodes.each do |k, v|

data/lib/porkadot/configs/addons.rb

@@ -0,0 +1,21 @@
+
+module Porkadot; module Configs
+  class Addons
+    include Porkadot::ConfigUtils
+
+    def initialize config
+      @config = config
+      @raw = config.raw.addons
+    end
+
+    def target_path
+      File.join(self.config.assets_dir, 'kubernetes', 'manifests', 'addons')
+    end
+
+    def target_secrets_path
+      File.join(self.config.secrets_root_dir, 'kubernetes', 'manifests', 'addons')
+    end
+
+  end
+end; end
+

data/lib/porkadot/configs/kubelet.rb

@@ -1,4 +1,30 @@
 module Porkadot; module Configs
+  class KubeletDefault
+    include Porkadot::ConfigUtils
+
+    def initialize config
+      @config = config
+      @raw = ::Porkadot::Raw.new
+    end
+
+    def target_path
+      File.join(self.config.assets_dir, 'kubelet-default')
+    end
+
+    def target_secrets_path
+      File.join(self.config.secrets_root_dir, 'kubelet-default')
+    end
+
+    def addon_path
+      File.join(self.target_path, 'addons')
+    end
+
+    def addon_secrets_path
+      File.join(self.target_secrets_path, 'addons')
+    end
+
+  end
+
   class Kubelet
     include Porkadot::ConfigUtils
     attr_reader :name

data/lib/porkadot/configs/kubernetes.rb

@@ -1,4 +1,3 @@
-
 module Porkadot; module Configs
   class Kubernetes
     include Porkadot::ConfigUtils
@@ -35,10 +34,6 @@ module Porkadot; module Configs
       File.join(self.target_path, 'manifests')
     end
 
-    def manifests_secrets_path
-      File.join(self.target_secrets_path, 'manifests')
-    end
-
     def control_plane_endpoint_host_and_port
       endpoint = self.config.k8s.control_plane_endpoint
       raise "kubernetes.control_plane_endpoint should not be nil" unless endpoint
@@ -128,7 +123,9 @@ module Porkadot; module Configs
           --requestheader-group-headers=X-Remote-Group
           --requestheader-username-headers=X-Remote-User
           --secure-port=#{self.bind_port}
+          --service-account-issuer=https://kubernetes.default.svc#{self.config.k8s.networking.dns_domain}
           --service-account-key-file=/etc/kubernetes/pki/kubernetes/sa.pub
+          --service-account-signing-key-file=/etc/kubernetes/pki/kubernetes/sa.key
           --service-cluster-ip-range=#{config.k8s.networking.service_subnet}
           --storage-backend=etcd3
           --tls-cert-file=/etc/kubernetes/pki/kubernetes/apiserver.crt
@@ -194,9 +191,9 @@ module Porkadot; module Configs
           --cluster-signing-key-file=/etc/kubernetes/pki/kubernetes/ca.key
           --controllers=*,bootstrapsigner,tokencleaner
           --leader-elect=true
-          --node-cidr-mask-size=24
           --root-ca-file=/etc/kubernetes/pki/kubernetes/ca.crt
           --service-account-private-key-file=/etc/kubernetes/pki/kubernetes/sa.key
+          --service-cluster-ip-range=#{config.k8s.networking.service_subnet}
           --use-service-account-credentials=true
           --v=#{self.log_level}
         ).map {|i| i.split('=', 2)}.to_h
@@ -247,13 +244,35 @@ module Porkadot; module Configs
       end
 
       def kubernetes_ip
-        cluster_ip_range = IPAddr.new(self.service_subnet)
-        cluster_ip_range.to_range.first(2)[1].to_s
+        cluster_ip_range = IPAddr.new(self.default_service_subnet)
+        cluster_ip_range.to_range.first(2)[1]
       end
 
       def dns_ip
-        cluster_ip_range = IPAddr.new(self.service_subnet)
-        cluster_ip_range.to_range.first(11)[10].to_s
+        cluster_ip_range = IPAddr.new(self.default_service_subnet)
+        cluster_ip_range.to_range.first(11)[10]
+      end
+
+      def default_service_subnet
+        self.service_subnet.split(',')[0]
+      end
+
+      def pod_v4subnet
+        if ip = self._pod_subnet.find{ |net| net.ipv4? }
+          return "#{ip.to_s}/#{ip.prefix}"
+        end
+      end
+      alias enable_ipv4 pod_v4subnet
+
+      def pod_v6subnet
+        if ip = self._pod_subnet.find{ |net| net.ipv6? }
+          return "#{ip.to_s}/#{ip.prefix}"
+        end
+      end
+      alias enable_ipv6 pod_v6subnet
+
+      def _pod_subnet
+        self.pod_subnet.split(",").map{|net| IPAddr.new(net)}
       end
     end
   end

data/lib/porkadot/default.yaml

@@ -10,13 +10,25 @@ nodes: {}
 
 bootstrap: {}
 
-cni:
-  type: flannel
+addons:
+  enabled: [flannel, coredns, metallb, kubelet-rubber-stamp, storage-version-migrator]
+
   flannel:
     backend: vxlan
+    plugin_image_repository: rancher/mirrored-flannelcni-flannel-cni-plugin
+    plugin_image_tag: v1.0.1
+    daemon_image_repository: rancher/mirrored-flannelcni-flannel
+    daemon_image_tag: v0.17.0
+    resources:
+      requests:
+        cpu: "100m"
+        memory: "50Mi"
+      limits:
+        cpu: "100m"
+        memory: "50Mi"
+
+  coredns: {}
 
-lb:
-  type: metallb
   metallb:
     config: |
       address-pools:
@@ -25,20 +37,26 @@ lb:
         addresses:
         - 192.168.1.240-192.168.1.250
 
+  kubelet-rubber-stamp: {}
+
+  storage-version-migrator: {}
+
 etcd:
   image_repository: gcr.io/etcd-development/etcd
-  image_tag: v3.4.3
+  image_tag: v3.4.13
   extra_env: []
 
 kubernetes:
-  kubernetes_version: v1.19.6
+  kubernetes_version: v1.22.8
+  crictl_version: v1.22.0
   image_repository: k8s.gcr.io
 
   networking:
-    cni_version: v0.8.2
+    cni_version: v1.0.1
     service_subnet: '10.254.0.0/24'
     pod_subnet: '10.244.0.0/16'
     dns_domain: 'cluster.local'
+    additional_domains: []
 
   apiserver:
     bind_port: 6443

data/lib/porkadot/install/kubelet.rb

@@ -2,6 +2,7 @@ module Porkadot; module Install
   class KubeletList
     KUBE_TEMP = File.join(Porkadot::Install::KUBE_TEMP, 'kubelet')
     KUBE_SECRETS_TEMP = File.join(Porkadot::Install::KUBE_TEMP, '.kubelet')
+    KUBE_DEFAULT_TEMP = File.join(Porkadot::Install::KUBE_TEMP, '.default')
     include SSHKit::DSL
     attr_reader :global_config
     attr_reader :logger
@@ -40,6 +41,30 @@ module Porkadot; module Install
       end
     end
 
+    def setup_default hosts: nil, force: false
+      unless hosts
+        hosts = []
+        self.kubelets.each do |_, v|
+          hosts << v
+        end
+      end
+
+      on(hosts) do |host|
+        execute(:mkdir, '-p', Porkadot::Install::KUBE_TEMP)
+        if test("[ -d #{KUBE_TEMP} ]")
+          execute(:rm, '-rf', KUBE_TEMP)
+          execute(:rm, '-rf', KUBE_SECRETS_TEMP)
+        end
+        upload! host.global_config.kubelet_default.target_path, KUBE_TEMP, recursive: true
+        upload! host.global_config.kubelet_default.target_secrets_path, KUBE_SECRETS_TEMP, recursive: true
+        execute(:cp, '-r', KUBE_SECRETS_TEMP + '/*', KUBE_TEMP)
+
+        as user: 'root' do
+          execute(:bash, File.join(KUBE_TEMP, 'install.sh'))
+        end
+      end
+    end
+
     def install hosts: nil, force: false
       unless hosts
         hosts = []

data/lib/porkadot/install/kubernetes.rb

@@ -24,9 +24,10 @@ module Porkadot; module Install
         end
         upload! config.target_path, KUBE_TEMP, recursive: true
         upload! config.target_secrets_path, KUBE_SECRETS_TEMP, recursive: true
-        execute(:cp, '-r', KUBE_SECRETS_TEMP + '/*', KUBE_TEMP)
 
-        as user: 'root' do
+        # as user: 'root' do
+        with KUBECONFIG: File.join(KUBE_SECRETS_TEMP, 'kubeconfig.yaml') do
+          execute(:bash, File.join(KUBE_SECRETS_TEMP, 'install.secrets.sh'))
           execute(:bash, File.join(KUBE_TEMP, 'install.sh'))
         end
       end

data/lib/porkadot/version.rb

@@ -1,3 +1,3 @@
 module Porkadot
-  VERSION = "0.19.1"
+  VERSION = "0.22.2"
 end

data/lib/porkadot.rb

@@ -20,8 +20,7 @@ require 'porkadot/configs/kubernetes'
 require 'porkadot/configs/etcd'
 require 'porkadot/configs/bootstrap'
 require 'porkadot/configs/kubernetes'
-require 'porkadot/configs/loadbalancer'
-require 'porkadot/configs/cni'
+require 'porkadot/configs/addons'
 
 require 'porkadot/assets/certs'
 require 'porkadot/assets/kubelet'