porkadot 0.23.0 → 0.25.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8e2a062cd96fa6e9c56b2fd70f7d0dd4709265e1b5cf58057048cc19bf46868e
-  data.tar.gz: 4a18c93e458b1a822fe35b73d7af0b12804d4d4c174cbb8a3efc0daf30bbbc1e
+  metadata.gz: 91ee01d34fb9504cf2c2da60cc375102d212a88adfa83d5335ba07efaf1dc126
+  data.tar.gz: 88cc5cf3c93e0bf12d41a558a1d92e2fb62a86ea7824282e55d2eb66c9fc029b
 SHA512:
-  metadata.gz: fef441fe9dc698fa5e993ae9b7d5a4e6270590aa2f8fcdbb3bca4601266faed8d6b5c96f545d3347716915bdbe0f78ebc1898caef201c1be50cc006955dec44d
-  data.tar.gz: a7bfadba85de2c3d631ebab8f74c4afaa75fcfdabeccbae6d20d4f23e2817185efc7d4174d4e901e19da8dbabcf4bdf522e12706dbc025113e6b0480d9b32826
+  metadata.gz: ad4c075b775c38030b2f96a32d09a2a28a3eb10eba0c4bb96109ee997a3c26d4c3e11ef3e37dbffc41f3c63d158668242a14493002ec2bcc9b7485886eb2b62b
+  data.tar.gz: ae0f9beb775bea1601fc7189a5aff026592c88567bf5dc9be32a82068849d48da451c06fd0c56c7250c371a4ff4110d92187c00a6257a9df199751e1e6c1328d

data/hack/metallb/crds/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- https://github.com/metallb/metallb/config/crd/?ref=v0.13.9

data/hack/metallb/exclude-l2-config.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: metallb-excludel2
+  namespace: metallb-system
+data:
+  excludel2.yaml: |
+    announcedInterfacesToExclude: ["docker.*", "cbr.*", "dummy.*", "virbr.*", "lxcbr.*", "veth.*", "lo", "^cali.*", "^tunl.*", "flannel.*", "kube-ipvs.*", "cni.*", "^nodelocaldns.*"]

data/hack/metallb/kustomization.yaml

@@ -0,0 +1,10 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: metallb-system
+
+resources:
+- https://github.com/metallb/metallb/config/rbac/?ref=v0.13.9
+- https://github.com/metallb/metallb/config/controllers/?ref=v0.13.9
+- https://github.com/metallb/metallb/config/webhook/?ref=v0.13.9
+- exclude-l2-config.yaml

data/hack/update-kubelet-cert-approver.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+ROOT=$(dirname "${BASH_SOURCE}")/..
+ROOT=$(cd ${ROOT} && pwd)
+
+curl -L https://github.com/alex1989hu/kubelet-serving-cert-approver/raw/main/deploy/standalone-install.yaml > ${ROOT}/lib/porkadot/assets/kubernetes/manifests/addons/kubelet-serving-cert-approver/src.yaml.erb

data/hack/update-metallb.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+ROOT=$(dirname "${BASH_SOURCE}")/..
+ROOT=$(cd ${ROOT} && pwd)
+
+kubectl kustomize ${ROOT}/hack/metallb/crds > ${ROOT}/lib/porkadot/assets/kubernetes/manifests/addons/metallb/crds.yaml
+kubectl kustomize ${ROOT}/hack/metallb > ${ROOT}/lib/porkadot/assets/kubernetes/manifests/addons/metallb/metallb.yaml.erb

data/lib/porkadot/assets/bootstrap/manifests/kube-apiserver.bootstrap.yaml.erb

@@ -46,7 +46,7 @@ spec:
       periodSeconds: 1
       timeoutSeconds: 15
     startupProbe:
-      failureThreshold: 24
+      failureThreshold: 48
       httpGet:
         host: 127.0.0.1
         path: /livez

data/lib/porkadot/assets/etcd/etcd-server.yaml.erb

@@ -13,23 +13,27 @@ spec:
     image: <%= etcd.image_repository %>:<%= etcd.image_tag %>
     command:
     - /usr/local/bin/etcd
-    - --name=<%= config.member_name %>
     - --advertise-client-urls=<%= config.advertise_client_urls.join(',') %>
+    - --cert-file=/etc/etcd/pki/etcd.crt
+    - --client-cert-auth=true
+    - --data-dir=/var/lib/etcd
+    - --election-timeout=10000
+    - --experimental-initial-corrupt-check=true
+    - --experimental-watch-progress-notify-interval=5s
+    - --heartbeat-interval=1000
     - --initial-advertise-peer-urls=<%= config.advertise_peer_urls.join(',') %>
     - --initial-cluster=<%= config.initial_cluster.map{|k,v| "#{k}=#{v}"}.join(',') %>
+    - --key-file=/etc/etcd/pki/etcd.key
     - --listen-client-urls=<%= config.listen_client_urls.join(',') %>
     - --listen-peer-urls=<%= config.listen_peer_urls.join(',') %>
-    - --client-cert-auth=true
-    - --cert-file=/etc/etcd/pki/etcd.crt
-    - --key-file=/etc/etcd/pki/etcd.key
-    - --trusted-ca-file=/etc/etcd/pki/ca.crt
-    - --peer-client-cert-auth=true
+    - --listen-metrics-urls=<%= config.listen_metrics_urls.join(',') %>
+    - --name=<%= config.member_name %>
     - --peer-cert-file=/etc/etcd/pki/etcd.crt
+    - --peer-client-cert-auth=true
     - --peer-key-file=/etc/etcd/pki/etcd.key
     - --peer-trusted-ca-file=/etc/etcd/pki/ca.crt
-    - --data-dir=/var/lib/etcd
-    - --heartbeat-interval=1000
-    - --election-timeout=10000
+    - --snapshot-count=10000
+    - --trusted-ca-file=/etc/etcd/pki/ca.crt
     env:
 <%= u.to_yaml(etcd.extra_env, 4) -%>
     volumeMounts:
@@ -44,6 +48,10 @@ spec:
     - mountPath: /etc/etcd/pki
       name: etcd-certs-host
       readOnly: true
+  priorityClassName: system-node-critical
+  securityContext:
+    seccompProfile:
+      type: RuntimeDefault
   volumes:
   - hostPath:
       path: /var/lib/etcd

data/lib/porkadot/assets/etcd/etcd.env.erb

@@ -0,0 +1,4 @@
+export ETCDCTL_CACERT=/etc/etcd/pki/ca.crt
+export ETCDCTL_CERT=/etc/etcd/pki/etcd.crt
+export ETCDCTL_KEY=/etc/etcd/pki/etcd.key
+export ETCDCTL_ENDPOINTS=https://127.0.0.1:2379

data/lib/porkadot/assets/etcd/install.sh.erb

@@ -8,5 +8,6 @@ mkdir -p /etc/etcd/pki
 cp ${ROOT}/etcd.crt /etc/etcd/pki/
 cp ${ROOT}/etcd.key /etc/etcd/pki/
 cp ${ROOT}/ca.crt /etc/etcd/pki/
+cp ${ROOT}/etcd.env /etc/etcd/
 mkdir -p /etc/kubernetes/manifests
 cp ${ROOT}/etcd-server.yaml /etc/kubernetes/manifests/

data/lib/porkadot/assets/etcd.rb

@@ -56,6 +56,7 @@ module Porkadot; module Assets
       render_ca_crt
       render_etcd_crt
       render_erb 'etcd-server.yaml', etcd: global_config.etcd
+      render_erb 'etcd.env', etcd: global_config.etcd
       render_erb 'install.sh', etcd: global_config.etcd
     end
 

data/lib/porkadot/assets/kubelet/config.yaml.erb

@@ -1,39 +1 @@
-apiVersion: kubelet.config.k8s.io/v1beta1
-authentication:
-  anonymous:
-    enabled: false
-  webhook:
-    cacheTTL: 0s
-    enabled: true
-  x509:
-    clientCAFile: /etc/kubernetes/pki/ca.crt
-authorization:
-  mode: Webhook
-  webhook:
-    cacheAuthorizedTTL: 0s
-    cacheUnauthorizedTTL: 0s
-cgroupDriver: systemd
-clusterDNS:
-  - <%= global_config.k8s.networking.dns_ip %>
-clusterDomain: <%= global_config.k8s.networking.dns_domain %>
-cpuManagerReconcilePeriod: 0s
-evictionPressureTransitionPeriod: 0s
-fileCheckFrequency: 0s
-healthzBindAddress: 127.0.0.1
-healthzPort: 10248
-httpCheckFrequency: 0s
-imageMinimumGCAge: 0s
-kind: KubeletConfiguration
-nodeStatusReportFrequency: 0s
-nodeStatusUpdateFrequency: 0s
-rotateCertificates: true
-runtimeRequestTimeout: 0s
-staticPodPath: /etc/kubernetes/manifests
-streamingConnectionIdleTimeout: 0s
-syncFrequency: 0s
-volumeStatsAggPeriod: 0s
-serverTLSBootstrap: true
-featureGates:
-  CSIMigration: false
-
-# vim:filetype=yaml
+<%= config.kubelet_config.to_hash.to_yaml %>

data/lib/porkadot/assets/kubelet/initiatorname.iscsi.erb

@@ -0,0 +1 @@
+InitiatorName=iqn.2020-04.cloud.unstable:<%= config.hostname %>

data/lib/porkadot/assets/kubelet/kubelet.service.erb

@@ -5,18 +5,14 @@ Documentation=http://kubernetes.io/docs/
 [Service]
 EnvironmentFile=-/etc/default/kubelet
 ExecStart=/opt/bin/kubelet \
-    --container-runtime=remote \
     --container-runtime-endpoint=/run/containerd/containerd.sock \
     --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \
     --kubeconfig=/etc/kubernetes/kubelet.conf \
     --config=/var/lib/kubelet/config.yaml \
-    --network-plugin=cni \
-    --pod-infra-container-image=k8s.gcr.io/pause:3.4.1 \
     --hostname-override=<%= config.hostname %> \
-    --node-labels=<%= config.labels_string %> \
-    --register-with-taints=<%= config.taints_string %> \
-    --resolv-conf=/run/systemd/resolve/resolv.conf
+    --node-labels=<%= config.labels_string %>
 Restart=always
+
 StartLimitInterval=0
 RestartSec=10
 

data/lib/porkadot/assets/kubelet/metadata.json.erb

@@ -0,0 +1,5 @@
+{
+  "name": "<%= config.name %>",
+  "labels": <%= config.labels.to_json %>,
+  "annotations": <%= config.annotations.to_json %>
+}

data/lib/porkadot/assets/{kubelet → kubelet-default}/install-deps.sh.erb

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -euo pipefail
+
 architecture="arm64"
 case $(uname -m) in
     x86_64) architecture="amd64" ;;
@@ -28,7 +30,7 @@ rm -f /opt/bin/kubelet
 ln -s /opt/bin/kubelet-${RELEASE} /opt/bin/kubelet
 
 ETCD_VER="<%= global_config.etcd.image_tag.gsub(/\-\w+$/, '') %>"
-ETCD_URL=https://storage.googleapis.com/etcd/${ETCD_VER}/etcd-${ETCD_VER}-linux-${architecture}.tar.gz
+ETCD_URL=https://github.com/etcd-io/etcd/releases/download/v${ETCD_VER}/etcd-v${ETCD_VER}-linux-${architecture}.tar.gz
 ETCD_TMP=$(mktemp -d)
 
 curl -L ${ETCD_URL} -o ${ETCD_TMP}/etcd.tar.gz

data/lib/porkadot/assets/{kubelet → kubelet-default}/install-pkgs.sh.erb

@@ -41,9 +41,7 @@ net.ipv4.ip_forward                 = 1
 net.bridge.bridge-nf-call-iptables  = 1
 EOF
 
-cat <<EOF > /etc/iscsi/initiatorname.iscsi
-InitiatorName=iqn.2020-04.cloud.unstable:<%= config.hostname %>
-EOF
+cp ${ROOT}/initiatorname.iscsi /etc/iscsi/
 
 systemctl restart iscsid.service
 

data/lib/porkadot/assets/kubelet-default/install.sh.erb

@@ -4,11 +4,26 @@ set -eu
 export LC_ALL=C
 ROOT=$(dirname "${BASH_SOURCE}")
 
+export KUBERNETES_PATH="/etc/kubernetes"
+export KUBERNETES_PKI_PATH="${KUBERNETES_PATH}/pki"
+export KUBERNETES_MANIFESTS_PATH="${KUBERNETES_PATH}/manifests"
+export KUBELET_PATH="/var/lib/kubelet"
+
+mkdir -p ${KUBERNETES_PATH}
+mkdir -p ${KUBERNETES_PKI_PATH}
+mkdir -p ${KUBERNETES_MANIFESTS_PATH}
+mkdir -p ${KUBELET_PATH}
+
+cp ${ROOT}/bootstrap-kubelet.conf ${KUBERNETES_PATH}/
+cp ${ROOT}/bootstrap.* ${KUBERNETES_PKI_PATH}/
+cp ${ROOT}/ca.crt ${KUBERNETES_PKI_PATH}/
+cp ${ROOT}/config.yaml ${KUBELET_PATH}/
+cp ${ROOT}/kubelet.service /etc/systemd/system/
+
 # Install addons
-for addon in $(ls ${ROOT}/addons/); do
-  install_sh="${ROOT}/addons/${addon}/install.sh"
-  if [[ -f ${install_sh} ]]; then
-    echo "Install: ${install_sh}"
-    bash ${install_sh}
-  fi
-done
+bash ${ROOT}/setup-node.sh
+
+rm -f ${KUBERNETES_PATH}/kubelet.conf
+systemctl daemon-reload
+systemctl enable kubelet
+systemctl restart kubelet

data/lib/porkadot/assets/kubelet-default/setup-containerd.sh.erb

@@ -0,0 +1,22 @@
+#!/bin/bash
+set -eu
+export LC_ALL=C
+ROOT=$(dirname "${BASH_SOURCE}")
+
+mkdir -p /etc/containerd
+
+if [[ -f ${ROOT}/containerd/config.toml ]]; then
+  cp -rp ${ROOT}/containerd/config.toml /etc/containerd/config.toml
+else
+  containerd config default | tee /etc/containerd/config.toml
+
+  grep SystemdCgroup /etc/containerd/config.toml && :
+
+  if [[ $? == 0 ]]; then
+    sed -i -e "s/SystemdCgroup.*$/SystemdCgroup = true/" /etc/containerd/config.toml
+  else
+    sed -i -e "/containerd.runtimes.runc.options/a SystemdCgroup = true" /etc/containerd/config.toml
+  fi
+fi
+
+systemctl restart containerd

data/lib/porkadot/assets/kubelet-default/setup-node.sh.erb

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -eu
+export LC_ALL=C
+ROOT=$(dirname "${BASH_SOURCE}")
+
+export PORKADOT_METADATA=${ROOT}/metadata.json
+
+# Install addons
+for addon in $(ls ${ROOT}/addons/); do
+  install_sh="${ROOT}/addons/${addon}/install.sh"
+  if [[ -f ${install_sh} ]]; then
+    echo "Install: ${install_sh}"
+    bash ${install_sh}
+  fi
+done

data/lib/porkadot/assets/kubelet.rb

@@ -57,7 +57,19 @@ module Porkadot; module Assets
         FileUtils.mkdir_p(config.addon_secrets_path)
       end
 
+      render_ca_crt
+      render_erb 'setup-node.sh'
+      render_erb 'setup-containerd.sh'
       render_erb 'install.sh'
+      render_erb 'install-deps.sh'
+      render_erb 'install-pkgs.sh'
+    end
+
+    def render_ca_crt
+      logger.info "----> ca.crt"
+      open(config.ca_crt_path, 'w') do |out|
+        out.write self.certs.ca_cert(false).to_pem
+      end
     end
   end
 
@@ -92,11 +104,8 @@ module Porkadot; module Assets
       render_bootstrap_certs
       render_erb 'config.yaml'
       render_erb 'kubelet.service'
-      render_ca_crt
-      render_erb 'install.sh'
-      render_erb 'install-deps.sh'
-      render_erb 'install-pkgs.sh'
-      render_erb 'setup-containerd.sh'
+      render_erb 'initiatorname.iscsi'
+      render_erb 'metadata.json'
     end
 
     def render_bootstrap_certs
@@ -105,13 +114,6 @@ module Porkadot; module Assets
       self.bootstrap_cert(true)
     end
 
-    def render_ca_crt
-      logger.info "----> ca.crt"
-      open(config.ca_crt_path, 'w') do |out|
-        out.write self.certs.ca_cert(false).to_pem
-      end
-    end
-
     def bootstrap_key
       @bootstrap_key ||= certs.private_key(config.bootstrap_key_path)
       return @bootstrap_key

data/lib/porkadot/assets/kubernetes/install.sh.erb

@@ -5,10 +5,12 @@ export LC_ALL=C
 ROOT=$(dirname "${BASH_SOURCE}")
 KUBECTL_OPTS=${KUBECTL_OPTS:-""}
 
-KUBECTL_OPTS="${KUBECTL_OPTS} --server-side --force-conflicts --prune"
+KUBECTL_OPTS="${KUBECTL_OPTS} --prune"
 KUBECTL_OPTS="${KUBECTL_OPTS} -l kubernetes.unstable.cloud/installed-by=porkadot"
 <%- prune_allowlist.each do |a| -%>
 KUBECTL_OPTS="${KUBECTL_OPTS} --prune-whitelist=<%= a %>"
 <%- end -%>
 
+/opt/bin/kubectl apply --force-conflicts --server-side -R -f ${ROOT}/manifests/crds
+/opt/bin/kubectl wait --for condition=established --timeout=60s crd --all
 /opt/bin/kubectl apply ${KUBECTL_OPTS} -k ${ROOT}

data/lib/porkadot/assets/kubernetes/manifests/addons/coredns/coredns.yaml.erb

@@ -126,7 +126,7 @@ spec:
         kubernetes.io/os: linux
       containers:
       - name: coredns
-        image: k8s.gcr.io/coredns/coredns:v1.8.3
+        image: registry.k8s.io/coredns/coredns:v1.10.1
         imagePullPolicy: IfNotPresent
         resources:
           limits:

data/lib/porkadot/assets/kubernetes/manifests/addons/coredns/dns-horizontal-autoscaler.yaml.erb

@@ -84,7 +84,7 @@ spec:
         fsGroup: 65534
       containers:
       - name: autoscaler
-        image: k8s.gcr.io/cluster-proportional-autoscaler-amd64:1.7.1
+        image: registry.k8s.io/cluster-proportional-autoscaler-amd64:1.7.1
         resources:
             requests:
                 cpu: "20m"

data/lib/porkadot/assets/kubernetes/manifests/addons/flannel/flannel.yaml.erb

@@ -1,62 +1,13 @@
 <% cni = config.flannel -%>
 <% k8s = global_config.k8s -%>
 ---
-apiVersion: policy/v1beta1
-kind: PodSecurityPolicy
-metadata:
-  name: psp.flannel.unprivileged
-  annotations:
-    seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default
-    seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default
-    apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default
-    apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default
-spec:
-  privileged: false
-  volumes:
-  - configMap
-  - secret
-  - emptyDir
-  - hostPath
-  allowedHostPaths:
-  - pathPrefix: "/etc/cni/net.d"
-  - pathPrefix: "/etc/kube-flannel"
-  - pathPrefix: "/run/flannel"
-  readOnlyRootFilesystem: false
-  # Users and groups
-  runAsUser:
-    rule: RunAsAny
-  supplementalGroups:
-    rule: RunAsAny
-  fsGroup:
-    rule: RunAsAny
-  # Privilege Escalation
-  allowPrivilegeEscalation: false
-  defaultAllowPrivilegeEscalation: false
-  # Capabilities
-  allowedCapabilities: ['NET_ADMIN', 'NET_RAW']
-  defaultAddCapabilities: []
-  requiredDropCapabilities: []
-  # Host namespaces
-  hostPID: false
-  hostIPC: false
-  hostNetwork: true
-  hostPorts:
-  - min: 0
-    max: 65535
-  # SELinux
-  seLinux:
-    # SELinux is unused in CaaSP
-    rule: 'RunAsAny'
----
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
+  labels:
+    k8s-app: flannel
   name: flannel
 rules:
-- apiGroups: ['extensions']
-  resources: ['podsecuritypolicies']
-  verbs: ['use']
-  resourceNames: ['psp.flannel.unprivileged']
 - apiGroups:
   - ""
   resources:
@@ -68,6 +19,7 @@ rules:
   resources:
   - nodes
   verbs:
+  - get
   - list
   - watch
 - apiGroups:
@@ -80,6 +32,8 @@ rules:
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
+  labels:
+    k8s-app: flannel
   name: flannel
 roleRef:
   apiGroup: rbac.authorization.k8s.io
@@ -93,6 +47,8 @@ subjects:
 apiVersion: v1
 kind: ServiceAccount
 metadata:
+  labels:
+    k8s-app: flannel
   name: flannel
   namespace: kube-system
 ---
@@ -103,6 +59,7 @@ metadata:
   namespace: kube-system
   labels:
     tier: node
+    k8s-app: flannel
     app: flannel
 data:
   cni-conf.json: |
@@ -134,6 +91,7 @@ data:
       "EnableIPv6": true,
       "IPv6Network": "<%= k8s.networking.pod_v6subnet %>",
       <%- end -%>
+      "EnableNFTables": false,
       "Backend": {
         "Type": "<%= cni.backend %>"
       }
@@ -147,6 +105,7 @@ metadata:
   labels:
     tier: node
     app: flannel
+    k8s-app: flannel
 spec:
   selector:
     matchLabels:
@@ -221,6 +180,8 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
+        - name: EVENT_QUEUE_DEPTH
+          value: "5000"
         volumeMounts:
         - name: run
           mountPath: /run/flannel

data/lib/porkadot/assets/kubernetes/manifests/addons/kubelet-serving-cert-approver/kustomization.yaml.erb

@@ -0,0 +1,3 @@
+resources:
+- src.yaml
+