porkadot 0.2.2 → 0.19.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 958ab6006bc337cbefb89951fcc80af8f32d4799dcf6f7090a68c7901743d2df
-  data.tar.gz: 1b9b98c07b61d6d3ed29879e81983a65637228be7ca494d4cff45a34cd64d08e
+  metadata.gz: 834e1f31cbbf8c7c8766162945572512fc0311dbf772df008f85ef2a00b3ea3d
+  data.tar.gz: f453e7a4899673f08a550b69c41b30c908eb5bf0906a67eee79a21ccd5fc1dcd
 SHA512:
-  metadata.gz: bd2e2d802c35ace23dc60f202b8314974936b77a0b00c94a2013303b9dfd13519a0d673ee4fb2ef322e1d8d97f751db6d355ea9cf6e789219059c3028085cbe3
-  data.tar.gz: a4291b924ae9f280b2beaee9fd7cb2ed1c711cd64a1e6c11061a5b26477de5b9913c7abc9d2baaa88d6e402d0f585aec7b8bad888eaa19c16af7697cc3a57015
+  metadata.gz: e13576f10e90eb2d277302bfcfe7cecb7feb681d25070052c50b6060b95e0b455061d093d8ddde84a69c650e2fa6f2b3775e25913cf743023c2ee036bdef0764
+  data.tar.gz: 5714fefdd57b9683974ea42f6d111ac87b66f723221f5f0852c95b758d92cf19f6eac32c9ddd2e98e1f77dcbac4db70c735eddf4999cb39be71ad2717efb5d5d

data/hack/gen-storage-version-migrator.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -eu
+export LC_ALL=C
+ROOT=$(dirname "${BASH_SOURCE}")
+
+kustomize build ${ROOT}/storage-version-migrator | sed -e "s/NAMESPACE/kube-system/g" > ${ROOT}/../lib/porkadot/assets/kubernetes/manifests/storage-version-migrator.yaml.erb

data/hack/storage-version-migrator/kustomization.yaml

@@ -0,0 +1,77 @@
+namespace: kube-system
+
+resources:
+- https://github.com/kubernetes-sigs/kube-storage-version-migrator/manifests/?ref=acdee30ced218b79e39c6a701985e8cd8bd33824
+
+images:
+- name: REGISTRY/storage-version-migration-initializer:VERSION
+  newName: asia.gcr.io/k8s-artifacts-prod/storage-migrator/storage-version-migration-initializer
+  newTag: v0.0.3
+- name: REGISTRY/storage-version-migration-migrator:VERSION
+  newName: asia.gcr.io/k8s-artifacts-prod/storage-migrator/storage-version-migration-migrator
+  newTag: v0.0.3
+- name: REGISTRY/storage-version-migration-trigger:VERSION
+  newName: asia.gcr.io/k8s-artifacts-prod/storage-migrator/storage-version-migration-trigger
+  newTag: v0.0.3
+patchesJson6902:
+- target:
+    group: apps
+    version: v1
+    kind: Deployment
+    name: migrator
+    namespace: kube-system
+  patch: |-
+    - op: remove
+      path: /spec/template/spec/containers/0/livenessProbe
+    - op: add
+      path: /spec/template/spec/containers/0/command/-
+      value: --kubeconfig=/etc/migrator/kubeconfig
+- target:
+    group: apps
+    version: v1
+    kind: Deployment
+    name: trigger
+    namespace: kube-system
+  patch: |-
+    - op: remove
+      path: /spec/template/spec/containers/0/livenessProbe
+    - op: add
+      path: /spec/template/spec/containers/0/args
+      value: ["--kubeconfig=/etc/migrator/kubeconfig"]
+patchesStrategicMerge:
+- |-
+  apiVersion: apps/v1
+  kind: Deployment
+  metadata:
+    name: migrator
+    namespace: NAMESPACE
+  spec:
+    template:
+      spec:
+        containers:
+        - name: migrator
+          volumeMounts:
+          - mountPath: /etc/migrator
+            name: kubeconfig
+        volumes:
+        - name: kubeconfig
+          configMap:
+            name: kubeconfig-in-cluster-latest
+- |-
+  apiVersion: apps/v1
+  kind: Deployment
+  metadata:
+    name: trigger
+    namespace: NAMESPACE
+  spec:
+    template:
+      spec:
+        containers:
+        - name: trigger
+          volumeMounts:
+          - mountPath: /etc/migrator
+            name: kubeconfig
+        volumes:
+        - name: kubeconfig
+          configMap:
+            name: kubeconfig-in-cluster-latest

data/lib/porkadot/assets.rb

@@ -4,6 +4,15 @@ module Porkadot::Assets
       space = space.times.map{' '}.join('')
       text.lines.map{|line| "#{space}#{line}"}.join('')
     end
+
+    def to_yaml(obj, space=0)
+      h = Hashie::Mash.new({obj: obj})
+      h = h.to_hash
+      if h['obj'].size == 0
+        return ''
+      end
+      return self.indent(h['obj'].to_yaml(canonical: false, header: false).gsub(/---\n/, ''), space)
+    end
   end
 
   def render_erb file, opts={}

data/lib/porkadot/assets/bootstrap/manifests/kube-apiserver.bootstrap.yaml.erb

@@ -20,35 +20,9 @@ spec:
     image: <%= k8s.image_repository %>/kube-apiserver:<%= k8s.kubernetes_version %>
     command:
     - kube-apiserver
-    - --advertise-address=$(POD_IP)
-    - --allow-privileged
-    - --authorization-mode=Node,RBAC
-    - --bind-address=0.0.0.0
-    - --client-ca-file=/etc/kubernetes/secrets/kubernetes/ca.crt
-    - --enable-admission-plugins=NodeRestriction
-    - --enable-bootstrap-token-auth=true
-    - --etcd-cafile=/etc/kubernetes/secrets/etcd/ca.crt
-    - --etcd-certfile=/etc/kubernetes/secrets/etcd/etcd-client.crt
-    - --etcd-keyfile=/etc/kubernetes/secrets/etcd/etcd-client.key
-    - --etcd-servers=<%= global_config.etcd.advertise_client_urls.join(',') %>
-    - --kubelet-certificate-authority=/etc/kubernetes/secrets/kubernetes/ca.crt
-    - --kubelet-client-certificate=/etc/kubernetes/secrets/kubernetes/kubelet-client.crt
-    - --kubelet-client-key=/etc/kubernetes/secrets/kubernetes/kubelet-client.key
-    - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
-    - --proxy-client-cert-file=/etc/kubernetes/secrets/kubernetes/front-proxy-client.crt
-    - --proxy-client-key-file=/etc/kubernetes/secrets/kubernetes/front-proxy-client.key
-    - --requestheader-allowed-names=front-proxy-client
-    - --requestheader-client-ca-file=/etc/kubernetes/secrets/kubernetes/front-proxy-ca.crt
-    - --requestheader-extra-headers-prefix=X-Remote-Extra-
-    - --requestheader-group-headers=X-Remote-Group
-    - --requestheader-username-headers=X-Remote-User
-    - --secure-port=<%= k8s.apiserver.bind_port %>
-    - --service-account-key-file=/etc/kubernetes/secrets/kubernetes/sa.pub
-    - --service-cluster-ip-range=<%= k8s.networking.service_subnet %>
-    - --storage-backend=etcd3
-    - --tls-cert-file=/etc/kubernetes/secrets/kubernetes/apiserver.crt
-    - --tls-private-key-file=/etc/kubernetes/secrets/kubernetes/apiserver.key
-    - --v=2
+    <%- k8s.apiserver.args(bootstrap: true).each do |k, v| -%>
+    - <%= k %><% if v ;%>=<%= v %><%; end %>
+    <%- end -%>
     env:
     - name: POD_IP
       valueFrom:
@@ -64,7 +38,7 @@ spec:
     - mountPath: /usr/share/ca-certificates
       name: usr-share-ca-certificates
       readOnly: true
-    - mountPath: /etc/kubernetes/secrets
+    - mountPath: /etc/kubernetes/pki
       name: secrets
       readOnly: true
     - mountPath: /var/lock

data/lib/porkadot/assets/bootstrap/manifests/kube-controller-manager.bootstrap.yaml.erb

@@ -15,23 +15,17 @@ spec:
     image: <%= k8s.image_repository %>/kube-controller-manager:<%= k8s.kubernetes_version %>
     command:
     - kube-controller-manager
-    - --allocate-node-cidrs=true
-    - --cluster-cidr=<%= k8s.networking.pod_subnet %>
-    - --cluster-signing-cert-file=/etc/kubernetes/bootstrap/secrets/kubernetes/ca.crt
-    - --cluster-signing-key-file=/etc/kubernetes/bootstrap/secrets/kubernetes/ca.key
-    - --controllers=*,bootstrapsigner,tokencleaner
-    - --kubeconfig=/etc/kubernetes/bootstrap/kubeconfig-bootstrap.yaml
-    - --leader-elect=true
-    - --node-cidr-mask-size=24
-    - --root-ca-file=/etc/kubernetes/bootstrap/secrets/kubernetes/ca.crt
-    - --service-account-private-key-file=/etc/kubernetes/bootstrap/secrets/kubernetes/sa.key
-    - --use-service-account-credentials=true
-    - --v=2
+    <%- k8s.controller_manager.args(bootstrap: true).each do |k, v| -%>
+    - <%= k %><% if v ;%>=<%= v %><%; end %>
+    <%- end -%>
     volumeMounts:
     - name: var-run-kubernetes
       mountPath: /var/run/kubernetes
-    - name: kubernetes
-      mountPath: /etc/kubernetes
+    - name: kubernetes-secrets
+      mountPath: /etc/kubernetes/pki
+      readOnly: true
+    - name: kubernetes-bootstrap
+      mountPath: /etc/kubernetes/bootstrap
       readOnly: true
     - mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
       name: flexvolume-dir
@@ -48,9 +42,12 @@ spec:
   volumes:
   - name: var-run-kubernetes
     emptyDir: {}
-  - name: kubernetes
+  - name: kubernetes-secrets
+    hostPath:
+      path: /etc/kubernetes/bootstrap/secrets
+  - name: kubernetes-bootstrap
     hostPath:
-      path: /etc/kubernetes
+      path: /etc/kubernetes/bootstrap
   - hostPath:
       path: /etc/ssl/certs
       type: DirectoryOrCreate

data/lib/porkadot/assets/bootstrap/manifests/kube-proxy.bootstrap.yaml.erb

@@ -18,8 +18,9 @@ spec:
     imagePullPolicy: IfNotPresent
     command:
     - kube-proxy
-    - --config=/etc/kubernetes/bootstrap/kube-proxy-bootstrap.yaml
-    - --hostname-override=$(NODE_NAME)
+    <%- k8s.proxy.args(bootstrap: true).each do |k, v| -%>
+    - <%= k %><% if v ;%>=<%= v %><%; end %>
+    <%- end -%>
     env:
       - name: NODE_NAME
         valueFrom:

data/lib/porkadot/assets/bootstrap/manifests/kube-scheduler.bootstrap.yaml.erb

@@ -15,11 +15,9 @@ spec:
     image: <%= k8s.image_repository %>/kube-scheduler:<%= k8s.kubernetes_version %>
     command:
     - kube-scheduler
-    - --kubeconfig=/etc/kubernetes/bootstrap/kubeconfig-bootstrap.yaml
-    - --authentication-kubeconfig=/etc/kubernetes/bootstrap/kubeconfig-bootstrap.yaml
-    - --authorization-kubeconfig=/etc/kubernetes/bootstrap/kubeconfig-bootstrap.yaml
-    - --leader-elect=true
-    - --v=2
+    <%- k8s.scheduler.args(bootstrap: true).each do |k, v| -%>
+    - <%= k %><% if v ;%>=<%= v %><%; end %>
+    <%- end -%>
     volumeMounts:
     - name: kubernetes
       mountPath: /etc/kubernetes

data/lib/porkadot/assets/etcd/etcd-server.yaml.erb

@@ -30,6 +30,8 @@ spec:
     - --data-dir=/var/lib/etcd
     - --heartbeat-interval=1000
     - --election-timeout=10000
+    env:
+<%= u.to_yaml(etcd.extra_env, 4) -%>
     volumeMounts:
     - mountPath: /var/lib/etcd
       name: etcd

data/lib/porkadot/assets/kubelet.rb

@@ -64,6 +64,7 @@ module Porkadot; module Assets
       render_erb 'install.sh'
       render_erb 'install-deps.sh'
       render_erb 'install-pkgs.sh'
+      render_erb 'setup-containerd.sh'
     end
 
     def render_bootstrap_certs

data/lib/porkadot/assets/kubelet/config.yaml.erb

@@ -12,6 +12,7 @@ authorization:
   webhook:
     cacheAuthorizedTTL: 0s
     cacheUnauthorizedTTL: 0s
+cgroupDriver: systemd
 clusterDNS:
   - <%= global_config.k8s.networking.dns_ip %>
 clusterDomain: <%= global_config.k8s.networking.dns_domain %>
@@ -32,5 +33,7 @@ streamingConnectionIdleTimeout: 0s
 syncFrequency: 0s
 volumeStatsAggPeriod: 0s
 serverTLSBootstrap: true
+featureGates:
+  CSIMigration: false
 
 # vim:filetype=yaml

data/lib/porkadot/assets/kubelet/install-deps.sh.erb

@@ -26,3 +26,14 @@ curl -L https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin
 chmod +x /opt/bin/kubelet-${RELEASE}
 rm -f /opt/bin/kubelet
 ln -s /opt/bin/kubelet-${RELEASE} /opt/bin/kubelet
+
+ETCD_VER="<%= global_config.etcd.image_tag.gsub(/\-\w+$/, '') %>"
+ETCD_URL=https://storage.googleapis.com/etcd/${ETCD_VER}/etcd-${ETCD_VER}-linux-${architecture}.tar.gz
+ETCD_TMP=$(mktemp -d)
+
+curl -L ${ETCD_URL} -o ${ETCD_TMP}/etcd.tar.gz
+tar zxvf ${ETCD_TMP}/etcd.tar.gz -C ${ETCD_TMP}/ --strip-components=1
+chmod +x ${ETCD_TMP}/etcdctl
+rm -f /opt/bin/etcdctl
+mv ${ETCD_TMP}/etcdctl /opt/bin/etcdctl-${ETCD_VER}
+ln -s /opt/bin/etcdctl-${ETCD_VER} /opt/bin/etcdctl

data/lib/porkadot/assets/kubelet/install-pkgs.sh.erb

@@ -4,6 +4,7 @@ export LC_ALL=C
 ROOT=$(dirname "${BASH_SOURCE}")
 
 if type apt-get > /dev/null 2>&1 ;then
+  export DEBIAN_FRONTEND=noninteractive
   apt-get update
   apt-get install -y \
       ca-certificates \
@@ -22,12 +23,28 @@ if type apt-get > /dev/null 2>&1 ;then
       nfs-common \
       socat \
       udev \
-      util-linux
+      util-linux \
+      open-iscsi
 fi
 
+cat > /etc/modules-load.d/porkadot.conf <<EOF
+overlay
+br_netfilter
+EOF
+
+modprobe overlay
+modprobe br_netfilter
+
 cat <<EOF >  /etc/sysctl.d/k8s.conf
 net.bridge.bridge-nf-call-ip6tables = 1
-net.bridge.bridge-nf-call-iptables = 1
+net.ipv4.ip_forward                 = 1
+net.bridge.bridge-nf-call-iptables  = 1
 EOF
 
+cat <<EOF > /etc/iscsi/initiatorname.iscsi
+InitiatorName=iqn.2020-04.cloud.unstable:<%= config.hostname %>
+EOF
+
+systemctl restart iscsid.service
+
 sysctl --system

data/lib/porkadot/assets/kubelet/kubelet.service.erb

@@ -5,11 +5,13 @@ Documentation=http://kubernetes.io/docs/
 [Service]
 EnvironmentFile=-/etc/default/kubelet
 ExecStart=/opt/bin/kubelet \
+    --container-runtime=remote \
+    --container-runtime-endpoint=/run/containerd/containerd.sock \
     --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \
     --kubeconfig=/etc/kubernetes/kubelet.conf \
     --config=/var/lib/kubelet/config.yaml \
     --network-plugin=cni \
-    --pod-infra-container-image=k8s.gcr.io/pause:3.1 \
+    --pod-infra-container-image=k8s.gcr.io/pause:3.4.1 \
     --hostname-override=<%= config.hostname %> \
     --node-labels=<%= config.labels_string %> \
     --register-with-taints=<%= config.taints_string %> \

data/lib/porkadot/assets/kubelet/setup-containerd.sh.erb

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -eu
+export LC_ALL=C
+ROOT=$(dirname "${BASH_SOURCE}")
+
+mkdir -p /etc/containerd
+containerd config default | tee /etc/containerd/config.toml
+sed -i -e "/containerd.runtimes.runc.options/a SystemdCgroup = true" /etc/containerd/config.toml
+
+systemctl restart containerd

data/lib/porkadot/assets/kubernetes.rb

@@ -28,15 +28,18 @@ module Porkadot; module Assets
       render_erb 'manifests/porkadot.yaml'
       render_erb 'manifests/kubelet.yaml'
       render_erb "manifests/#{lb.type}.yaml"
+      render_secrets_erb "manifests/#{lb.type}.secrets.yaml"
       render_erb "manifests/#{cni.type}.yaml"
+      render_erb "manifests/coredns.yaml"
+      render_erb "manifests/dns-horizontal-autoscaler.yaml"
       render_erb "manifests/kube-apiserver.yaml"
       render_secrets_erb "manifests/kube-apiserver.secrets.yaml"
       render_erb "manifests/kube-proxy.yaml"
       render_erb "manifests/kube-scheduler.yaml"
       render_erb "manifests/kube-controller-manager.yaml"
       render_secrets_erb "manifests/kube-controller-manager.secrets.yaml"
-      render_erb "manifests/pod-checkpointer.yaml"
       render_erb "manifests/kubelet-rubber-stamp.yaml"
+      render_erb "manifests/storage-version-migrator.yaml"
       render_erb 'install.sh'
     end
 

data/lib/porkadot/assets/kubernetes/manifests/coredns.yaml.erb

@@ -0,0 +1,209 @@
+<% k8s = global_config.k8s -%>
+# __MACHINE_GENERATED_WARNING__
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: coredns
+  namespace: kube-system
+  labels:
+      kubernetes.io/cluster-service: "true"
+      addonmanager.kubernetes.io/mode: Reconcile
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    kubernetes.io/bootstrapping: rbac-defaults
+    addonmanager.kubernetes.io/mode: Reconcile
+  name: system:coredns
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  - services
+  - pods
+  - namespaces
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+- apiGroups:
+  - discovery.k8s.io
+  resources:
+  - endpointslices
+  verbs:
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  annotations:
+    rbac.authorization.kubernetes.io/autoupdate: "true"
+  labels:
+    kubernetes.io/bootstrapping: rbac-defaults
+    addonmanager.kubernetes.io/mode: EnsureExists
+  name: system:coredns
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:coredns
+subjects:
+- kind: ServiceAccount
+  name: coredns
+  namespace: kube-system
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: coredns
+  namespace: kube-system
+  labels:
+      addonmanager.kubernetes.io/mode: EnsureExists
+data:
+  Corefile: |
+    .:53 {
+        errors
+        health {
+            lameduck 5s
+        }
+        ready
+        kubernetes <%= k8s.networking.dns_domain %>  in-addr.arpa ip6.arpa {
+            pods insecure
+            fallthrough in-addr.arpa ip6.arpa
+            ttl 30
+        }
+        prometheus :9153
+        forward . /etc/resolv.conf
+        cache 30
+        loop
+        reload
+        loadbalance
+    }
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: coredns
+  namespace: kube-system
+  labels:
+    k8s-app: kube-dns
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+    kubernetes.io/name: "CoreDNS"
+spec:
+  # replicas: not specified here:
+  # 1. In order to make Addon Manager do not reconcile this replicas parameter.
+  # 2. Default is 1.
+  # 3. Will be tuned in real time if DNS horizontal auto-scaling is turned on.
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
+  selector:
+    matchLabels:
+      k8s-app: kube-dns
+  template:
+    metadata:
+      labels:
+        k8s-app: kube-dns
+      annotations:
+        seccomp.security.alpha.kubernetes.io/pod: 'runtime/default'
+    spec:
+      priorityClassName: system-cluster-critical
+      serviceAccountName: coredns
+      tolerations:
+        - key: "CriticalAddonsOnly"
+          operator: "Exists"
+      nodeSelector:
+        kubernetes.io/os: linux
+      containers:
+      - name: coredns
+        image: k8s.gcr.io/coredns/coredns:v1.8.3
+        imagePullPolicy: IfNotPresent
+        resources:
+          limits:
+            memory: 170Mi
+          requests:
+            cpu: 100m
+            memory: 70Mi
+        args: [ "-conf", "/etc/coredns/Corefile" ]
+        volumeMounts:
+        - name: config-volume
+          mountPath: /etc/coredns
+          readOnly: true
+        ports:
+        - containerPort: 53
+          name: dns
+          protocol: UDP
+        - containerPort: 53
+          name: dns-tcp
+          protocol: TCP
+        - containerPort: 9153
+          name: metrics
+          protocol: TCP
+        livenessProbe:
+          httpGet:
+            path: /health
+            port: 8080
+            scheme: HTTP
+          initialDelaySeconds: 60
+          timeoutSeconds: 5
+          successThreshold: 1
+          failureThreshold: 5
+        readinessProbe:
+          httpGet:
+            path: /ready
+            port: 8181
+            scheme: HTTP
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            add:
+            - NET_BIND_SERVICE
+            drop:
+            - all
+          readOnlyRootFilesystem: true
+      dnsPolicy: Default
+      volumes:
+        - name: config-volume
+          configMap:
+            name: coredns
+            items:
+            - key: Corefile
+              path: Corefile
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: kube-dns
+  namespace: kube-system
+  annotations:
+    prometheus.io/port: "9153"
+    prometheus.io/scrape: "true"
+  labels:
+    k8s-app: kube-dns
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+    kubernetes.io/name: "CoreDNS"
+spec:
+  selector:
+    k8s-app: kube-dns
+  clusterIP: <%= k8s.networking.dns_ip %>
+  ports:
+  - name: dns
+    port: 53
+    protocol: UDP
+  - name: dns-tcp
+    port: 53
+    protocol: TCP
+  - name: metrics
+    port: 9153
+    protocol: TCP