porkadot 0.18.0 → 0.20.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d57037f96d15fcabbd441ec706449775c89984f0b6019b1f15f863ccc970360a
-  data.tar.gz: 8c03689d67687fde6012cda7f0cea22e0f1c8b7b96a64942649d43e76560aaa2
+  metadata.gz: dab74c5a6f0fd112b9ec994a67fe85c2bbe02c478ab68c95dcf8c998d9ddb64f
+  data.tar.gz: 272069efcb1b5bea219c54b4a4b0ec432bc215352fa786adcfc7923fdf73ae75
 SHA512:
-  metadata.gz: 9a941712075f648d17b5e6a75de10ce83974945ffb7feafcd630d4695d7ba7fab501d63e4a965319957ddf6676aa893e5035b01a2b93711ce1759153e35fc0d8
-  data.tar.gz: cb9b0ff915cea06c91a8da7fcdd894bee593bda0ec2efe3593a4b9dd5030b42269e5ba1f29230a77b076629651e387995cd2b1022906a66848cd74b84585c713
+  metadata.gz: eea11e276a5d6d95b231f3dce02b36eeb040533902c03202273b66ac1ad1d6dbe52349a0a5bfc6a69e0a5cf920dee49e871dcc2b8174c5e2ea3c8460fff4ec67
+  data.tar.gz: fa3c81ddc0dc1330ad4ea579b27557bd7316b3ccec0f1cd08c11a1cd107ecb4a2b472d71502811e88addd185a72f7b5b1fd0355ded8929a6a1de0299054d1778

data/hack/gen-storage-version-migrator.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -eu
+export LC_ALL=C
+ROOT=$(dirname "${BASH_SOURCE}")
+
+kustomize build ${ROOT}/storage-version-migrator | sed -e "s/NAMESPACE/kube-system/g" > ${ROOT}/../lib/porkadot/assets/kubernetes/manifests/storage-version-migrator.yaml.erb

data/hack/storage-version-migrator/kustomization.yaml

@@ -0,0 +1,77 @@
+namespace: kube-system
+
+resources:
+- https://github.com/kubernetes-sigs/kube-storage-version-migrator/manifests/?ref=acdee30ced218b79e39c6a701985e8cd8bd33824
+
+images:
+- name: REGISTRY/storage-version-migration-initializer:VERSION
+  newName: asia.gcr.io/k8s-artifacts-prod/storage-migrator/storage-version-migration-initializer
+  newTag: v0.0.3
+- name: REGISTRY/storage-version-migration-migrator:VERSION
+  newName: asia.gcr.io/k8s-artifacts-prod/storage-migrator/storage-version-migration-migrator
+  newTag: v0.0.3
+- name: REGISTRY/storage-version-migration-trigger:VERSION
+  newName: asia.gcr.io/k8s-artifacts-prod/storage-migrator/storage-version-migration-trigger
+  newTag: v0.0.3
+patchesJson6902:
+- target:
+    group: apps
+    version: v1
+    kind: Deployment
+    name: migrator
+    namespace: kube-system
+  patch: |-
+    - op: remove
+      path: /spec/template/spec/containers/0/livenessProbe
+    - op: add
+      path: /spec/template/spec/containers/0/command/-
+      value: --kubeconfig=/etc/migrator/kubeconfig
+- target:
+    group: apps
+    version: v1
+    kind: Deployment
+    name: trigger
+    namespace: kube-system
+  patch: |-
+    - op: remove
+      path: /spec/template/spec/containers/0/livenessProbe
+    - op: add
+      path: /spec/template/spec/containers/0/args
+      value: ["--kubeconfig=/etc/migrator/kubeconfig"]
+patchesStrategicMerge:
+- |-
+  apiVersion: apps/v1
+  kind: Deployment
+  metadata:
+    name: migrator
+    namespace: NAMESPACE
+  spec:
+    template:
+      spec:
+        containers:
+        - name: migrator
+          volumeMounts:
+          - mountPath: /etc/migrator
+            name: kubeconfig
+        volumes:
+        - name: kubeconfig
+          configMap:
+            name: kubeconfig-in-cluster-latest
+- |-
+  apiVersion: apps/v1
+  kind: Deployment
+  metadata:
+    name: trigger
+    namespace: NAMESPACE
+  spec:
+    template:
+      spec:
+        containers:
+        - name: trigger
+          volumeMounts:
+          - mountPath: /etc/migrator
+            name: kubeconfig
+        volumes:
+        - name: kubeconfig
+          configMap:
+            name: kubeconfig-in-cluster-latest

data/lib/porkadot/assets/bootstrap/manifests/kube-apiserver.bootstrap.yaml.erb

@@ -20,35 +20,9 @@ spec:
     image: <%= k8s.image_repository %>/kube-apiserver:<%= k8s.kubernetes_version %>
     command:
     - kube-apiserver
-    - --advertise-address=$(POD_IP)
-    - --allow-privileged
-    - --authorization-mode=Node,RBAC
-    - --bind-address=0.0.0.0
-    - --client-ca-file=/etc/kubernetes/secrets/kubernetes/ca.crt
-    - --enable-admission-plugins=NodeRestriction
-    - --enable-bootstrap-token-auth=true
-    - --etcd-cafile=/etc/kubernetes/secrets/etcd/ca.crt
-    - --etcd-certfile=/etc/kubernetes/secrets/etcd/etcd-client.crt
-    - --etcd-keyfile=/etc/kubernetes/secrets/etcd/etcd-client.key
-    - --etcd-servers=<%= global_config.etcd.advertise_client_urls.join(',') %>
-    - --kubelet-certificate-authority=/etc/kubernetes/secrets/kubernetes/ca.crt
-    - --kubelet-client-certificate=/etc/kubernetes/secrets/kubernetes/kubelet-client.crt
-    - --kubelet-client-key=/etc/kubernetes/secrets/kubernetes/kubelet-client.key
-    - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
-    - --proxy-client-cert-file=/etc/kubernetes/secrets/kubernetes/front-proxy-client.crt
-    - --proxy-client-key-file=/etc/kubernetes/secrets/kubernetes/front-proxy-client.key
-    - --requestheader-allowed-names=front-proxy-client
-    - --requestheader-client-ca-file=/etc/kubernetes/secrets/kubernetes/front-proxy-ca.crt
-    - --requestheader-extra-headers-prefix=X-Remote-Extra-
-    - --requestheader-group-headers=X-Remote-Group
-    - --requestheader-username-headers=X-Remote-User
-    - --secure-port=<%= k8s.apiserver.bind_port %>
-    - --service-account-key-file=/etc/kubernetes/secrets/kubernetes/sa.pub
-    - --service-cluster-ip-range=<%= k8s.networking.service_subnet %>
-    - --storage-backend=etcd3
-    - --tls-cert-file=/etc/kubernetes/secrets/kubernetes/apiserver.crt
-    - --tls-private-key-file=/etc/kubernetes/secrets/kubernetes/apiserver.key
-    - --v=2
+    <%- k8s.apiserver.args(bootstrap: true).each do |k, v| -%>
+    - <%= k %><% if v ;%>=<%= v %><%; end %>
+    <%- end -%>
     env:
     - name: POD_IP
       valueFrom:
@@ -64,7 +38,7 @@ spec:
     - mountPath: /usr/share/ca-certificates
       name: usr-share-ca-certificates
       readOnly: true
-    - mountPath: /etc/kubernetes/secrets
+    - mountPath: /etc/kubernetes/pki
       name: secrets
       readOnly: true
     - mountPath: /var/lock

data/lib/porkadot/assets/bootstrap/manifests/kube-controller-manager.bootstrap.yaml.erb

@@ -15,23 +15,17 @@ spec:
     image: <%= k8s.image_repository %>/kube-controller-manager:<%= k8s.kubernetes_version %>
     command:
     - kube-controller-manager
-    - --allocate-node-cidrs=true
-    - --cluster-cidr=<%= k8s.networking.pod_subnet %>
-    - --cluster-signing-cert-file=/etc/kubernetes/bootstrap/secrets/kubernetes/ca.crt
-    - --cluster-signing-key-file=/etc/kubernetes/bootstrap/secrets/kubernetes/ca.key
-    - --controllers=*,bootstrapsigner,tokencleaner
-    - --kubeconfig=/etc/kubernetes/bootstrap/kubeconfig-bootstrap.yaml
-    - --leader-elect=true
-    - --node-cidr-mask-size=24
-    - --root-ca-file=/etc/kubernetes/bootstrap/secrets/kubernetes/ca.crt
-    - --service-account-private-key-file=/etc/kubernetes/bootstrap/secrets/kubernetes/sa.key
-    - --use-service-account-credentials=true
-    - --v=2
+    <%- k8s.controller_manager.args(bootstrap: true).each do |k, v| -%>
+    - <%= k %><% if v ;%>=<%= v %><%; end %>
+    <%- end -%>
     volumeMounts:
     - name: var-run-kubernetes
       mountPath: /var/run/kubernetes
-    - name: kubernetes
-      mountPath: /etc/kubernetes
+    - name: kubernetes-secrets
+      mountPath: /etc/kubernetes/pki
+      readOnly: true
+    - name: kubernetes-bootstrap
+      mountPath: /etc/kubernetes/bootstrap
       readOnly: true
     - mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
       name: flexvolume-dir
@@ -48,9 +42,12 @@ spec:
   volumes:
   - name: var-run-kubernetes
     emptyDir: {}
-  - name: kubernetes
+  - name: kubernetes-secrets
+    hostPath:
+      path: /etc/kubernetes/bootstrap/secrets
+  - name: kubernetes-bootstrap
     hostPath:
-      path: /etc/kubernetes
+      path: /etc/kubernetes/bootstrap
   - hostPath:
       path: /etc/ssl/certs
       type: DirectoryOrCreate

data/lib/porkadot/assets/bootstrap/manifests/kube-proxy.bootstrap.yaml.erb

@@ -18,8 +18,9 @@ spec:
     imagePullPolicy: IfNotPresent
     command:
     - kube-proxy
-    - --config=/etc/kubernetes/bootstrap/kube-proxy-bootstrap.yaml
-    - --hostname-override=$(NODE_NAME)
+    <%- k8s.proxy.args(bootstrap: true).each do |k, v| -%>
+    - <%= k %><% if v ;%>=<%= v %><%; end %>
+    <%- end -%>
     env:
       - name: NODE_NAME
         valueFrom:

data/lib/porkadot/assets/bootstrap/manifests/kube-scheduler.bootstrap.yaml.erb

@@ -15,11 +15,9 @@ spec:
     image: <%= k8s.image_repository %>/kube-scheduler:<%= k8s.kubernetes_version %>
     command:
     - kube-scheduler
-    - --kubeconfig=/etc/kubernetes/bootstrap/kubeconfig-bootstrap.yaml
-    - --authentication-kubeconfig=/etc/kubernetes/bootstrap/kubeconfig-bootstrap.yaml
-    - --authorization-kubeconfig=/etc/kubernetes/bootstrap/kubeconfig-bootstrap.yaml
-    - --leader-elect=true
-    - --v=2
+    <%- k8s.scheduler.args(bootstrap: true).each do |k, v| -%>
+    - <%= k %><% if v ;%>=<%= v %><%; end %>
+    <%- end -%>
     volumeMounts:
     - name: kubernetes
       mountPath: /etc/kubernetes

data/lib/porkadot/assets/kubelet.rb

@@ -64,6 +64,7 @@ module Porkadot; module Assets
       render_erb 'install.sh'
       render_erb 'install-deps.sh'
       render_erb 'install-pkgs.sh'
+      render_erb 'setup-containerd.sh'
     end
 
     def render_bootstrap_certs

data/lib/porkadot/assets/kubelet/config.yaml.erb

@@ -12,6 +12,7 @@ authorization:
   webhook:
     cacheAuthorizedTTL: 0s
     cacheUnauthorizedTTL: 0s
+cgroupDriver: systemd
 clusterDNS:
   - <%= global_config.k8s.networking.dns_ip %>
 clusterDomain: <%= global_config.k8s.networking.dns_domain %>

data/lib/porkadot/assets/kubelet/install-pkgs.sh.erb

@@ -4,6 +4,7 @@ export LC_ALL=C
 ROOT=$(dirname "${BASH_SOURCE}")
 
 if type apt-get > /dev/null 2>&1 ;then
+  export DEBIAN_FRONTEND=noninteractive
   apt-get update
   apt-get install -y \
       ca-certificates \
@@ -26,9 +27,18 @@ if type apt-get > /dev/null 2>&1 ;then
       open-iscsi
 fi
 
+cat > /etc/modules-load.d/porkadot.conf <<EOF
+overlay
+br_netfilter
+EOF
+
+modprobe overlay
+modprobe br_netfilter
+
 cat <<EOF >  /etc/sysctl.d/k8s.conf
 net.bridge.bridge-nf-call-ip6tables = 1
-net.bridge.bridge-nf-call-iptables = 1
+net.ipv4.ip_forward                 = 1
+net.bridge.bridge-nf-call-iptables  = 1
 EOF
 
 cat <<EOF > /etc/iscsi/initiatorname.iscsi

data/lib/porkadot/assets/kubelet/kubelet.service.erb

@@ -5,11 +5,13 @@ Documentation=http://kubernetes.io/docs/
 [Service]
 EnvironmentFile=-/etc/default/kubelet
 ExecStart=/opt/bin/kubelet \
+    --container-runtime=remote \
+    --container-runtime-endpoint=/run/containerd/containerd.sock \
     --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \
     --kubeconfig=/etc/kubernetes/kubelet.conf \
     --config=/var/lib/kubelet/config.yaml \
     --network-plugin=cni \
-    --pod-infra-container-image=k8s.gcr.io/pause:3.1 \
+    --pod-infra-container-image=k8s.gcr.io/pause:3.4.1 \
     --hostname-override=<%= config.hostname %> \
     --node-labels=<%= config.labels_string %> \
     --register-with-taints=<%= config.taints_string %> \

data/lib/porkadot/assets/kubelet/setup-containerd.sh.erb

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -eu
+export LC_ALL=C
+ROOT=$(dirname "${BASH_SOURCE}")
+
+mkdir -p /etc/containerd
+containerd config default | tee /etc/containerd/config.toml
+sed -i -e "/containerd.runtimes.runc.options/a SystemdCgroup = true" /etc/containerd/config.toml
+
+systemctl restart containerd

data/lib/porkadot/assets/kubernetes.rb

@@ -28,6 +28,7 @@ module Porkadot; module Assets
       render_erb 'manifests/porkadot.yaml'
       render_erb 'manifests/kubelet.yaml'
       render_erb "manifests/#{lb.type}.yaml"
+      render_secrets_erb "manifests/#{lb.type}.secrets.yaml"
       render_erb "manifests/#{cni.type}.yaml"
       render_erb "manifests/coredns.yaml"
       render_erb "manifests/dns-horizontal-autoscaler.yaml"
@@ -37,9 +38,9 @@ module Porkadot; module Assets
       render_erb "manifests/kube-scheduler.yaml"
       render_erb "manifests/kube-controller-manager.yaml"
       render_secrets_erb "manifests/kube-controller-manager.secrets.yaml"
-      render_erb "manifests/pod-checkpointer.yaml"
       render_erb "manifests/kubelet-rubber-stamp.yaml"
       render_erb "manifests/storage-version-migrator.yaml"
+      render_secrets_erb "kubeconfig.yaml"
       render_erb 'install.sh'
     end
 

data/lib/porkadot/assets/kubernetes/kubeconfig.yaml.erb

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Config
+clusters:
+- name: kubernetes
+  cluster:
+    certificate-authority-data: <%= certs.kubernetes.to_base64(:ca_cert) %>
+    server: https://127.0.0.1:<%= global_config.k8s.apiserver.bind_port %>
+users:
+- name: admin
+  user:
+    client-certificate-data: <%= certs.kubernetes.to_base64(:client_cert) %>
+    client-key-data: <%= certs.kubernetes.to_base64(:client_key) %>
+contexts:
+- context:
+    cluster: kubernetes
+    user: admin
+  name: admin-context
+current-context: admin-context
+

data/lib/porkadot/assets/kubernetes/manifests/coredns.yaml.erb

@@ -34,6 +34,13 @@ rules:
   - nodes
   verbs:
   - get
+- apiGroups:
+  - discovery.k8s.io
+  resources:
+  - endpointslices
+  verbs:
+  - list
+  - watch
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -119,7 +126,7 @@ spec:
         kubernetes.io/os: linux
       containers:
       - name: coredns
-        image: k8s.gcr.io/coredns:1.6.7
+        image: k8s.gcr.io/coredns/coredns:v1.8.3
         imagePullPolicy: IfNotPresent
         resources:
           limits:

data/lib/porkadot/assets/kubernetes/manifests/flannel.yaml.erb

@@ -11,14 +11,14 @@ metadata:
 spec:
   privileged: false
   volumes:
-    - configMap
-    - secret
-    - emptyDir
-    - hostPath
+  - configMap
+  - secret
+  - emptyDir
+  - hostPath
   allowedHostPaths:
-    - pathPrefix: "/etc/cni/net.d"
-    - pathPrefix: "/etc/kube-flannel"
-    - pathPrefix: "/run/flannel"
+  - pathPrefix: "/etc/cni/net.d"
+  - pathPrefix: "/etc/kube-flannel"
+  - pathPrefix: "/run/flannel"
   readOnlyRootFilesystem: false
   # Users and groups
   runAsUser:
@@ -31,7 +31,7 @@ spec:
   allowPrivilegeEscalation: false
   defaultAllowPrivilegeEscalation: false
   # Capabilities
-  allowedCapabilities: ['NET_ADMIN']
+  allowedCapabilities: ['NET_ADMIN', 'NET_RAW']
   defaultAddCapabilities: []
   requiredDropCapabilities: []
   # Host namespaces
@@ -47,36 +47,36 @@ spec:
     rule: 'RunAsAny'
 ---
 kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   name: flannel
 rules:
-  - apiGroups: ['extensions']
-    resources: ['podsecuritypolicies']
-    verbs: ['use']
-    resourceNames: ['psp.flannel.unprivileged']
-  - apiGroups:
-      - ""
-    resources:
-      - pods
-    verbs:
-      - get
-  - apiGroups:
-      - ""
-    resources:
-      - nodes
-    verbs:
-      - list
-      - watch
-  - apiGroups:
-      - ""
-    resources:
-      - nodes/status
-    verbs:
-      - patch
+- apiGroups: ['extensions']
+  resources: ['podsecuritypolicies']
+  verbs: ['use']
+  resourceNames: ['psp.flannel.unprivileged']
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - nodes/status
+  verbs:
+  - patch
 ---
 kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   name: flannel
 roleRef:
@@ -134,7 +134,7 @@ data:
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
-  name: kube-flannel-ds-amd64
+  name: kube-flannel-ds
   namespace: kube-system
   labels:
     tier: node
@@ -153,23 +153,20 @@ spec:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
-              - matchExpressions:
-                  - key: kubernetes.io/os
-                    operator: In
-                    values:
-                      - linux
-                  - key: kubernetes.io/arch
-                    operator: In
-                    values:
-                      - amd64
+            - matchExpressions:
+              - key: kubernetes.io/os
+                operator: In
+                values:
+                - linux
       hostNetwork: true
+      priorityClassName: system-node-critical
       tolerations:
       - operator: Exists
         effect: NoSchedule
       serviceAccountName: flannel
       initContainers:
       - name: install-cni
-        image: quay.io/coreos/flannel:v0.12.0-amd64
+        image: quay.io/coreos/flannel:v0.14.0
         command:
         - cp
         args:
@@ -183,7 +180,7 @@ spec:
           mountPath: /etc/kube-flannel/
       containers:
       - name: kube-flannel
-        image: quay.io/coreos/flannel:v0.12.0-amd64
+        image: quay.io/coreos/flannel:v0.14.0
         command:
         - /opt/bin/flanneld
         args:
@@ -199,7 +196,7 @@ spec:
         securityContext:
           privileged: false
           capabilities:
-            add: ["NET_ADMIN"]
+            add: ["NET_ADMIN", "NET_RAW"]
         env:
         - name: POD_NAME
           valueFrom:
@@ -215,388 +212,12 @@ spec:
         - name: flannel-cfg
           mountPath: /etc/kube-flannel/
       volumes:
-        - name: run
-          hostPath:
-            path: /run/flannel
-        - name: cni
-          hostPath:
-            path: /etc/cni/net.d
-        - name: flannel-cfg
-          configMap:
-            name: kube-flannel-cfg
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: kube-flannel-ds-arm64
-  namespace: kube-system
-  labels:
-    tier: node
-    app: flannel
-spec:
-  selector:
-    matchLabels:
-      app: flannel
-  template:
-    metadata:
-      labels:
-        tier: node
-        app: flannel
-    spec:
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-              - matchExpressions:
-                  - key: kubernetes.io/os
-                    operator: In
-                    values:
-                      - linux
-                  - key: kubernetes.io/arch
-                    operator: In
-                    values:
-                      - arm64
-      hostNetwork: true
-      tolerations:
-      - operator: Exists
-        effect: NoSchedule
-      serviceAccountName: flannel
-      initContainers:
-      - name: install-cni
-        image: quay.io/coreos/flannel:v0.12.0-arm64
-        command:
-        - cp
-        args:
-        - -f
-        - /etc/kube-flannel/cni-conf.json
-        - /etc/cni/net.d/10-flannel.conflist
-        volumeMounts:
-        - name: cni
-          mountPath: /etc/cni/net.d
-        - name: flannel-cfg
-          mountPath: /etc/kube-flannel/
-      containers:
-      - name: kube-flannel
-        image: quay.io/coreos/flannel:v0.12.0-arm64
-        command:
-        - /opt/bin/flanneld
-        args:
-        - --ip-masq
-        - --kube-subnet-mgr
-        resources:
-          requests:
-            cpu: "100m"
-            memory: "50Mi"
-          limits:
-            cpu: "100m"
-            memory: "50Mi"
-        securityContext:
-          privileged: false
-          capabilities:
-             add: ["NET_ADMIN"]
-        env:
-        - name: POD_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-        - name: POD_NAMESPACE
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.namespace
-        volumeMounts:
-        - name: run
-          mountPath: /run/flannel
-        - name: flannel-cfg
-          mountPath: /etc/kube-flannel/
-      volumes:
-        - name: run
-          hostPath:
-            path: /run/flannel
-        - name: cni
-          hostPath:
-            path: /etc/cni/net.d
-        - name: flannel-cfg
-          configMap:
-            name: kube-flannel-cfg
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: kube-flannel-ds-arm
-  namespace: kube-system
-  labels:
-    tier: node
-    app: flannel
-spec:
-  selector:
-    matchLabels:
-      app: flannel
-  template:
-    metadata:
-      labels:
-        tier: node
-        app: flannel
-    spec:
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-              - matchExpressions:
-                  - key: kubernetes.io/os
-                    operator: In
-                    values:
-                      - linux
-                  - key: kubernetes.io/arch
-                    operator: In
-                    values:
-                      - arm
-      hostNetwork: true
-      tolerations:
-      - operator: Exists
-        effect: NoSchedule
-      serviceAccountName: flannel
-      initContainers:
-      - name: install-cni
-        image: quay.io/coreos/flannel:v0.12.0-arm
-        command:
-        - cp
-        args:
-        - -f
-        - /etc/kube-flannel/cni-conf.json
-        - /etc/cni/net.d/10-flannel.conflist
-        volumeMounts:
-        - name: cni
-          mountPath: /etc/cni/net.d
-        - name: flannel-cfg
-          mountPath: /etc/kube-flannel/
-      containers:
-      - name: kube-flannel
-        image: quay.io/coreos/flannel:v0.12.0-arm
-        command:
-        - /opt/bin/flanneld
-        args:
-        - --ip-masq
-        - --kube-subnet-mgr
-        resources:
-          requests:
-            cpu: "100m"
-            memory: "50Mi"
-          limits:
-            cpu: "100m"
-            memory: "50Mi"
-        securityContext:
-          privileged: false
-          capabilities:
-             add: ["NET_ADMIN"]
-        env:
-        - name: POD_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-        - name: POD_NAMESPACE
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.namespace
-        volumeMounts:
-        - name: run
-          mountPath: /run/flannel
-        - name: flannel-cfg
-          mountPath: /etc/kube-flannel/
-      volumes:
-        - name: run
-          hostPath:
-            path: /run/flannel
-        - name: cni
-          hostPath:
-            path: /etc/cni/net.d
-        - name: flannel-cfg
-          configMap:
-            name: kube-flannel-cfg
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: kube-flannel-ds-ppc64le
-  namespace: kube-system
-  labels:
-    tier: node
-    app: flannel
-spec:
-  selector:
-    matchLabels:
-      app: flannel
-  template:
-    metadata:
-      labels:
-        tier: node
-        app: flannel
-    spec:
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-              - matchExpressions:
-                  - key: kubernetes.io/os
-                    operator: In
-                    values:
-                      - linux
-                  - key: kubernetes.io/arch
-                    operator: In
-                    values:
-                      - ppc64le
-      hostNetwork: true
-      tolerations:
-      - operator: Exists
-        effect: NoSchedule
-      serviceAccountName: flannel
-      initContainers:
-      - name: install-cni
-        image: quay.io/coreos/flannel:v0.12.0-ppc64le
-        command:
-        - cp
-        args:
-        - -f
-        - /etc/kube-flannel/cni-conf.json
-        - /etc/cni/net.d/10-flannel.conflist
-        volumeMounts:
-        - name: cni
-          mountPath: /etc/cni/net.d
-        - name: flannel-cfg
-          mountPath: /etc/kube-flannel/
-      containers:
-      - name: kube-flannel
-        image: quay.io/coreos/flannel:v0.12.0-ppc64le
-        command:
-        - /opt/bin/flanneld
-        args:
-        - --ip-masq
-        - --kube-subnet-mgr
-        resources:
-          requests:
-            cpu: "100m"
-            memory: "50Mi"
-          limits:
-            cpu: "100m"
-            memory: "50Mi"
-        securityContext:
-          privileged: false
-          capabilities:
-             add: ["NET_ADMIN"]
-        env:
-        - name: POD_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-        - name: POD_NAMESPACE
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.namespace
-        volumeMounts:
-        - name: run
-          mountPath: /run/flannel
-        - name: flannel-cfg
-          mountPath: /etc/kube-flannel/
-      volumes:
-        - name: run
-          hostPath:
-            path: /run/flannel
-        - name: cni
-          hostPath:
-            path: /etc/cni/net.d
-        - name: flannel-cfg
-          configMap:
-            name: kube-flannel-cfg
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: kube-flannel-ds-s390x
-  namespace: kube-system
-  labels:
-    tier: node
-    app: flannel
-spec:
-  selector:
-    matchLabels:
-      app: flannel
-  template:
-    metadata:
-      labels:
-        tier: node
-        app: flannel
-    spec:
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-              - matchExpressions:
-                  - key: kubernetes.io/os
-                    operator: In
-                    values:
-                      - linux
-                  - key: kubernetes.io/arch
-                    operator: In
-                    values:
-                      - s390x
-      hostNetwork: true
-      tolerations:
-      - operator: Exists
-        effect: NoSchedule
-      serviceAccountName: flannel
-      initContainers:
-      - name: install-cni
-        image: quay.io/coreos/flannel:v0.12.0-s390x
-        command:
-        - cp
-        args:
-        - -f
-        - /etc/kube-flannel/cni-conf.json
-        - /etc/cni/net.d/10-flannel.conflist
-        volumeMounts:
-        - name: cni
-          mountPath: /etc/cni/net.d
-        - name: flannel-cfg
-          mountPath: /etc/kube-flannel/
-      containers:
-      - name: kube-flannel
-        image: quay.io/coreos/flannel:v0.12.0-s390x
-        command:
-        - /opt/bin/flanneld
-        args:
-        - --ip-masq
-        - --kube-subnet-mgr
-        resources:
-          requests:
-            cpu: "100m"
-            memory: "50Mi"
-          limits:
-            cpu: "100m"
-            memory: "50Mi"
-        securityContext:
-          privileged: false
-          capabilities:
-             add: ["NET_ADMIN"]
-        env:
-        - name: POD_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-        - name: POD_NAMESPACE
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.namespace
-        volumeMounts:
-        - name: run
-          mountPath: /run/flannel
-        - name: flannel-cfg
-          mountPath: /etc/kube-flannel/
-      volumes:
-        - name: run
-          hostPath:
-            path: /run/flannel
-        - name: cni
-          hostPath:
-            path: /etc/cni/net.d
-        - name: flannel-cfg
-          configMap:
-            name: kube-flannel-cfg
+      - name: run
+        hostPath:
+          path: /run/flannel
+      - name: cni
+        hostPath:
+          path: /etc/cni/net.d
+      - name: flannel-cfg
+        configMap:
+          name: kube-flannel-cfg