porkadot 0.1.0

data/lib/porkadot/configs/etcd.rb

@@ -0,0 +1,95 @@
+
+module Porkadot; module Configs
+  class Etcd
+    include Porkadot::ConfigUtils
+
+    def initialize config
+      @config = config
+      @raw = config.raw.etcd
+    end
+
+    def advertise_client_urls
+      urls = []
+      config.etcd_nodes.each do |_, v|
+        urls += v.advertise_client_urls
+      end
+      return urls
+    end
+
+  end
+
+  class EtcdNode
+    include Porkadot::ConfigUtils
+    include Porkadot::Configs::CertsUtils
+    attr_reader :kubelet
+    attr_reader :name
+
+    def initialize config, name, raw
+      @config = config
+      @kubelet = config.nodes[name]
+      @name = name
+      @raw = raw || ::Porkadot::Raw.new
+    end
+
+    def member_name
+      return (self.raw.labels && self.raw.labels[Porkadot::ETCD_MEMBER_LABEL]) || self.raw.hostname || self.name
+    end
+
+    def member_address
+      return (self.raw.labels && self.raw.labels[Porkadot::ETCD_ADDRESS_LABEL]) || self.raw.hostname || self.name
+    end
+
+    def advertise_client_urls
+     ["https://#{member_address}:2379"]
+    end
+
+    def advertise_peer_urls
+      ["https://#{member_address}:2380"]
+    end
+
+    def listen_client_urls
+      self.advertise_client_urls + ["https://127.0.0.1:2379"]
+    end
+
+    def listen_peer_urls
+      self.advertise_peer_urls
+    end
+
+    def initial_cluster
+      return {}.tap do |rtn|
+        self.config.etcd_nodes.each do |_, v|
+          rtn[v.member_name] = "https://#{v.member_address}:2380"
+        end
+      end
+    end
+
+    def additional_sans
+      sans = []
+      [self.member_name, self.member_address].each do |san|
+        if self.ipaddr?(san)
+          sans << "IP:#{san}"
+        else
+          sans << "DNS:#{san}"
+        end
+      end
+      return sans
+    end
+
+    def target_path
+      File.join(self.kubelet.addon_path, 'etcd')
+    end
+
+    def ca_crt_path
+      File.join(self.target_path, 'ca.crt')
+    end
+
+    def etcd_key_path
+      File.join(self.target_path, 'etcd.key')
+    end
+
+    def etcd_crt_path
+      File.join(self.target_path, 'etcd.crt')
+    end
+
+  end
+end; end

data/lib/porkadot/configs/kubelet.rb

@@ -0,0 +1,61 @@
+module Porkadot; module Configs
+  class Kubelet
+    include Porkadot::ConfigUtils
+    attr_reader :name
+    attr_reader :connection
+
+    def initialize config, name, raw
+      @config = config
+      @name = name
+      @raw = raw || ::Porkadot::Raw.new
+      hostname = @raw.hostname || name
+      con = { hostname: hostname }
+      gcon = config.connection.to_hash
+      lcon = @raw.connection ? @raw.connection.to_hash : {}
+      @connection = ::Porkadot::Raw.new(con.rmerge(gcon.rmerge(lcon)))
+    end
+
+    def apiserver?
+      self.raw.labels && self.raw.labels.include?(Porkadot::K8S_MASTER_LABEL)
+    end
+
+    def control_plane_endpoint
+      (self.raw.kubernetes && self.raw.kubernetes.control_plane_endpoint) || self.config.k8s.control_plane_endpoint
+    end
+
+    def labels_string
+      return '' unless self.raw.labels
+      return self.raw.labels.map{|v| v.compact.join('=')}.join(',')
+    end
+
+    def taints_string
+      return '' unless self.raw.taints
+      return self.raw.taints.map{|v| v.compact.join('=')}.join(',')
+    end
+
+    def hostname
+      self.raw.hostname || self.name
+    end
+
+    def target_path
+      File.join(self.config.assets_dir, 'kubelet', name)
+    end
+
+    def addon_path
+      File.join(self.target_path, 'addons')
+    end
+
+    def ca_crt_path
+      File.join(self.target_path, 'ca.crt')
+    end
+
+    def bootstrap_key_path
+      File.join(self.target_path, 'bootstrap.key')
+    end
+
+    def bootstrap_cert_path
+      File.join(self.target_path, 'bootstrap.crt')
+    end
+  end
+end; end
+

data/lib/porkadot/configs/kubernetes.rb

@@ -0,0 +1,223 @@
+
+module Porkadot; module Configs
+  class Kubernetes
+    include Porkadot::ConfigUtils
+    attr_reader :networking
+    attr_reader :proxy
+    attr_reader :apiserver
+    attr_reader :controller_manager
+    attr_reader :scheduler
+
+    def initialize config
+      @config = config
+      @raw = config.raw.kubernetes
+
+      @networking = Networking.new(config)
+      @proxy = Proxy.new(config)
+      @apiserver = Apiserver.new(config)
+      @controller_manager = ControllerManager.new(config)
+      @scheduler = Scheduler.new(config)
+    end
+
+    def cluster_name
+      self.raw.cluster_name || 'porkadot'
+    end
+
+    def target_path
+      File.join(self.config.assets_dir, 'kubernetes')
+    end
+
+    def manifests_path
+      File.join(self.target_path, 'manifests')
+    end
+
+    def control_plane_endpoint_host_and_port
+      endpoint = self.config.k8s.control_plane_endpoint
+      raise "kubernetes.control_plane_endpoint should not be nil" unless endpoint
+      index = endpoint.rindex(':')
+      return [endpoint[0, index], endpoint[index+1, 6]]
+    end
+
+    module Component
+      RECOMMENDED_LABEL_PREFIX = 'app.kubernetes.io'
+      def labels
+        self.instance_labels.merge({
+          "#{RECOMMENDED_LABEL_PREFIX}/name": self.component_name,
+          "#{RECOMMENDED_LABEL_PREFIX}/component": self.component_name,
+          "#{RECOMMENDED_LABEL_PREFIX}/instance": "#{self.component_name}-porkadot",
+          "#{RECOMMENDED_LABEL_PREFIX}/version": self.config.k8s.kubernetes_version,
+          "#{RECOMMENDED_LABEL_PREFIX}/part-of": 'kubernetes',
+          "#{RECOMMENDED_LABEL_PREFIX}/managed-by": 'porkadot',
+        })
+      end
+
+      def instance_labels
+        {
+
+          "#{RECOMMENDED_LABEL_PREFIX}/component": self.component_name,
+          "#{RECOMMENDED_LABEL_PREFIX}/instance": "#{self.component_name}-porkadot",
+          "#{RECOMMENDED_LABEL_PREFIX}/managed-by": 'porkadot',
+        }
+      end
+
+      def args
+        extra = {}
+        if self.extra_args
+          extra = self.extra_args.map{|i| i.split('=', 2)}.to_h
+        end
+        return self.default_args.merge(extra)
+      end
+
+      def log_level
+        config.kubernetes.log_level || raw.log_level || 2
+      end
+
+    end
+
+    class Apiserver
+      include Porkadot::ConfigUtils
+      include Component
+
+      def initialize config
+        @config = config
+        @raw = config.raw.kubernetes.apiserver
+      end
+
+      def component_name
+        'kube-apiserver'
+      end
+
+      def default_args
+        return %W(
+          --advertise-address=$(POD_IP)
+          --allow-privileged=true
+          --authorization-mode=Node,RBAC
+          --bind-address=0.0.0.0
+          --client-ca-file=/etc/kubernetes/pki/kubernetes/ca.crt
+          --enable-bootstrap-token-auth=true
+          --etcd-cafile=/etc/kubernetes/pki/etcd/ca.crt
+          --etcd-certfile=/etc/kubernetes/pki/etcd/etcd-client.crt
+          --etcd-keyfile=/etc/kubernetes/pki/etcd/etcd-client.key
+          --etcd-servers=#{config.etcd.advertise_client_urls.join(',')}
+          --kubelet-certificate-authority=/etc/kubernetes/pki/kubernetes/ca.crt
+          --kubelet-client-certificate=/etc/kubernetes/pki/kubernetes/kubelet-client.crt
+          --kubelet-client-key=/etc/kubernetes/pki/kubernetes/kubelet-client.key
+          --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
+          --proxy-client-cert-file=/etc/kubernetes/pki/kubernetes/front-proxy-client.crt
+          --proxy-client-key-file=/etc/kubernetes/pki/kubernetes/front-proxy-client.key
+          --requestheader-allowed-names=aggregator-client
+          --requestheader-client-ca-file=/etc/kubernetes/pki/kubernetes/front-proxy-ca.crt
+          --requestheader-extra-headers-prefix=X-Remote-Extra-
+          --requestheader-group-headers=X-Remote-Group
+          --requestheader-username-headers=X-Remote-User
+          --secure-port=#{self.bind_port}
+          --service-account-key-file=/etc/kubernetes/pki/kubernetes/sa.pub
+          --service-cluster-ip-range=#{config.k8s.networking.service_subnet}
+          --storage-backend=etcd3
+          --tls-cert-file=/etc/kubernetes/pki/kubernetes/apiserver.crt
+          --tls-private-key-file=/etc/kubernetes/pki/kubernetes/apiserver.key
+          --v=#{self.log_level}
+        ).map {|i| i.split('=', 2)}.to_h
+      end
+    end
+
+    class Scheduler
+      include Porkadot::ConfigUtils
+      include Component
+
+      def initialize config
+        @config = config
+        @raw = config.raw.kubernetes.scheduler
+      end
+
+      def component_name
+        'kube-scheduler'
+      end
+
+      def default_args
+        return %W(
+          --leader-elect=true
+          --v=#{self.log_level}
+        ).map {|i| i.split('=', 2)}.to_h
+      end
+    end
+
+    class ControllerManager
+      include Porkadot::ConfigUtils
+      include Component
+
+      def initialize config
+        @config = config
+        @raw = config.raw.kubernetes.controller_manager
+      end
+
+      def component_name
+        'kube-controller-manager'
+      end
+
+      def default_args
+        return %W(
+          --allocate-node-cidrs=true
+          --cluster-cidr=#{config.k8s.networking.pod_subnet}
+          --cluster-signing-cert-file=/etc/kubernetes/pki/kubernetes/ca.crt
+          --cluster-signing-key-file=/etc/kubernetes/pki/kubernetes/ca.key
+          --controllers=*,bootstrapsigner,tokencleaner
+          --leader-elect=true
+          --node-cidr-mask-size=24
+          --root-ca-file=/etc/kubernetes/pki/kubernetes/ca.crt
+          --service-account-private-key-file=/etc/kubernetes/pki/kubernetes/sa.key
+          --use-service-account-credentials=true
+          --v=#{self.log_level}
+        ).map {|i| i.split('=', 2)}.to_h
+      end
+    end
+
+    class Proxy
+      include Porkadot::ConfigUtils
+      include Component
+
+      def initialize config
+        @config = config
+        @raw = config.raw.kubernetes.proxy
+      end
+
+      def proxy_config kubeconfig=nil
+        self.raw.config['clusterCIDR'] = config.k8s.networking.pod_subnet
+        if kubeconfig
+          self.raw.config['clientConnection']['kubeconfig'] = kubeconfig
+        end
+        self.raw.config.to_hash.to_yaml
+      end
+
+      def component_name
+        'kube-proxy'
+      end
+
+      def default_args
+        return %W(
+          --config=/var/lib/kube-proxy/config.conf
+          --hostname-override=$(NODE_NAME)
+        ).map {|i| i.split('=', 2)}.to_h
+      end
+    end
+
+    class Networking
+      include Porkadot::ConfigUtils
+
+      def initialize config
+        @config = config
+        @raw = config.raw.kubernetes.networking
+      end
+
+      def kubernetes_ip
+        cluster_ip_range = IPAddr.new(self.service_subnet)
+        cluster_ip_range.to_range.first(2)[1].to_s
+      end
+
+      def dns_ip
+        cluster_ip_range = IPAddr.new(self.service_subnet)
+        cluster_ip_range.to_range.first(11)[10].to_s
+      end
+    end
+  end
+end; end

data/lib/porkadot/configs/loadbalancer.rb

@@ -0,0 +1,26 @@
+
+module Porkadot; module Configs
+  class Lb
+    include Porkadot::ConfigUtils
+    attr_reader :type
+
+    def initialize config
+      @config = config
+      @type = config.raw.lb.type
+      @raw = config.raw.lb.send(config.raw.lb.type.to_sym)
+    end
+
+    def target_path
+      File.join(self.config.assets_dir, 'kubernetes')
+    end
+
+    def manifests_path
+      File.join(self.target_path, 'manifests')
+    end
+
+    def lb_config
+      return self.raw.config
+    end
+
+  end
+end; end

data/lib/porkadot/const.rb

@@ -0,0 +1,8 @@
+
+module Porkadot
+  ROOT = File.expand_path("../..", __FILE__)
+
+  K8S_MASTER_LABEL = "k8s.unstable.cloud/master"
+  ETCD_MEMBER_LABEL = "etcd.unstable.cloud/member"
+  ETCD_ADDRESS_LABEL = "etcd.unstable.cloud/address"
+end

data/lib/porkadot/default.yaml

@@ -0,0 +1,123 @@
+local:
+  assets_dir: ./assets
+
+connection:
+  user: ubuntu
+  port: 22
+  keys: ["~/.ssh/id_rsa", "~/.ssh/id_dsa"]
+
+nodes: {}
+
+bootstrap: {}
+
+cni:
+  type: flannel
+  flannel:
+    backend: vxlan
+
+lb:
+  type: metallb
+  metallb:
+    config: |
+      address-pools:
+      - name: default
+        protocol: layer2
+        addresses:
+        - 192.168.1.240-192.168.1.250
+
+etcd:
+  image_repository: gcr.io/etcd-development/etcd
+  image_tag: v3.3.10
+
+kubernetes:
+  kubernetes_version: v1.17.3
+  image_repository: k8s.gcr.io
+
+  networking:
+    cni_version: v0.8.2
+    service_subnet: '10.254.0.0/24'
+    pod_subnet: '10.244.0.0/16'
+    dns_domain: 'cluster.local'
+
+  apiserver:
+    bind_port: 6443
+
+  scheduler: {}
+
+  controller_manager: {}
+
+  proxy:
+    config:
+      apiVersion: kubeproxy.config.k8s.io/v1alpha1
+      kind: KubeProxyConfiguration
+      bindAddress: 0.0.0.0
+      clientConnection:
+        acceptContentTypes: ""
+        burst: 10
+        contentType: application/vnd.kubernetes.protobuf
+        kubeconfig: /var/lib/kube-proxy/kubeconfig.conf
+        qps: 5
+      # clusterCIDR: ${KUBE_CLUSTER_CIDR} # will be dynamically set
+      configSyncPeriod: 15m0s
+      conntrack:
+        maxPerCore: 32768
+        min: 131072
+        tcpCloseWaitTimeout: 1h0m0s
+        tcpEstablishedTimeout: 24h0m0s
+      enableProfiling: false
+      healthzBindAddress: 0.0.0.0:10256
+      hostnameOverride: ""
+      iptables:
+        masqueradeAll: false
+        masqueradeBit: 14
+        minSyncPeriod: 0s
+        syncPeriod: 30s
+      ipvs:
+        excludeCIDRs: null
+        minSyncPeriod: 0s
+        scheduler: ""
+        syncPeriod: 30s
+      metricsBindAddress: 127.0.0.1:10249
+      mode: "iptables"
+      nodePortAddresses: null
+      oomScoreAdj: -999
+      portRange: ""
+      udpIdleTimeout: 250ms
+
+  kubelet:
+    config:
+      apiVersion: kubelet.config.k8s.io/v1beta1
+      kind: KubeletConfiguration
+      authentication:
+        anonymous:
+          enabled: false
+        webhook:
+          cacheTTL: 0s
+          enabled: true
+        x509:
+          clientCAFile: /etc/kubernetes/pki/ca.crt
+      authorization:
+        mode: Webhook
+        webhook:
+          cacheAuthorizedTTL: 0s
+          cacheUnauthorizedTTL: 0s
+      cgroupDriver: cgroupfs
+      clusterDNS: []
+      clusterDomain: cluster.local
+      cpuManagerReconcilePeriod: 0s
+      evictionPressureTransitionPeriod: 0s
+      fileCheckFrequency: 0s
+      healthzBindAddress: 127.0.0.1
+      healthzPort: 10248
+      httpCheckFrequency: 0s
+      imageMinimumGCAge: 0s
+      nodeStatusReportFrequency: 0s
+      nodeStatusUpdateFrequency: 0s
+      resolvConf: /run/systemd/resolve/resolv.conf
+      rotateCertificates: true
+      runtimeRequestTimeout: 0s
+      serverTLSBootstrap: true
+      staticPodPath: /etc/kubernetes/manifests
+      streamingConnectionIdleTimeout: 0s
+      syncFrequency: 0s
+      volumeStatsAggPeriod: 0s

data/lib/porkadot/install/base.rb

@@ -0,0 +1,5 @@
+require 'sshkit/dsl'
+
+module Porkadot; module Install
+  KUBE_TEMP = './kube_temp'
+end; end

data/lib/porkadot/install/bootstrap.rb

@@ -0,0 +1,76 @@
+module Porkadot; module Install
+  class Bootstrap
+    KUBE_TEMP = File.join(Porkadot::Install::KUBE_TEMP, 'bootstrap')
+    include SSHKit::DSL
+    attr_reader :global_config
+    attr_reader :config
+    attr_reader :logger
+    attr_reader :host
+
+    def initialize global_config
+      @global_config = global_config
+      @config = global_config.bootstrap
+      @logger = global_config.logger
+      @host = Porkadot::Install::Kubelet.new(self.config.kubelet_config)
+    end
+
+    def install
+      global_config = self.global_config
+      config = self.config
+      on(host) do |host|
+        execute(:mkdir, '-p', Porkadot::Install::KUBE_TEMP)
+        if test("[ -d #{KUBE_TEMP} ]")
+          execute(:rm, '-rf', KUBE_TEMP)
+        end
+        upload! config.target_path, KUBE_TEMP, recursive: true
+
+        as user: 'root' do
+          execute(:bash, File.join(KUBE_TEMP, 'install.sh'))
+        end
+
+       endpoint = "https://127.0.0.1:#{global_config.k8s.apiserver.bind_port}/healthz"
+       info "Start to wait for Bootstrapping Kubernetes API: #{endpoint}"
+        while !test('curl', '-skf', endpoint)
+          info "Still wating for Bootstrapping Kubernetes API..."
+          sleep 5
+        end
+      end
+    end
+
+    def cleanup
+      global_config = self.global_config
+      config = self.config
+      on(host) do |host|
+        execute(:mkdir, '-p', Porkadot::Install::KUBE_TEMP)
+        if test("[ -d #{KUBE_TEMP} ]")
+          execute(:rm, '-rf', KUBE_TEMP)
+        end
+        upload! config.target_path, KUBE_TEMP, recursive: true
+
+        global_config.nodes.each do |k, node|
+          if node.apiserver?
+            endpoint = "https://#{node.hostname}:#{global_config.k8s.apiserver.bind_port}/healthz"
+            info "Start to wait api node #{node.hostname}"
+            while !test('curl', '-skf', endpoint)
+              info "Still waiting for API node: #{node.hostname}"
+              sleep 5
+            end
+          end
+        end
+
+        endpoint = "https://#{global_config.k8s.control_plane_endpoint}/healthz"
+        info "Start to wait api endpoint"
+        while !test('curl', '-skf', endpoint)
+          info "Still waiting for API: #{endpoint}"
+          sleep 5
+        end
+
+        as user: 'root' do
+          execute(:bash, File.join(KUBE_TEMP, 'cleanup.sh'))
+        end
+      end
+    end
+
+  end
+
+end; end

data/lib/porkadot/install/kubelet.rb

@@ -0,0 +1,63 @@
+module Porkadot; module Install
+  class KubeletList
+    KUBE_TEMP = File.join(Porkadot::Install::KUBE_TEMP, 'kubelet')
+    include SSHKit::DSL
+    attr_reader :global_config
+    attr_reader :logger
+    attr_reader :kubelets
+
+    def initialize global_config
+      @global_config = global_config
+      @logger = global_config.logger
+      @kubelets = {}
+      global_config.nodes.each do |k, config|
+        @kubelets[k] = Kubelet.new(config)
+      end
+    end
+
+    def install hosts: nil, force: false
+      unless hosts
+        hosts = []
+        self.kubelets.each do |_, v|
+          hosts << v
+        end
+      end
+
+      on(hosts) do |host|
+        execute(:mkdir, '-p', Porkadot::Install::KUBE_TEMP)
+        if test("[ -d #{KUBE_TEMP} ]")
+          execute(:rm, '-rf', KUBE_TEMP)
+        end
+        upload! host.config.target_path, KUBE_TEMP, recursive: true
+
+        as user: 'root' do
+          unless test("[ -f /opt/bin/kubelet-#{host.global_config.k8s.kubernetes_version} ]") && !force
+            execute(:bash, File.join(KUBE_TEMP, 'install-deps.sh'))
+          end
+          execute(:bash, File.join(KUBE_TEMP, 'install-pkgs.sh'))
+          execute(:bash, File.join(KUBE_TEMP, 'install.sh'))
+        end
+      end
+    end
+
+    def [](name)
+      self.kubelets[name]
+    end
+  end
+
+  class Kubelet < SSHKit::Host
+    attr_reader :global_config
+    attr_reader :config
+    attr_reader :logger
+    attr_reader :connection
+
+    def initialize config
+      @config = config
+      @logger = config.logger
+      @global_config = config.config
+      @connection = config.connection.to_hash(symbolize_keys: true)
+      super(@connection)
+    end
+
+  end
+end; end