hetzner-k3s 0.5.0 → 0.5.4

data/lib/hetzner/k3s/cluster.rb

@@ -1,20 +1,20 @@
-require 'thread'
+# frozen_string_literal: true
+
 require 'net/ssh'
-require "securerandom"
-require "base64"
+require 'securerandom'
+require 'base64'
 require 'timeout'
-require "subprocess"
-
-require_relative "../infra/client"
-require_relative "../infra/firewall"
-require_relative "../infra/network"
-require_relative "../infra/ssh_key"
-require_relative "../infra/server"
-require_relative "../infra/load_balancer"
-require_relative "../infra/placement_group"
+require 'subprocess'
 
-require_relative "../utils"
+require_relative '../infra/client'
+require_relative '../infra/firewall'
+require_relative '../infra/network'
+require_relative '../infra/ssh_key'
+require_relative '../infra/server'
+require_relative '../infra/load_balancer'
+require_relative '../infra/placement_group'
 
+require_relative '../utils'
 
 class Cluster
   include Utils
@@ -26,19 +26,25 @@ class Cluster
 
   def create(configuration:)
     @configuration = configuration
-    @cluster_name = configuration.dig("cluster_name")
-    @kubeconfig_path = File.expand_path(configuration.dig("kubeconfig_path"))
-    @public_ssh_key_path = File.expand_path(configuration.dig("public_ssh_key_path"))
-    private_ssh_key_path = configuration.dig("private_ssh_key_path")
-    @private_ssh_key_path = File.expand_path(private_ssh_key_path) if private_ssh_key_path
-    @k3s_version = configuration.dig("k3s_version")
-    @masters_config = configuration.dig("masters")
+    @cluster_name = configuration['cluster_name']
+    @kubeconfig_path = File.expand_path(configuration['kubeconfig_path'])
+    @public_ssh_key_path = File.expand_path(configuration['public_ssh_key_path'])
+    private_ssh_key_path = configuration['private_ssh_key_path']
+    @private_ssh_key_path = private_ssh_key_path && File.expand_path(private_ssh_key_path)
+    @k3s_version = configuration['k3s_version']
+    @masters_config = configuration['masters']
     @worker_node_pools = find_worker_node_pools(configuration)
-    @location = configuration.dig("location")
-    @verify_host_key = configuration.fetch("verify_host_key", false)
+    @masters_location = configuration['location']
+    @verify_host_key = configuration.fetch('verify_host_key', false)
     @servers = []
-    @networks = configuration.dig("ssh_allowed_networks")
-    @enable_ipsec_encryption = configuration.fetch("enable_ipsec_encryption", false)
+    @networks = configuration['ssh_allowed_networks']
+    @enable_encryption = configuration.fetch('enable_encryption', false)
+    @kube_api_server_args = configuration.fetch('kube_api_server_args', [])
+    @kube_scheduler_args = configuration.fetch('kube_scheduler_args', [])
+    @kube_controller_manager_args = configuration.fetch('kube_controller_manager_args', [])
+    @kube_cloud_controller_manager_args = configuration.fetch('kube_cloud_controller_manager_args', [])
+    @kubelet_args = configuration.fetch('kubelet_args', [])
+    @kube_proxy_args = configuration.fetch('kube_proxy_args', [])
 
     create_resources
 
@@ -52,17 +58,20 @@ class Cluster
   end
 
   def delete(configuration:)
-    @cluster_name = configuration.dig("cluster_name")
-    @kubeconfig_path = File.expand_path(configuration.dig("kubeconfig_path"))
-    @public_ssh_key_path = File.expand_path(configuration.dig("public_ssh_key_path"))
+    @configuration = configuration
+    @cluster_name = configuration['cluster_name']
+    @kubeconfig_path = File.expand_path(configuration['kubeconfig_path'])
+    @public_ssh_key_path = File.expand_path(configuration['public_ssh_key_path'])
+    @masters_config = configuration['masters']
+    @worker_node_pools = find_worker_node_pools(configuration)
 
     delete_resources
   end
 
   def upgrade(configuration:, new_k3s_version:, config_file:)
     @configuration = configuration
-    @cluster_name = configuration.dig("cluster_name")
-    @kubeconfig_path = File.expand_path(configuration.dig("kubeconfig_path"))
+    @cluster_name = configuration['cluster_name']
+    @kubeconfig_path = File.expand_path(configuration['kubeconfig_path'])
     @new_k3s_version = new_k3s_version
     @config_file = config_file
 
@@ -71,472 +80,561 @@ class Cluster
 
   private
 
-    def find_worker_node_pools(configuration)
-      configuration.fetch("worker_node_pools", [])
-    end
+  attr_accessor :servers
 
-    attr_accessor :servers
+  attr_reader :hetzner_client, :cluster_name, :kubeconfig_path, :k3s_version,
+              :masters_config, :worker_node_pools,
+              :masters_location, :public_ssh_key_path,
+              :hetzner_token, :new_k3s_version, :configuration,
+              :config_file, :verify_host_key, :networks, :private_ssh_key_path,
+              :enable_encryption, :kube_api_server_args, :kube_scheduler_args,
+              :kube_controller_manager_args, :kube_cloud_controller_manager_args,
+              :kubelet_args, :kube_proxy_args
 
-    attr_reader :hetzner_client, :cluster_name, :kubeconfig_path, :k3s_version,
-                :masters_config, :worker_node_pools,
-                :location, :public_ssh_key_path,
-                :hetzner_token, :tls_sans, :new_k3s_version, :configuration,
-                :config_file, :verify_host_key, :networks, :private_ssh_key_path,
-                :configuration, :enable_ipsec_encryption
+  def find_worker_node_pools(configuration)
+    configuration.fetch('worker_node_pools', [])
+  end
+
+  def latest_k3s_version
+    response = HTTP.get('https://api.github.com/repos/k3s-io/k3s/tags').body
+    JSON.parse(response).first['name']
+  end
 
+  def create_resources
+    create_servers
+    create_load_balancer if masters.size > 1
+  end
 
-    def latest_k3s_version
-      response = HTTP.get("https://api.github.com/repos/k3s-io/k3s/tags").body
-      JSON.parse(response).first["name"]
+  def delete_placement_groups
+    Hetzner::PlacementGroup.new(hetzner_client:, cluster_name:).delete
+
+    worker_node_pools.each do |pool|
+      pool_name = pool['name']
+      Hetzner::PlacementGroup.new(hetzner_client:, cluster_name:, pool_name:).delete
     end
+  end
 
-    def create_resources
-      master_instance_type = masters_config["instance_type"]
-      masters_count = masters_config["instance_count"]
-
-      placement_group_id = Hetzner::PlacementGroup.new(
-        hetzner_client: hetzner_client,
-        cluster_name: cluster_name
-      ).create
-
-      firewall_id = Hetzner::Firewall.new(
-        hetzner_client: hetzner_client,
-        cluster_name: cluster_name
-      ).create(ha: (masters_count > 1), networks: networks)
-
-      network_id = Hetzner::Network.new(
-        hetzner_client: hetzner_client,
-        cluster_name: cluster_name
-      ).create(location: location)
-
-      ssh_key_id = Hetzner::SSHKey.new(
-        hetzner_client: hetzner_client,
-        cluster_name: cluster_name
-      ).create(public_ssh_key_path: public_ssh_key_path)
-
-      server_configs = []
-
-      masters_count.times do |i|
-        server_configs << {
-          location: location,
-          instance_type: master_instance_type,
-          instance_id: "master#{i+1}",
-          firewall_id: firewall_id,
-          network_id: network_id,
-          ssh_key_id: ssh_key_id,
-          placement_group_id: placement_group_id,
-          image: image,
-          additional_packages: additional_packages,
-        }
-      end
+  def delete_resources
+    Hetzner::LoadBalancer.new(hetzner_client:, cluster_name:).delete(high_availability: (masters.size > 1))
 
-      if masters_count > 1
-        Hetzner::LoadBalancer.new(
-          hetzner_client: hetzner_client,
-          cluster_name: cluster_name
-        ).create(location: location, network_id: network_id)
-      end
+    Hetzner::Firewall.new(hetzner_client:, cluster_name:).delete(all_servers)
 
-      worker_node_pools.each do |worker_node_pool|
-        worker_node_pool_name = worker_node_pool["name"]
-        worker_instance_type = worker_node_pool["instance_type"]
-        worker_count = worker_node_pool["instance_count"]
-
-        worker_count.times do |i|
-          server_configs << {
-            location: location,
-            instance_type: worker_instance_type,
-            instance_id: "pool-#{worker_node_pool_name}-worker#{i+1}",
-            firewall_id: firewall_id,
-            network_id: network_id,
-            ssh_key_id: ssh_key_id,
-            placement_group_id: placement_group_id,
-            image: image,
-            additional_packages: additional_packages,
-          }
-        end
-      end
+    Hetzner::Network.new(hetzner_client:, cluster_name:).delete
 
-      threads = server_configs.map do |server_config|
-        Thread.new do
-          servers << Hetzner::Server.new(hetzner_client: hetzner_client, cluster_name: cluster_name).create(**server_config)
-        end
-      end
+    Hetzner::SSHKey.new(hetzner_client:, cluster_name:).delete(public_ssh_key_path:)
 
-      threads.each(&:join) unless threads.empty?
+    delete_placement_groups
+    delete_servers
+  end
 
-      while servers.size != server_configs.size
-        sleep 1
-      end
+  def upgrade_cluster
+    worker_upgrade_concurrency = workers.size - 1
+    worker_upgrade_concurrency = 1 if worker_upgrade_concurrency.zero?
+
+    cmd = <<~BASH
+      kubectl apply -f - <<-EOF
+        apiVersion: upgrade.cattle.io/v1
+        kind: Plan
+        metadata:
+          name: k3s-server
+          namespace: system-upgrade
+          labels:
+            k3s-upgrade: server
+        spec:
+          concurrency: 1
+          version: #{new_k3s_version}
+          nodeSelector:
+            matchExpressions:
+              - {key: node-role.kubernetes.io/master, operator: In, values: ["true"]}
+          serviceAccountName: system-upgrade
+          tolerations:
+          - key: "CriticalAddonsOnly"
+            operator: "Equal"
+            value: "true"
+            effect: "NoExecute"
+          cordon: true
+          upgrade:
+            image: rancher/k3s-upgrade
+      EOF
+    BASH
+
+    run cmd, kubeconfig_path: kubeconfig_path
+
+    cmd = <<~BASH
+      kubectl apply -f - <<-EOF
+        apiVersion: upgrade.cattle.io/v1
+        kind: Plan
+        metadata:
+          name: k3s-agent
+          namespace: system-upgrade
+          labels:
+            k3s-upgrade: agent
+        spec:
+          concurrency: #{worker_upgrade_concurrency}
+          version: #{new_k3s_version}
+          nodeSelector:
+            matchExpressions:
+              - {key: node-role.kubernetes.io/master, operator: NotIn, values: ["true"]}
+          serviceAccountName: system-upgrade
+          prepare:
+            image: rancher/k3s-upgrade
+            args: ["prepare", "k3s-server"]
+          cordon: true
+          upgrade:
+            image: rancher/k3s-upgrade
+      EOF
+    BASH
 
-      puts
-      threads = servers.map do |server|
-        Thread.new { wait_for_ssh server }
-      end
+    run cmd, kubeconfig_path: kubeconfig_path
 
-      threads.each(&:join) unless threads.empty?
-    end
+    puts 'Upgrade will now start. Run `watch kubectl get nodes` to see the nodes being upgraded. This should take a few minutes for a small cluster.'
+    puts 'The API server may be briefly unavailable during the upgrade of the controlplane.'
+
+    configuration['k3s_version'] = new_k3s_version
+
+    File.write(config_file, configuration.to_yaml)
+  end
+
+  def master_script(master)
+    server = master == first_master ? ' --cluster-init ' : " --server https://#{api_server_ip}:6443 "
+    flannel_interface = find_flannel_interface(master)
+    flannel_wireguard = enable_encryption ? ' --flannel-backend=wireguard ' : ' '
+    extra_args = "#{kube_api_server_args_list} #{kube_scheduler_args_list} #{kube_controller_manager_args_list} #{kube_cloud_controller_manager_args_list} #{kubelet_args_list} #{kube_proxy_args_list}"
+    taint = schedule_workloads_on_masters? ? ' ' : ' --node-taint CriticalAddonsOnly=true:NoExecute '
+
+    <<~SCRIPT
+      curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION="#{k3s_version}" K3S_TOKEN="#{k3s_token}" INSTALL_K3S_EXEC="server \
+        --disable-cloud-controller \
+        --disable servicelb \
+        --disable traefik \
+        --disable local-storage \
+        --disable metrics-server \
+        --write-kubeconfig-mode=644 \
+        --node-name="$(hostname -f)" \
+        --cluster-cidr=10.244.0.0/16 \
+        --etcd-expose-metrics=true \
+        #{flannel_wireguard} \
+        --kube-controller-manager-arg="address=0.0.0.0" \
+        --kube-controller-manager-arg="bind-address=0.0.0.0" \
+        --kube-proxy-arg="metrics-bind-address=0.0.0.0" \
+        --kube-scheduler-arg="address=0.0.0.0" \
+        --kube-scheduler-arg="bind-address=0.0.0.0" \
+        #{taint} #{extra_args} \
+        --kubelet-arg="cloud-provider=external" \
+        --advertise-address=$(hostname -I | awk '{print $2}') \
+        --node-ip=$(hostname -I | awk '{print $2}') \
+        --node-external-ip=$(hostname -I | awk '{print $1}') \
+        --flannel-iface=#{flannel_interface} \
+        #{server} #{tls_sans}" sh -
+    SCRIPT
+  end
+
+  def worker_script(worker)
+    flannel_interface = find_flannel_interface(worker)
+
+    <<~BASH
+      curl -sfL https://get.k3s.io | K3S_TOKEN="#{k3s_token}" INSTALL_K3S_VERSION="#{k3s_version}" K3S_URL=https://#{first_master_private_ip}:6443 INSTALL_K3S_EXEC="agent \
+        --node-name="$(hostname -f)" \
+        --kubelet-arg="cloud-provider=external" \
+        --node-ip=$(hostname -I | awk '{print $2}') \
+        --node-external-ip=$(hostname -I | awk '{print $1}') \
+        --flannel-iface=#{flannel_interface}" sh -
+    BASH
+  end
+
+  def deploy_kubernetes
+    puts
+    puts "Deploying k3s to first master (#{first_master['name']})..."
 
-    def delete_resources
-      Hetzner::PlacementGroup.new(
-        hetzner_client: hetzner_client,
-        cluster_name: cluster_name
-      ).delete
-
-      Hetzner::LoadBalancer.new(
-        hetzner_client: hetzner_client,
-        cluster_name: cluster_name
-      ).delete(ha: (masters.size > 1))
-
-      Hetzner::Firewall.new(
-        hetzner_client: hetzner_client,
-        cluster_name: cluster_name
-      ).delete(all_servers)
-
-      Hetzner::Network.new(
-        hetzner_client: hetzner_client,
-        cluster_name: cluster_name
-      ).delete
-
-      Hetzner::SSHKey.new(
-        hetzner_client: hetzner_client,
-        cluster_name: cluster_name
-      ).delete(public_ssh_key_path: public_ssh_key_path)
-
-      threads = all_servers.map do |server|
+    ssh first_master, master_script(first_master), print_output: true
+
+    puts
+    puts '...k3s has been deployed to first master.'
+
+    save_kubeconfig
+
+    if masters.size > 1
+      threads = masters[1..].map do |master|
         Thread.new do
-          Hetzner::Server.new(hetzner_client: hetzner_client, cluster_name: cluster_name).delete(server_name: server["name"])
+          puts
+          puts "Deploying k3s to master #{master['name']}..."
+
+          ssh master, master_script(master), print_output: true
+
+          puts
+          puts "...k3s has been deployed to master #{master['name']}."
         end
       end
 
       threads.each(&:join) unless threads.empty?
     end
 
-    def upgrade_cluster
-      worker_upgrade_concurrency = workers.size - 1
-      worker_upgrade_concurrency = 1 if worker_upgrade_concurrency == 0
-
-      cmd = <<~EOS
-        kubectl apply -f - <<-EOF
-          apiVersion: upgrade.cattle.io/v1
-          kind: Plan
-          metadata:
-            name: k3s-server
-            namespace: system-upgrade
-            labels:
-              k3s-upgrade: server
-          spec:
-            concurrency: 1
-            version: #{new_k3s_version}
-            nodeSelector:
-              matchExpressions:
-                - {key: node-role.kubernetes.io/master, operator: In, values: ["true"]}
-            serviceAccountName: system-upgrade
-            tolerations:
-            - key: "CriticalAddonsOnly"
-              operator: "Equal"
-              value: "true"
-              effect: "NoExecute"
-            cordon: true
-            upgrade:
-              image: rancher/k3s-upgrade
-        EOF
-      EOS
-
-      run cmd, kubeconfig_path: kubeconfig_path
-
-      cmd = <<~EOS
-        kubectl apply -f - <<-EOF
-          apiVersion: upgrade.cattle.io/v1
-          kind: Plan
-          metadata:
-            name: k3s-agent
-            namespace: system-upgrade
-            labels:
-              k3s-upgrade: agent
-          spec:
-            concurrency: #{worker_upgrade_concurrency}
-            version: #{new_k3s_version}
-            nodeSelector:
-              matchExpressions:
-                - {key: node-role.kubernetes.io/master, operator: NotIn, values: ["true"]}
-            serviceAccountName: system-upgrade
-            prepare:
-              image: rancher/k3s-upgrade
-              args: ["prepare", "k3s-server"]
-            cordon: true
-            upgrade:
-              image: rancher/k3s-upgrade
-        EOF
-      EOS
-
-      run cmd, kubeconfig_path: kubeconfig_path
-
-      puts "Upgrade will now start. Run `watch kubectl get nodes` to see the nodes being upgraded. This should take a few minutes for a small cluster."
-      puts "The API server may be briefly unavailable during the upgrade of the controlplane."
-
-      configuration["k3s_version"] = new_k3s_version
-
-      File.write(config_file, configuration.to_yaml)
-    end
+    threads = workers.map do |worker|
+      Thread.new do
+        puts
+        puts "Deploying k3s to worker (#{worker['name']})..."
 
+        ssh worker, worker_script(worker), print_output: true
 
-    def master_script(master)
-      server = master == first_master ? " --cluster-init " : " --server https://#{first_master_private_ip}:6443 "
-      flannel_interface = find_flannel_interface(master)
-      flannel_ipsec = enable_ipsec_encryption ? " --flannel-backend=ipsec " : " "
-
-      taint = schedule_workloads_on_masters? ? " " : " --node-taint CriticalAddonsOnly=true:NoExecute "
-
-      <<~EOF
-        curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION="#{k3s_version}" K3S_TOKEN="#{k3s_token}" INSTALL_K3S_EXEC="server \
-          --disable-cloud-controller \
-          --disable servicelb \
-          --disable traefik \
-          --disable local-storage \
-          --disable metrics-server \
-          --write-kubeconfig-mode=644 \
-          --node-name="$(hostname -f)" \
-          --cluster-cidr=10.244.0.0/16 \
-          --etcd-expose-metrics=true \
-          #{flannel_ipsec} \
-          --kube-controller-manager-arg="address=0.0.0.0" \
-          --kube-controller-manager-arg="bind-address=0.0.0.0" \
-          --kube-proxy-arg="metrics-bind-address=0.0.0.0" \
-          --kube-scheduler-arg="address=0.0.0.0" \
-          --kube-scheduler-arg="bind-address=0.0.0.0" \
-          #{taint} \
-          --kubelet-arg="cloud-provider=external" \
-          --advertise-address=$(hostname -I | awk '{print $2}') \
-          --node-ip=$(hostname -I | awk '{print $2}') \
-          --node-external-ip=$(hostname -I | awk '{print $1}') \
-          --flannel-iface=#{flannel_interface} \
-          #{server} #{tls_sans}" sh -
-      EOF
+        puts
+        puts "...k3s has been deployed to worker (#{worker['name']})."
+      end
     end
 
-    def worker_script(worker)
-      flannel_interface = find_flannel_interface(worker)
+    threads.each(&:join) unless threads.empty?
+  end
 
-      <<~EOF
-        curl -sfL https://get.k3s.io | K3S_TOKEN="#{k3s_token}" INSTALL_K3S_VERSION="#{k3s_version}" K3S_URL=https://#{first_master_private_ip}:6443 INSTALL_K3S_EXEC="agent \
-          --node-name="$(hostname -f)" \
-          --kubelet-arg="cloud-provider=external" \
-          --node-ip=$(hostname -I | awk '{print $2}') \
-          --node-external-ip=$(hostname -I | awk '{print $1}') \
-          --flannel-iface=#{flannel_interface}" sh -
+  def deploy_cloud_controller_manager
+    check_kubectl
+
+    puts
+    puts 'Deploying Hetzner Cloud Controller Manager...'
+
+    cmd = <<~BASH
+      kubectl apply -f - <<-EOF
+        apiVersion: "v1"
+        kind: "Secret"
+        metadata:
+          namespace: 'kube-system'
+          name: 'hcloud'
+        stringData:
+          network: "#{cluster_name}"
+          token: "#{hetzner_token}"
       EOF
-    end
+    BASH
 
-    def deploy_kubernetes
-      puts
-      puts "Deploying k3s to first master (#{first_master["name"]})..."
+    run cmd, kubeconfig_path: kubeconfig_path
 
-      ssh first_master, master_script(first_master), print_output: true
+    cmd = 'kubectl apply -f https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/latest/download/ccm-networks.yaml'
 
-      puts
-      puts "...k3s has been deployed to first master."
+    run cmd, kubeconfig_path: kubeconfig_path
 
-      save_kubeconfig
+    puts '...Cloud Controller Manager deployed'
+  end
 
-      if masters.size > 1
-        threads = masters[1..-1].map do |master|
-          Thread.new do
-            puts
-            puts "Deploying k3s to master #{master["name"]}..."
+  def deploy_system_upgrade_controller
+    check_kubectl
 
-            ssh master, master_script(master), print_output: true
+    puts
+    puts 'Deploying k3s System Upgrade Controller...'
 
-            puts
-            puts "...k3s has been deployed to master #{master["name"]}."
-          end
-        end
+    cmd = 'kubectl apply -f https://github.com/rancher/system-upgrade-controller/releases/download/v0.8.1/system-upgrade-controller.yaml'
 
-        threads.each(&:join) unless threads.empty?
-      end
+    run cmd, kubeconfig_path: kubeconfig_path
 
-      threads = workers.map do |worker|
-        Thread.new do
-          puts
-          puts "Deploying k3s to worker (#{worker["name"]})..."
+    puts '...k3s System Upgrade Controller deployed'
+  end
 
-          ssh worker, worker_script(worker), print_output: true
+  def deploy_csi_driver
+    check_kubectl
+
+    puts
+    puts 'Deploying Hetzner CSI Driver...'
+
+    cmd = <<~BASH
+      kubectl apply -f - <<-EOF
+        apiVersion: "v1"
+        kind: "Secret"
+        metadata:
+          namespace: 'kube-system'
+          name: 'hcloud-csi'
+        stringData:
+          token: "#{hetzner_token}"
+      EOF
+    BASH
 
-          puts
-          puts "...k3s has been deployed to worker (#{worker["name"]})."
-        end
-      end
+    run cmd, kubeconfig_path: kubeconfig_path
 
-      threads.each(&:join) unless threads.empty?
-    end
+    cmd = 'kubectl apply -f https://raw.githubusercontent.com/hetznercloud/csi-driver/v1.6.0/deploy/kubernetes/hcloud-csi.yml'
 
-    def deploy_cloud_controller_manager
-      check_kubectl
+    run cmd, kubeconfig_path: kubeconfig_path
 
-      puts
-      puts "Deploying Hetzner Cloud Controller Manager..."
+    puts '...CSI Driver deployed'
+  end
 
-      cmd = <<~EOS
-        kubectl apply -f - <<-EOF
-          apiVersion: "v1"
-          kind: "Secret"
-          metadata:
-            namespace: 'kube-system'
-            name: 'hcloud'
-          stringData:
-            network: "#{cluster_name}"
-            token: "#{hetzner_token}"
-        EOF
-      EOS
+  def find_flannel_interface(server)
+    if ssh(server, 'lscpu | grep Vendor') =~ /Intel/
+      'ens10'
+    else
+      'enp7s0'
+    end
+  end
+
+  def all_servers
+    @all_servers ||= hetzner_client.get('/servers?sort=created:desc')['servers'].select do |server|
+      belongs_to_cluster?(server) == true
+    end
+  end
 
-      run cmd, kubeconfig_path: kubeconfig_path
+  def masters
+    @masters ||= all_servers.select { |server| server['name'] =~ /master\d+\Z/ }.sort { |a, b| a['name'] <=> b['name'] }
+  end
 
-      cmd = "kubectl apply -f https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/latest/download/ccm-networks.yaml"
+  def workers
+    @workers = all_servers.select { |server| server['name'] =~ /worker\d+\Z/ }.sort { |a, b| a['name'] <=> b['name'] }
+  end
 
-      run cmd, kubeconfig_path: kubeconfig_path
+  def k3s_token
+    @k3s_token ||= begin
+      token = ssh(first_master, '{ TOKEN=$(< /var/lib/rancher/k3s/server/node-token); } 2> /dev/null; echo $TOKEN')
 
-      puts "...Cloud Controller Manager deployed"
+      if token.empty?
+        SecureRandom.hex
+      else
+        token.split(':').last
+      end
     end
+  end
 
-    def deploy_system_upgrade_controller
-      check_kubectl
+  def first_master_private_ip
+    @first_master_private_ip ||= first_master['private_net'][0]['ip']
+  end
 
-      puts
-      puts "Deploying k3s System Upgrade Controller..."
+  def first_master
+    masters.first
+  end
 
-      cmd = "kubectl apply -f https://github.com/rancher/system-upgrade-controller/releases/download/v0.8.1/system-upgrade-controller.yaml"
+  def api_server_ip
+    return @api_server_ip if @api_server_ip
+
+    @api_server_ip = if masters.size > 1
+                       load_balancer_name = "#{cluster_name}-api"
+                       load_balancer = hetzner_client.get('/load_balancers')['load_balancers'].detect do |lb|
+                         lb['name'] == load_balancer_name
+                       end
+                       load_balancer['public_net']['ipv4']['ip']
+                     else
+                       first_master_public_ip
+                     end
+  end
 
-      run cmd, kubeconfig_path: kubeconfig_path
+  def tls_sans
+    sans = " --tls-san=#{api_server_ip} "
 
-      puts "...k3s System Upgrade Controller deployed"
+    masters.each do |master|
+      master_private_ip = master['private_net'][0]['ip']
+      sans << " --tls-san=#{master_private_ip} "
     end
 
-    def deploy_csi_driver
-      check_kubectl
+    sans
+  end
 
-      puts
-      puts "Deploying Hetzner CSI Driver..."
+  def first_master_public_ip
+    @first_master_public_ip ||= first_master.dig('public_net', 'ipv4', 'ip')
+  end
 
-      cmd = <<~EOS
-        kubectl apply -f - <<-EOF
-          apiVersion: "v1"
-          kind: "Secret"
-          metadata:
-            namespace: 'kube-system'
-            name: 'hcloud-csi'
-          stringData:
-            token: "#{hetzner_token}"
-        EOF
-      EOS
+  def save_kubeconfig
+    kubeconfig = ssh(first_master, 'cat /etc/rancher/k3s/k3s.yaml')
+                 .gsub('127.0.0.1', api_server_ip)
+                 .gsub('default', cluster_name)
 
-      run cmd, kubeconfig_path: kubeconfig_path
+    File.write(kubeconfig_path, kubeconfig)
 
-      cmd = "kubectl apply -f https://raw.githubusercontent.com/hetznercloud/csi-driver/v1.6.0/deploy/kubernetes/hcloud-csi.yml"
+    FileUtils.chmod 'go-r', kubeconfig_path
+  end
 
-      run cmd, kubeconfig_path: kubeconfig_path
+  def belongs_to_cluster?(server)
+    server.dig('labels', 'cluster') == cluster_name
+  end
 
-      puts "...CSI Driver deployed"
-    end
+  def schedule_workloads_on_masters?
+    schedule_workloads_on_masters = configuration['schedule_workloads_on_masters']
+    schedule_workloads_on_masters ? !!schedule_workloads_on_masters : false
+  end
 
-    def find_flannel_interface(server)
-      if ssh(server, "lscpu | grep Vendor") =~ /Intel/
-        "ens10"
-      else
-        "enp7s0"
-      end
+  def image
+    configuration['image'] || 'ubuntu-20.04'
+  end
+
+  def additional_packages
+    configuration['additional_packages'] || []
+  end
+
+  def check_kubectl
+    return if which('kubectl')
+
+    puts 'Please ensure kubectl is installed and in your PATH.'
+    exit 1
+  end
+
+  def placement_group_id(pool_name = nil)
+    @placement_groups ||= {}
+    @placement_groups[pool_name || '__masters__'] ||= Hetzner::PlacementGroup.new(hetzner_client:, cluster_name:, pool_name:).create
+  end
+
+  def master_instance_type
+    @master_instance_type ||= masters_config['instance_type']
+  end
+
+  def masters_count
+    @masters_count ||= masters_config['instance_count']
+  end
+
+  def firewall_id
+    @firewall_id ||= Hetzner::Firewall.new(hetzner_client:, cluster_name:).create(high_availability: (masters_count > 1), networks:)
+  end
+
+  def network_id
+    @network_id ||= Hetzner::Network.new(hetzner_client:, cluster_name:).create(location: masters_location)
+  end
+
+  def ssh_key_id
+    @ssh_key_id ||= Hetzner::SSHKey.new(hetzner_client:, cluster_name:).create(public_ssh_key_path:)
+  end
+
+  def master_definitions_for_create
+    definitions = []
+
+    masters_count.times do |i|
+      definitions << {
+        instance_type: master_instance_type,
+        instance_id: "master#{i + 1}",
+        location: masters_location,
+        placement_group_id:,
+        firewall_id:,
+        network_id:,
+        ssh_key_id:,
+        image:,
+        additional_packages:
+      }
     end
 
-    def all_servers
-      @all_servers ||= hetzner_client.get("/servers?sort=created:desc")["servers"].select{ |server| belongs_to_cluster?(server) == true }
+    definitions
+  end
+
+  def master_definitions_for_delete
+    definitions = []
+
+    masters_count.times do |i|
+      definitions << {
+        instance_type: master_instance_type,
+        instance_id: "master#{i + 1}"
+      }
     end
 
-    def masters
-      @masters ||= all_servers.select{ |server| server["name"] =~ /master\d+\Z/ }.sort{ |a, b| a["name"] <=> b["name"] }
+    definitions
+  end
+
+  def worker_node_pool_definitions(worker_node_pool)
+    worker_node_pool_name = worker_node_pool['name']
+    worker_instance_type = worker_node_pool['instance_type']
+    worker_count = worker_node_pool['instance_count']
+    worker_location = worker_node_pool['location'] || masters_location
+
+    definitions = []
+
+    worker_count.times do |i|
+      definitions << {
+        instance_type: worker_instance_type,
+        instance_id: "pool-#{worker_node_pool_name}-worker#{i + 1}",
+        placement_group_id: placement_group_id(worker_node_pool_name),
+        location: worker_location,
+        firewall_id:,
+        network_id:,
+        ssh_key_id:,
+        image:,
+        additional_packages:
+      }
     end
 
-    def workers
-      @workers = all_servers.select{ |server| server["name"] =~ /worker\d+\Z/ }.sort{ |a, b| a["name"] <=> b["name"] }
+    definitions
+  end
+
+  def create_load_balancer
+    Hetzner::LoadBalancer.new(hetzner_client:, cluster_name:).create(location:, network_id:)
+  end
+
+  def server_configs
+    return @server_configs if @server_configs
+
+    @server_configs = master_definitions_for_create
+
+    worker_node_pools.each do |worker_node_pool|
+      @server_configs += worker_node_pool_definitions(worker_node_pool)
     end
 
-    def k3s_token
-      @k3s_token ||= begin
-        token = ssh(first_master, "{ TOKEN=$(< /var/lib/rancher/k3s/server/node-token); } 2> /dev/null; echo $TOKEN")
+    @server_configs
+  end
 
-        if token.empty?
-          SecureRandom.hex
-        else
-          token.split(":").last
-        end
+  def create_servers
+    servers = []
+
+    threads = server_configs.map do |server_config|
+      Thread.new do
+        servers << Hetzner::Server.new(hetzner_client:, cluster_name:).create(**server_config)
       end
     end
 
-    def first_master_private_ip
-      @first_master_private_ip ||= first_master["private_net"][0]["ip"]
-    end
+    threads.each(&:join) unless threads.empty?
+
+    sleep 1 while servers.size != server_configs.size
 
-    def first_master
-      masters.first
+    wait_for_servers(servers)
+  end
+
+  def wait_for_servers(servers)
+    threads = servers.map do |server|
+      Thread.new { wait_for_ssh server }
     end
 
-    def api_server_ip
-      return @api_server_ip if @api_server_ip
+    threads.each(&:join) unless threads.empty?
+  end
 
-      @api_server_ip = if masters.size > 1
-        load_balancer_name = "#{cluster_name}-api"
-        load_balancer = hetzner_client.get("/load_balancers")["load_balancers"].detect{ |load_balancer| load_balancer["name"] == load_balancer_name }
-        load_balancer["public_net"]["ipv4"]["ip"]
-      else
-        first_master_public_ip
+  def delete_servers
+    threads = all_servers.map do |server|
+      Thread.new do
+        Hetzner::Server.new(hetzner_client:, cluster_name:).delete(server_name: server['name'])
       end
     end
 
-    def tls_sans
-      sans = " --tls-san=#{api_server_ip} "
+    threads.each(&:join) unless threads.empty?
+  end
 
-      masters.each do |master|
-        master_private_ip = master["private_net"][0]["ip"]
-        sans << " --tls-san=#{master_private_ip} "
-      end
+  def kube_api_server_args_list
+    return '' if kube_api_server_args.empty?
 
-      sans
-    end
+    kube_api_server_args.map do |arg|
+      " --kube-apiserver-arg=\"#{arg}\" "
+    end.join
+  end
 
-    def first_master_public_ip
-      @first_master_public_ip ||= first_master.dig("public_net", "ipv4", "ip")
-    end
+  def kube_scheduler_args_list
+    return '' if kube_scheduler_args.empty?
 
-    def save_kubeconfig
-      kubeconfig = ssh(first_master, "cat /etc/rancher/k3s/k3s.yaml").
-        gsub("127.0.0.1", api_server_ip).
-        gsub("default", cluster_name)
+    kube_scheduler_args.map do |arg|
+      " --kube-scheduler-arg=\"#{arg}\" "
+    end.join
+  end
 
-      File.write(kubeconfig_path, kubeconfig)
+  def kube_controller_manager_args_list
+    return '' if kube_controller_manager_args.empty?
 
-      FileUtils.chmod "go-r", kubeconfig_path
-    end
+    kube_controller_manager_args.map do |arg|
+      " --kube-controller-manager-arg=\"#{arg}\" "
+    end.join
+  end
 
-    def belongs_to_cluster?(server)
-      server.dig("labels", "cluster") == cluster_name
-    end
+  def kube_cloud_controller_manager_args_list
+    return '' if kube_cloud_controller_manager_args.empty?
 
-    def schedule_workloads_on_masters?
-      schedule_workloads_on_masters = configuration.dig("schedule_workloads_on_masters")
-      schedule_workloads_on_masters ? !!schedule_workloads_on_masters : false
-    end
+    kube_cloud_controller_manager_args.map do |arg|
+      " --kube-cloud-controller-manager-arg=\"#{arg}\" "
+    end.join
+  end
 
-    def image
-      configuration.dig("image") || "ubuntu-20.04"
-    end
+  def kubelet_args_list
+    return '' if kubelet_args.empty?
 
-    def additional_packages
-      configuration.dig("additional_packages") || []
-    end
+    kubelet_args.map do |arg|
+      " --kubelet-arg=\"#{arg}\" "
+    end.join
+  end
 
-    def check_kubectl
-      unless which("kubectl")
-        puts "Please ensure kubectl is installed and in your PATH."
-        exit 1
-      end
-    end
+  def kube_proxy_args_list
+    return '' if kube_proxy_args.empty?
 
+    kube_api_server_args.map do |arg|
+      " --kube-proxy-arg=\"#{arg}\" "
+    end.join
+  end
 end