hetzner-k3s 0.4.7 → 0.5.1

data/lib/hetzner/k3s/cluster.rb

@@ -1,22 +1,24 @@
-require 'thread'
+# frozen_string_literal: true
+
 require 'net/ssh'
-require "securerandom"
-require "base64"
-require "k8s-ruby"
+require 'securerandom'
+require 'base64'
 require 'timeout'
+require 'subprocess'
 
-require_relative "../infra/client"
-require_relative "../infra/firewall"
-require_relative "../infra/network"
-require_relative "../infra/ssh_key"
-require_relative "../infra/server"
-require_relative "../infra/load_balancer"
-require_relative "../infra/placement_group"
-
-require_relative "../k3s/client_patch"
+require_relative '../infra/client'
+require_relative '../infra/firewall'
+require_relative '../infra/network'
+require_relative '../infra/ssh_key'
+require_relative '../infra/server'
+require_relative '../infra/load_balancer'
+require_relative '../infra/placement_group'
 
+require_relative '../utils'
 
 class Cluster
+  include Utils
+
   def initialize(hetzner_client:, hetzner_token:)
     @hetzner_client = hetzner_client
     @hetzner_token = hetzner_token
@@ -24,18 +26,19 @@ class Cluster
 
   def create(configuration:)
     @configuration = configuration
-    @cluster_name = configuration.dig("cluster_name")
-    @kubeconfig_path = File.expand_path(configuration.dig("kubeconfig_path"))
-    @public_ssh_key_path = File.expand_path(configuration.dig("public_ssh_key_path"))
-    private_ssh_key_path = configuration.dig("private_ssh_key_path")
-    @private_ssh_key_path = File.expand_path(private_ssh_key_path) if private_ssh_key_path
-    @k3s_version = configuration.dig("k3s_version")
-    @masters_config = configuration.dig("masters")
+    @cluster_name = configuration['cluster_name']
+    @kubeconfig_path = File.expand_path(configuration['kubeconfig_path'])
+    @public_ssh_key_path = File.expand_path(configuration['public_ssh_key_path'])
+    private_ssh_key_path = configuration['private_ssh_key_path']
+    @private_ssh_key_path = private_ssh_key_path && File.expand_path(private_ssh_key_path)
+    @k3s_version = configuration['k3s_version']
+    @masters_config = configuration['masters']
     @worker_node_pools = find_worker_node_pools(configuration)
-    @location = configuration.dig("location")
-    @verify_host_key = configuration.fetch("verify_host_key", false)
+    @location = configuration['location']
+    @verify_host_key = configuration.fetch('verify_host_key', false)
     @servers = []
-    @networks = configuration.dig("ssh_allowed_networks")
+    @networks = configuration['ssh_allowed_networks']
+    @enable_ipsec_encryption = configuration.fetch('enable_ipsec_encryption', false)
 
     create_resources
 
@@ -49,17 +52,20 @@ class Cluster
   end
 
   def delete(configuration:)
-    @cluster_name = configuration.dig("cluster_name")
-    @kubeconfig_path = File.expand_path(configuration.dig("kubeconfig_path"))
-    @public_ssh_key_path = File.expand_path(configuration.dig("public_ssh_key_path"))
+    @configuration = configuration
+    @cluster_name = configuration['cluster_name']
+    @kubeconfig_path = File.expand_path(configuration['kubeconfig_path'])
+    @public_ssh_key_path = File.expand_path(configuration['public_ssh_key_path'])
+    @masters_config = configuration['masters']
+    @worker_node_pools = find_worker_node_pools(configuration)
 
     delete_resources
   end
 
   def upgrade(configuration:, new_k3s_version:, config_file:)
     @configuration = configuration
-    @cluster_name = configuration.dig("cluster_name")
-    @kubeconfig_path = File.expand_path(configuration.dig("kubeconfig_path"))
+    @cluster_name = configuration['cluster_name']
+    @kubeconfig_path = File.expand_path(configuration['kubeconfig_path'])
     @new_k3s_version = new_k3s_version
     @config_file = config_file
 
@@ -68,597 +74,510 @@ class Cluster
 
   private
 
-    def find_worker_node_pools(configuration)
-      configuration.fetch("worker_node_pools", [])
-    end
-
-    attr_accessor :servers
-
-    attr_reader :hetzner_client, :cluster_name, :kubeconfig_path, :k3s_version,
-                :masters_config, :worker_node_pools,
-                :location, :public_ssh_key_path, :kubernetes_client,
-                :hetzner_token, :tls_sans, :new_k3s_version, :configuration,
-                :config_file, :verify_host_key, :networks, :private_ssh_key_path, :configuration
-
-
-    def latest_k3s_version
-      response = HTTP.get("https://api.github.com/repos/k3s-io/k3s/tags").body
-      JSON.parse(response).first["name"]
-    end
-
-    def create_resources
-      master_instance_type = masters_config["instance_type"]
-      masters_count = masters_config["instance_count"]
-
-      placement_group_id = Hetzner::PlacementGroup.new(
-        hetzner_client: hetzner_client,
-        cluster_name: cluster_name
-      ).create
-
-      firewall_id = Hetzner::Firewall.new(
-        hetzner_client: hetzner_client,
-        cluster_name: cluster_name
-      ).create(ha: (masters_count > 1), networks: networks)
-
-      network_id = Hetzner::Network.new(
-        hetzner_client: hetzner_client,
-        cluster_name: cluster_name
-      ).create(location: location)
-
-      ssh_key_id = Hetzner::SSHKey.new(
-        hetzner_client: hetzner_client,
-        cluster_name: cluster_name
-      ).create(public_ssh_key_path: public_ssh_key_path)
-
-      server_configs = []
-
-      masters_count.times do |i|
-        server_configs << {
-          location: location,
-          instance_type: master_instance_type,
-          instance_id: "master#{i+1}",
-          firewall_id: firewall_id,
-          network_id: network_id,
-          ssh_key_id: ssh_key_id,
-          placement_group_id: placement_group_id,
-          image: image
-        }
-      end
-
-      if masters_count > 1
-        Hetzner::LoadBalancer.new(
-          hetzner_client: hetzner_client,
-          cluster_name: cluster_name
-        ).create(location: location, network_id: network_id)
-      end
-
-      worker_node_pools.each do |worker_node_pool|
-        worker_node_pool_name = worker_node_pool["name"]
-        worker_instance_type = worker_node_pool["instance_type"]
-        worker_count = worker_node_pool["instance_count"]
-
-        worker_count.times do |i|
-          server_configs << {
-            location: location,
-            instance_type: worker_instance_type,
-            instance_id: "pool-#{worker_node_pool_name}-worker#{i+1}",
-            firewall_id: firewall_id,
-            network_id: network_id,
-            ssh_key_id: ssh_key_id,
-            placement_group_id: placement_group_id,
-            image: image
-          }
-        end
-      end
-
-      threads = server_configs.map do |server_config|
-        Thread.new do
-          servers << Hetzner::Server.new(hetzner_client: hetzner_client, cluster_name: cluster_name).create(server_config)
-        end
-      end
+  attr_accessor :servers
 
-      threads.each(&:join) unless threads.empty?
+  attr_reader :hetzner_client, :cluster_name, :kubeconfig_path, :k3s_version,
+              :masters_config, :worker_node_pools,
+              :location, :public_ssh_key_path,
+              :hetzner_token, :new_k3s_version, :configuration,
+              :config_file, :verify_host_key, :networks, :private_ssh_key_path,
+              :enable_ipsec_encryption
 
-      while servers.size != server_configs.size
-        sleep 1
-      end
+  def find_worker_node_pools(configuration)
+    configuration.fetch('worker_node_pools', [])
+  end
 
-      puts
-      threads = servers.map do |server|
-        Thread.new { wait_for_ssh server }
-      end
+  def latest_k3s_version
+    response = HTTP.get('https://api.github.com/repos/k3s-io/k3s/tags').body
+    JSON.parse(response).first['name']
+  end
 
-      threads.each(&:join) unless threads.empty?
-    end
+  def create_resources
+    create_servers
+    create_load_balancer if masters.size > 1
+  end
 
-    def delete_resources
-      Hetzner::PlacementGroup.new(
-        hetzner_client: hetzner_client,
-        cluster_name: cluster_name
-      ).delete
-
-      Hetzner::LoadBalancer.new(
-        hetzner_client: hetzner_client,
-        cluster_name: cluster_name
-      ).delete(ha: (masters.size > 1))
-
-      Hetzner::Firewall.new(
-        hetzner_client: hetzner_client,
-        cluster_name: cluster_name
-      ).delete(all_servers)
-
-      Hetzner::Network.new(
-        hetzner_client: hetzner_client,
-        cluster_name: cluster_name
-      ).delete
-
-      Hetzner::SSHKey.new(
-        hetzner_client: hetzner_client,
-        cluster_name: cluster_name
-      ).delete(public_ssh_key_path: public_ssh_key_path)
-
-      threads = all_servers.map do |server|
-        Thread.new do
-          Hetzner::Server.new(hetzner_client: hetzner_client, cluster_name: cluster_name).delete(server_name: server["name"])
-        end
-      end
+  def delete_placement_groups
+    Hetzner::PlacementGroup.new(hetzner_client:, cluster_name:).delete
 
-      threads.each(&:join) unless threads.empty?
+    worker_node_pools.each do |pool|
+      pool_name = pool['name']
+      Hetzner::PlacementGroup.new(hetzner_client:, cluster_name:, pool_name:).delete
     end
+  end
 
-    def upgrade_cluster
-      resources = K8s::Resource.from_files(ugrade_plan_manifest_path)
+  def delete_resources
+    Hetzner::LoadBalancer.new(hetzner_client:, cluster_name:).delete(high_availability: (masters.size > 1))
 
-      begin
-        kubernetes_client.api("upgrade.cattle.io/v1").resource("plans").get("k3s-server", namespace: "system-upgrade")
+    Hetzner::Firewall.new(hetzner_client:, cluster_name:).delete(all_servers)
 
-        puts "Aborting - an upgrade is already in progress."
+    Hetzner::Network.new(hetzner_client:, cluster_name:).delete
 
-      rescue K8s::Error::NotFound
-        resources.each do |resource|
-          kubernetes_client.create_resource(resource)
-        end
+    Hetzner::SSHKey.new(hetzner_client:, cluster_name:).delete(public_ssh_key_path:)
 
-        puts "Upgrade will now start. Run `watch kubectl get nodes` to see the nodes being upgraded. This should take a few minutes for a small cluster."
-        puts "The API server may be briefly unavailable during the upgrade of the controlplane."
+    delete_placement_groups
+    delete_servers
+  end
 
-        configuration["k3s_version"] = new_k3s_version
+  def upgrade_cluster
+    worker_upgrade_concurrency = workers.size - 1
+    worker_upgrade_concurrency = 1 if worker_upgrade_concurrency.zero?
 
-        File.write(config_file, configuration.to_yaml)
-      end
-    end
+    cmd = <<~BASH
+      kubectl apply -f - <<-EOF
+        apiVersion: upgrade.cattle.io/v1
+        kind: Plan
+        metadata:
+          name: k3s-server
+          namespace: system-upgrade
+          labels:
+            k3s-upgrade: server
+        spec:
+          concurrency: 1
+          version: #{new_k3s_version}
+          nodeSelector:
+            matchExpressions:
+              - {key: node-role.kubernetes.io/master, operator: In, values: ["true"]}
+          serviceAccountName: system-upgrade
+          tolerations:
+          - key: "CriticalAddonsOnly"
+            operator: "Equal"
+            value: "true"
+            effect: "NoExecute"
+          cordon: true
+          upgrade:
+            image: rancher/k3s-upgrade
+      EOF
+    BASH
 
+    run cmd, kubeconfig_path: kubeconfig_path
 
-    def master_script(master)
-      server = master == first_master ? " --cluster-init " : " --server https://#{first_master_private_ip}:6443 "
-      flannel_interface = find_flannel_interface(master)
-
-      taint = schedule_workloads_on_masters? ? " " : " --node-taint CriticalAddonsOnly=true:NoExecute "
-
-      <<~EOF
-        curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION="#{k3s_version}" K3S_TOKEN="#{k3s_token}" INSTALL_K3S_EXEC="server \
-          --disable-cloud-controller \
-          --disable servicelb \
-          --disable traefik \
-          --disable local-storage \
-          --disable metrics-server \
-          --write-kubeconfig-mode=644 \
-          --node-name="$(hostname -f)" \
-          --cluster-cidr=10.244.0.0/16 \
-          --etcd-expose-metrics=true \
-          --kube-controller-manager-arg="address=0.0.0.0" \
-          --kube-controller-manager-arg="bind-address=0.0.0.0" \
-          --kube-proxy-arg="metrics-bind-address=0.0.0.0" \
-          --kube-scheduler-arg="address=0.0.0.0" \
-          --kube-scheduler-arg="bind-address=0.0.0.0" \
-          #{taint} \
-          --kubelet-arg="cloud-provider=external" \
-          --advertise-address=$(hostname -I | awk '{print $2}') \
-          --node-ip=$(hostname -I | awk '{print $2}') \
-          --node-external-ip=$(hostname -I | awk '{print $1}') \
-          --flannel-iface=#{flannel_interface} \
-          #{server} #{tls_sans}" sh -
+    cmd = <<~BASH
+      kubectl apply -f - <<-EOF
+        apiVersion: upgrade.cattle.io/v1
+        kind: Plan
+        metadata:
+          name: k3s-agent
+          namespace: system-upgrade
+          labels:
+            k3s-upgrade: agent
+        spec:
+          concurrency: #{worker_upgrade_concurrency}
+          version: #{new_k3s_version}
+          nodeSelector:
+            matchExpressions:
+              - {key: node-role.kubernetes.io/master, operator: NotIn, values: ["true"]}
+          serviceAccountName: system-upgrade
+          prepare:
+            image: rancher/k3s-upgrade
+            args: ["prepare", "k3s-server"]
+          cordon: true
+          upgrade:
+            image: rancher/k3s-upgrade
       EOF
-    end
+    BASH
 
-    def worker_script(worker)
-      flannel_interface = find_flannel_interface(worker)
+    run cmd, kubeconfig_path: kubeconfig_path
 
-      <<~EOF
-        curl -sfL https://get.k3s.io | K3S_TOKEN="#{k3s_token}" INSTALL_K3S_VERSION="#{k3s_version}" K3S_URL=https://#{first_master_private_ip}:6443 INSTALL_K3S_EXEC="agent \
-          --node-name="$(hostname -f)" \
-          --kubelet-arg="cloud-provider=external" \
-          --node-ip=$(hostname -I | awk '{print $2}') \
-          --node-external-ip=$(hostname -I | awk '{print $1}') \
-          --flannel-iface=#{flannel_interface}" sh -
-      EOF
-    end
+    puts 'Upgrade will now start. Run `watch kubectl get nodes` to see the nodes being upgraded. This should take a few minutes for a small cluster.'
+    puts 'The API server may be briefly unavailable during the upgrade of the controlplane.'
 
-    def deploy_kubernetes
-      puts
-      puts "Deploying k3s to first master (#{first_master["name"]})..."
+    configuration['k3s_version'] = new_k3s_version
 
-      ssh first_master, master_script(first_master), print_output: true
+    File.write(config_file, configuration.to_yaml)
+  end
 
-      puts
-      puts "...k3s has been deployed to first master."
+  def master_script(master)
+    server = master == first_master ? ' --cluster-init ' : " --server https://#{first_master_private_ip}:6443 "
+    flannel_interface = find_flannel_interface(master)
+    flannel_ipsec = enable_ipsec_encryption ? ' --flannel-backend=ipsec ' : ' '
+
+    taint = schedule_workloads_on_masters? ? ' ' : ' --node-taint CriticalAddonsOnly=true:NoExecute '
+
+    <<~SCRIPT
+      curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION="#{k3s_version}" K3S_TOKEN="#{k3s_token}" INSTALL_K3S_EXEC="server \
+        --disable-cloud-controller \
+        --disable servicelb \
+        --disable traefik \
+        --disable local-storage \
+        --disable metrics-server \
+        --write-kubeconfig-mode=644 \
+        --node-name="$(hostname -f)" \
+        --cluster-cidr=10.244.0.0/16 \
+        --etcd-expose-metrics=true \
+        #{flannel_ipsec} \
+        --kube-controller-manager-arg="address=0.0.0.0" \
+        --kube-controller-manager-arg="bind-address=0.0.0.0" \
+        --kube-proxy-arg="metrics-bind-address=0.0.0.0" \
+        --kube-scheduler-arg="address=0.0.0.0" \
+        --kube-scheduler-arg="bind-address=0.0.0.0" \
+        #{taint} \
+        --kubelet-arg="cloud-provider=external" \
+        --advertise-address=$(hostname -I | awk '{print $2}') \
+        --node-ip=$(hostname -I | awk '{print $2}') \
+        --node-external-ip=$(hostname -I | awk '{print $1}') \
+        --flannel-iface=#{flannel_interface} \
+        #{server} #{tls_sans}" sh -
+    SCRIPT
+  end
 
-      save_kubeconfig
+  def worker_script(worker)
+    flannel_interface = find_flannel_interface(worker)
+
+    <<~BASH
+      curl -sfL https://get.k3s.io | K3S_TOKEN="#{k3s_token}" INSTALL_K3S_VERSION="#{k3s_version}" K3S_URL=https://#{first_master_private_ip}:6443 INSTALL_K3S_EXEC="agent \
+        --node-name="$(hostname -f)" \
+        --kubelet-arg="cloud-provider=external" \
+        --node-ip=$(hostname -I | awk '{print $2}') \
+        --node-external-ip=$(hostname -I | awk '{print $1}') \
+        --flannel-iface=#{flannel_interface}" sh -
+    BASH
+  end
 
-      if masters.size > 1
-        threads = masters[1..-1].map do |master|
-          Thread.new do
-            puts
-            puts "Deploying k3s to master #{master["name"]}..."
+  def deploy_kubernetes
+    puts
+    puts "Deploying k3s to first master (#{first_master['name']})..."
 
-            ssh master, master_script(master), print_output: true
+    ssh first_master, master_script(first_master), print_output: true
 
-            puts
-            puts "...k3s has been deployed to master #{master["name"]}."
-          end
-        end
+    puts
+    puts '...k3s has been deployed to first master.'
 
-        threads.each(&:join) unless threads.empty?
-      end
+    save_kubeconfig
 
-      threads = workers.map do |worker|
+    if masters.size > 1
+      threads = masters[1..].map do |master|
         Thread.new do
           puts
-          puts "Deploying k3s to worker (#{worker["name"]})..."
+          puts "Deploying k3s to master #{master['name']}..."
 
-          ssh worker, worker_script(worker), print_output: true
+          ssh master, master_script(master), print_output: true
 
           puts
-          puts "...k3s has been deployed to worker (#{worker["name"]})."
+          puts "...k3s has been deployed to master #{master['name']}."
         end
       end
 
       threads.each(&:join) unless threads.empty?
     end
 
-    def deploy_cloud_controller_manager
-      puts
-      puts "Deploying Hetzner Cloud Controller Manager..."
-
-      begin
-        kubernetes_client.api("v1").resource("secrets").get("hcloud", namespace: "kube-system")
-
-      rescue K8s::Error::NotFound
-        secret = K8s::Resource.new(
-          apiVersion: "v1",
-          kind: "Secret",
-          metadata: {
-            namespace: 'kube-system',
-            name: 'hcloud',
-          },
-          data: {
-            network: Base64.encode64(cluster_name),
-            token: Base64.encode64(hetzner_token)
-          }
-        )
-
-        kubernetes_client.api('v1').resource('secrets').create_resource(secret)
-      end
-
+    threads = workers.map do |worker|
+      Thread.new do
+        puts
+        puts "Deploying k3s to worker (#{worker['name']})..."
 
-      manifest = fetch_manifest("https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/latest/download/ccm-networks.yaml")
+        ssh worker, worker_script(worker), print_output: true
 
-      File.write("/tmp/cloud-controller-manager.yaml", manifest)
+        puts
+        puts "...k3s has been deployed to worker (#{worker['name']})."
+      end
+    end
 
-      resources = K8s::Resource.from_files("/tmp/cloud-controller-manager.yaml")
+    threads.each(&:join) unless threads.empty?
+  end
 
-      begin
-        kubernetes_client.api("apps/v1").resource("deployments").get("hcloud-cloud-controller-manager", namespace: "kube-system")
+  def deploy_cloud_controller_manager
+    check_kubectl
 
-        resources.each do |resource|
-          kubernetes_client.update_resource(resource)
-        end
+    puts
+    puts 'Deploying Hetzner Cloud Controller Manager...'
 
-      rescue K8s::Error::NotFound
-        resources.each do |resource|
-          kubernetes_client.create_resource(resource)
-        end
+    cmd = <<~BASH
+      kubectl apply -f - <<-EOF
+        apiVersion: "v1"
+        kind: "Secret"
+        metadata:
+          namespace: 'kube-system'
+          name: 'hcloud'
+        stringData:
+          network: "#{cluster_name}"
+          token: "#{hetzner_token}"
+      EOF
+    BASH
 
-      end
+    run cmd, kubeconfig_path: kubeconfig_path
 
-      puts "...Cloud Controller Manager deployed"
-    rescue Excon::Error::Socket
-      retry
-    end
+    cmd = 'kubectl apply -f https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/latest/download/ccm-networks.yaml'
 
-    def fetch_manifest(url)
-      retries ||= 1
-      HTTP.follow.get(url).body
-    rescue
-      retry if (retries += 1) <= 10
-    end
+    run cmd, kubeconfig_path: kubeconfig_path
 
-    def deploy_system_upgrade_controller
-      puts
-      puts "Deploying k3s System Upgrade Controller..."
+    puts '...Cloud Controller Manager deployed'
+  end
 
-      manifest = HTTP.follow.get("https://github.com/rancher/system-upgrade-controller/releases/download/v0.8.0/system-upgrade-controller.yaml").body
+  def deploy_system_upgrade_controller
+    check_kubectl
 
-      File.write("/tmp/system-upgrade-controller.yaml", manifest)
+    puts
+    puts 'Deploying k3s System Upgrade Controller...'
 
-      resources = K8s::Resource.from_files("/tmp/system-upgrade-controller.yaml")
+    cmd = 'kubectl apply -f https://github.com/rancher/system-upgrade-controller/releases/download/v0.8.1/system-upgrade-controller.yaml'
 
-      begin
-        kubernetes_client.api("apps/v1").resource("deployments").get("system-upgrade-controller", namespace: "system-upgrade")
+    run cmd, kubeconfig_path: kubeconfig_path
 
-        resources.each do |resource|
-          kubernetes_client.update_resource(resource)
-        end
+    puts '...k3s System Upgrade Controller deployed'
+  end
 
-      rescue K8s::Error::NotFound
-        resources.each do |resource|
-          kubernetes_client.create_resource(resource)
-        end
+  def deploy_csi_driver
+    check_kubectl
 
-      end
+    puts
+    puts 'Deploying Hetzner CSI Driver...'
 
-      puts "...k3s System Upgrade Controller deployed"
-    rescue Excon::Error::Socket
-      retry
-    end
+    cmd = <<~BASH
+      kubectl apply -f - <<-EOF
+        apiVersion: "v1"
+        kind: "Secret"
+        metadata:
+          namespace: 'kube-system'
+          name: 'hcloud-csi'
+        stringData:
+          token: "#{hetzner_token}"
+      EOF
+    BASH
 
-    def deploy_csi_driver
-      puts
-      puts "Deploying Hetzner CSI Driver..."
-
-      begin
-        kubernetes_client.api("v1").resource("secrets").get("hcloud-csi", namespace: "kube-system")
-
-      rescue K8s::Error::NotFound
-        secret = K8s::Resource.new(
-          apiVersion: "v1",
-          kind: "Secret",
-          metadata: {
-            namespace: 'kube-system',
-            name: 'hcloud-csi',
-          },
-          data: {
-            token: Base64.encode64(hetzner_token)
-          }
-        )
-
-        kubernetes_client.api('v1').resource('secrets').create_resource(secret)
-      end
+    run cmd, kubeconfig_path: kubeconfig_path
 
+    cmd = 'kubectl apply -f https://raw.githubusercontent.com/hetznercloud/csi-driver/v1.6.0/deploy/kubernetes/hcloud-csi.yml'
 
-      manifest = HTTP.follow.get("https://raw.githubusercontent.com/hetznercloud/csi-driver/v1.6.0/deploy/kubernetes/hcloud-csi.yml").body
+    run cmd, kubeconfig_path: kubeconfig_path
 
-      File.write("/tmp/csi-driver.yaml", manifest)
+    puts '...CSI Driver deployed'
+  end
 
-      resources = K8s::Resource.from_files("/tmp/csi-driver.yaml")
+  def find_flannel_interface(server)
+    if ssh(server, 'lscpu | grep Vendor') =~ /Intel/
+      'ens10'
+    else
+      'enp7s0'
+    end
+  end
 
-      begin
-        kubernetes_client.api("apps/v1").resource("daemonsets").get("hcloud-csi-node", namespace: "kube-system")
+  def all_servers
+    @all_servers ||= hetzner_client.get('/servers?sort=created:desc')['servers'].select do |server|
+      belongs_to_cluster?(server) == true
+    end
+  end
 
+  def masters
+    @masters ||= all_servers.select { |server| server['name'] =~ /master\d+\Z/ }.sort { |a, b| a['name'] <=> b['name'] }
+  end
 
-        resources.each do |resource|
-          begin
-            kubernetes_client.update_resource(resource)
-          rescue K8s::Error::Invalid => e
-            raise e unless e.message =~ /must be specified/i
-          end
-        end
+  def workers
+    @workers = all_servers.select { |server| server['name'] =~ /worker\d+\Z/ }.sort { |a, b| a['name'] <=> b['name'] }
+  end
 
-      rescue K8s::Error::NotFound
-        resources.each do |resource|
-          kubernetes_client.create_resource(resource)
-        end
+  def k3s_token
+    @k3s_token ||= begin
+      token = ssh(first_master, '{ TOKEN=$(< /var/lib/rancher/k3s/server/node-token); } 2> /dev/null; echo $TOKEN')
 
+      if token.empty?
+        SecureRandom.hex
+      else
+        token.split(':').last
       end
-
-      puts "...CSI Driver deployed"
-    rescue Excon::Error::Socket
-      retry
     end
+  end
 
-    def wait_for_ssh(server)
-      Timeout::timeout(5) do
-        server_name = server["name"]
+  def first_master_private_ip
+    @first_master_private_ip ||= first_master['private_net'][0]['ip']
+  end
 
-        puts "Waiting for server #{server_name} to be up..."
+  def first_master
+    masters.first
+  end
 
-        loop do
-          result = ssh(server, "echo UP")
-          break if result == "UP"
-        end
+  def api_server_ip
+    return @api_server_ip if @api_server_ip
+
+    @api_server_ip = if masters.size > 1
+                       load_balancer_name = "#{cluster_name}-api"
+                       load_balancer = hetzner_client.get('/load_balancers')['load_balancers'].detect do |lb|
+                         lb['name'] == load_balancer_name
+                       end
+                       load_balancer['public_net']['ipv4']['ip']
+                     else
+                       first_master_public_ip
+                     end
+  end
 
-        puts "...server #{server_name} is now up."
-      end
-    rescue Errno::ENETUNREACH, Errno::EHOSTUNREACH, Timeout::Error, IOError
-      retry
+  def tls_sans
+    sans = " --tls-san=#{api_server_ip} "
+
+    masters.each do |master|
+      master_private_ip = master['private_net'][0]['ip']
+      sans << " --tls-san=#{master_private_ip} "
     end
 
-    def ssh(server, command, print_output: false)
-      public_ip = server.dig("public_net", "ipv4", "ip")
-      output = ""
+    sans
+  end
 
-      params = { verify_host_key: (verify_host_key ? :always : :never) }
+  def first_master_public_ip
+    @first_master_public_ip ||= first_master.dig('public_net', 'ipv4', 'ip')
+  end
 
-      if private_ssh_key_path
-        params[:keys] = [private_ssh_key_path]
-      end
+  def save_kubeconfig
+    kubeconfig = ssh(first_master, 'cat /etc/rancher/k3s/k3s.yaml')
+                 .gsub('127.0.0.1', api_server_ip)
+                 .gsub('default', cluster_name)
 
-      Net::SSH.start(public_ip, "root", params) do |session|
-        session.exec!(command) do |channel, stream, data|
-          output << data
-          puts data if print_output
-        end
-      end
-      output.chop
-    rescue Net::SSH::Disconnect => e
-      retry unless e.message =~ /Too many authentication failures/
-    rescue Net::SSH::ConnectionTimeout, Errno::ECONNREFUSED, Errno::ENETUNREACH, Errno::EHOSTUNREACH
-      retry
-    rescue Net::SSH::AuthenticationFailed
-      puts
-      puts "Cannot continue: SSH authentication failed. Please ensure that the private SSH key is correct."
-      exit 1
-    rescue Net::SSH::HostKeyMismatch
-      puts
-      puts "Cannot continue: Unable to SSH into server with IP #{public_ip} because the existing fingerprint in the known_hosts file does not match that of the actual host key."
-      puts "This is due to a security check but can also happen when creating a new server that gets assigned the same IP address as another server you've owned in the past."
-      puts "If are sure no security is being violated here and you're just creating new servers, you can eiher remove the relevant lines from your known_hosts (see IPs from the cloud console) or disable host key verification by setting the option 'verify_host_key' to false in the configuration file for the cluster."
-      exit 1
-    end
+    File.write(kubeconfig_path, kubeconfig)
 
-    def kubernetes_client
-      return @kubernetes_client if @kubernetes_client
+    FileUtils.chmod 'go-r', kubeconfig_path
+  end
 
-      config_hash = YAML.load_file(kubeconfig_path)
-      config_hash['current-context'] = cluster_name
-      @kubernetes_client = K8s::Client.config(K8s::Config.new(config_hash))
-    end
+  def belongs_to_cluster?(server)
+    server.dig('labels', 'cluster') == cluster_name
+  end
 
-    def find_flannel_interface(server)
-      if ssh(server, "lscpu | grep Vendor") =~ /Intel/
-        "ens10"
-      else
-        "enp7s0"
-      end
-    end
+  def schedule_workloads_on_masters?
+    schedule_workloads_on_masters = configuration['schedule_workloads_on_masters']
+    schedule_workloads_on_masters ? !!schedule_workloads_on_masters : false
+  end
 
-    def all_servers
-      @all_servers ||= hetzner_client.get("/servers")["servers"].select{ |server| belongs_to_cluster?(server) == true }
-    end
+  def image
+    configuration['image'] || 'ubuntu-20.04'
+  end
 
-    def masters
-      @masters ||= all_servers.select{ |server| server["name"] =~ /master\d+\Z/ }.sort{ |a, b| a["name"] <=> b["name"] }
-    end
+  def additional_packages
+    configuration['additional_packages'] || []
+  end
 
-    def workers
-      @workers = all_servers.select{ |server| server["name"] =~ /worker\d+\Z/ }.sort{ |a, b| a["name"] <=> b["name"] }
-    end
+  def check_kubectl
+    return if which('kubectl')
 
-    def k3s_token
-      @k3s_token ||= begin
-        token = ssh(first_master, "{ TOKEN=$(< /var/lib/rancher/k3s/server/node-token); } 2> /dev/null; echo $TOKEN")
+    puts 'Please ensure kubectl is installed and in your PATH.'
+    exit 1
+  end
 
-        if token.empty?
-          SecureRandom.hex
-        else
-          token.split(":").last
-        end
-      end
-    end
+  def placement_group_id(pool_name = nil)
+    @placement_groups ||= {}
+    @placement_groups[pool_name || '__masters__'] ||= Hetzner::PlacementGroup.new(hetzner_client:, cluster_name:, pool_name:).create
+  end
 
-    def first_master_private_ip
-      @first_master_private_ip ||= first_master["private_net"][0]["ip"]
-    end
+  def master_instance_type
+    @master_instance_type ||= masters_config['instance_type']
+  end
 
-    def first_master
-      masters.first
-    end
+  def masters_count
+    @masters_count ||= masters_config['instance_count']
+  end
 
-    def api_server_ip
-      return @api_server_ip if @api_server_ip
+  def firewall_id
+    @firewall_id ||= Hetzner::Firewall.new(hetzner_client:, cluster_name:).create(high_availability: (masters_count > 1), networks:)
+  end
 
-      @api_server_ip = if masters.size > 1
-        load_balancer_name = "#{cluster_name}-api"
-        load_balancer = hetzner_client.get("/load_balancers")["load_balancers"].detect{ |load_balancer| load_balancer["name"] == load_balancer_name }
-        load_balancer["public_net"]["ipv4"]["ip"]
-      else
-        first_master_public_ip
-      end
+  def network_id
+    @network_id ||= Hetzner::Network.new(hetzner_client:, cluster_name:).create(location:)
+  end
+
+  def ssh_key_id
+    @ssh_key_id ||= Hetzner::SSHKey.new(hetzner_client:, cluster_name:).create(public_ssh_key_path:)
+  end
+
+  def master_definitions_for_create
+    definitions = []
+
+    masters_count.times do |i|
+      definitions << {
+        instance_type: master_instance_type,
+        instance_id: "master#{i + 1}",
+        placement_group_id:,
+        location:,
+        firewall_id:,
+        network_id:,
+        ssh_key_id:,
+        image:,
+        additional_packages:
+      }
     end
 
-    def tls_sans
-      sans = " --tls-san=#{api_server_ip} "
+    definitions
+  end
 
-      masters.each do |master|
-        master_private_ip = master["private_net"][0]["ip"]
-        sans << " --tls-san=#{master_private_ip} "
-      end
+  def master_definitions_for_delete
+    definitions = []
 
-      sans
+    masters_count.times do |i|
+      definitions << {
+        instance_type: master_instance_type,
+        instance_id: "master#{i + 1}"
+      }
     end
 
-    def first_master_public_ip
-      @first_master_public_ip ||= first_master.dig("public_net", "ipv4", "ip")
+    definitions
+  end
+
+  def worker_node_pool_definitions(worker_node_pool)
+    worker_node_pool_name = worker_node_pool['name']
+    worker_instance_type = worker_node_pool['instance_type']
+    worker_count = worker_node_pool['instance_count']
+
+    definitions = []
+
+    worker_count.times do |i|
+      definitions << {
+        instance_type: worker_instance_type,
+        instance_id: "pool-#{worker_node_pool_name}-worker#{i + 1}",
+        placement_group_id: placement_group_id(worker_node_pool_name),
+        location:,
+        firewall_id:,
+        network_id:,
+        ssh_key_id:,
+        image:,
+        additional_packages:
+      }
     end
 
-    def save_kubeconfig
-      kubeconfig = ssh(first_master, "cat /etc/rancher/k3s/k3s.yaml").
-        gsub("127.0.0.1", api_server_ip).
-        gsub("default", cluster_name)
+    definitions
+  end
+
+  def create_load_balancer
+    Hetzner::LoadBalancer.new(hetzner_client:, cluster_name:).create(location:, network_id:)
+  end
+
+  def server_configs
+    return @server_configs if @server_configs
 
-      File.write(kubeconfig_path, kubeconfig)
+    @server_configs = master_definitions_for_create
 
-      FileUtils.chmod "go-r", kubeconfig_path
+    worker_node_pools.each do |worker_node_pool|
+      @server_configs += worker_node_pool_definitions(worker_node_pool)
     end
 
-    def ugrade_plan_manifest_path
-      worker_upgrade_concurrency = workers.size - 1
-      worker_upgrade_concurrency = 1 if worker_upgrade_concurrency == 0
+    @server_configs
+  end
 
-      manifest = <<~EOF
-        apiVersion: upgrade.cattle.io/v1
-        kind: Plan
-        metadata:
-          name: k3s-server
-          namespace: system-upgrade
-          labels:
-            k3s-upgrade: server
-        spec:
-          concurrency: 1
-          version: #{new_k3s_version}
-          nodeSelector:
-            matchExpressions:
-              - {key: node-role.kubernetes.io/master, operator: In, values: ["true"]}
-          serviceAccountName: system-upgrade
-          tolerations:
-          - key: "CriticalAddonsOnly"
-            operator: "Equal"
-            value: "true"
-            effect: "NoExecute"
-          cordon: true
-          upgrade:
-            image: rancher/k3s-upgrade
-        ---
-        apiVersion: upgrade.cattle.io/v1
-        kind: Plan
-        metadata:
-          name: k3s-agent
-          namespace: system-upgrade
-          labels:
-            k3s-upgrade: agent
-        spec:
-          concurrency: #{worker_upgrade_concurrency}
-          version: #{new_k3s_version}
-          nodeSelector:
-            matchExpressions:
-              - {key: node-role.kubernetes.io/master, operator: NotIn, values: ["true"]}
-          serviceAccountName: system-upgrade
-          prepare:
-            image: rancher/k3s-upgrade
-            args: ["prepare", "k3s-server"]
-          cordon: true
-          upgrade:
-            image: rancher/k3s-upgrade
-      EOF
+  def create_servers
+    servers = []
+
+    threads = server_configs.map do |server_config|
+      Thread.new do
+        servers << Hetzner::Server.new(hetzner_client:, cluster_name:).create(**server_config)
+      end
+    end
 
-      temp_file_path = "/tmp/k3s-upgrade-plan.yaml"
+    threads.each(&:join) unless threads.empty?
 
-      File.write(temp_file_path, manifest)
+    sleep 1 while servers.size != server_configs.size
 
-      temp_file_path
-    end
+    wait_for_servers(servers)
+  end
 
-    def belongs_to_cluster?(server)
-      server.dig("labels", "cluster") == cluster_name
+  def wait_for_servers(servers)
+    threads = servers.map do |server|
+      Thread.new { wait_for_ssh server }
     end
 
-    def schedule_workloads_on_masters?
-      schedule_workloads_on_masters = configuration.dig("schedule_workloads_on_masters")
-      schedule_workloads_on_masters ? !!schedule_workloads_on_masters : false
-    end
+    threads.each(&:join) unless threads.empty?
+  end
 
-    def image
-      configuration.dig("image") || "ubuntu-20.04"
+  def delete_servers
+    threads = all_servers.map do |server|
+      Thread.new do
+        Hetzner::Server.new(hetzner_client:, cluster_name:).delete(server_name: server['name'])
+      end
     end
 
+    threads.each(&:join) unless threads.empty?
+  end
 end