hetzner-k3s 0.1.0

data/lib/hetzner/infra/ssh_key.rb

@@ -0,0 +1,57 @@
+module Hetzner
+  class SSHKey
+    def initialize(hetzner_client:, cluster_name:)
+      @hetzner_client = hetzner_client
+      @cluster_name = cluster_name
+    end
+
+    def create(ssh_key_path:)
+      @ssh_key_path = ssh_key_path
+
+      puts
+
+      if ssh_key = find_ssh_key
+        puts "SSH key already exists, skipping."
+        puts
+        return ssh_key["id"]
+      end
+
+      puts "Creating SSH key..."
+
+      response = hetzner_client.post("/ssh_keys", ssh_key_config).body
+
+      puts "...SSH key created."
+      puts
+
+      JSON.parse(response)["ssh_key"]["id"]
+    end
+
+    def delete
+      if ssh_key = find_ssh_key
+        puts "Deleting ssh_key..."
+        hetzner_client.delete("/ssh_keys", ssh_key["id"])
+        puts "...ssh_key deleted."
+      else
+        puts "SSH key no longer exists, skipping."
+      end
+
+      puts
+    end
+
+    private
+
+      attr_reader :hetzner_client, :cluster_name, :ssh_key_path
+
+      def ssh_key_config
+        {
+          name: cluster_name,
+          public_key: File.read(ssh_key_path)
+        }
+      end
+
+      def find_ssh_key
+        hetzner_client.get("/ssh_keys")["ssh_keys"].detect{ |ssh_key| ssh_key["name"] == cluster_name }
+      end
+
+  end
+end

data/lib/hetzner/k3s/cli.rb

@@ -0,0 +1,303 @@
+require "thor"
+require "http"
+require "sshkey"
+
+require_relative "cluster"
+
+module Hetzner
+  module K3s
+    class CLI < Thor
+      def self.exit_on_failure?
+        true
+      end
+
+      desc "create-cluster", "Create a k3s cluster in Hetzner Cloud"
+      option :config_file, required: true
+
+      def create_cluster
+        validate_config_file :create
+
+        Cluster.new(hetzner_client: hetzner_client).create configuration: configuration
+      end
+
+      desc "delete-cluster", "Delete an existing k3s cluster in Hetzner Cloud"
+      option :config_file, required: true
+
+      def delete_cluster
+        validate_config_file :delete
+        Cluster.new(hetzner_client: hetzner_client).delete configuration: configuration
+      end
+
+      desc "upgrade-cluster", "Upgrade an existing k3s cluster in Hetzner Cloud to a new version"
+      option :config_file, required: true
+      option :new_k3s_version, required: true
+      option :force, default: "false"
+
+      def upgrade_cluster
+        validate_config_file :upgrade
+        Cluster.new(hetzner_client: hetzner_client).upgrade configuration: configuration, new_k3s_version: options[:new_k3s_version], config_file: options[:config_file]
+      end
+
+      desc "releases", "List available k3s releases"
+      def releases
+        find_available_releases.each do |release|
+          puts release
+        end
+      end
+
+      private
+
+        attr_reader :configuration, :hetzner_client, :k3s_version
+        attr_accessor :errors, :used_server_types
+
+        def validate_config_file(action)
+          config_file_path = options[:config_file]
+
+          if File.exists?(config_file_path)
+            begin
+              @configuration = YAML.load_file(options[:config_file])
+              raise "invalid" unless configuration.is_a? Hash
+            rescue
+              puts "Please ensure that the config file is a correct YAML manifest."
+              return
+            end
+          else
+            puts "Please specify a correct path for the config file."
+            return
+          end
+
+          @errors = []
+          @used_server_types = []
+
+          validate_token
+          validate_cluster_name
+          validate_kubeconfig_path
+
+          case action
+          when :create
+            validate_ssh_key
+            validate_location
+            validate_k3s_version
+            validate_masters
+            validate_worker_node_pools
+            validate_all_nodes_must_be_of_same_series
+          when :delete
+            validate_kubeconfig_path_must_exist
+          when :upgrade
+            validate_kubeconfig_path_must_exist
+            validate_new_k3s_version
+            validate_new_k3s_version_must_be_more_recent
+          end
+
+          errors.flatten!
+
+          unless errors.empty?
+            puts "Some information in the configuration file requires your attention:"
+            errors.each do |error|
+              puts " - #{error}"
+            end
+
+            exit 1
+          end
+        end
+
+        def validate_token
+          token = configuration.dig("hetzner_token")
+          @hetzner_client = Hetzner::Client.new(token: token)
+          hetzner_client.get("/locations")
+        rescue
+          errors << "Invalid Hetzner Cloid token"
+        end
+
+        def validate_cluster_name
+          errors << "Cluster name is an invalid format" unless configuration["cluster_name"] =~ /\A([A-Za-z0-9\-\_]+)\Z/
+        end
+
+        def validate_kubeconfig_path
+          path = File.expand_path(configuration.dig("kubeconfig_path"))
+          errors << "kubeconfig path cannot be a directory" and return if File.directory? path
+
+          directory = File.dirname(path)
+          errors << "Directory #{directory} doesn't exist" unless File.exists? directory
+        rescue
+          errors << "Invalid path for the kubeconfig"
+        end
+
+        def validate_ssh_key
+          path = File.expand_path(configuration.dig("ssh_key_path"))
+          errors << "Invalid Public SSH key path" and return unless File.exists? path
+
+          key = File.read(path)
+          errors << "Public SSH key is invalid" unless ::SSHKey.valid_ssh_public_key? key
+        rescue
+          errors << "Invalid Public SSH key path"
+        end
+
+        def validate_kubeconfig_path_must_exist
+          path = File.expand_path configuration.dig("kubeconfig_path")
+          errors << "kubeconfig path is invalid" and return unless File.exists? path
+          errors << "kubeconfig path cannot be a directory" if File.directory? path
+        rescue
+          errors << "Invalid kubeconfig path"
+        end
+
+        def server_types
+          @server_types ||= hetzner_client.get("/server_types")["server_types"].map{ |server_type| server_type["name"] }
+        rescue
+          @errors << "Cannot fetch server types with Hetzner API, please try again later"
+          false
+        end
+
+        def locations
+          @locations ||= hetzner_client.get("/locations")["locations"].map{ |location| location["name"] }
+        rescue
+          @errors << "Cannot fetch locations with Hetzner API, please try again later"
+          false
+        end
+
+        def validate_location
+          errors << "Invalid location - available locations: nbg1 (Nuremberg, Germany), fsn1 (Falkenstein, Germany), hel1 (Helsinki, Finland)" unless locations.include? configuration.dig("location")
+        end
+
+        def find_available_releases
+          @available_releases ||= begin
+            response = HTTP.get("https://api.github.com/repos/k3s-io/k3s/tags").body
+            JSON.parse(response).map { |hash| hash["name"] }
+          end
+        rescue
+          errors << "Cannot fetch the releases with Hetzner API, please try again later"
+        end
+
+        def validate_k3s_version
+          k3s_version = configuration.dig("k3s_version")
+          available_releases = find_available_releases
+          errors << "Invalid k3s version" unless available_releases.include? k3s_version
+        end
+
+        def validate_new_k3s_version
+          new_k3s_version = options[:new_k3s_version]
+          available_releases = find_available_releases
+          errors << "The new k3s version is invalid" unless available_releases.include? new_k3s_version
+        end
+
+        def validate_masters
+          masters_pool = nil
+
+          begin
+            masters_pool = configuration.dig("masters")
+          rescue
+            errors << "Invalid masters configuration"
+            return
+          end
+
+          if masters_pool.nil?
+            errors << "Invalid masters configuration"
+            return
+          end
+
+          validate_instance_group masters_pool, workers: false
+        end
+
+        def validate_worker_node_pools
+          worker_node_pools = nil
+
+          begin
+            worker_node_pools = configuration.dig("worker_node_pools")
+          rescue
+            errors << "Invalid node pools configuration"
+            return
+          end
+
+          if !worker_node_pools.is_a? Array
+            errors << "Invalid node pools configuration"
+          elsif worker_node_pools.size == 0
+            errors << "At least one node pool is required in order to schedule workloads"
+          elsif worker_node_pools.map{ |worker_node_pool| worker_node_pool["name"]}.uniq.size != worker_node_pools.size
+            errors << "Each node pool must have an unique name"
+          elsif server_types
+            worker_node_pools.each do |worker_node_pool|
+              validate_instance_group worker_node_pool
+            end
+          end
+        end
+
+        def validate_all_nodes_must_be_of_same_series
+          series = used_server_types.map{ |used_server_type| used_server_type[0..1]}
+          errors << "Master and worker node pools must all be of the same server series for networking to function properly (available series: cx, cp, ccx)" unless series.uniq.size == 1
+        end
+
+        def validate_new_k3s_version_must_be_more_recent
+          return if options[:force] == "true"
+          return unless kubernetes_client
+
+          begin
+            Timeout::timeout(5) do
+              servers = kubernetes_client.api("v1").resource("nodes").list
+
+              if servers.size == 0
+                errors << "The cluster seems to have no nodes, nothing to upgrade"
+              else
+                available_releases = find_available_releases
+
+                current_k3s_version = servers.first.dig(:status, :nodeInfo, :kubeletVersion)
+                current_k3s_version_index = available_releases.index(current_k3s_version) || 1000
+
+                new_k3s_version = options[:new_k3s_version]
+                new_k3s_version_index = available_releases.index(new_k3s_version) || 1000
+
+                unless new_k3s_version_index < current_k3s_version_index
+                  errors << "The new k3s version must be more recent than the current one"
+                end
+              end
+            end
+
+          rescue Timeout::Error
+            puts "Cannot upgrade: Unable to fetch nodes from Kubernetes API. Is the cluster online?"
+          end
+        end
+
+        def validate_instance_group(instance_group, workers: true)
+          instance_group_errors = []
+
+          instance_group_type = workers ? "Worker mode pool #{instance_group["name"]}" : "Masters pool"
+
+          unless !workers || instance_group["name"] =~ /\A([A-Za-z0-9\-\_]+)\Z/
+            instance_group_errors << "#{instance_group_type} has an invalid name"
+          end
+
+          unless instance_group.is_a? Hash
+            instance_group_errors << "#{instance_group_type} is in an invalid format"
+          end
+
+          unless server_types.include?(instance_group["instance_type"])
+            instance_group_errors << "#{instance_group_type} has an invalid instance type"
+          end
+
+          if instance_group["instance_count"].is_a? Integer
+            if instance_group["instance_count"] < 1
+              instance_group_errors << "#{instance_group_type} must have at least one node"
+            elsif !workers
+              instance_group_errors << "Masters count must equal to 1 for non-HA clusters or an odd number (recommended 3) for an HA cluster" unless instance_group["instance_count"].odd?
+            end
+          else
+            instance_group_errors << "#{instance_group_type} has an invalid instance count"
+          end
+
+          used_server_types << instance_group["instance_type"]
+
+          errors << instance_group_errors
+        end
+
+        def kubernetes_client
+          return @kubernetes_client if @kubernetes_client
+
+          config_hash = YAML.load_file(File.expand_path(configuration["kubeconfig_path"]))
+          config_hash['current-context'] = configuration["cluster_name"]
+          @kubernetes_client = K8s::Client.config(K8s::Config.new(config_hash))
+        rescue
+          errors << "Cannot connect to the Kubernetes cluster"
+          false
+        end
+    end
+  end
+end

data/lib/hetzner/k3s/client_patch.rb

@@ -0,0 +1,38 @@
+module K8s
+  class ResourceClient
+    def initialize(transport, api_client, api_resource, namespace: nil, resource_class: K8s::Resource)
+      @transport = transport
+      @api_client = api_client
+      @api_resource = api_resource
+      @namespace = namespace
+      @resource_class = resource_class
+
+      if @api_resource.name.include? '/'
+        @resource, @subresource = @api_resource.name.split('/', 2)
+      else
+        @resource = @api_resource.name
+        @subresource = nil
+      end
+
+      # fail "Resource #{api_resource.name} is not namespaced" unless api_resource.namespaced || !namespace
+    end
+
+    def path(name = nil, subresource: @subresource, namespace: @namespace)
+      namespace_part = namespace ? ['namespaces', namespace] : []
+
+      if namespaced?
+        if name && subresource
+          @api_client.path(*namespace_part, @resource, name, subresource)
+        elsif name
+          @api_client.path(*namespace_part, @resource, name)
+        else namespaced?
+          @api_client.path(*namespace_part, @resource)
+        end
+      elsif name
+        @api_client.path(@resource, name)
+      else
+        @api_client.path(@resource)
+      end
+    end
+  end
+end

data/lib/hetzner/k3s/cluster.rb

@@ -0,0 +1,609 @@
+require 'thread'
+require 'net/ssh'
+require "securerandom"
+require "base64"
+require "k8s-ruby"
+require 'timeout'
+
+require_relative "../infra/client"
+require_relative "../infra/firewall"
+require_relative "../infra/network"
+require_relative "../infra/ssh_key"
+require_relative "../infra/server"
+require_relative "../infra/load_balancer"
+
+require_relative "../k3s/client_patch"
+
+
+class Cluster
+  def initialize(hetzner_client:)
+    @hetzner_client = hetzner_client
+  end
+
+  def create(configuration:)
+    @hetzner_token = configuration.dig("hetzner_token")
+    @cluster_name = configuration.dig("cluster_name")
+    @kubeconfig_path = File.expand_path(configuration.dig("kubeconfig_path"))
+    @ssh_key_path = File.expand_path(configuration.dig("ssh_key_path"))
+    @k3s_version = configuration.dig("k3s_version")
+    @masters_config = configuration.dig("masters")
+    @worker_node_pools = configuration.dig("worker_node_pools")
+    @location = configuration.dig("location")
+    @flannel_interface = find_flannel_interface(configuration.dig("masters")["instance_type"])
+    @servers = []
+
+    create_resources
+
+    deploy_kubernetes
+
+    sleep 10
+
+    deploy_cloud_controller_manager
+    deploy_csi_driver
+    deploy_system_upgrade_controller
+  end
+
+  def delete(configuration:)
+    @cluster_name = configuration.dig("cluster_name")
+    @kubeconfig_path = File.expand_path(configuration.dig("kubeconfig_path"))
+
+    delete_resources
+  end
+
+  def upgrade(configuration:, new_k3s_version:, config_file:)
+    @configuration = configuration
+    @cluster_name = configuration.dig("cluster_name")
+    @kubeconfig_path = File.expand_path(configuration.dig("kubeconfig_path"))
+    @new_k3s_version = new_k3s_version
+    @config_file = config_file
+
+    upgrade_cluster
+  end
+
+  private
+
+    attr_accessor :servers
+
+    attr_reader :hetzner_client, :cluster_name, :kubeconfig_path, :k3s_version,
+                :masters_config, :worker_node_pools,
+                :location, :flannel_interface, :ssh_key_path, :kubernetes_client,
+                :hetzner_token, :tls_sans, :new_k3s_version, :configuration,
+                :config_file
+
+
+    def latest_k3s_version
+      response = HTTP.get("https://api.github.com/repos/k3s-io/k3s/tags").body
+      JSON.parse(response).first["name"]
+    end
+
+    def create_resources
+      firewall_id = Hetzner::Firewall.new(
+        hetzner_client: hetzner_client,
+        cluster_name: cluster_name
+      ).create
+
+      network_id = Hetzner::Network.new(
+        hetzner_client: hetzner_client,
+        cluster_name: cluster_name
+      ).create
+
+      ssh_key_id = Hetzner::SSHKey.new(
+        hetzner_client: hetzner_client,
+        cluster_name: cluster_name
+      ).create(ssh_key_path: ssh_key_path)
+
+      server_configs = []
+
+      master_instance_type = masters_config["instance_type"]
+      masters_count = masters_config["instance_count"]
+
+      masters_count.times do |i|
+        server_configs << {
+          location: location,
+          instance_type: master_instance_type,
+          instance_id: "master#{i+1}",
+          firewall_id: firewall_id,
+          network_id: network_id,
+          ssh_key_id: ssh_key_id
+        }
+      end
+
+      if masters_count > 1
+        Hetzner::LoadBalancer.new(
+          hetzner_client: hetzner_client,
+          cluster_name: cluster_name
+        ).create(location: location, network_id: network_id)
+      end
+
+      worker_node_pools.each do |worker_node_pool|
+        worker_node_pool_name = worker_node_pool["name"]
+        worker_instance_type = worker_node_pool["instance_type"]
+        worker_count = worker_node_pool["instance_count"]
+
+        worker_count.times do |i|
+          server_configs << {
+            location: location,
+            instance_type: worker_instance_type,
+            instance_id: "pool-#{worker_node_pool_name}-worker#{i+1}",
+            firewall_id: firewall_id,
+            network_id: network_id,
+            ssh_key_id: ssh_key_id
+          }
+        end
+      end
+
+      threads = server_configs.map do |server_config|
+        Thread.new do
+          servers << Hetzner::Server.new(hetzner_client: hetzner_client, cluster_name: cluster_name).create(server_config)
+        end
+      end
+
+      threads.each(&:join)
+
+      puts
+      threads = servers.map do |server|
+        Thread.new { wait_for_ssh server }
+      end
+
+      threads.each(&:join)
+    end
+
+    def delete_resources
+      begin
+        Timeout::timeout(5) do
+          servers = kubernetes_client.api("v1").resource("nodes").list
+
+          threads = servers.map do |node|
+            Thread.new do
+              Hetzner::Server.new(hetzner_client: hetzner_client, cluster_name: cluster_name).delete(server_name: node.metadata[:name])
+            end
+          end
+
+          threads.each(&:join)
+        end
+      rescue Timeout::Error
+        puts "Unable to fetch nodes from Kubernetes API. Is the cluster online?"
+      end
+
+      puts
+
+      sleep 5 # give time for the servers to actually be deleted
+
+      Hetzner::Firewall.new(
+        hetzner_client: hetzner_client,
+        cluster_name: cluster_name
+      ).delete
+
+      Hetzner::Network.new(
+        hetzner_client: hetzner_client,
+        cluster_name: cluster_name
+      ).delete
+
+      Hetzner::SSHKey.new(
+        hetzner_client: hetzner_client,
+        cluster_name: cluster_name
+      ).delete
+
+      Hetzner::LoadBalancer.new(
+        hetzner_client: hetzner_client,
+        cluster_name: cluster_name
+      ).delete
+
+    end
+
+    def upgrade_cluster
+      resources = K8s::Resource.from_files(ugrade_plan_manifest_path)
+
+      begin
+        kubernetes_client.api("upgrade.cattle.io/v1").resource("plans").get("k3s-server", namespace: "system-upgrade")
+
+        puts "Aborting - an upgrade is already in progress."
+
+      rescue K8s::Error::NotFound
+        resources.each do |resource|
+          kubernetes_client.create_resource(resource)
+        end
+
+        puts "Upgrade will now start. Run `watch kubectl get nodes` to see the nodes being upgraded. This should take a few minutes for a small cluster."
+        puts "The API server may be briefly unavailable during the upgrade of the controlplane."
+
+        configuration["k3s_version"] = new_k3s_version
+
+        File.write(config_file, configuration.to_yaml)
+      end
+    end
+
+
+    def master_script(master)
+      server = master == first_master ? " --cluster-init " : " --server https://#{first_master_private_ip}:6443 "
+
+      <<~EOF
+      curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION="#{k3s_version}" K3S_TOKEN="#{k3s_token}" INSTALL_K3S_EXEC="server \
+        --disable-cloud-controller \
+        --disable servicelb \
+        --disable traefik \
+        --disable local-storage \
+        --disable metrics-server \
+        --write-kubeconfig-mode=644 \
+        --node-name="$(hostname -f)" \
+        --cluster-cidr=10.244.0.0/16 \
+        --etcd-expose-metrics=true \
+        --kube-controller-manager-arg="address=0.0.0.0" \
+        --kube-controller-manager-arg="bind-address=0.0.0.0" \
+        --kube-proxy-arg="metrics-bind-address=0.0.0.0" \
+        --kube-scheduler-arg="address=0.0.0.0" \
+        --kube-scheduler-arg="bind-address=0.0.0.0" \
+        --node-taint CriticalAddonsOnly=true:NoExecute \
+        --kubelet-arg="cloud-provider=external" \
+        --node-ip=$(hostname -I | awk '{print $2}') \
+        --node-external-ip=$(hostname -I | awk '{print $1}') \
+        --flannel-iface=#{flannel_interface} \
+        #{server} #{tls_sans}" sh -
+      EOF
+    end
+
+    def worker_script
+      <<~EOF
+      curl -sfL https://get.k3s.io | K3S_TOKEN="#{k3s_token}" INSTALL_K3S_VERSION="#{k3s_version}" K3S_URL=https://#{first_master_private_ip}:6443 INSTALL_K3S_EXEC="agent \
+        --node-name="$(hostname -f)" \
+        --kubelet-arg="cloud-provider=external" \
+        --node-ip=$(hostname -I | awk '{print $2}') \
+        --node-external-ip=$(hostname -I | awk '{print $1}') \
+        --flannel-iface=#{flannel_interface}" sh -
+      EOF
+    end
+
+    def deploy_kubernetes
+      puts
+      puts "Deploying k3s to first master (#{first_master["name"]})..."
+
+      ssh first_master, master_script(first_master), print_output: true
+
+      puts
+      puts "...k3s has been deployed to first master."
+
+      save_kubeconfig
+
+      if masters.size > 1
+        threads = masters[1..-1].map do |master|
+          Thread.new do
+            puts
+            puts "Deploying k3s to master #{master["name"]}..."
+
+            ssh master, master_script(master), print_output: true
+
+            puts
+            puts "...k3s has been deployed to master #{master["name"]}."
+          end
+        end
+
+        threads.each(&:join)
+      end
+
+      threads = workers.map do |worker|
+        Thread.new do
+          puts
+          puts "Deploying k3s to worker (#{worker["name"]})..."
+
+          ssh worker, worker_script, print_output: true
+
+          puts
+          puts "...k3s has been deployed to worker (#{worker["name"]})."
+        end
+      end
+
+      threads.each(&:join)
+    end
+
+    def deploy_cloud_controller_manager
+      puts
+      puts "Deploying Hetzner Cloud Controller Manager..."
+
+      begin
+        kubernetes_client.api("v1").resource("secrets").get("hcloud", namespace: "kube-system")
+
+      rescue K8s::Error::NotFound
+        secret = K8s::Resource.new(
+          apiVersion: "v1",
+          kind: "Secret",
+          metadata: {
+            namespace: 'kube-system',
+            name: 'hcloud',
+          },
+          data: {
+            network: Base64.encode64(cluster_name),
+            token: Base64.encode64(hetzner_token)
+          }
+        )
+
+        kubernetes_client.api('v1').resource('secrets').create_resource(secret)
+      end
+
+
+      manifest = HTTP.follow.get("https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/latest/download/ccm-networks.yaml").body
+
+      File.write("/tmp/cloud-controller-manager.yaml", manifest)
+
+      resources = K8s::Resource.from_files("/tmp/cloud-controller-manager.yaml")
+
+      begin
+        kubernetes_client.api("apps/v1").resource("deployments").get("hcloud-cloud-controller-manager", namespace: "kube-system")
+
+        resources.each do |resource|
+          kubernetes_client.update_resource(resource)
+        end
+
+      rescue K8s::Error::NotFound
+        resources.each do |resource|
+          kubernetes_client.create_resource(resource)
+        end
+
+      end
+
+      puts "...Cloud Controller Manager deployed"
+    rescue Excon::Error::Socket
+      retry
+    end
+
+    def deploy_system_upgrade_controller
+      puts
+      puts "Deploying k3s System Upgrade Controller..."
+
+      manifest = HTTP.follow.get("https://github.com/rancher/system-upgrade-controller/releases/download/v0.7.3/system-upgrade-controller.yaml").body
+
+      File.write("/tmp/system-upgrade-controller.yaml", manifest)
+
+      resources = K8s::Resource.from_files("/tmp/system-upgrade-controller.yaml")
+
+      begin
+        kubernetes_client.api("apps/v1").resource("deployments").get("system-upgrade-controller", namespace: "system-upgrade")
+
+        resources.each do |resource|
+          kubernetes_client.update_resource(resource)
+        end
+
+      rescue K8s::Error::NotFound
+        resources.each do |resource|
+          kubernetes_client.create_resource(resource)
+        end
+
+      end
+
+      puts "...k3s System Upgrade Controller deployed"
+    rescue Excon::Error::Socket
+      retry
+    end
+
+    def deploy_csi_driver
+      puts
+      puts "Deploying Hetzner CSI Driver..."
+
+      begin
+        kubernetes_client.api("v1").resource("secrets").get("hcloud-csi", namespace: "kube-system")
+
+      rescue K8s::Error::NotFound
+        secret = K8s::Resource.new(
+          apiVersion: "v1",
+          kind: "Secret",
+          metadata: {
+            namespace: 'kube-system',
+            name: 'hcloud-csi',
+          },
+          data: {
+            token: Base64.encode64(hetzner_token)
+          }
+        )
+
+        kubernetes_client.api('v1').resource('secrets').create_resource(secret)
+      end
+
+
+      manifest = HTTP.follow.get("https://raw.githubusercontent.com/hetznercloud/csi-driver/v1.5.3/deploy/kubernetes/hcloud-csi.yml").body
+
+      File.write("/tmp/csi-driver.yaml", manifest)
+
+      resources = K8s::Resource.from_files("/tmp/csi-driver.yaml")
+
+      begin
+        kubernetes_client.api("apps/v1").resource("daemonsets").get("hcloud-csi-node", namespace: "kube-system")
+
+
+        resources.each do |resource|
+          begin
+            kubernetes_client.update_resource(resource)
+          rescue K8s::Error::Invalid => e
+            raise e unless e.message =~ /must be specified/i
+          end
+        end
+
+      rescue K8s::Error::NotFound
+        resources.each do |resource|
+          kubernetes_client.create_resource(resource)
+        end
+
+      end
+
+      puts "...CSI Driver deployed"
+    rescue Excon::Error::Socket
+      retry
+    end
+
+    def wait_for_ssh(server)
+      server_name = server["name"]
+
+      puts "Waiting for server #{server_name} to be up..."
+
+      loop do
+        result = ssh(server, "echo UP")
+        break if result == "UP"
+      end
+
+      puts "...server #{server_name} is now up."
+    rescue Errno::ENETUNREACH
+      retry
+    end
+
+    def ssh(server, command, print_output: false)
+      public_ip = server.dig("public_net", "ipv4", "ip")
+      output = ""
+
+      Net::SSH.start(public_ip, "root") do |session|
+        session.exec!(command) do |channel, stream, data|
+          output << data
+          puts data if print_output
+        end
+      end
+
+      output.chop
+    rescue Net::SSH::ConnectionTimeout
+      retry
+    rescue Net::SSH::Disconnect => e
+      retry unless e.message =~ /Too many authentication failures/
+    rescue Errno::ECONNREFUSED
+      retry
+    end
+
+    def kubernetes_client
+      return @kubernetes_client if @kubernetes_client
+
+      config_hash = YAML.load_file(kubeconfig_path)
+      config_hash['current-context'] = cluster_name
+      @kubernetes_client = K8s::Client.config(K8s::Config.new(config_hash))
+    end
+
+    def find_flannel_interface(server_type)
+      case server_type[0..1]
+      when "cp"
+        "enp7s0"
+      when "cc"
+        "enp7s0"
+      when "cx"
+        "ens10"
+      end
+    end
+
+    def all_servers
+      @all_servers ||= hetzner_client.get("/servers")["servers"]
+    end
+
+    def masters
+      @masters ||= all_servers.select{ |server| server["name"] =~ /master\d+\Z/ }.sort{ |a, b| a["name"] <=> b["name"] }
+    end
+
+    def workers
+      @workers = all_servers.select{ |server| server["name"] =~ /worker\d+\Z/ }.sort{ |a, b| a["name"] <=> b["name"] }
+    end
+
+    def k3s_token
+      @k3s_token ||= begin
+        token = ssh(first_master, "{ TOKEN=$(< /var/lib/rancher/k3s/server/node-token); } 2> /dev/null; echo $TOKEN")
+
+        if token.empty?
+          SecureRandom.hex
+        else
+          token.split(":").last
+        end
+      end
+    end
+
+    def first_master_private_ip
+      @first_master_private_ip ||= first_master["private_net"][0]["ip"]
+    end
+
+    def first_master
+      masters.first
+    end
+
+    def api_server_ip
+      return @api_server_ip if @api_server_ip
+
+      @api_server_ip = if masters.size > 1
+        load_balancer_name = "#{cluster_name}-api"
+        load_balancer = hetzner_client.get("/load_balancers")["load_balancers"].detect{ |load_balancer| load_balancer["name"] == load_balancer_name }
+        load_balancer["public_net"]["ipv4"]["ip"]
+      else
+        first_master_public_ip
+      end
+    end
+
+    def tls_sans
+      sans = " --tls-san=#{api_server_ip} "
+
+      masters.each do |master|
+        master_private_ip = master["private_net"][0]["ip"]
+        sans << " --tls-san=#{master_private_ip} "
+      end
+
+      sans
+    end
+
+    def first_master_public_ip
+      @first_master_public_ip ||= first_master.dig("public_net", "ipv4", "ip")
+    end
+
+    def save_kubeconfig
+      kubeconfig = ssh(first_master, "cat /etc/rancher/k3s/k3s.yaml").
+        gsub("127.0.0.1", api_server_ip).
+        gsub("default", cluster_name)
+
+      File.write(kubeconfig_path, kubeconfig)
+    end
+
+    def ugrade_plan_manifest_path
+      worker_upgrade_concurrency = workers.size - 1
+      worker_upgrade_concurrency = 1 if worker_upgrade_concurrency == 0
+
+      manifest = <<~EOF
+        apiVersion: upgrade.cattle.io/v1
+        kind: Plan
+        metadata:
+          name: k3s-server
+          namespace: system-upgrade
+          labels:
+            k3s-upgrade: server
+        spec:
+          concurrency: 1
+          version: #{new_k3s_version}
+          nodeSelector:
+            matchExpressions:
+              - {key: node-role.kubernetes.io/master, operator: In, values: ["true"]}
+          serviceAccountName: system-upgrade
+          tolerations:
+          - key: "CriticalAddonsOnly"
+            operator: "Equal"
+            value: "true"
+            effect: "NoExecute"
+          cordon: true
+          upgrade:
+            image: rancher/k3s-upgrade
+        ---
+        apiVersion: upgrade.cattle.io/v1
+        kind: Plan
+        metadata:
+          name: k3s-agent
+          namespace: system-upgrade
+          labels:
+            k3s-upgrade: agent
+        spec:
+          concurrency: #{worker_upgrade_concurrency}
+          version: #{new_k3s_version}
+          nodeSelector:
+            matchExpressions:
+              - {key: node-role.kubernetes.io/master, operator: NotIn, values: ["true"]}
+          serviceAccountName: system-upgrade
+          prepare:
+            image: rancher/k3s-upgrade
+            args: ["prepare", "k3s-server"]
+          cordon: true
+          upgrade:
+            image: rancher/k3s-upgrade
+      EOF
+
+      temp_file_path = "/tmp/k3s-upgrade-plan.yaml"
+
+      File.write(temp_file_path, manifest)
+
+      temp_file_path
+    end
+
+end