hetzner-k3s 0.1.0 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 600b092e02f2bca4fd0be7e830a13913d2bbf82d2e3b98226ab52a2b5df4e859
-  data.tar.gz: f595dda56d1ca9aeaa611a77d82c168a63d300a83c5f3e8edc44b3da5790d46a
+  metadata.gz: 109610e9f4d807bac091471880141069488865da1024cee8bb0479846c772165
+  data.tar.gz: d4f882d488ecc94d6f7234fc122c4dc6241dfc8ce946cbce1855619353296917
 SHA512:
-  metadata.gz: 6024a0b99ecc6d97d56e50f39e9e89f9cd2d92db7261604162ec37628df6bd4b942f50be64c4f05d9a745e55a3e18e567a759f20626671e81746c4160a280a9a
-  data.tar.gz: 17f2095befd0035555adc902dc52baed71f77bd82ef26ecb9bff3df5e7b5ec14068cdd16923fd6a2297e22badaab922b3fbc745939285e41fed8699922e26017
+  metadata.gz: 6cd9649eef2f75f616f9dc5bc2bb86d2b10e0e227137adb6683c502b0ed42a0888f59f361e088edece98eb91c0ee11c596c8734c00865081564cb1539cb537ea
+  data.tar.gz: e1569ef139160e15daa78566cb48e71bf99999d5e6f19a98f30fc73b984bc7fb5fe0a208a4594c7d8c4606ffd6ea509409cc31cd48818739be1ed83be914c9b0

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    hetzner-k3s (0.1.0)
+    hetzner-k3s (0.3.1)
       http
       k8s-ruby
       net-ssh

data/README.md

@@ -44,6 +44,7 @@ cluster_name: test
 kubeconfig_path: "./kubeconfig"
 k3s_version: v1.21.3+k3s1
 ssh_key_path: "~/.ssh/id_rsa.pub"
+verify_host_key: false
 location: nbg1
 masters:
   instance_type: cpx21
@@ -53,7 +54,7 @@ worker_node_pools:
   instance_type: cpx21
   instance_count: 4
 - name: big
-  instance_type: cp321
+  instance_type: cpx31
   instance_count: 2
 ```
 
@@ -74,6 +75,8 @@ curl \
 ```
 
 
+Note: the option `verify_host_key` is by default set to `false` to disable host key verification. This is because sometimes when creating new servers, Hetzner may assign IP addresses that were previously used by other servers you owned in the past. Therefore the host key verification would fail. If you set this option to `true` and this happens, the tool won't be able to continue creating the cluster until you resolve the issue with one of the suggestions it will give you.
+
 Finally, to create the cluster run:
 
 ```bash
@@ -208,6 +211,23 @@ The other annotations should be self explanatory. You can find a list of the ava
 
 Once the cluster is ready you can create persistent volumes out of the box with the default storage class `hcloud-volumes`, since the Hetzner CSI driver is installed automatically. This will use Hetzner's block storage (based on Ceph so it's replicated and highly available) for your persistent volumes. Note that the minimum size of a volume is 10Gi. If you specify a smaller size for a volume, the volume will be created with a capacity of 10Gi anyway.
 
+
+## changelog
+
+- 0.3.2
+  - Configure DNS to use Cloudflare's resolver instead of Hetzner's, since Hetzner's resolvers are not always reliable
+
+- 0.3.1
+  - Allow enabling/disabling the host key verification
+
+- 0.3.0
+  - Handle case when an SSH key with the given fingerprint already exists in the Hetzner project
+  - Handle a timeout of 5 seconds for requests to the Hetzner API
+  - Retry waiting for server to be up when timeouts/host-unreachable errors occur
+  - Ignore known_hosts entry to prevent errors when recreating servers with IPs that have been used previously
+
+- 0.2.0
+  - Allow mixing servers of different series Intel/AMD
 ## Contributing and support
 
 Please create a PR if you want to propose any changes, or open an issue if you are having trouble with the tool - I will do my best to help if I can.
@@ -218,4 +238,4 @@ The gem is available as open source under the terms of the [MIT License](https:/
 
 ## Code of Conduct
 
-Everyone interacting in the hetzner-k3s project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/vitobotta/k3s/blob/master/CODE_OF_CONDUCT.md).
+Everyone interacting in the hetzner-k3s project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/vitobotta/hetzner-k3s/blob/main/CODE_OF_CONDUCT.md).

data/cluster_config.yaml.example

@@ -4,6 +4,7 @@ cluster_name: test
 kubeconfig_path: "../kubeconfig"
 k3s_version: v1.21.3+k3s1
 ssh_key_path: "~/.ssh/id_rsa.pub"
+verify_host_key: false
 location: nbg1
 masters:
   instance_type: cpx21

data/lib/hetzner/infra/client.rb

@@ -9,15 +9,21 @@ module Hetzner
     end
 
     def get(path)
-      JSON.parse HTTP.headers(headers).get(BASE_URI + path).body
+      make_request do
+        JSON.parse HTTP.headers(headers).get(BASE_URI + path).body
+      end
     end
 
     def post(path, data)
-      HTTP.headers(headers).post(BASE_URI + path, json: data)
+      make_request do
+        HTTP.headers(headers).post(BASE_URI + path, json: data)
+      end
     end
 
     def delete(path, id)
-      HTTP.headers(headers).delete(BASE_URI + path + "/" + id.to_s)
+      make_request do
+        HTTP.headers(headers).delete(BASE_URI + path + "/" + id.to_s)
+      end
     end
 
     private
@@ -28,5 +34,13 @@ module Hetzner
           "Content-Type": "application/json"
         }
       end
+
+      def make_request &block
+        Timeout::timeout(5) do
+          block.call
+        end
+      rescue Timeout::Error
+        retry
+      end
   end
 end

data/lib/hetzner/infra/load_balancer.rb

@@ -26,12 +26,12 @@ module Hetzner
       JSON.parse(response)["load_balancer"]["id"]
     end
 
-    def delete
+    def delete(ha:)
       if load_balancer = find_load_balancer
-        puts "Deleting API load balancer..."
+        puts "Deleting API load balancer..." unless ha
         hetzner_client.delete("/load_balancers", load_balancer["id"])
-        puts "...API load balancer deleted."
-      else
+        puts "...API load balancer deleted." unless ha
+      elsif ha
         puts "API load balancer no longer exists, skipping."
       end
 

data/lib/hetzner/infra/server.rb

@@ -74,7 +74,12 @@ module Hetzner
             - sed -i 's/[#]*PermitRootLogin yes/PermitRootLogin prohibit-password/g' /etc/ssh/sshd_config
             - sed -i 's/[#]*PasswordAuthentication yes/PasswordAuthentication no/g' /etc/ssh/sshd_config
             - systemctl restart sshd
-        EOS
+            - systemctl stop systemd-resolved
+            - systemctl disable systemd-resolved
+            - rm /etc/resolv.conf
+            - echo "nameserver 1.1.1.1" > /etc/resolv.conf
+            - echo "nameserver 1.0.0.1" >> /etc/resolv.conf
+            EOS
       end
 
   end

data/lib/hetzner/infra/ssh_key.rb

@@ -26,11 +26,17 @@ module Hetzner
       JSON.parse(response)["ssh_key"]["id"]
     end
 
-    def delete
+    def delete(ssh_key_path:)
+      @ssh_key_path = ssh_key_path
+
       if ssh_key = find_ssh_key
-        puts "Deleting ssh_key..."
-        hetzner_client.delete("/ssh_keys", ssh_key["id"])
-        puts "...ssh_key deleted."
+        if ssh_key["name"] == cluster_name
+          puts "Deleting ssh_key..."
+          hetzner_client.delete("/ssh_keys", ssh_key["id"])
+          puts "...ssh_key deleted."
+        else
+          puts "The SSH key existed before creating the cluster, so I won't delete it."
+        end
       else
         puts "SSH key no longer exists, skipping."
       end
@@ -42,15 +48,33 @@ module Hetzner
 
       attr_reader :hetzner_client, :cluster_name, :ssh_key_path
 
+      def public_key
+        @public_key ||= File.read(ssh_key_path).chop
+      end
+
       def ssh_key_config
         {
           name: cluster_name,
-          public_key: File.read(ssh_key_path)
+          public_key: public_key
         }
       end
 
+      def fingerprint
+        @fingerprint ||= ::SSHKey.fingerprint(public_key)
+      end
+
       def find_ssh_key
-        hetzner_client.get("/ssh_keys")["ssh_keys"].detect{ |ssh_key| ssh_key["name"] == cluster_name }
+        key = hetzner_client.get("/ssh_keys")["ssh_keys"].detect do |ssh_key|
+          ssh_key["fingerprint"] == fingerprint
+        end
+
+        unless key
+          key = hetzner_client.get("/ssh_keys")["ssh_keys"].detect do |ssh_key|
+            ssh_key["name"] == cluster_name
+          end
+        end
+
+        key
       end
 
   end

data/lib/hetzner/k3s/cli.rb

@@ -80,7 +80,7 @@ module Hetzner
             validate_k3s_version
             validate_masters
             validate_worker_node_pools
-            validate_all_nodes_must_be_of_same_series
+            validate_verify_host_key
           when :delete
             validate_kubeconfig_path_must_exist
           when :upgrade
@@ -221,11 +221,6 @@ module Hetzner
           end
         end
 
-        def validate_all_nodes_must_be_of_same_series
-          series = used_server_types.map{ |used_server_type| used_server_type[0..1]}
-          errors << "Master and worker node pools must all be of the same server series for networking to function properly (available series: cx, cp, ccx)" unless series.uniq.size == 1
-        end
-
         def validate_new_k3s_version_must_be_more_recent
           return if options[:force] == "true"
           return unless kubernetes_client
@@ -298,6 +293,12 @@ module Hetzner
           errors << "Cannot connect to the Kubernetes cluster"
           false
         end
+
+
+        def validate_verify_host_key
+          return unless [true, false].include?(configuration.fetch("ssh_key_path", false))
+          errors << "Please set the verify_host_key option to either true or false"
+        end
     end
   end
 end

data/lib/hetzner/k3s/cluster.rb

@@ -29,7 +29,7 @@ class Cluster
     @masters_config = configuration.dig("masters")
     @worker_node_pools = configuration.dig("worker_node_pools")
     @location = configuration.dig("location")
-    @flannel_interface = find_flannel_interface(configuration.dig("masters")["instance_type"])
+    @verify_host_key = configuration.fetch("verify_host_key", false)
     @servers = []
 
     create_resources
@@ -46,6 +46,7 @@ class Cluster
   def delete(configuration:)
     @cluster_name = configuration.dig("cluster_name")
     @kubeconfig_path = File.expand_path(configuration.dig("kubeconfig_path"))
+    @ssh_key_path = File.expand_path(configuration.dig("ssh_key_path"))
 
     delete_resources
   end
@@ -66,9 +67,9 @@ class Cluster
 
     attr_reader :hetzner_client, :cluster_name, :kubeconfig_path, :k3s_version,
                 :masters_config, :worker_node_pools,
-                :location, :flannel_interface, :ssh_key_path, :kubernetes_client,
+                :location, :ssh_key_path, :kubernetes_client,
                 :hetzner_token, :tls_sans, :new_k3s_version, :configuration,
-                :config_file
+                :config_file, :verify_host_key
 
 
     def latest_k3s_version
@@ -138,17 +139,18 @@ class Cluster
         end
       end
 
-      threads.each(&:join)
+      threads.each(&:join) unless threads.empty?
 
       puts
       threads = servers.map do |server|
         Thread.new { wait_for_ssh server }
       end
 
-      threads.each(&:join)
+      threads.each(&:join) unless threads.empty?
     end
 
     def delete_resources
+      # Deleting nodes defined according to Kubernetes first
       begin
         Timeout::timeout(5) do
           servers = kubernetes_client.api("v1").resource("nodes").list
@@ -159,12 +161,23 @@ class Cluster
             end
           end
 
-          threads.each(&:join)
+          threads.each(&:join) unless threads.empty?
         end
-      rescue Timeout::Error
+      rescue Timeout::Error, Excon::Error::Socket
         puts "Unable to fetch nodes from Kubernetes API. Is the cluster online?"
       end
 
+      # Deleting nodes defined in the config file just in case there are leftovers i.e. nodes that
+      # were not part of the cluster for some reason
+
+      threads = all_servers.map do |server|
+        Thread.new do
+          Hetzner::Server.new(hetzner_client: hetzner_client, cluster_name: cluster_name).delete(server_name: server["name"])
+        end
+      end
+
+      threads.each(&:join) unless threads.empty?
+
       puts
 
       sleep 5 # give time for the servers to actually be deleted
@@ -182,12 +195,12 @@ class Cluster
       Hetzner::SSHKey.new(
         hetzner_client: hetzner_client,
         cluster_name: cluster_name
-      ).delete
+      ).delete(ssh_key_path: ssh_key_path)
 
       Hetzner::LoadBalancer.new(
         hetzner_client: hetzner_client,
         cluster_name: cluster_name
-      ).delete
+      ).delete(ha: (masters.size > 1))
 
     end
 
@@ -216,6 +229,7 @@ class Cluster
 
     def master_script(master)
       server = master == first_master ? " --cluster-init " : " --server https://#{first_master_private_ip}:6443 "
+      flannel_interface = find_flannel_interface(master)
 
       <<~EOF
       curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION="#{k3s_version}" K3S_TOKEN="#{k3s_token}" INSTALL_K3S_EXEC="server \
@@ -242,7 +256,9 @@ class Cluster
       EOF
     end
 
-    def worker_script
+    def worker_script(worker)
+      flannel_interface = find_flannel_interface(worker)
+
       <<~EOF
       curl -sfL https://get.k3s.io | K3S_TOKEN="#{k3s_token}" INSTALL_K3S_VERSION="#{k3s_version}" K3S_URL=https://#{first_master_private_ip}:6443 INSTALL_K3S_EXEC="agent \
         --node-name="$(hostname -f)" \
@@ -277,7 +293,7 @@ class Cluster
           end
         end
 
-        threads.each(&:join)
+        threads.each(&:join) unless threads.empty?
       end
 
       threads = workers.map do |worker|
@@ -285,14 +301,14 @@ class Cluster
           puts
           puts "Deploying k3s to worker (#{worker["name"]})..."
 
-          ssh worker, worker_script, print_output: true
+          ssh worker, worker_script(worker), print_output: true
 
           puts
           puts "...k3s has been deployed to worker (#{worker["name"]})."
         end
       end
 
-      threads.each(&:join)
+      threads.each(&:join) unless threads.empty?
     end
 
     def deploy_cloud_controller_manager
@@ -429,17 +445,19 @@ class Cluster
     end
 
     def wait_for_ssh(server)
-      server_name = server["name"]
+      Timeout::timeout(5) do
+        server_name = server["name"]
 
-      puts "Waiting for server #{server_name} to be up..."
+        puts "Waiting for server #{server_name} to be up..."
 
-      loop do
-        result = ssh(server, "echo UP")
-        break if result == "UP"
-      end
+        loop do
+          result = ssh(server, "echo UP")
+          break if result == "UP"
+        end
 
-      puts "...server #{server_name} is now up."
-    rescue Errno::ENETUNREACH
+        puts "...server #{server_name} is now up."
+      end
+    rescue Errno::ENETUNREACH, Errno::EHOSTUNREACH, Timeout::Error
       retry
     end
 
@@ -447,20 +465,23 @@ class Cluster
       public_ip = server.dig("public_net", "ipv4", "ip")
       output = ""
 
-      Net::SSH.start(public_ip, "root") do |session|
+      Net::SSH.start(public_ip, "root", verify_host_key: (verify_host_key ? :always : :never)) do |session|
         session.exec!(command) do |channel, stream, data|
           output << data
           puts data if print_output
         end
       end
-
       output.chop
-    rescue Net::SSH::ConnectionTimeout
-      retry
     rescue Net::SSH::Disconnect => e
       retry unless e.message =~ /Too many authentication failures/
-    rescue Errno::ECONNREFUSED
+    rescue Net::SSH::ConnectionTimeout, Errno::ECONNREFUSED, Errno::ENETUNREACH, Errno::EHOSTUNREACH
       retry
+    rescue Net::SSH::HostKeyMismatch
+      puts
+      puts "Cannot continue: Unable to SSH into server with IP #{public_ip} because the existing fingerprint in the known_hosts file does not match that of the actual host key."
+      puts "This is due to a security check but can also happen when creating a new server that gets assigned the same IP address as another server you've owned in the past."
+      puts "If are sure no security is being violated here and you're just creating new servers, you can eiher remove the relevant lines from your known_hosts (see IPs from the cloud console) or disable host key verification by setting the option 'verify_host_key' to false in the configuration file for the cluster."
+      exit 1
     end
 
     def kubernetes_client
@@ -471,14 +492,11 @@ class Cluster
       @kubernetes_client = K8s::Client.config(K8s::Config.new(config_hash))
     end
 
-    def find_flannel_interface(server_type)
-      case server_type[0..1]
-      when "cp"
-        "enp7s0"
-      when "cc"
-        "enp7s0"
-      when "cx"
+    def find_flannel_interface(server)
+      if ssh(server, "lscpu | grep Vendor") =~ /Intel/
         "ens10"
+      else
+        "enp7s0"
       end
     end
 

data/lib/hetzner/k3s/version.rb

@@ -1,5 +1,5 @@
 module Hetzner
   module K3s
-    VERSION = "0.1.0"
+    VERSION = "0.3.2"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hetzner-k3s
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.3.2
 platform: ruby
 authors:
 - Vito Botta
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-08-07 00:00:00.000000000 Z
+date: 2021-08-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor