hetzner-k3s 0.3.6 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2b1b2acb7f0649eac91750409418446c2b1ffea2b1b7a32b1f9f8a6fd2d50c5b
-  data.tar.gz: 457994e28a028e8a1052e20a68ab653bb41fdf5fb88160f43e51b3de49bc8710
+  metadata.gz: 6ee4a4ac2c31ebff805ee20edc3658ffe64be32e50b524ee4af3646e3ffc3a3c
+  data.tar.gz: 8cbc33a2a696b19c8e614932d1daa7fa9beddaf9d69dd8377d909cb382e40f87
 SHA512:
-  metadata.gz: 743edefe6fdaa9ebf9dc2236e4ce672f44d10bcbe954bda7a1560c4166cacf12824eb6c994dfe635543746c402696d981a22224852bb25a7df7558c81bdbbff3
-  data.tar.gz: 506488a2edac4c9e7b68d09e8b8dd8a970993dc8a87ba7d6bf3200856f4384ebe90362ea2fe337e2cdb4f43cb951b7c79c2a9a26154ed278b1704c369b88e85a
+  metadata.gz: ff2ca466abbd198b3bc76c8854113d90033fb606e9f11152ecf6d079564ee4dcbdab359a5b17229770a7dc531a9674b211d079a2204596efe3ec5b67157bf82e
+  data.tar.gz: a6a16c64b0ada5c4d1a740894df09a9f41ed0110b06cfd5629b1971c64836a5fd38bceebb7898432b275cb677779606ecd780b917d4095eb161987d26c0eecc0

data/Dockerfile

@@ -3,6 +3,8 @@ FROM ruby:2.7.4-alpine
 RUN apk update --no-cache \
   && apk add build-base git openssh-client
 
+COPY . .
+
 RUN gem install hetzner-k3s
 
 COPY entrypoint.sh /entrypoint.sh

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    hetzner-k3s (0.3.6)
+    hetzner-k3s (0.4.0)
       bcrypt_pbkdf
       ed25519
       http

data/README.md

@@ -38,7 +38,7 @@ This will install the `hetzner-k3s` executable in your PATH.
 Alternatively, if you don't want to set up a Ruby runtime but have Docker installed, you can use a container. Run the following from inside the directory where you have the config file for the cluster (described in the next section):
 
 ```bash
-docker run --rm -it -v ${PWD}:/cluster -v ${HOME}/.ssh:/tmp/.ssh vitobotta/hetzner-k3s create-cluster --config-file /cluster/test.yaml
+docker run --rm -it -v ${PWD}:/cluster -v ${HOME}/.ssh:/tmp/.ssh vitobotta/hetzner-k3s:v0.3.8 create-cluster --config-file /cluster/test.yaml
 ```
 
 Replace `test.yaml` with the name of your config file.
@@ -54,6 +54,8 @@ cluster_name: test
 kubeconfig_path: "./kubeconfig"
 k3s_version: v1.21.3+k3s1
 ssh_key_path: "~/.ssh/id_rsa.pub"
+ssh_allowed_networks:
+  - 0.0.0.0/0
 verify_host_key: false
 location: nbg1
 masters:
@@ -72,6 +74,11 @@ It should hopefully be self explanatory; you can run `hetzner-k3s releases` to s
 
 If you are using Docker, then set `kubeconfig_path` to `/cluster/kubeconfig` so that the kubeconfig is created in the same directory where your config file is.
 
+If you don't want to specify the Hetzner token in the config file (for example if you want to use the tool with CI), then you can use the `HCLOUD_TOKEN` environment variable instead, which has predecence.
+
+**Important**: The tool assignes the label `cluster` to each server it creates, with the clsuter name you specify in the config file, as the value. So please ensure you don't create unrelated servers in the same project having
+the label `cluster=<cluster name>`, because otherwise they will be deleted if you delete the cluster. I recommend you create a separate Hetzner project for each cluster, see note at the end of this README for more details.
+
 
 If you set `masters.instance_count` to 1 then the tool will create a non highly available control plane; for production clusters you may want to set it to a number greater than 1. This number must be odd to avoid split brain issues with etcd and the recommended number is 3.
 
@@ -225,8 +232,28 @@ The other annotations should be self explanatory. You can find a list of the ava
 Once the cluster is ready you can create persistent volumes out of the box with the default storage class `hcloud-volumes`, since the Hetzner CSI driver is installed automatically. This will use Hetzner's block storage (based on Ceph so it's replicated and highly available) for your persistent volumes. Note that the minimum size of a volume is 10Gi. If you specify a smaller size for a volume, the volume will be created with a capacity of 10Gi anyway.
 
 
+## Keeping a project per cluster
+
+I recommend that you create a separate Hetzner project for each cluster, because otherwise multiple clusters will attempt to create overlapping routes. I will make the pod cidr configurable in the future to avoid this, but I still recommend keeping clusters separated from each other. This way, if you want to delete a cluster with all the resources created for it, you can just delete the project.
+
+
 ## changelog
 
+- 0.4.0
+  - Ensure the masters are removed from the API load balancer before deleting the load balancer
+  - Ensure the servers are removed from the firewall before deleting it
+  - Allow using an environment variable to specify the Hetzner token
+  - Allow restricting SSH access to the nodes to specific networks
+
+- 0.3.9
+  - Add command "version" to print the version of the tool in use
+
+- 0.3.8
+  - Fix: added a check on a label to ensure that only servers that belong to the cluster are deleted from the project
+
+- 0.3.7
+  - Ensure that the cluster name only contains lowercase letters, digits and dashes for compatibility with the cloud controller manager
+
 - 0.3.6
   - Retry SSH commands when IO errors occur
 

data/lib/hetzner/infra/firewall.rb

@@ -5,7 +5,9 @@ module Hetzner
       @cluster_name = cluster_name
     end
 
-    def create
+    def create(ha:, networks:)
+      @ha = ha
+      @networks = networks
       puts
 
       if firewall = find_firewall
@@ -16,16 +18,21 @@ module Hetzner
 
       puts "Creating firewall..."
 
-      response = hetzner_client.post("/firewalls", firewall_config).body
+      response = hetzner_client.post("/firewalls", create_firewall_config).body
       puts "...firewall created."
       puts
 
       JSON.parse(response)["firewall"]["id"]
     end
 
-    def delete
+    def delete(servers)
       if firewall = find_firewall
         puts "Deleting firewall..."
+
+        servers.each do |server|
+          hetzner_client.post("/firewalls/#{firewall["id"]}/actions/remove_from_resources", remove_targets_config(server["id"]))
+        end
+
         hetzner_client.delete("/firewalls", firewall["id"])
         puts "...firewall deleted."
       else
@@ -37,64 +44,79 @@ module Hetzner
 
     private
 
-      attr_reader :hetzner_client, :cluster_name, :firewall
+      attr_reader :hetzner_client, :cluster_name, :firewall, :ha, :networks
+
+      def create_firewall_config
+        rules = [
+          {
+            "description": "Allow port 22 (SSH)",
+            "direction": "in",
+            "protocol": "tcp",
+            "port": "22",
+            "source_ips": networks,
+            "destination_ips": []
+          },
+          {
+            "description": "Allow ICMP (ping)",
+            "direction": "in",
+            "protocol": "icmp",
+            "port": nil,
+            "source_ips": [
+              "0.0.0.0/0",
+              "::/0"
+            ],
+            "destination_ips": []
+          },
+          {
+            "description": "Allow all TCP traffic between nodes on the private network",
+            "direction": "in",
+            "protocol": "tcp",
+            "port": "any",
+            "source_ips": [
+              "10.0.0.0/16"
+            ],
+            "destination_ips": []
+          },
+          {
+            "description": "Allow all UDP traffic between nodes on the private network",
+            "direction": "in",
+            "protocol": "udp",
+            "port": "any",
+            "source_ips": [
+              "10.0.0.0/16"
+            ],
+            "destination_ips": []
+          }
+        ]
+
+        unless ha
+          rules << {
+            "description": "Allow port 6443 (Kubernetes API server)",
+            "direction": "in",
+            "protocol": "tcp",
+            "port": "6443",
+            "source_ips": [
+              "0.0.0.0/0",
+              "::/0"
+            ],
+            "destination_ips": []
+          }
+        end
 
-      def firewall_config
         {
           name: cluster_name,
-          rules: [
-            {
-              "description": "Allow port 22 (SSH)",
-              "direction": "in",
-              "protocol": "tcp",
-              "port": "22",
-              "source_ips": [
-                "0.0.0.0/0",
-                "::/0"
-              ],
-              "destination_ips": []
-            },
-            {
-              "description": "Allow ICMP (ping)",
-              "direction": "in",
-              "protocol": "icmp",
-              "port": nil,
-              "source_ips": [
-                "0.0.0.0/0",
-                "::/0"
-              ],
-              "destination_ips": []
-            },
-            {
-              "description": "Allow port 6443 (Kubernetes API server)",
-              "direction": "in",
-              "protocol": "tcp",
-              "port": "6443",
-              "source_ips": [
-                "0.0.0.0/0",
-                "::/0"
-              ],
-              "destination_ips": []
-            },
-            {
-              "description": "Allow all TCP traffic between nodes on the private network",
-              "direction": "in",
-              "protocol": "tcp",
-              "port": "any",
-              "source_ips": [
-                "10.0.0.0/16"
-              ],
-              "destination_ips": []
-            },
+          rules: rules
+        }
+      end
+
+      def remove_targets_config(server_id)
+        {
+          "remove_from": [
             {
-              "description": "Allow all UDP traffic between nodes on the private network",
-              "direction": "in",
-              "protocol": "udp",
-              "port": "any",
-              "source_ips": [
-                "10.0.0.0/16"
-              ],
-              "destination_ips": []
+              "server": {
+                "id": server_id
+              },
+              "type": "server"
             }
           ]
         }

data/lib/hetzner/infra/load_balancer.rb

@@ -19,7 +19,7 @@ module Hetzner
 
       puts "Creating API load_balancer..."
 
-      response = hetzner_client.post("/load_balancers", load_balancer_config).body
+      response = hetzner_client.post("/load_balancers", create_load_balancer_config).body
       puts "...API load balancer created."
       puts
 
@@ -29,6 +29,9 @@ module Hetzner
     def delete(ha:)
       if load_balancer = find_load_balancer
         puts "Deleting API load balancer..." unless ha
+
+        hetzner_client.post("/load_balancers/#{load_balancer["id"]}/actions/remove_target", remove_targets_config)
+
         hetzner_client.delete("/load_balancers", load_balancer["id"])
         puts "...API load balancer deleted." unless ha
       elsif ha
@@ -46,7 +49,7 @@ module Hetzner
         "#{cluster_name}-api"
       end
 
-      def load_balancer_config
+      def create_load_balancer_config
         {
           "algorithm": {
             "type": "round_robin"
@@ -76,6 +79,15 @@ module Hetzner
         }
       end
 
+      def remove_targets_config
+        {
+          "label_selector": {
+            "selector": "cluster=#{cluster_name},role=master"
+          },
+          "type": "label_selector"
+        }
+      end
+
       def find_load_balancer
         hetzner_client.get("/load_balancers")["load_balancers"].detect{ |load_balancer| load_balancer["name"] == load_balancer_name }
       end

data/lib/hetzner/k3s/cli.rb

@@ -1,8 +1,11 @@
 require "thor"
 require "http"
 require "sshkey"
+require 'ipaddr'
+require 'open-uri'
 
 require_relative "cluster"
+require_relative "version"
 
 module Hetzner
   module K3s
@@ -11,13 +14,18 @@ module Hetzner
         true
       end
 
+      desc "version", "Print the version"
+      def version
+        puts Hetzner::K3s::VERSION
+      end
+
       desc "create-cluster", "Create a k3s cluster in Hetzner Cloud"
       option :config_file, required: true
 
       def create_cluster
         validate_config_file :create
 
-        Cluster.new(hetzner_client: hetzner_client).create configuration: configuration
+        Cluster.new(hetzner_client: hetzner_client, hetzner_token: find_hetzner_token).create configuration: configuration
       end
 
       desc "delete-cluster", "Delete an existing k3s cluster in Hetzner Cloud"
@@ -25,7 +33,7 @@ module Hetzner
 
       def delete_cluster
         validate_config_file :delete
-        Cluster.new(hetzner_client: hetzner_client).delete configuration: configuration
+        Cluster.new(hetzner_client: hetzner_client, hetzner_token: find_hetzner_token).delete configuration: configuration
       end
 
       desc "upgrade-cluster", "Upgrade an existing k3s cluster in Hetzner Cloud to a new version"
@@ -35,7 +43,7 @@ module Hetzner
 
       def upgrade_cluster
         validate_config_file :upgrade
-        Cluster.new(hetzner_client: hetzner_client).upgrade configuration: configuration, new_k3s_version: options[:new_k3s_version], config_file: options[:config_file]
+        Cluster.new(hetzner_client: hetzner_client, hetzner_token: find_hetzner_token).upgrade configuration: configuration, new_k3s_version: options[:new_k3s_version], config_file: options[:config_file]
       end
 
       desc "releases", "List available k3s releases"
@@ -76,6 +84,7 @@ module Hetzner
           case action
           when :create
             validate_ssh_key
+            validate_ssh_allowed_networks
             validate_location
             validate_k3s_version
             validate_masters
@@ -101,16 +110,31 @@ module Hetzner
           end
         end
 
+        def valid_token?
+          return @valid unless @valid.nil?
+
+          begin
+            token = find_hetzner_token
+            @hetzner_client = Hetzner::Client.new(token: token)
+            response = hetzner_client.get("/locations")
+            error_code = response.dig("error", "code")
+            @valid = if error_code and error_code.size > 0
+              false
+            else
+              true
+            end
+          rescue
+            @valid = false
+          end
+        end
+
         def validate_token
-          token = configuration.dig("hetzner_token")
-          @hetzner_client = Hetzner::Client.new(token: token)
-          hetzner_client.get("/locations")
-        rescue
-          errors << "Invalid Hetzner Cloid token"
+          errors << "Invalid Hetzner Cloud token" unless valid_token?
         end
 
         def validate_cluster_name
-          errors << "Cluster name is an invalid format" unless configuration["cluster_name"] =~ /\A([A-Za-z0-9\-\_]+)\Z/
+          errors << "Cluster name is an invalid format (only lowercase letters, digits and dashes are allowed)" unless configuration["cluster_name"] =~ /\A[a-z\d-]+\z/
+          errors << "Ensure that the cluster name starts with a normal letter" unless configuration["cluster_name"] =~ /\A[a-z]+.*\z/
         end
 
         def validate_kubeconfig_path
@@ -142,6 +166,7 @@ module Hetzner
         end
 
         def server_types
+          return [] unless valid_token?
           @server_types ||= hetzner_client.get("/server_types")["server_types"].map{ |server_type| server_type["name"] }
         rescue
           @errors << "Cannot fetch server types with Hetzner API, please try again later"
@@ -149,13 +174,15 @@ module Hetzner
         end
 
         def locations
+          return [] unless valid_token?
           @locations ||= hetzner_client.get("/locations")["locations"].map{ |location| location["name"] }
         rescue
           @errors << "Cannot fetch locations with Hetzner API, please try again later"
-          false
+          []
         end
 
         def validate_location
+          return if locations.empty? && !valid_token?
           errors << "Invalid location - available locations: nbg1 (Nuremberg, Germany), fsn1 (Falkenstein, Germany), hel1 (Helsinki, Finland)" unless locations.include? configuration.dig("location")
         end
 
@@ -264,7 +291,7 @@ module Hetzner
             instance_group_errors << "#{instance_group_type} is in an invalid format"
           end
 
-          unless server_types.include?(instance_group["instance_type"])
+          unless !valid_token? or server_types.include?(instance_group["instance_type"])
             instance_group_errors << "#{instance_group_type} has an invalid instance type"
           end
 
@@ -289,16 +316,62 @@ module Hetzner
           config_hash = YAML.load_file(File.expand_path(configuration["kubeconfig_path"]))
           config_hash['current-context'] = configuration["cluster_name"]
           @kubernetes_client = K8s::Client.config(K8s::Config.new(config_hash))
-        rescue
           errors << "Cannot connect to the Kubernetes cluster"
           false
         end
 
-
         def validate_verify_host_key
           return unless [true, false].include?(configuration.fetch("ssh_key_path", false))
           errors << "Please set the verify_host_key option to either true or false"
         end
+
+        def find_hetzner_token
+          @token = ENV["HCLOUD_TOKEN"]
+          return @token if @token
+          @token = configuration.dig("hetzner_token")
+        end
+
+        def validate_ssh_allowed_networks
+          networks ||= configuration.dig("ssh_allowed_networks")
+
+          if networks.nil? or networks.empty?
+            errors << "At least one network/IP range must be specified for SSH access"
+            return
+          end
+
+          invalid_networks = networks.reject do |network|
+            IPAddr.new(network) rescue false
+          end
+
+          unless invalid_networks.empty?
+            invalid_networks.each do |network|
+              errors << "The network #{network} is an invalid range"
+            end
+          end
+
+          invalid_ranges = networks.reject do |network|
+            network.include? "/"
+          end
+
+          unless invalid_ranges.empty?
+            invalid_ranges.each do |network|
+              errors << "Please use the CIDR notation for the networks to avoid ambiguity"
+            end
+          end
+
+          return unless invalid_networks.empty?
+
+          current_ip = URI.open('http://whatismyip.akamai.com').read
+
+          current_ip_networks = networks.detect do |network|
+            IPAddr.new(network).include?(current_ip) rescue false
+          end
+
+          unless current_ip_networks
+            errors << "Your current IP #{current_ip} is not included into any of the networks you've specified, so we won't be able to SSH into the nodes"
+          end
+        end
+
     end
   end
 end

data/lib/hetzner/k3s/cluster.rb

@@ -16,12 +16,12 @@ require_relative "../k3s/client_patch"
 
 
 class Cluster
-  def initialize(hetzner_client:)
+  def initialize(hetzner_client:, hetzner_token:)
     @hetzner_client = hetzner_client
+    @hetzner_token = hetzner_token
   end
 
   def create(configuration:)
-    @hetzner_token = configuration.dig("hetzner_token")
     @cluster_name = configuration.dig("cluster_name")
     @kubeconfig_path = File.expand_path(configuration.dig("kubeconfig_path"))
     @ssh_key_path = File.expand_path(configuration.dig("ssh_key_path"))
@@ -31,6 +31,7 @@ class Cluster
     @location = configuration.dig("location")
     @verify_host_key = configuration.fetch("verify_host_key", false)
     @servers = []
+    @networks = configuration.dig("ssh_allowed_networks")
 
     create_resources
 
@@ -69,7 +70,7 @@ class Cluster
                 :masters_config, :worker_node_pools,
                 :location, :ssh_key_path, :kubernetes_client,
                 :hetzner_token, :tls_sans, :new_k3s_version, :configuration,
-                :config_file, :verify_host_key
+                :config_file, :verify_host_key, :networks
 
 
     def latest_k3s_version
@@ -78,10 +79,13 @@ class Cluster
     end
 
     def create_resources
+      master_instance_type = masters_config["instance_type"]
+      masters_count = masters_config["instance_count"]
+
       firewall_id = Hetzner::Firewall.new(
         hetzner_client: hetzner_client,
         cluster_name: cluster_name
-      ).create
+      ).create(ha: (masters_count > 1), networks: networks)
 
       network_id = Hetzner::Network.new(
         hetzner_client: hetzner_client,
@@ -95,9 +99,6 @@ class Cluster
 
       server_configs = []
 
-      master_instance_type = masters_config["instance_type"]
-      masters_count = masters_config["instance_count"]
-
       masters_count.times do |i|
         server_configs << {
           location: location,
@@ -150,42 +151,15 @@ class Cluster
     end
 
     def delete_resources
-      # Deleting nodes defined according to Kubernetes first
-      begin
-        Timeout::timeout(5) do
-          servers = kubernetes_client.api("v1").resource("nodes").list
-
-          threads = servers.map do |node|
-            Thread.new do
-              Hetzner::Server.new(hetzner_client: hetzner_client, cluster_name: cluster_name).delete(server_name: node.metadata[:name])
-            end
-          end
-
-          threads.each(&:join) unless threads.empty?
-        end
-      rescue Timeout::Error, Excon::Error::Socket
-        puts "Unable to fetch nodes from Kubernetes API. Is the cluster online?"
-      end
-
-      # Deleting nodes defined in the config file just in case there are leftovers i.e. nodes that
-      # were not part of the cluster for some reason
-
-      threads = all_servers.map do |server|
-        Thread.new do
-          Hetzner::Server.new(hetzner_client: hetzner_client, cluster_name: cluster_name).delete(server_name: server["name"])
-        end
-      end
-
-      threads.each(&:join) unless threads.empty?
-
-      puts
-
-      sleep 5 # give time for the servers to actually be deleted
+      Hetzner::LoadBalancer.new(
+        hetzner_client: hetzner_client,
+        cluster_name: cluster_name
+      ).delete(ha: (masters.size > 1))
 
       Hetzner::Firewall.new(
         hetzner_client: hetzner_client,
         cluster_name: cluster_name
-      ).delete
+      ).delete(all_servers)
 
       Hetzner::Network.new(
         hetzner_client: hetzner_client,
@@ -197,11 +171,13 @@ class Cluster
         cluster_name: cluster_name
       ).delete(ssh_key_path: ssh_key_path)
 
-      Hetzner::LoadBalancer.new(
-        hetzner_client: hetzner_client,
-        cluster_name: cluster_name
-      ).delete(ha: (masters.size > 1))
+      threads = all_servers.map do |server|
+        Thread.new do
+          Hetzner::Server.new(hetzner_client: hetzner_client, cluster_name: cluster_name).delete(server_name: server["name"])
+        end
+      end
 
+      threads.each(&:join) unless threads.empty?
     end
 
     def upgrade_cluster
@@ -249,6 +225,7 @@ class Cluster
         --kube-scheduler-arg="bind-address=0.0.0.0" \
         --node-taint CriticalAddonsOnly=true:NoExecute \
         --kubelet-arg="cloud-provider=external" \
+        --advertise-address=$(hostname -I | awk '{print $2}') \
         --node-ip=$(hostname -I | awk '{print $2}') \
         --node-external-ip=$(hostname -I | awk '{print $1}') \
         --flannel-iface=#{flannel_interface} \
@@ -501,7 +478,7 @@ class Cluster
     end
 
     def all_servers
-      @all_servers ||= hetzner_client.get("/servers")["servers"]
+      @all_servers ||= hetzner_client.get("/servers")["servers"].select{ |server| belongs_to_cluster?(server) == true }
     end
 
     def masters
@@ -624,4 +601,8 @@ class Cluster
       temp_file_path
     end
 
+    def belongs_to_cluster?(server)
+      server.dig("labels", "cluster") == cluster_name
+    end
+
 end

data/lib/hetzner/k3s/version.rb

@@ -1,5 +1,5 @@
 module Hetzner
   module K3s
-    VERSION = "0.3.6"
+    VERSION = "0.4.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hetzner-k3s
 version: !ruby/object:Gem::Version
-  version: 0.3.6
+  version: 0.4.0
 platform: ruby
 authors:
 - Vito Botta
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-08-18 00:00:00.000000000 Z
+date: 2021-08-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor