hetzner-k3s 0.5.1 → 0.5.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c236d440e3e99f656c62b8ced99ac899199e435ca41e9675c4adeb326a87c5d8
-  data.tar.gz: fd8da58d3c4cc3b54018c1ff261f8606391e8f524b974660fc5984fcd10397f7
+  metadata.gz: 54bad08b660435283978314edf5d2823fd98a72958051615efc8ec5b24ce0b88
+  data.tar.gz: 653edb24301d237515b4abfae4538893f40f4707d462c3344f083a082d0fe0e8
 SHA512:
-  metadata.gz: 8ebe7779e0f79ec500c1e0d230212cd568ef62e94a3dd2d936214f03f48baf8fc937a683b7bf4478b66a3dc0a7b02ee001a6fe3e542ad401f0add4ee27a68b03
-  data.tar.gz: dae117d8d1e3babf392814baf372009d1558113673a4968b7526a0a4e3a06b8eea034edaa11452047286721efc2a37af3e8214f2dbeafde73d864493feae5626
+  metadata.gz: 93536c6ddf9091ed6c148388e4f29dd2630fd5daf03dd7feafe218cc1dad97ebc3bf3db01799126a9fed70003b2b57db1eb73a81ad807362459c95190527684e
+  data.tar.gz: 6bf703828e75363b7ba03ec413b4eff9ab97ca06c2cd8d4f82baca05fc2c32e43d86f0137f1f8f5af0dc4455b7edf157f74bda1621869beb89432058a78b0dcd

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    hetzner-k3s (0.5.0)
+    hetzner-k3s (0.5.5)
       bcrypt_pbkdf
       ed25519
       http

data/README.md

@@ -14,6 +14,7 @@ Using this tool, creating a highly available k3s cluster with 3 masters for the
 - installing the [Hetzner CSI Driver](https://github.com/hetznercloud/csi-driver) to provision persistent volumes using Hetzner's block storage
 - installing the [Rancher System Upgrade Controller](https://github.com/rancher/system-upgrade-controller) to make upgrades to a newer version of k3s easy and quick
 
+See roadmap [here](https://github.com/vitobotta/hetzner-k3s/projects/1) for the features planned or in progress.
 
 ## Requirements
 
@@ -25,7 +26,7 @@ All that is needed to use this tool is
 
 ## Installation
 
-Once you have the Ruby runtime up and running (2.7.2 or newer), you just need to install the gem:
+Once you have the Ruby runtime up and running (3.1.0 or newer), you just need to install the gem:
 
 ```bash
 gem install hetzner-k3s
@@ -38,7 +39,7 @@ This will install the `hetzner-k3s` executable in your PATH.
 Alternatively, if you don't want to set up a Ruby runtime but have Docker installed, you can use a container. Run the following from inside the directory where you have the config file for the cluster (described in the next section):
 
 ```bash
-docker run --rm -it -v ${PWD}:/cluster -v ${HOME}/.ssh:/tmp/.ssh vitobotta/hetzner-k3s:v0.5.1 create-cluster --config-file /cluster/test.yaml
+docker run --rm -it -v ${PWD}:/cluster -v ${HOME}/.ssh:/tmp/.ssh vitobotta/hetzner-k3s:v0.5.5 create-cluster --config-file /cluster/test.yaml
 ```
 
 Replace `test.yaml` with the name of your config file.
@@ -72,7 +73,26 @@ worker_node_pools:
   instance_count: 2
 additional_packages:
 - somepackage
-enable_ipsec_encryption: true
+enable_encryption: true
+# kube_api_server_args:
+# - arg1
+# - ...
+# kube_scheduler_args:
+# - arg1
+# - ...
+# kube_controller_manager_args:
+# - arg1
+# - ...
+# kube_cloud_controller_manager_args:
+# - arg1
+# - ...
+# kubelet_args:
+# - arg1
+# - ...
+# kube_proxy_args:
+# - arg1
+# - ...
+
 ```
 
 It should hopefully be self explanatory; you can run `hetzner-k3s releases` to see a list of the available releases from the most recent to the oldest available.
@@ -255,95 +275,6 @@ Once the cluster is ready you can create persistent volumes out of the box with
 I recommend that you create a separate Hetzner project for each cluster, because otherwise multiple clusters will attempt to create overlapping routes. I will make the pod cidr configurable in the future to avoid this, but I still recommend keeping clusters separated from each other. This way, if you want to delete a cluster with all the resources created for it, you can just delete the project.
 
 
-## changelog
-
-- 0.5.1
-  - Each node pool gets its own placement group. This is to minimize issues due to the max 10 nodes limitation for a single node group. A validation has also been added to limit pools to 10 nodes each because of this.
-
-- 0.5.0
-  - Allow installing additional packages when creating the servers
-  - Allow enabling ipsec encryption
-
-- 0.4.9
-  - Ensure the program always exits with exit code 1 if the config file fails validation
-  - Upgrade System Upgrade Controller to 0.8.1
-  - Remove dependency on unmaintained gem k8s-ruby
-  - Make the gem compatible with Ruby 3.1.0
-
-- 0.4.8
-  - Increase timeout with API requests to 30 seconds
-  - Limit number of retries for API requests to 3
-  - Ensure all version tags are listed for k3s (thanks @janosmiko)
-
-- 0.4.7
-  - Made it possible to specify a custom image/snapshot for the servers
-
-- 0.4.6
-  - Added a check to abort gracefully when for some reason one or more servers are not created, for example due to temporary problems with the Hetzner API.
-
-- 0.4.5
-  - Fix network creation (bug introduced in the previous version)
-
-- 0.4.4
-  - Add support for the new Ashburn, Virginia (USA) location
-  - Automatically use a placement group so that the instances are all created on different physical hosts for high availability
-
-- 0.4.3
-  - Fix an issue with SSH key creation
-
-- 0.4.2
-  - Update Hetzner CSI driver to v1.6.0
-  - Update System Upgrade Controller to v0.8.0
-
-- 0.4.1
-  - Allow to optionally specify the path of the private SSH key
-  - Set correct permissions for the kubeconfig file
-  - Retry fetching manifests a few times to allow for temporary network issues
-  - Allow to optionally schedule workloads on masters
-  - Allow clusters with no worker node pools if scheduling is enabled for the masters
-
-- 0.4.0
-  - Ensure the masters are removed from the API load balancer before deleting the load balancer
-  - Ensure the servers are removed from the firewall before deleting it
-  - Allow using an environment variable to specify the Hetzner token
-  - Allow restricting SSH access to the nodes to specific networks
-  - Do not open the port 6443 on the nodes if a load balancer is created for an HA cluster
-
-- 0.3.9
-  - Add command "version" to print the version of the tool in use
-
-- 0.3.8
-  - Fix: added a check on a label to ensure that only servers that belong to the cluster are deleted from the project
-
-- 0.3.7
-  - Ensure that the cluster name only contains lowercase letters, digits and dashes for compatibility with the cloud controller manager
-
-- 0.3.6
-  - Retry SSH commands when IO errors occur
-
-- 0.3.5
-  - Add descriptions for firewall rules
-
-- 0.3.4
-  - Added Docker support
-
-- 0.3.3
-  - Add some gems required on Linux
-
-- 0.3.2
-  - Configure DNS to use Cloudflare's resolver instead of Hetzner's, since Hetzner's resolvers are not always reliable
-
-- 0.3.1
-  - Allow enabling/disabling the host key verification
-
-- 0.3.0
-  - Handle case when an SSH key with the given fingerprint already exists in the Hetzner project
-  - Handle a timeout of 5 seconds for requests to the Hetzner API
-  - Retry waiting for server to be up when timeouts/host-unreachable errors occur
-  - Ignore known_hosts entry to prevent errors when recreating servers with IPs that have been used previously
-
-- 0.2.0
-  - Allow mixing servers of different series Intel/AMD
 ## Contributing and support
 
 Please create a PR if you want to propose any changes, or open an issue if you are having trouble with the tool - I will do my best to help if I can.

data/bin/build.sh

@@ -6,9 +6,9 @@ set -e
 
 IMAGE="vitobotta/hetzner-k3s"
 
-docker build -t ${IMAGE}:v0.5.1 \
+docker build -t ${IMAGE}:v0.5.5 \
   --platform=linux/amd64 \
-  --cache-from ${IMAGE}:v0.5.0 \
+  --cache-from ${IMAGE}:v0.5.4 \
   --build-arg BUILDKIT_INLINE_CACHE=1 .
 
-docker push vitobotta/hetzner-k3s:v0.5.1
+docker push vitobotta/hetzner-k3s:v0.5.5

data/lib/hetzner/infra/server.rb

@@ -81,7 +81,7 @@ module Hetzner
     end
 
     def user_data
-      packages = ['fail2ban']
+      packages = ['fail2ban', 'wireguard']
       packages += additional_packages if additional_packages
       packages = "'#{packages.join("', '")}'"
 

data/lib/hetzner/k3s/cli.rb

@@ -178,11 +178,16 @@ module Hetzner
         []
       end
 
-      def validate_location
+      def valid_location?(location)
         return if locations.empty? && !valid_token?
-        return if locations.include? configuration['location']
 
-        errors << 'Invalid location - available locations: nbg1 (Nuremberg, Germany), fsn1 (Falkenstein, Germany), hel1 (Helsinki, Finland) or ash (Ashburn, Virginia, USA)'
+        locations.include? location
+      end
+
+      def validate_masters_location
+        return if valid_location?(configuration['location'])
+
+        errors << 'Invalid location for master nodes - valid locations: nbg1 (Nuremberg, Germany), fsn1 (Falkenstein, Germany), hel1 (Helsinki, Finland) or ash (Ashburn, Virginia, USA)'
       end
 
       def available_releases
@@ -261,6 +266,14 @@ module Hetzner
 
         instance_group_errors << "#{instance_group_type} has an invalid instance type" unless !valid_token? || server_types.include?(instance_group['instance_type'])
 
+        if workers
+          location = instance_group.fetch('location', configuration['location'])
+          instance_group_errors << "#{instance_group_type} has an invalid location - valid locations: nbg1 (Nuremberg, Germany), fsn1 (Falkenstein, Germany), hel1 (Helsinki, Finland) or ash (Ashburn, Virginia, USA)" unless valid_location?(location)
+
+          in_network_zone = configuration['location'] == 'ash' ? location == 'ash' : location != 'ash'
+          instance_group_errors << "#{instance_group_type} must be in the same network zone as the masters. If the masters are located in Ashburn, all the node pools must be located in Ashburn too, otherwise none of the node pools should be located in Ashburn." unless in_network_zone
+        end
+
         if instance_group['instance_count'].is_a? Integer
           if instance_group['instance_count'] < 1
             instance_group_errors << "#{instance_group_type} must have at least one node"
@@ -343,12 +356,18 @@ module Hetzner
         validate_public_ssh_key
         validate_private_ssh_key
         validate_ssh_allowed_networks
-        validate_location
+        validate_masters_location
         validate_k3s_version
         validate_masters
         validate_worker_node_pools
         validate_verify_host_key
         validate_additional_packages
+        validate_kube_api_server_args
+        validate_kube_scheduler_args
+        validate_kube_controller_manager_args
+        validate_kube_cloud_controller_manager_args
+        validate_kubelet_args
+        validate_kube_proxy_args
       end
 
       def validate_upgrade
@@ -375,6 +394,48 @@ module Hetzner
           exit 1
         end
       end
+
+      def validate_kube_api_server_args
+        kube_api_server_args = configuration['kube_api_server_args']
+        return unless kube_api_server_args
+
+        errors << 'kube_api_server_args must be an array of arguments' unless kube_api_server_args.is_a? Array
+      end
+
+      def validate_kube_scheduler_args
+        kube_scheduler_args = configuration['kube_scheduler_args']
+        return unless kube_scheduler_args
+
+        errors << 'kube_scheduler_args must be an array of arguments' unless kube_scheduler_args.is_a? Array
+      end
+
+      def validate_kube_controller_manager_args
+        kube_controller_manager_args = configuration['kube_controller_manager_args']
+        return unless kube_controller_manager_args
+
+        errors << 'kube_controller_manager_args must be an array of arguments' unless kube_controller_manager_args.is_a? Array
+      end
+
+      def validate_kube_cloud_controller_manager_args
+        kube_cloud_controller_manager_args = configuration['kube_cloud_controller_manager_args']
+        return unless kube_cloud_controller_manager_args
+
+        errors << 'kube_cloud_controller_manager_args must be an array of arguments' unless kube_cloud_controller_manager_args.is_a? Array
+      end
+
+      def validate_kubelet_args
+        kubelet_args = configuration['kubelet_args']
+        return unless kubelet_args
+
+        errors << 'kubelet_args must be an array of arguments' unless kubelet_args.is_a? Array
+      end
+
+      def validate_kube_proxy_args
+        kube_proxy_args = configuration['kube_proxy_args']
+        return unless kube_proxy_args
+
+        errors << 'kube_proxy_args must be an array of arguments' unless kube_proxy_args.is_a? Array
+      end
     end
   end
 end

data/lib/hetzner/k3s/cluster.rb

@@ -34,11 +34,17 @@ class Cluster
     @k3s_version = configuration['k3s_version']
     @masters_config = configuration['masters']
     @worker_node_pools = find_worker_node_pools(configuration)
-    @location = configuration['location']
+    @masters_location = configuration['location']
     @verify_host_key = configuration.fetch('verify_host_key', false)
     @servers = []
     @networks = configuration['ssh_allowed_networks']
-    @enable_ipsec_encryption = configuration.fetch('enable_ipsec_encryption', false)
+    @enable_encryption = configuration.fetch('enable_encryption', false)
+    @kube_api_server_args = configuration.fetch('kube_api_server_args', [])
+    @kube_scheduler_args = configuration.fetch('kube_scheduler_args', [])
+    @kube_controller_manager_args = configuration.fetch('kube_controller_manager_args', [])
+    @kube_cloud_controller_manager_args = configuration.fetch('kube_cloud_controller_manager_args', [])
+    @kubelet_args = configuration.fetch('kubelet_args', [])
+    @kube_proxy_args = configuration.fetch('kube_proxy_args', [])
 
     create_resources
 
@@ -78,10 +84,12 @@ class Cluster
 
   attr_reader :hetzner_client, :cluster_name, :kubeconfig_path, :k3s_version,
               :masters_config, :worker_node_pools,
-              :location, :public_ssh_key_path,
+              :masters_location, :public_ssh_key_path,
               :hetzner_token, :new_k3s_version, :configuration,
               :config_file, :verify_host_key, :networks, :private_ssh_key_path,
-              :enable_ipsec_encryption
+              :enable_encryption, :kube_api_server_args, :kube_scheduler_args,
+              :kube_controller_manager_args, :kube_cloud_controller_manager_args,
+              :kubelet_args, :kube_proxy_args
 
   def find_worker_node_pools(configuration)
     configuration.fetch('worker_node_pools', [])
@@ -188,10 +196,10 @@ class Cluster
   end
 
   def master_script(master)
-    server = master == first_master ? ' --cluster-init ' : " --server https://#{first_master_private_ip}:6443 "
+    server = master == first_master ? ' --cluster-init ' : " --server https://#{api_server_ip}:6443 "
     flannel_interface = find_flannel_interface(master)
-    flannel_ipsec = enable_ipsec_encryption ? ' --flannel-backend=ipsec ' : ' '
-
+    flannel_wireguard = enable_encryption ? ' --flannel-backend=wireguard ' : ' '
+    extra_args = "#{kube_api_server_args_list} #{kube_scheduler_args_list} #{kube_controller_manager_args_list} #{kube_cloud_controller_manager_args_list} #{kubelet_args_list} #{kube_proxy_args_list}"
     taint = schedule_workloads_on_masters? ? ' ' : ' --node-taint CriticalAddonsOnly=true:NoExecute '
 
     <<~SCRIPT
@@ -205,13 +213,13 @@ class Cluster
         --node-name="$(hostname -f)" \
         --cluster-cidr=10.244.0.0/16 \
         --etcd-expose-metrics=true \
-        #{flannel_ipsec} \
+        #{flannel_wireguard} \
         --kube-controller-manager-arg="address=0.0.0.0" \
         --kube-controller-manager-arg="bind-address=0.0.0.0" \
         --kube-proxy-arg="metrics-bind-address=0.0.0.0" \
         --kube-scheduler-arg="address=0.0.0.0" \
         --kube-scheduler-arg="bind-address=0.0.0.0" \
-        #{taint} \
+        #{taint} #{extra_args} \
         --kubelet-arg="cloud-provider=external" \
         --advertise-address=$(hostname -I | awk '{print $2}') \
         --node-ip=$(hostname -I | awk '{print $2}') \
@@ -467,7 +475,7 @@ class Cluster
   end
 
   def network_id
-    @network_id ||= Hetzner::Network.new(hetzner_client:, cluster_name:).create(location:)
+    @network_id ||= Hetzner::Network.new(hetzner_client:, cluster_name:).create(location: masters_location)
   end
 
   def ssh_key_id
@@ -481,8 +489,8 @@ class Cluster
       definitions << {
         instance_type: master_instance_type,
         instance_id: "master#{i + 1}",
+        location: masters_location,
         placement_group_id:,
-        location:,
         firewall_id:,
         network_id:,
         ssh_key_id:,
@@ -511,6 +519,7 @@ class Cluster
     worker_node_pool_name = worker_node_pool['name']
     worker_instance_type = worker_node_pool['instance_type']
     worker_count = worker_node_pool['instance_count']
+    worker_location = worker_node_pool['location'] || masters_location
 
     definitions = []
 
@@ -519,7 +528,7 @@ class Cluster
         instance_type: worker_instance_type,
         instance_id: "pool-#{worker_node_pool_name}-worker#{i + 1}",
         placement_group_id: placement_group_id(worker_node_pool_name),
-        location:,
+        location: worker_location,
         firewall_id:,
         network_id:,
         ssh_key_id:,
@@ -580,4 +589,52 @@ class Cluster
 
     threads.each(&:join) unless threads.empty?
   end
+
+  def kube_api_server_args_list
+    return '' if kube_api_server_args.empty?
+
+    kube_api_server_args.map do |arg|
+      " --kube-apiserver-arg=\"#{arg}\" "
+    end.join
+  end
+
+  def kube_scheduler_args_list
+    return '' if kube_scheduler_args.empty?
+
+    kube_scheduler_args.map do |arg|
+      " --kube-scheduler-arg=\"#{arg}\" "
+    end.join
+  end
+
+  def kube_controller_manager_args_list
+    return '' if kube_controller_manager_args.empty?
+
+    kube_controller_manager_args.map do |arg|
+      " --kube-controller-manager-arg=\"#{arg}\" "
+    end.join
+  end
+
+  def kube_cloud_controller_manager_args_list
+    return '' if kube_cloud_controller_manager_args.empty?
+
+    kube_cloud_controller_manager_args.map do |arg|
+      " --kube-cloud-controller-manager-arg=\"#{arg}\" "
+    end.join
+  end
+
+  def kubelet_args_list
+    return '' if kubelet_args.empty?
+
+    kubelet_args.map do |arg|
+      " --kubelet-arg=\"#{arg}\" "
+    end.join
+  end
+
+  def kube_proxy_args_list
+    return '' if kube_proxy_args.empty?
+
+    kube_api_server_args.map do |arg|
+      " --kube-proxy-arg=\"#{arg}\" "
+    end.join
+  end
 end

data/lib/hetzner/k3s/version.rb

@@ -2,6 +2,6 @@
 
 module Hetzner
   module K3s
-    VERSION = '0.5.1'
+    VERSION = '0.5.5'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hetzner-k3s
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.5.5
 platform: ruby
 authors:
 - Vito Botta
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-02-06 00:00:00.000000000 Z
+date: 2022-02-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bcrypt_pbkdf