hetzner-k3s 0.6.2 → 0.6.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/README.md +5 -5
- data/lib/hetzner/infra/client.rb +4 -4
- data/lib/hetzner/infra/firewall.rb +1 -2
- data/lib/hetzner/infra/load_balancer.rb +1 -1
- data/lib/hetzner/infra/server.rb +28 -24
- data/lib/hetzner/k3s/cluster.rb +62 -526
- data/lib/hetzner/k3s/version.rb +1 -1
- data/lib/hetzner/kubernetes/client.rb +475 -0
- data/lib/hetzner/utils.rb +4 -4
- metadata +2 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 2bbaf7bd5387cc92f0725308a42f982df7b47596d5d1a3d50693cf3b6a741359
|
|
4
|
+
data.tar.gz: 0617b66a6ca8299c5c5ada69c586a532e915657c857c6f6821cd81d29c17a774
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: a457da09e05ac40da2d0b40de65193f16ecd42ae926f6d59271ca9fedc77587faf2bb90b4c4a62e326e509076b0f44e84062fb6c72536e5e8c1ce4b716d56813
|
|
7
|
+
data.tar.gz: '0628cf98daa772f50251257b626ca148420d1a8853618652c9fd7dd6750694393d9902ea684c786029e0db3bcf8d01729aa217e9fc3c070067fbbe6afb395dd8'
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
|
@@ -58,13 +58,13 @@ Before using the tool, be sure to have kubectl installed as it's required to ins
|
|
|
58
58
|
#### With Homebrew
|
|
59
59
|
|
|
60
60
|
```bash
|
|
61
|
-
brew install vitobotta/tap/
|
|
61
|
+
brew install vitobotta/tap/hetzner_k3s
|
|
62
62
|
```
|
|
63
63
|
|
|
64
64
|
#### Binary installation (Intel)
|
|
65
65
|
|
|
66
66
|
```bash
|
|
67
|
-
wget https://github.com/vitobotta/hetzner-k3s/releases/download/v0.6.
|
|
67
|
+
wget https://github.com/vitobotta/hetzner-k3s/releases/download/v0.6.3/hetzner-k3s-mac-amd64
|
|
68
68
|
chmod +x hetzner-k3s-mac-x64
|
|
69
69
|
sudo mv hetzner-k3s-mac-x64 /usr/local/bin/hetzner-k3s
|
|
70
70
|
```
|
|
@@ -72,7 +72,7 @@ sudo mv hetzner-k3s-mac-x64 /usr/local/bin/hetzner-k3s
|
|
|
72
72
|
#### Binary installation (Apple Silicon/M1)
|
|
73
73
|
|
|
74
74
|
```bash
|
|
75
|
-
wget https://github.com/vitobotta/hetzner-k3s/releases/download/v0.6.
|
|
75
|
+
wget https://github.com/vitobotta/hetzner-k3s/releases/download/v0.6.3/hetzner-k3s-mac-arm64
|
|
76
76
|
chmod +x hetzner-k3s-mac-arm
|
|
77
77
|
sudo mv hetzner-k3s-mac-arm /usr/local/bin/hetzner-k3s
|
|
78
78
|
```
|
|
@@ -82,7 +82,7 @@ NOTE: currently the ARM version still requires [Rosetta](https://support.apple.c
|
|
|
82
82
|
### Linux
|
|
83
83
|
|
|
84
84
|
```bash
|
|
85
|
-
wget https://github.com/vitobotta/hetzner-k3s/releases/download/v0.6.
|
|
85
|
+
wget https://github.com/vitobotta/hetzner-k3s/releases/download/v0.6.3/hetzner-k3s-linux-x86_64
|
|
86
86
|
chmod +x hetzner-k3s-linux-x86_64
|
|
87
87
|
sudo mv hetzner-k3s-linux-x86_64 /usr/local/bin/hetzner-k3s
|
|
88
88
|
```
|
|
@@ -107,7 +107,7 @@ Alternatively, if you don't want to set up a Ruby runtime but have Docker instal
|
|
|
107
107
|
docker run --rm -it \
|
|
108
108
|
-v ${PWD}:/cluster \
|
|
109
109
|
-v ${HOME}/.ssh:/tmp/.ssh \
|
|
110
|
-
vitobotta/hetzner-k3s:v0.6.
|
|
110
|
+
vitobotta/hetzner-k3s:v0.6.3 \
|
|
111
111
|
create-cluster \
|
|
112
112
|
--config-file /cluster/test.yaml
|
|
113
113
|
```
|
data/lib/hetzner/infra/client.rb
CHANGED
|
@@ -6,10 +6,6 @@ module Hetzner
|
|
|
6
6
|
|
|
7
7
|
attr_reader :token
|
|
8
8
|
|
|
9
|
-
def initialize(token:)
|
|
10
|
-
@token = token
|
|
11
|
-
end
|
|
12
|
-
|
|
13
9
|
def get(path)
|
|
14
10
|
make_request do
|
|
15
11
|
JSON.parse HTTParty.get(BASE_URI + path, headers: headers).body
|
|
@@ -30,6 +26,10 @@ module Hetzner
|
|
|
30
26
|
|
|
31
27
|
private
|
|
32
28
|
|
|
29
|
+
def initialize(token:)
|
|
30
|
+
@token = token
|
|
31
|
+
end
|
|
32
|
+
|
|
33
33
|
def headers
|
|
34
34
|
{
|
|
35
35
|
'Authorization' => "Bearer #{@token}",
|
|
@@ -33,8 +33,7 @@ module Hetzner
|
|
|
33
33
|
puts 'Deleting firewall...'
|
|
34
34
|
|
|
35
35
|
servers.each do |server|
|
|
36
|
-
hetzner_client.post("/firewalls/#{firewall['id']}/actions/remove_from_resources",
|
|
37
|
-
remove_targets_config(server['id']))
|
|
36
|
+
hetzner_client.post("/firewalls/#{firewall['id']}/actions/remove_from_resources", remove_targets_config(server['id']))
|
|
38
37
|
end
|
|
39
38
|
|
|
40
39
|
hetzner_client.delete('/firewalls', firewall['id'])
|
|
@@ -33,8 +33,8 @@ module Hetzner
|
|
|
33
33
|
puts 'Deleting API load balancer...' unless high_availability
|
|
34
34
|
|
|
35
35
|
hetzner_client.post("/load_balancers/#{load_balancer['id']}/actions/remove_target", remove_targets_config)
|
|
36
|
-
|
|
37
36
|
hetzner_client.delete('/load_balancers', load_balancer['id'])
|
|
37
|
+
|
|
38
38
|
puts '...API load balancer deleted.' unless high_availability
|
|
39
39
|
elsif high_availability
|
|
40
40
|
puts 'API load balancer no longer exists, skipping.'
|
data/lib/hetzner/infra/server.rb
CHANGED
|
@@ -59,30 +59,6 @@ module Hetzner
|
|
|
59
59
|
end
|
|
60
60
|
|
|
61
61
|
def user_data
|
|
62
|
-
packages = %w[fail2ban wireguard]
|
|
63
|
-
packages += additional_packages if additional_packages
|
|
64
|
-
packages = "'#{packages.join("', '")}'"
|
|
65
|
-
|
|
66
|
-
post_create_commands = [
|
|
67
|
-
'crontab -l > /etc/cron_bkp',
|
|
68
|
-
'echo "@reboot echo true > /etc/ready" >> /etc/cron_bkp',
|
|
69
|
-
'crontab /etc/cron_bkp',
|
|
70
|
-
'sed -i \'s/[#]*PermitRootLogin yes/PermitRootLogin prohibit-password/g\' /etc/ssh/sshd_config',
|
|
71
|
-
'sed -i \'s/[#]*PasswordAuthentication yes/PasswordAuthentication no/g\' /etc/ssh/sshd_config',
|
|
72
|
-
'systemctl restart sshd',
|
|
73
|
-
'systemctl stop systemd-resolved',
|
|
74
|
-
'systemctl disable systemd-resolved',
|
|
75
|
-
'rm /etc/resolv.conf',
|
|
76
|
-
'echo \'nameserver 1.1.1.1\' > /etc/resolv.conf',
|
|
77
|
-
'echo \'nameserver 1.0.0.1\' >> /etc/resolv.conf'
|
|
78
|
-
]
|
|
79
|
-
|
|
80
|
-
post_create_commands += additional_post_create_commands if additional_post_create_commands
|
|
81
|
-
|
|
82
|
-
post_create_commands += ['shutdown -r now'] if post_create_commands.grep(/shutdown|reboot/).grep_v(/@reboot/).empty?
|
|
83
|
-
|
|
84
|
-
post_create_commands = " - #{post_create_commands.join("\n - ")}"
|
|
85
|
-
|
|
86
62
|
<<~YAML
|
|
87
63
|
#cloud-config
|
|
88
64
|
packages: [#{packages}]
|
|
@@ -125,5 +101,33 @@ module Hetzner
|
|
|
125
101
|
|
|
126
102
|
JSON.parse(response_body)['server']
|
|
127
103
|
end
|
|
104
|
+
|
|
105
|
+
def post_create_commands
|
|
106
|
+
commands = [
|
|
107
|
+
'crontab -l > /etc/cron_bkp',
|
|
108
|
+
'echo "@reboot echo true > /etc/ready" >> /etc/cron_bkp',
|
|
109
|
+
'crontab /etc/cron_bkp',
|
|
110
|
+
'sed -i \'s/[#]*PermitRootLogin yes/PermitRootLogin prohibit-password/g\' /etc/ssh/sshd_config',
|
|
111
|
+
'sed -i \'s/[#]*PasswordAuthentication yes/PasswordAuthentication no/g\' /etc/ssh/sshd_config',
|
|
112
|
+
'systemctl restart sshd',
|
|
113
|
+
'systemctl stop systemd-resolved',
|
|
114
|
+
'systemctl disable systemd-resolved',
|
|
115
|
+
'rm /etc/resolv.conf',
|
|
116
|
+
'echo \'nameserver 1.1.1.1\' > /etc/resolv.conf',
|
|
117
|
+
'echo \'nameserver 1.0.0.1\' >> /etc/resolv.conf'
|
|
118
|
+
]
|
|
119
|
+
|
|
120
|
+
commands += additional_post_create_commands if additional_post_create_commands
|
|
121
|
+
|
|
122
|
+
commands << 'shutdown -r now' if commands.grep(/shutdown|reboot/).grep_v(/@reboot/).empty?
|
|
123
|
+
|
|
124
|
+
" - #{commands.join("\n - ")}"
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def packages
|
|
128
|
+
packages = %w[fail2ban wireguard]
|
|
129
|
+
packages += additional_packages if additional_packages
|
|
130
|
+
"'#{packages.join("', '")}'"
|
|
131
|
+
end
|
|
128
132
|
end
|
|
129
133
|
end
|
data/lib/hetzner/k3s/cluster.rb
CHANGED
|
@@ -14,6 +14,8 @@ require_relative '../infra/server'
|
|
|
14
14
|
require_relative '../infra/load_balancer'
|
|
15
15
|
require_relative '../infra/placement_group'
|
|
16
16
|
|
|
17
|
+
require_relative '../kubernetes/client'
|
|
18
|
+
|
|
17
19
|
require_relative '../utils'
|
|
18
20
|
|
|
19
21
|
class Cluster
|
|
@@ -25,38 +27,18 @@ class Cluster
|
|
|
25
27
|
|
|
26
28
|
def create
|
|
27
29
|
@cluster_name = configuration['cluster_name']
|
|
28
|
-
@kubeconfig_path = File.expand_path(configuration['kubeconfig_path'])
|
|
29
|
-
@public_ssh_key_path = File.expand_path(configuration['public_ssh_key_path'])
|
|
30
|
-
private_ssh_key_path = configuration['private_ssh_key_path']
|
|
31
|
-
@private_ssh_key_path = private_ssh_key_path && File.expand_path(private_ssh_key_path)
|
|
32
|
-
@k3s_version = configuration['k3s_version']
|
|
33
30
|
@masters_config = configuration['masters']
|
|
34
31
|
@worker_node_pools = find_worker_node_pools(configuration)
|
|
35
32
|
@masters_location = configuration['location']
|
|
36
|
-
@verify_host_key = configuration.fetch('verify_host_key', false)
|
|
37
33
|
@servers = []
|
|
38
34
|
@ssh_networks = configuration['ssh_allowed_networks']
|
|
39
35
|
@api_networks = configuration['api_allowed_networks']
|
|
40
|
-
@
|
|
41
|
-
@
|
|
42
|
-
@kube_scheduler_args = configuration.fetch('kube_scheduler_args', [])
|
|
43
|
-
@kube_controller_manager_args = configuration.fetch('kube_controller_manager_args', [])
|
|
44
|
-
@kube_cloud_controller_manager_args = configuration.fetch('kube_cloud_controller_manager_args', [])
|
|
45
|
-
@kubelet_args = configuration.fetch('kubelet_args', [])
|
|
46
|
-
@kube_proxy_args = configuration.fetch('kube_proxy_args', [])
|
|
36
|
+
@private_ssh_key_path = File.expand_path(configuration['private_ssh_key_path'])
|
|
37
|
+
@public_ssh_key_path = File.expand_path(configuration['public_ssh_key_path'])
|
|
47
38
|
|
|
48
39
|
create_resources
|
|
49
40
|
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
sleep 10
|
|
53
|
-
|
|
54
|
-
label_nodes
|
|
55
|
-
taint_nodes
|
|
56
|
-
|
|
57
|
-
deploy_cloud_controller_manager
|
|
58
|
-
deploy_csi_driver
|
|
59
|
-
deploy_system_upgrade_controller
|
|
41
|
+
kubernetes_client.deploy(masters: masters, workers: workers, master_definitions: master_definitions_for_create, worker_definitions: workers_definitions_for_marking)
|
|
60
42
|
end
|
|
61
43
|
|
|
62
44
|
def delete
|
|
@@ -75,378 +57,26 @@ class Cluster
|
|
|
75
57
|
@new_k3s_version = new_k3s_version
|
|
76
58
|
@config_file = config_file
|
|
77
59
|
|
|
78
|
-
|
|
60
|
+
kubernetes_client.upgrade
|
|
79
61
|
end
|
|
80
62
|
|
|
81
63
|
private
|
|
82
64
|
|
|
83
65
|
attr_accessor :servers
|
|
84
66
|
|
|
85
|
-
attr_reader :configuration, :cluster_name, :kubeconfig_path,
|
|
67
|
+
attr_reader :configuration, :cluster_name, :kubeconfig_path,
|
|
86
68
|
:masters_config, :worker_node_pools,
|
|
87
|
-
:masters_location, :public_ssh_key_path,
|
|
69
|
+
:masters_location, :private_ssh_key_path, :public_ssh_key_path,
|
|
88
70
|
:hetzner_token, :new_k3s_version,
|
|
89
|
-
:config_file, :
|
|
90
|
-
:
|
|
91
|
-
:kube_controller_manager_args, :kube_cloud_controller_manager_args,
|
|
92
|
-
:kubelet_args, :kube_proxy_args, :api_networks
|
|
71
|
+
:config_file, :ssh_networks,
|
|
72
|
+
:api_networks
|
|
93
73
|
|
|
94
74
|
def find_worker_node_pools(configuration)
|
|
95
75
|
configuration.fetch('worker_node_pools', [])
|
|
96
76
|
end
|
|
97
77
|
|
|
98
|
-
def
|
|
99
|
-
|
|
100
|
-
JSON.parse(response).first['name']
|
|
101
|
-
end
|
|
102
|
-
|
|
103
|
-
def create_resources
|
|
104
|
-
create_servers
|
|
105
|
-
create_load_balancer if masters.size > 1
|
|
106
|
-
end
|
|
107
|
-
|
|
108
|
-
def delete_placement_groups
|
|
109
|
-
Hetzner::PlacementGroup.new(hetzner_client: hetzner_client, cluster_name: cluster_name).delete
|
|
110
|
-
|
|
111
|
-
worker_node_pools.each do |pool|
|
|
112
|
-
pool_name = pool['name']
|
|
113
|
-
Hetzner::PlacementGroup.new(hetzner_client: hetzner_client, cluster_name: cluster_name, pool_name: pool_name).delete
|
|
114
|
-
end
|
|
115
|
-
end
|
|
116
|
-
|
|
117
|
-
def delete_resources
|
|
118
|
-
Hetzner::LoadBalancer.new(hetzner_client: hetzner_client, cluster_name: cluster_name).delete(high_availability: (masters.size > 1))
|
|
119
|
-
|
|
120
|
-
Hetzner::Firewall.new(hetzner_client: hetzner_client, cluster_name: cluster_name).delete(all_servers)
|
|
121
|
-
|
|
122
|
-
Hetzner::Network.new(hetzner_client: hetzner_client, cluster_name: cluster_name, existing_network: existing_network).delete
|
|
123
|
-
|
|
124
|
-
Hetzner::SSHKey.new(hetzner_client: hetzner_client, cluster_name: cluster_name).delete(public_ssh_key_path: public_ssh_key_path)
|
|
125
|
-
|
|
126
|
-
delete_placement_groups
|
|
127
|
-
delete_servers
|
|
128
|
-
end
|
|
129
|
-
|
|
130
|
-
def upgrade_cluster
|
|
131
|
-
worker_upgrade_concurrency = workers.size - 1
|
|
132
|
-
worker_upgrade_concurrency = 1 if worker_upgrade_concurrency.zero?
|
|
133
|
-
|
|
134
|
-
cmd = <<~BASH
|
|
135
|
-
kubectl apply -f - <<-EOF
|
|
136
|
-
apiVersion: upgrade.cattle.io/v1
|
|
137
|
-
kind: Plan
|
|
138
|
-
metadata:
|
|
139
|
-
name: k3s-server
|
|
140
|
-
namespace: system-upgrade
|
|
141
|
-
labels:
|
|
142
|
-
k3s-upgrade: server
|
|
143
|
-
spec:
|
|
144
|
-
concurrency: 1
|
|
145
|
-
version: #{new_k3s_version}
|
|
146
|
-
nodeSelector:
|
|
147
|
-
matchExpressions:
|
|
148
|
-
- {key: node-role.kubernetes.io/master, operator: In, values: ["true"]}
|
|
149
|
-
serviceAccountName: system-upgrade
|
|
150
|
-
tolerations:
|
|
151
|
-
- key: "CriticalAddonsOnly"
|
|
152
|
-
operator: "Equal"
|
|
153
|
-
value: "true"
|
|
154
|
-
effect: "NoExecute"
|
|
155
|
-
cordon: true
|
|
156
|
-
upgrade:
|
|
157
|
-
image: rancher/k3s-upgrade
|
|
158
|
-
EOF
|
|
159
|
-
BASH
|
|
160
|
-
|
|
161
|
-
run cmd, kubeconfig_path: kubeconfig_path
|
|
162
|
-
|
|
163
|
-
cmd = <<~BASH
|
|
164
|
-
kubectl apply -f - <<-EOF
|
|
165
|
-
apiVersion: upgrade.cattle.io/v1
|
|
166
|
-
kind: Plan
|
|
167
|
-
metadata:
|
|
168
|
-
name: k3s-agent
|
|
169
|
-
namespace: system-upgrade
|
|
170
|
-
labels:
|
|
171
|
-
k3s-upgrade: agent
|
|
172
|
-
spec:
|
|
173
|
-
concurrency: #{worker_upgrade_concurrency}
|
|
174
|
-
version: #{new_k3s_version}
|
|
175
|
-
nodeSelector:
|
|
176
|
-
matchExpressions:
|
|
177
|
-
- {key: node-role.kubernetes.io/master, operator: NotIn, values: ["true"]}
|
|
178
|
-
serviceAccountName: system-upgrade
|
|
179
|
-
prepare:
|
|
180
|
-
image: rancher/k3s-upgrade
|
|
181
|
-
args: ["prepare", "k3s-server"]
|
|
182
|
-
cordon: true
|
|
183
|
-
upgrade:
|
|
184
|
-
image: rancher/k3s-upgrade
|
|
185
|
-
EOF
|
|
186
|
-
BASH
|
|
187
|
-
|
|
188
|
-
run cmd, kubeconfig_path: kubeconfig_path
|
|
189
|
-
|
|
190
|
-
puts 'Upgrade will now start. Run `watch kubectl get nodes` to see the nodes being upgraded. This should take a few minutes for a small cluster.'
|
|
191
|
-
puts 'The API server may be briefly unavailable during the upgrade of the controlplane.'
|
|
192
|
-
|
|
193
|
-
updated_configuration = configuration.raw
|
|
194
|
-
updated_configuration['k3s_version'] = new_k3s_version
|
|
195
|
-
|
|
196
|
-
File.write(config_file, updated_configuration.to_yaml)
|
|
197
|
-
end
|
|
198
|
-
|
|
199
|
-
def master_script(master)
|
|
200
|
-
server = master == first_master ? ' --cluster-init ' : " --server https://#{api_server_ip}:6443 "
|
|
201
|
-
flannel_interface = find_flannel_interface(master)
|
|
202
|
-
|
|
203
|
-
available_k3s_releases = Hetzner::Configuration.available_releases
|
|
204
|
-
wireguard_native_min_version_index = available_k3s_releases.find_index('v1.23.6+k3s1')
|
|
205
|
-
selected_version_index = available_k3s_releases.find_index(k3s_version)
|
|
206
|
-
|
|
207
|
-
flannel_wireguard = if enable_encryption
|
|
208
|
-
if selected_version_index >= wireguard_native_min_version_index
|
|
209
|
-
' --flannel-backend=wireguard-native '
|
|
210
|
-
else
|
|
211
|
-
' --flannel-backend=wireguard '
|
|
212
|
-
end
|
|
213
|
-
else
|
|
214
|
-
' '
|
|
215
|
-
end
|
|
216
|
-
|
|
217
|
-
extra_args = "#{kube_api_server_args_list} #{kube_scheduler_args_list} #{kube_controller_manager_args_list} #{kube_cloud_controller_manager_args_list} #{kubelet_args_list} #{kube_proxy_args_list}"
|
|
218
|
-
taint = schedule_workloads_on_masters? ? ' ' : ' --node-taint CriticalAddonsOnly=true:NoExecute '
|
|
219
|
-
|
|
220
|
-
<<~SCRIPT
|
|
221
|
-
curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION="#{k3s_version}" K3S_TOKEN="#{k3s_token}" INSTALL_K3S_EXEC="server \
|
|
222
|
-
--disable-cloud-controller \
|
|
223
|
-
--disable servicelb \
|
|
224
|
-
--disable traefik \
|
|
225
|
-
--disable local-storage \
|
|
226
|
-
--disable metrics-server \
|
|
227
|
-
--write-kubeconfig-mode=644 \
|
|
228
|
-
--node-name="$(hostname -f)" \
|
|
229
|
-
--cluster-cidr=10.244.0.0/16 \
|
|
230
|
-
--etcd-expose-metrics=true \
|
|
231
|
-
#{flannel_wireguard} \
|
|
232
|
-
--kube-controller-manager-arg="bind-address=0.0.0.0" \
|
|
233
|
-
--kube-proxy-arg="metrics-bind-address=0.0.0.0" \
|
|
234
|
-
--kube-scheduler-arg="bind-address=0.0.0.0" \
|
|
235
|
-
#{taint} #{extra_args} \
|
|
236
|
-
--kubelet-arg="cloud-provider=external" \
|
|
237
|
-
--advertise-address=$(hostname -I | awk '{print $2}') \
|
|
238
|
-
--node-ip=$(hostname -I | awk '{print $2}') \
|
|
239
|
-
--node-external-ip=$(hostname -I | awk '{print $1}') \
|
|
240
|
-
--flannel-iface=#{flannel_interface} \
|
|
241
|
-
#{server} #{tls_sans}" sh -
|
|
242
|
-
SCRIPT
|
|
243
|
-
end
|
|
244
|
-
|
|
245
|
-
def worker_script(worker)
|
|
246
|
-
flannel_interface = find_flannel_interface(worker)
|
|
247
|
-
|
|
248
|
-
<<~BASH
|
|
249
|
-
curl -sfL https://get.k3s.io | K3S_TOKEN="#{k3s_token}" INSTALL_K3S_VERSION="#{k3s_version}" K3S_URL=https://#{first_master_private_ip}:6443 INSTALL_K3S_EXEC="agent \
|
|
250
|
-
--node-name="$(hostname -f)" \
|
|
251
|
-
--kubelet-arg="cloud-provider=external" \
|
|
252
|
-
--node-ip=$(hostname -I | awk '{print $2}') \
|
|
253
|
-
--node-external-ip=$(hostname -I | awk '{print $1}') \
|
|
254
|
-
--flannel-iface=#{flannel_interface}" sh -
|
|
255
|
-
BASH
|
|
256
|
-
end
|
|
257
|
-
|
|
258
|
-
def deploy_kubernetes
|
|
259
|
-
puts
|
|
260
|
-
puts "Deploying k3s to first master (#{first_master['name']})..."
|
|
261
|
-
|
|
262
|
-
ssh first_master, master_script(first_master), print_output: true
|
|
263
|
-
|
|
264
|
-
puts
|
|
265
|
-
puts '...k3s has been deployed to first master.'
|
|
266
|
-
|
|
267
|
-
save_kubeconfig
|
|
268
|
-
|
|
269
|
-
if masters.size > 1
|
|
270
|
-
threads = masters[1..].map do |master|
|
|
271
|
-
Thread.new do
|
|
272
|
-
puts
|
|
273
|
-
puts "Deploying k3s to master #{master['name']}..."
|
|
274
|
-
|
|
275
|
-
ssh master, master_script(master), print_output: true
|
|
276
|
-
|
|
277
|
-
puts
|
|
278
|
-
puts "...k3s has been deployed to master #{master['name']}."
|
|
279
|
-
end
|
|
280
|
-
end
|
|
281
|
-
|
|
282
|
-
threads.each(&:join) unless threads.empty?
|
|
283
|
-
end
|
|
284
|
-
|
|
285
|
-
threads = workers.map do |worker|
|
|
286
|
-
Thread.new do
|
|
287
|
-
puts
|
|
288
|
-
puts "Deploying k3s to worker (#{worker['name']})..."
|
|
289
|
-
|
|
290
|
-
ssh worker, worker_script(worker), print_output: true
|
|
291
|
-
|
|
292
|
-
puts
|
|
293
|
-
puts "...k3s has been deployed to worker (#{worker['name']})."
|
|
294
|
-
end
|
|
295
|
-
end
|
|
296
|
-
|
|
297
|
-
threads.each(&:join) unless threads.empty?
|
|
298
|
-
end
|
|
299
|
-
|
|
300
|
-
def label_nodes
|
|
301
|
-
check_kubectl
|
|
302
|
-
|
|
303
|
-
if master_definitions_for_create.first[:labels]
|
|
304
|
-
master_labels = master_definitions_for_create.first[:labels].map{ |k, v| "#{k}=#{v}" }.join(' ')
|
|
305
|
-
master_node_names = []
|
|
306
|
-
|
|
307
|
-
master_definitions_for_create.each do |master|
|
|
308
|
-
master_node_names << "#{configuration['cluster_name']}-#{master[:instance_type]}-#{master[:instance_id]}"
|
|
309
|
-
end
|
|
310
|
-
|
|
311
|
-
master_node_names = master_node_names.join(' ')
|
|
312
|
-
|
|
313
|
-
cmd = "kubectl label --overwrite nodes #{master_node_names} #{master_labels}"
|
|
314
|
-
|
|
315
|
-
run cmd, kubeconfig_path: kubeconfig_path
|
|
316
|
-
end
|
|
317
|
-
|
|
318
|
-
workers = []
|
|
319
|
-
|
|
320
|
-
worker_node_pools.each do |worker_node_pool|
|
|
321
|
-
workers += worker_node_pool_definitions(worker_node_pool)
|
|
322
|
-
end
|
|
323
|
-
|
|
324
|
-
return unless workers.any?
|
|
325
|
-
|
|
326
|
-
workers.each do |worker|
|
|
327
|
-
next unless worker[:labels]
|
|
328
|
-
|
|
329
|
-
worker_labels = worker[:labels].map{ |k, v| "#{k}=#{v}" }.join(' ')
|
|
330
|
-
worker_node_name = "#{configuration['cluster_name']}-#{worker[:instance_type]}-#{worker[:instance_id]}"
|
|
331
|
-
|
|
332
|
-
cmd = "kubectl label --overwrite nodes #{worker_node_name} #{worker_labels}"
|
|
333
|
-
|
|
334
|
-
run cmd, kubeconfig_path: kubeconfig_path
|
|
335
|
-
end
|
|
336
|
-
end
|
|
337
|
-
|
|
338
|
-
def taint_nodes
|
|
339
|
-
check_kubectl
|
|
340
|
-
|
|
341
|
-
if master_definitions_for_create.first[:taints]
|
|
342
|
-
master_taints = master_definitions_for_create.first[:taints].map{ |k, v| "#{k}=#{v}" }.join(' ')
|
|
343
|
-
master_node_names = []
|
|
344
|
-
|
|
345
|
-
master_definitions_for_create.each do |master|
|
|
346
|
-
master_node_names << "#{configuration['cluster_name']}-#{master[:instance_type]}-#{master[:instance_id]}"
|
|
347
|
-
end
|
|
348
|
-
|
|
349
|
-
master_node_names = master_node_names.join(' ')
|
|
350
|
-
|
|
351
|
-
cmd = "kubectl taint --overwrite nodes #{master_node_names} #{master_taints}"
|
|
352
|
-
|
|
353
|
-
run cmd, kubeconfig_path: kubeconfig_path
|
|
354
|
-
end
|
|
355
|
-
|
|
356
|
-
workers = []
|
|
357
|
-
|
|
358
|
-
worker_node_pools.each do |worker_node_pool|
|
|
359
|
-
workers += worker_node_pool_definitions(worker_node_pool)
|
|
360
|
-
end
|
|
361
|
-
|
|
362
|
-
return unless workers.any?
|
|
363
|
-
|
|
364
|
-
workers.each do |worker|
|
|
365
|
-
next unless worker[:taints]
|
|
366
|
-
|
|
367
|
-
worker_taints = worker[:taints].map{ |k, v| "#{k}=#{v}" }.join(' ')
|
|
368
|
-
worker_node_name = "#{configuration['cluster_name']}-#{worker[:instance_type]}-#{worker[:instance_id]}"
|
|
369
|
-
|
|
370
|
-
cmd = "kubectl taint --overwrite nodes #{worker_node_name} #{worker_taints}"
|
|
371
|
-
|
|
372
|
-
run cmd, kubeconfig_path: kubeconfig_path
|
|
373
|
-
end
|
|
374
|
-
end
|
|
375
|
-
|
|
376
|
-
def deploy_cloud_controller_manager
|
|
377
|
-
check_kubectl
|
|
378
|
-
|
|
379
|
-
puts
|
|
380
|
-
puts 'Deploying Hetzner Cloud Controller Manager...'
|
|
381
|
-
|
|
382
|
-
cmd = <<~BASH
|
|
383
|
-
kubectl apply -f - <<-EOF
|
|
384
|
-
apiVersion: "v1"
|
|
385
|
-
kind: "Secret"
|
|
386
|
-
metadata:
|
|
387
|
-
namespace: 'kube-system'
|
|
388
|
-
name: 'hcloud'
|
|
389
|
-
stringData:
|
|
390
|
-
network: "#{existing_network || cluster_name}"
|
|
391
|
-
token: "#{configuration.hetzner_token}"
|
|
392
|
-
EOF
|
|
393
|
-
BASH
|
|
394
|
-
|
|
395
|
-
run cmd, kubeconfig_path: kubeconfig_path
|
|
396
|
-
|
|
397
|
-
cmd = 'kubectl apply -f https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/latest/download/ccm-networks.yaml'
|
|
398
|
-
|
|
399
|
-
run cmd, kubeconfig_path: kubeconfig_path
|
|
400
|
-
|
|
401
|
-
puts '...Cloud Controller Manager deployed'
|
|
402
|
-
end
|
|
403
|
-
|
|
404
|
-
def deploy_system_upgrade_controller
|
|
405
|
-
check_kubectl
|
|
406
|
-
|
|
407
|
-
puts
|
|
408
|
-
puts 'Deploying k3s System Upgrade Controller...'
|
|
409
|
-
|
|
410
|
-
cmd = 'kubectl apply -f https://github.com/rancher/system-upgrade-controller/releases/download/v0.9.1/system-upgrade-controller.yaml'
|
|
411
|
-
|
|
412
|
-
run cmd, kubeconfig_path: kubeconfig_path
|
|
413
|
-
|
|
414
|
-
puts '...k3s System Upgrade Controller deployed'
|
|
415
|
-
end
|
|
416
|
-
|
|
417
|
-
def deploy_csi_driver
|
|
418
|
-
check_kubectl
|
|
419
|
-
|
|
420
|
-
puts
|
|
421
|
-
puts 'Deploying Hetzner CSI Driver...'
|
|
422
|
-
|
|
423
|
-
cmd = <<~BASH
|
|
424
|
-
kubectl apply -f - <<-EOF
|
|
425
|
-
apiVersion: "v1"
|
|
426
|
-
kind: "Secret"
|
|
427
|
-
metadata:
|
|
428
|
-
namespace: 'kube-system'
|
|
429
|
-
name: 'hcloud-csi'
|
|
430
|
-
stringData:
|
|
431
|
-
token: "#{configuration.hetzner_token}"
|
|
432
|
-
EOF
|
|
433
|
-
BASH
|
|
434
|
-
|
|
435
|
-
run cmd, kubeconfig_path: kubeconfig_path
|
|
436
|
-
|
|
437
|
-
cmd = 'kubectl apply -f https://raw.githubusercontent.com/hetznercloud/csi-driver/master/deploy/kubernetes/hcloud-csi.yml'
|
|
438
|
-
|
|
439
|
-
run cmd, kubeconfig_path: kubeconfig_path
|
|
440
|
-
|
|
441
|
-
puts '...CSI Driver deployed'
|
|
442
|
-
end
|
|
443
|
-
|
|
444
|
-
def find_flannel_interface(server)
|
|
445
|
-
if ssh(server, 'lscpu | grep Vendor') =~ /Intel/
|
|
446
|
-
'ens10'
|
|
447
|
-
else
|
|
448
|
-
'enp7s0'
|
|
449
|
-
end
|
|
78
|
+
def belongs_to_cluster?(server)
|
|
79
|
+
server.dig('labels', 'cluster') == cluster_name
|
|
450
80
|
end
|
|
451
81
|
|
|
452
82
|
def all_servers
|
|
@@ -463,74 +93,6 @@ class Cluster
|
|
|
463
93
|
@workers = all_servers.select { |server| server['name'] =~ /worker\d+\Z/ }.sort { |a, b| a['name'] <=> b['name'] }
|
|
464
94
|
end
|
|
465
95
|
|
|
466
|
-
def k3s_token
|
|
467
|
-
@k3s_token ||= begin
|
|
468
|
-
token = ssh(first_master, '{ TOKEN=$(< /var/lib/rancher/k3s/server/node-token); } 2> /dev/null; echo $TOKEN')
|
|
469
|
-
|
|
470
|
-
if token.empty?
|
|
471
|
-
SecureRandom.hex
|
|
472
|
-
else
|
|
473
|
-
token.split(':').last
|
|
474
|
-
end
|
|
475
|
-
end
|
|
476
|
-
end
|
|
477
|
-
|
|
478
|
-
def first_master_private_ip
|
|
479
|
-
@first_master_private_ip ||= first_master['private_net'][0]['ip']
|
|
480
|
-
end
|
|
481
|
-
|
|
482
|
-
def first_master
|
|
483
|
-
masters.first
|
|
484
|
-
end
|
|
485
|
-
|
|
486
|
-
def api_server_ip
|
|
487
|
-
return @api_server_ip if @api_server_ip
|
|
488
|
-
|
|
489
|
-
@api_server_ip = if masters.size > 1
|
|
490
|
-
load_balancer_name = "#{cluster_name}-api"
|
|
491
|
-
load_balancer = hetzner_client.get('/load_balancers')['load_balancers'].detect do |lb|
|
|
492
|
-
lb['name'] == load_balancer_name
|
|
493
|
-
end
|
|
494
|
-
load_balancer['public_net']['ipv4']['ip']
|
|
495
|
-
else
|
|
496
|
-
first_master_public_ip
|
|
497
|
-
end
|
|
498
|
-
end
|
|
499
|
-
|
|
500
|
-
def tls_sans
|
|
501
|
-
sans = " --tls-san=#{api_server_ip} "
|
|
502
|
-
|
|
503
|
-
masters.each do |master|
|
|
504
|
-
master_private_ip = master['private_net'][0]['ip']
|
|
505
|
-
sans += " --tls-san=#{master_private_ip} "
|
|
506
|
-
end
|
|
507
|
-
|
|
508
|
-
sans
|
|
509
|
-
end
|
|
510
|
-
|
|
511
|
-
def first_master_public_ip
|
|
512
|
-
@first_master_public_ip ||= first_master.dig('public_net', 'ipv4', 'ip')
|
|
513
|
-
end
|
|
514
|
-
|
|
515
|
-
def save_kubeconfig
|
|
516
|
-
kubeconfig = ssh(first_master, 'cat /etc/rancher/k3s/k3s.yaml')
|
|
517
|
-
.gsub('127.0.0.1', api_server_ip)
|
|
518
|
-
.gsub('default', cluster_name)
|
|
519
|
-
|
|
520
|
-
File.write(kubeconfig_path, kubeconfig)
|
|
521
|
-
|
|
522
|
-
FileUtils.chmod 'go-r', kubeconfig_path
|
|
523
|
-
end
|
|
524
|
-
|
|
525
|
-
def belongs_to_cluster?(server)
|
|
526
|
-
server.dig('labels', 'cluster') == cluster_name
|
|
527
|
-
end
|
|
528
|
-
|
|
529
|
-
def schedule_workloads_on_masters?
|
|
530
|
-
schedule_workloads_on_masters = configuration['schedule_workloads_on_masters']
|
|
531
|
-
schedule_workloads_on_masters ? !!schedule_workloads_on_masters : false
|
|
532
|
-
end
|
|
533
|
-
|
|
534
96
|
def image
|
|
535
97
|
configuration['image'] || 'ubuntu-20.04'
|
|
536
98
|
end
|
|
@@ -543,34 +105,19 @@ class Cluster
|
|
|
543
105
|
configuration['post_create_commands'] || []
|
|
544
106
|
end
|
|
545
107
|
|
|
546
|
-
def check_kubectl
|
|
547
|
-
return if which('kubectl')
|
|
548
|
-
|
|
549
|
-
puts 'Please ensure kubectl is installed and in your PATH.'
|
|
550
|
-
exit 1
|
|
551
|
-
end
|
|
552
|
-
|
|
553
|
-
def placement_group_id(pool_name = nil)
|
|
554
|
-
@placement_groups ||= {}
|
|
555
|
-
@placement_groups[pool_name || '__masters__'] ||= Hetzner::PlacementGroup.new(hetzner_client: hetzner_client, cluster_name: cluster_name, pool_name: pool_name).create
|
|
556
|
-
end
|
|
557
|
-
|
|
558
108
|
def master_instance_type
|
|
559
109
|
@master_instance_type ||= masters_config['instance_type']
|
|
560
110
|
end
|
|
561
111
|
|
|
562
|
-
def master_labels
|
|
563
|
-
@master_labels ||= masters_config['labels']
|
|
564
|
-
end
|
|
565
|
-
|
|
566
|
-
def master_taints
|
|
567
|
-
@master_taints ||= masters_config['taints']
|
|
568
|
-
end
|
|
569
|
-
|
|
570
112
|
def masters_count
|
|
571
113
|
@masters_count ||= masters_config['instance_count']
|
|
572
114
|
end
|
|
573
115
|
|
|
116
|
+
def placement_group_id(pool_name = nil)
|
|
117
|
+
@placement_groups ||= {}
|
|
118
|
+
@placement_groups[pool_name || '__masters__'] ||= Hetzner::PlacementGroup.new(hetzner_client: hetzner_client, cluster_name: cluster_name, pool_name: pool_name).create
|
|
119
|
+
end
|
|
120
|
+
|
|
574
121
|
def firewall_id
|
|
575
122
|
@firewall_id ||= Hetzner::Firewall.new(hetzner_client: hetzner_client, cluster_name: cluster_name).create(high_availability: (masters_count > 1), ssh_networks: ssh_networks, api_networks: api_networks)
|
|
576
123
|
end
|
|
@@ -598,8 +145,8 @@ class Cluster
|
|
|
598
145
|
image: image,
|
|
599
146
|
additional_packages: additional_packages,
|
|
600
147
|
additional_post_create_commands: additional_post_create_commands,
|
|
601
|
-
labels:
|
|
602
|
-
taints:
|
|
148
|
+
labels: masters_config['labels'],
|
|
149
|
+
taints: masters_config['taints']
|
|
603
150
|
}
|
|
604
151
|
end
|
|
605
152
|
|
|
@@ -649,10 +196,6 @@ class Cluster
|
|
|
649
196
|
definitions
|
|
650
197
|
end
|
|
651
198
|
|
|
652
|
-
def create_load_balancer
|
|
653
|
-
Hetzner::LoadBalancer.new(hetzner_client: hetzner_client, cluster_name: cluster_name).create(location: masters_location, network_id: network_id)
|
|
654
|
-
end
|
|
655
|
-
|
|
656
199
|
def server_configs
|
|
657
200
|
return @server_configs if @server_configs
|
|
658
201
|
|
|
@@ -665,6 +208,47 @@ class Cluster
|
|
|
665
208
|
@server_configs
|
|
666
209
|
end
|
|
667
210
|
|
|
211
|
+
def hetzner_client
|
|
212
|
+
configuration.hetzner_client
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
def kubernetes_client
|
|
216
|
+
@kubernetes_client ||= Kubernetes::Client.new(configuration: configuration)
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
def workers_definitions_for_marking
|
|
220
|
+
worker_node_pools.map do |worker_node_pool|
|
|
221
|
+
worker_node_pool_definitions(worker_node_pool)
|
|
222
|
+
end.flatten
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
def create_resources
|
|
226
|
+
create_servers
|
|
227
|
+
create_load_balancer if masters.size > 1
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
def delete_placement_groups
|
|
231
|
+
Hetzner::PlacementGroup.new(hetzner_client: hetzner_client, cluster_name: cluster_name).delete
|
|
232
|
+
|
|
233
|
+
worker_node_pools.each do |pool|
|
|
234
|
+
pool_name = pool['name']
|
|
235
|
+
Hetzner::PlacementGroup.new(hetzner_client: hetzner_client, cluster_name: cluster_name, pool_name: pool_name).delete
|
|
236
|
+
end
|
|
237
|
+
end
|
|
238
|
+
|
|
239
|
+
def delete_resources
|
|
240
|
+
Hetzner::LoadBalancer.new(hetzner_client: hetzner_client, cluster_name: cluster_name).delete(high_availability: (masters.size > 1))
|
|
241
|
+
|
|
242
|
+
Hetzner::Firewall.new(hetzner_client: hetzner_client, cluster_name: cluster_name).delete(all_servers)
|
|
243
|
+
|
|
244
|
+
Hetzner::Network.new(hetzner_client: hetzner_client, cluster_name: cluster_name, existing_network: existing_network).delete
|
|
245
|
+
|
|
246
|
+
Hetzner::SSHKey.new(hetzner_client: hetzner_client, cluster_name: cluster_name).delete(public_ssh_key_path: public_ssh_key_path)
|
|
247
|
+
|
|
248
|
+
delete_placement_groups
|
|
249
|
+
delete_servers
|
|
250
|
+
end
|
|
251
|
+
|
|
668
252
|
def create_servers
|
|
669
253
|
servers = []
|
|
670
254
|
|
|
@@ -701,56 +285,8 @@ class Cluster
|
|
|
701
285
|
threads.each(&:join) unless threads.empty?
|
|
702
286
|
end
|
|
703
287
|
|
|
704
|
-
def
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
kube_api_server_args.map do |arg|
|
|
708
|
-
" --kube-apiserver-arg=\"#{arg}\" "
|
|
709
|
-
end.join
|
|
710
|
-
end
|
|
711
|
-
|
|
712
|
-
def kube_scheduler_args_list
|
|
713
|
-
return '' if kube_scheduler_args.empty?
|
|
714
|
-
|
|
715
|
-
kube_scheduler_args.map do |arg|
|
|
716
|
-
" --kube-scheduler-arg=\"#{arg}\" "
|
|
717
|
-
end.join
|
|
718
|
-
end
|
|
719
|
-
|
|
720
|
-
def kube_controller_manager_args_list
|
|
721
|
-
return '' if kube_controller_manager_args.empty?
|
|
722
|
-
|
|
723
|
-
kube_controller_manager_args.map do |arg|
|
|
724
|
-
" --kube-controller-manager-arg=\"#{arg}\" "
|
|
725
|
-
end.join
|
|
726
|
-
end
|
|
727
|
-
|
|
728
|
-
def kube_cloud_controller_manager_args_list
|
|
729
|
-
return '' if kube_cloud_controller_manager_args.empty?
|
|
730
|
-
|
|
731
|
-
kube_cloud_controller_manager_args.map do |arg|
|
|
732
|
-
" --kube-cloud-controller-manager-arg=\"#{arg}\" "
|
|
733
|
-
end.join
|
|
734
|
-
end
|
|
735
|
-
|
|
736
|
-
def kubelet_args_list
|
|
737
|
-
return '' if kubelet_args.empty?
|
|
738
|
-
|
|
739
|
-
kubelet_args.map do |arg|
|
|
740
|
-
" --kubelet-arg=\"#{arg}\" "
|
|
741
|
-
end.join
|
|
742
|
-
end
|
|
743
|
-
|
|
744
|
-
def kube_proxy_args_list
|
|
745
|
-
return '' if kube_proxy_args.empty?
|
|
746
|
-
|
|
747
|
-
kube_api_server_args.map do |arg|
|
|
748
|
-
" --kube-proxy-arg=\"#{arg}\" "
|
|
749
|
-
end.join
|
|
750
|
-
end
|
|
751
|
-
|
|
752
|
-
def hetzner_client
|
|
753
|
-
configuration.hetzner_client
|
|
288
|
+
def create_load_balancer
|
|
289
|
+
Hetzner::LoadBalancer.new(hetzner_client: hetzner_client, cluster_name: cluster_name).create(location: masters_location, network_id: network_id)
|
|
754
290
|
end
|
|
755
291
|
|
|
756
292
|
def existing_network
|
data/lib/hetzner/k3s/version.rb
CHANGED
|
@@ -0,0 +1,475 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative '../utils'
|
|
4
|
+
|
|
5
|
+
module Kubernetes
|
|
6
|
+
class Client
|
|
7
|
+
include Utils
|
|
8
|
+
|
|
9
|
+
def initialize(configuration:)
|
|
10
|
+
@configuration = configuration
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def deploy(masters:, workers:, master_definitions:, worker_definitions:)
|
|
14
|
+
@masters = masters
|
|
15
|
+
@workers = workers
|
|
16
|
+
@master_definitions = master_definitions
|
|
17
|
+
@worker_definitions = worker_definitions
|
|
18
|
+
|
|
19
|
+
@kube_api_server_args = configuration.fetch('kube_api_server_args', [])
|
|
20
|
+
@kube_scheduler_args = configuration.fetch('kube_scheduler_args', [])
|
|
21
|
+
@kube_controller_manager_args = configuration.fetch('kube_controller_manager_args', [])
|
|
22
|
+
@kube_cloud_controller_manager_args = configuration.fetch('kube_cloud_controller_manager_args', [])
|
|
23
|
+
@kubelet_args = configuration.fetch('kubelet_args', [])
|
|
24
|
+
@kube_proxy_args = configuration.fetch('kube_proxy_args', [])
|
|
25
|
+
@private_ssh_key_path = File.expand_path(configuration['private_ssh_key_path'])
|
|
26
|
+
@public_ssh_key_path = File.expand_path(configuration['public_ssh_key_path'])
|
|
27
|
+
@cluster_name = configuration['cluster_name']
|
|
28
|
+
|
|
29
|
+
set_up_k3s
|
|
30
|
+
|
|
31
|
+
update_nodes
|
|
32
|
+
|
|
33
|
+
post_setup_deployments
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def upgrade
|
|
37
|
+
worker_upgrade_concurrency = workers.size - 1
|
|
38
|
+
worker_upgrade_concurrency = 1 if worker_upgrade_concurrency.zero?
|
|
39
|
+
|
|
40
|
+
cmd = <<~BASH
|
|
41
|
+
kubectl apply -f - <<-EOF
|
|
42
|
+
apiVersion: upgrade.cattle.io/v1
|
|
43
|
+
kind: Plan
|
|
44
|
+
metadata:
|
|
45
|
+
name: k3s-server
|
|
46
|
+
namespace: system-upgrade
|
|
47
|
+
labels:
|
|
48
|
+
k3s-upgrade: server
|
|
49
|
+
spec:
|
|
50
|
+
concurrency: 1
|
|
51
|
+
version: #{new_k3s_version}
|
|
52
|
+
nodeSelector:
|
|
53
|
+
matchExpressions:
|
|
54
|
+
- {key: node-role.kubernetes.io/master, operator: In, values: ["true"]}
|
|
55
|
+
serviceAccountName: system-upgrade
|
|
56
|
+
tolerations:
|
|
57
|
+
- key: "CriticalAddonsOnly"
|
|
58
|
+
operator: "Equal"
|
|
59
|
+
value: "true"
|
|
60
|
+
effect: "NoExecute"
|
|
61
|
+
cordon: true
|
|
62
|
+
upgrade:
|
|
63
|
+
image: rancher/k3s-upgrade
|
|
64
|
+
EOF
|
|
65
|
+
BASH
|
|
66
|
+
|
|
67
|
+
run cmd, kubeconfig_path: kubeconfig_path
|
|
68
|
+
|
|
69
|
+
cmd = <<~BASH
|
|
70
|
+
kubectl apply -f - <<-EOF
|
|
71
|
+
apiVersion: upgrade.cattle.io/v1
|
|
72
|
+
kind: Plan
|
|
73
|
+
metadata:
|
|
74
|
+
name: k3s-agent
|
|
75
|
+
namespace: system-upgrade
|
|
76
|
+
labels:
|
|
77
|
+
k3s-upgrade: agent
|
|
78
|
+
spec:
|
|
79
|
+
concurrency: #{worker_upgrade_concurrency}
|
|
80
|
+
version: #{new_k3s_version}
|
|
81
|
+
nodeSelector:
|
|
82
|
+
matchExpressions:
|
|
83
|
+
- {key: node-role.kubernetes.io/master, operator: NotIn, values: ["true"]}
|
|
84
|
+
serviceAccountName: system-upgrade
|
|
85
|
+
prepare:
|
|
86
|
+
image: rancher/k3s-upgrade
|
|
87
|
+
args: ["prepare", "k3s-server"]
|
|
88
|
+
cordon: true
|
|
89
|
+
upgrade:
|
|
90
|
+
image: rancher/k3s-upgrade
|
|
91
|
+
EOF
|
|
92
|
+
BASH
|
|
93
|
+
|
|
94
|
+
run cmd, kubeconfig_path: kubeconfig_path
|
|
95
|
+
|
|
96
|
+
puts 'Upgrade will now start. Run `watch kubectl get nodes` to see the nodes being upgraded. This should take a few minutes for a small cluster.'
|
|
97
|
+
puts 'The API server may be briefly unavailable during the upgrade of the controlplane.'
|
|
98
|
+
|
|
99
|
+
updated_configuration = configuration.raw
|
|
100
|
+
updated_configuration['k3s_version'] = new_k3s_version
|
|
101
|
+
|
|
102
|
+
File.write(config_file, updated_configuration.to_yaml)
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
private
|
|
106
|
+
|
|
107
|
+
attr_reader :configuration, :masters, :workers, :kube_api_server_args, :kube_scheduler_args,
|
|
108
|
+
:kube_controller_manager_args, :kube_cloud_controller_manager_args, :kubelet_args, :kube_proxy_args,
|
|
109
|
+
:private_ssh_key_path, :public_ssh_key_path, :master_definitions, :worker_definitions, :cluster_name
|
|
110
|
+
|
|
111
|
+
def set_up_k3s
|
|
112
|
+
set_up_first_master
|
|
113
|
+
set_up_additional_masters
|
|
114
|
+
set_up_workers
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
def set_up_first_master
|
|
118
|
+
puts
|
|
119
|
+
puts "Deploying k3s to first master (#{first_master['name']})..."
|
|
120
|
+
|
|
121
|
+
ssh first_master, master_install_script(first_master), print_output: true
|
|
122
|
+
|
|
123
|
+
puts
|
|
124
|
+
puts 'Waiting for the control plane to be ready...'
|
|
125
|
+
|
|
126
|
+
sleep 10
|
|
127
|
+
|
|
128
|
+
puts
|
|
129
|
+
puts '...k3s has been deployed to first master.'
|
|
130
|
+
|
|
131
|
+
save_kubeconfig
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
def set_up_additional_masters
|
|
135
|
+
return unless masters.size > 1
|
|
136
|
+
|
|
137
|
+
threads = masters[1..].map do |master|
|
|
138
|
+
Thread.new do
|
|
139
|
+
puts
|
|
140
|
+
puts "Deploying k3s to master #{master['name']}..."
|
|
141
|
+
|
|
142
|
+
ssh master, master_install_script(master), print_output: true
|
|
143
|
+
|
|
144
|
+
puts
|
|
145
|
+
puts "...k3s has been deployed to master #{master['name']}."
|
|
146
|
+
end
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
threads.each(&:join) unless threads.empty?
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
def set_up_workers
|
|
153
|
+
threads = workers.map do |worker|
|
|
154
|
+
Thread.new do
|
|
155
|
+
puts
|
|
156
|
+
puts "Deploying k3s to worker (#{worker['name']})..."
|
|
157
|
+
|
|
158
|
+
ssh worker, worker_install_script(worker), print_output: true
|
|
159
|
+
|
|
160
|
+
puts
|
|
161
|
+
puts "...k3s has been deployed to worker (#{worker['name']})."
|
|
162
|
+
end
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
threads.each(&:join) unless threads.empty?
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
def post_setup_deployments
|
|
169
|
+
deploy_cloud_controller_manager
|
|
170
|
+
deploy_csi_driver
|
|
171
|
+
deploy_system_upgrade_controller
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
def update_nodes
|
|
175
|
+
mark_nodes mark_type: :labels
|
|
176
|
+
mark_nodes mark_type: :taints
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
def first_master
|
|
180
|
+
masters.first
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
def kube_api_server_args_list
|
|
184
|
+
return '' if kube_api_server_args.empty?
|
|
185
|
+
|
|
186
|
+
kube_api_server_args.map do |arg|
|
|
187
|
+
" --kube-apiserver-arg=\"#{arg}\" "
|
|
188
|
+
end.join
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
def kube_scheduler_args_list
|
|
192
|
+
return '' if kube_scheduler_args.empty?
|
|
193
|
+
|
|
194
|
+
kube_scheduler_args.map do |arg|
|
|
195
|
+
" --kube-scheduler-arg=\"#{arg}\" "
|
|
196
|
+
end.join
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
def kube_controller_manager_args_list
|
|
200
|
+
return '' if kube_controller_manager_args.empty?
|
|
201
|
+
|
|
202
|
+
kube_controller_manager_args.map do |arg|
|
|
203
|
+
" --kube-controller-manager-arg=\"#{arg}\" "
|
|
204
|
+
end.join
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
def kube_cloud_controller_manager_args_list
|
|
208
|
+
return '' if kube_cloud_controller_manager_args.empty?
|
|
209
|
+
|
|
210
|
+
kube_cloud_controller_manager_args.map do |arg|
|
|
211
|
+
" --kube-cloud-controller-manager-arg=\"#{arg}\" "
|
|
212
|
+
end.join
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
def kubelet_args_list
|
|
216
|
+
return '' if kubelet_args.empty?
|
|
217
|
+
|
|
218
|
+
kubelet_args.map do |arg|
|
|
219
|
+
" --kubelet-arg=\"#{arg}\" "
|
|
220
|
+
end.join
|
|
221
|
+
end
|
|
222
|
+
|
|
223
|
+
def kube_proxy_args_list
|
|
224
|
+
return '' if kube_proxy_args.empty?
|
|
225
|
+
|
|
226
|
+
kube_api_server_args.map do |arg|
|
|
227
|
+
" --kube-proxy-arg=\"#{arg}\" "
|
|
228
|
+
end.join
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
def api_server_ip
|
|
232
|
+
return @api_server_ip if @api_server_ip
|
|
233
|
+
|
|
234
|
+
@api_server_ip = if masters.size > 1
|
|
235
|
+
load_balancer_name = "#{cluster_name}-api"
|
|
236
|
+
load_balancer = hetzner_client.get('/load_balancers')['load_balancers'].detect do |lb|
|
|
237
|
+
lb['name'] == load_balancer_name
|
|
238
|
+
end
|
|
239
|
+
load_balancer['public_net']['ipv4']['ip']
|
|
240
|
+
else
|
|
241
|
+
first_master_public_ip
|
|
242
|
+
end
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
def master_install_script(master)
|
|
246
|
+
server = master == first_master ? ' --cluster-init ' : " --server https://#{api_server_ip}:6443 "
|
|
247
|
+
flannel_interface = find_flannel_interface(master)
|
|
248
|
+
enable_encryption = configuration.fetch('enable_encryption', false)
|
|
249
|
+
flannel_wireguard = if enable_encryption
|
|
250
|
+
if Gem::Version.new(k3s_version.scan(/\Av(.*)\+.*\Z/).flatten.first) >= Gem::Version.new('1.23.6')
|
|
251
|
+
' --flannel-backend=wireguard-native '
|
|
252
|
+
else
|
|
253
|
+
' --flannel-backend=wireguard '
|
|
254
|
+
end
|
|
255
|
+
else
|
|
256
|
+
' '
|
|
257
|
+
end
|
|
258
|
+
|
|
259
|
+
extra_args = "#{kube_api_server_args_list} #{kube_scheduler_args_list} #{kube_controller_manager_args_list} #{kube_cloud_controller_manager_args_list} #{kubelet_args_list} #{kube_proxy_args_list}"
|
|
260
|
+
taint = schedule_workloads_on_masters? ? ' ' : ' --node-taint CriticalAddonsOnly=true:NoExecute '
|
|
261
|
+
|
|
262
|
+
<<~SCRIPT
|
|
263
|
+
curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION="#{k3s_version}" K3S_TOKEN="#{k3s_token}" INSTALL_K3S_EXEC="server \
|
|
264
|
+
--disable-cloud-controller \
|
|
265
|
+
--disable servicelb \
|
|
266
|
+
--disable traefik \
|
|
267
|
+
--disable local-storage \
|
|
268
|
+
--disable metrics-server \
|
|
269
|
+
--write-kubeconfig-mode=644 \
|
|
270
|
+
--node-name="$(hostname -f)" \
|
|
271
|
+
--cluster-cidr=10.244.0.0/16 \
|
|
272
|
+
--etcd-expose-metrics=true \
|
|
273
|
+
#{flannel_wireguard} \
|
|
274
|
+
--kube-controller-manager-arg="bind-address=0.0.0.0" \
|
|
275
|
+
--kube-proxy-arg="metrics-bind-address=0.0.0.0" \
|
|
276
|
+
--kube-scheduler-arg="bind-address=0.0.0.0" \
|
|
277
|
+
#{taint} #{extra_args} \
|
|
278
|
+
--kubelet-arg="cloud-provider=external" \
|
|
279
|
+
--advertise-address=$(hostname -I | awk '{print $2}') \
|
|
280
|
+
--node-ip=$(hostname -I | awk '{print $2}') \
|
|
281
|
+
--node-external-ip=$(hostname -I | awk '{print $1}') \
|
|
282
|
+
--flannel-iface=#{flannel_interface} \
|
|
283
|
+
#{server} #{tls_sans}" sh -
|
|
284
|
+
SCRIPT
|
|
285
|
+
end
|
|
286
|
+
|
|
287
|
+
def worker_install_script(worker)
|
|
288
|
+
flannel_interface = find_flannel_interface(worker)
|
|
289
|
+
|
|
290
|
+
<<~BASH
|
|
291
|
+
curl -sfL https://get.k3s.io | K3S_TOKEN="#{k3s_token}" INSTALL_K3S_VERSION="#{k3s_version}" K3S_URL=https://#{first_master_private_ip}:6443 INSTALL_K3S_EXEC="agent \
|
|
292
|
+
--node-name="$(hostname -f)" \
|
|
293
|
+
--kubelet-arg="cloud-provider=external" \
|
|
294
|
+
--node-ip=$(hostname -I | awk '{print $2}') \
|
|
295
|
+
--node-external-ip=$(hostname -I | awk '{print $1}') \
|
|
296
|
+
--flannel-iface=#{flannel_interface}" sh -
|
|
297
|
+
BASH
|
|
298
|
+
end
|
|
299
|
+
|
|
300
|
+
def find_flannel_interface(server)
|
|
301
|
+
if ssh(server, 'lscpu | grep Vendor') =~ /Intel/
|
|
302
|
+
'ens10'
|
|
303
|
+
else
|
|
304
|
+
'enp7s0'
|
|
305
|
+
end
|
|
306
|
+
end
|
|
307
|
+
|
|
308
|
+
def hetzner_client
|
|
309
|
+
configuration.hetzner_client
|
|
310
|
+
end
|
|
311
|
+
|
|
312
|
+
def first_master_public_ip
|
|
313
|
+
@first_master_public_ip ||= first_master.dig('public_net', 'ipv4', 'ip')
|
|
314
|
+
end
|
|
315
|
+
|
|
316
|
+
def save_kubeconfig
|
|
317
|
+
kubeconfig = ssh(first_master, 'cat /etc/rancher/k3s/k3s.yaml')
|
|
318
|
+
.gsub('127.0.0.1', api_server_ip)
|
|
319
|
+
.gsub('default', configuration['cluster_name'])
|
|
320
|
+
|
|
321
|
+
File.write(kubeconfig_path, kubeconfig)
|
|
322
|
+
|
|
323
|
+
FileUtils.chmod 'go-r', kubeconfig_path
|
|
324
|
+
end
|
|
325
|
+
|
|
326
|
+
def kubeconfig_path
|
|
327
|
+
@kubeconfig_path ||= File.expand_path(configuration['kubeconfig_path'])
|
|
328
|
+
end
|
|
329
|
+
|
|
330
|
+
def schedule_workloads_on_masters?
|
|
331
|
+
schedule_workloads_on_masters = configuration['schedule_workloads_on_masters']
|
|
332
|
+
schedule_workloads_on_masters ? !!schedule_workloads_on_masters : false
|
|
333
|
+
end
|
|
334
|
+
|
|
335
|
+
def k3s_version
|
|
336
|
+
@k3s_version ||= configuration['k3s_version']
|
|
337
|
+
end
|
|
338
|
+
|
|
339
|
+
def k3s_token
|
|
340
|
+
@k3s_token ||= begin
|
|
341
|
+
token = ssh(first_master, '{ TOKEN=$(< /var/lib/rancher/k3s/server/node-token); } 2> /dev/null; echo $TOKEN')
|
|
342
|
+
|
|
343
|
+
if token.empty?
|
|
344
|
+
SecureRandom.hex
|
|
345
|
+
else
|
|
346
|
+
token.split(':').last
|
|
347
|
+
end
|
|
348
|
+
end
|
|
349
|
+
end
|
|
350
|
+
|
|
351
|
+
def tls_sans
|
|
352
|
+
sans = " --tls-san=#{api_server_ip} "
|
|
353
|
+
|
|
354
|
+
masters.each do |master|
|
|
355
|
+
master_private_ip = master['private_net'][0]['ip']
|
|
356
|
+
sans += " --tls-san=#{master_private_ip} "
|
|
357
|
+
end
|
|
358
|
+
|
|
359
|
+
sans
|
|
360
|
+
end
|
|
361
|
+
|
|
362
|
+
def mark_nodes(mark_type:)
|
|
363
|
+
check_kubectl
|
|
364
|
+
|
|
365
|
+
action = mark_type == :labels ? 'label' : 'taint'
|
|
366
|
+
|
|
367
|
+
if master_definitions.first[mark_type]
|
|
368
|
+
master_labels = master_definitions.first[mark_type].map { |k, v| "#{k}=#{v}" }.join(' ')
|
|
369
|
+
master_node_names = []
|
|
370
|
+
|
|
371
|
+
master_definitions.each do |master|
|
|
372
|
+
master_node_names << "#{configuration['cluster_name']}-#{master[:instance_type]}-#{master[:instance_id]}"
|
|
373
|
+
end
|
|
374
|
+
|
|
375
|
+
master_node_names = master_node_names.join(' ')
|
|
376
|
+
|
|
377
|
+
cmd = "kubectl #{action} --overwrite nodes #{master_node_names} #{master_labels}"
|
|
378
|
+
|
|
379
|
+
run cmd, kubeconfig_path: kubeconfig_path
|
|
380
|
+
end
|
|
381
|
+
|
|
382
|
+
return unless worker_definitions.any?
|
|
383
|
+
|
|
384
|
+
worker_definitions.each do |worker|
|
|
385
|
+
next unless worker[mark_type]
|
|
386
|
+
|
|
387
|
+
worker_labels = worker[mark_type].map { |k, v| "#{k}=#{v}" }.join(' ')
|
|
388
|
+
worker_node_name = "#{configuration['cluster_name']}-#{worker[:instance_type]}-#{worker[:instance_id]}"
|
|
389
|
+
|
|
390
|
+
cmd = "kubectl #{action} --overwrite nodes #{worker_node_name} #{worker_labels}"
|
|
391
|
+
|
|
392
|
+
run cmd, kubeconfig_path: kubeconfig_path
|
|
393
|
+
end
|
|
394
|
+
end
|
|
395
|
+
|
|
396
|
+
def deploy_cloud_controller_manager
|
|
397
|
+
check_kubectl
|
|
398
|
+
|
|
399
|
+
puts
|
|
400
|
+
puts 'Deploying Hetzner Cloud Controller Manager...'
|
|
401
|
+
|
|
402
|
+
cmd = <<~BASH
|
|
403
|
+
kubectl apply -f - <<-EOF
|
|
404
|
+
apiVersion: "v1"
|
|
405
|
+
kind: "Secret"
|
|
406
|
+
metadata:
|
|
407
|
+
namespace: 'kube-system'
|
|
408
|
+
name: 'hcloud'
|
|
409
|
+
stringData:
|
|
410
|
+
network: "#{configuration['existing_network'] || cluster_name}"
|
|
411
|
+
token: "#{configuration.hetzner_token}"
|
|
412
|
+
EOF
|
|
413
|
+
BASH
|
|
414
|
+
|
|
415
|
+
run cmd, kubeconfig_path: kubeconfig_path
|
|
416
|
+
|
|
417
|
+
cmd = 'kubectl apply -f https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/latest/download/ccm-networks.yaml'
|
|
418
|
+
|
|
419
|
+
run cmd, kubeconfig_path: kubeconfig_path
|
|
420
|
+
|
|
421
|
+
puts '...Cloud Controller Manager deployed'
|
|
422
|
+
end
|
|
423
|
+
|
|
424
|
+
def deploy_system_upgrade_controller
|
|
425
|
+
check_kubectl
|
|
426
|
+
|
|
427
|
+
puts
|
|
428
|
+
puts 'Deploying k3s System Upgrade Controller...'
|
|
429
|
+
|
|
430
|
+
cmd = 'kubectl apply -f https://github.com/rancher/system-upgrade-controller/releases/download/v0.9.1/system-upgrade-controller.yaml'
|
|
431
|
+
|
|
432
|
+
run cmd, kubeconfig_path: kubeconfig_path
|
|
433
|
+
|
|
434
|
+
puts '...k3s System Upgrade Controller deployed'
|
|
435
|
+
end
|
|
436
|
+
|
|
437
|
+
def deploy_csi_driver
|
|
438
|
+
check_kubectl
|
|
439
|
+
|
|
440
|
+
puts
|
|
441
|
+
puts 'Deploying Hetzner CSI Driver...'
|
|
442
|
+
|
|
443
|
+
cmd = <<~BASH
|
|
444
|
+
kubectl apply -f - <<-EOF
|
|
445
|
+
apiVersion: "v1"
|
|
446
|
+
kind: "Secret"
|
|
447
|
+
metadata:
|
|
448
|
+
namespace: 'kube-system'
|
|
449
|
+
name: 'hcloud-csi'
|
|
450
|
+
stringData:
|
|
451
|
+
token: "#{configuration.hetzner_token}"
|
|
452
|
+
EOF
|
|
453
|
+
BASH
|
|
454
|
+
|
|
455
|
+
run cmd, kubeconfig_path: kubeconfig_path
|
|
456
|
+
|
|
457
|
+
cmd = 'kubectl apply -f https://raw.githubusercontent.com/hetznercloud/csi-driver/master/deploy/kubernetes/hcloud-csi.yml'
|
|
458
|
+
|
|
459
|
+
run cmd, kubeconfig_path: kubeconfig_path
|
|
460
|
+
|
|
461
|
+
puts '...CSI Driver deployed'
|
|
462
|
+
end
|
|
463
|
+
|
|
464
|
+
def check_kubectl
|
|
465
|
+
return if which('kubectl')
|
|
466
|
+
|
|
467
|
+
puts 'Please ensure kubectl is installed and in your PATH.'
|
|
468
|
+
exit 1
|
|
469
|
+
end
|
|
470
|
+
|
|
471
|
+
def first_master_private_ip
|
|
472
|
+
@first_master_private_ip ||= first_master['private_net'][0]['ip']
|
|
473
|
+
end
|
|
474
|
+
end
|
|
475
|
+
end
|
data/lib/hetzner/utils.rb
CHANGED
|
@@ -85,10 +85,6 @@ module Utils
|
|
|
85
85
|
end
|
|
86
86
|
end
|
|
87
87
|
output.chop
|
|
88
|
-
# rescue StandardError => e
|
|
89
|
-
# p [e.class, e.message]
|
|
90
|
-
# retries += 1
|
|
91
|
-
# retry unless retries > 15 || e.message =~ /Bad file descriptor/
|
|
92
88
|
rescue Timeout::Error, IOError, Errno::EBADF
|
|
93
89
|
retries += 1
|
|
94
90
|
retry unless retries > 15
|
|
@@ -109,4 +105,8 @@ module Utils
|
|
|
109
105
|
MESSAGE
|
|
110
106
|
exit 1
|
|
111
107
|
end
|
|
108
|
+
|
|
109
|
+
def verify_host_key
|
|
110
|
+
@verify_host_key ||= configuration.fetch('verify_host_key', false)
|
|
111
|
+
end
|
|
112
112
|
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: hetzner-k3s
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.6.
|
|
4
|
+
version: 0.6.3
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Vito Botta
|
|
@@ -155,6 +155,7 @@ files:
|
|
|
155
155
|
- lib/hetzner/k3s/cluster.rb
|
|
156
156
|
- lib/hetzner/k3s/configuration.rb
|
|
157
157
|
- lib/hetzner/k3s/version.rb
|
|
158
|
+
- lib/hetzner/kubernetes/client.rb
|
|
158
159
|
- lib/hetzner/utils.rb
|
|
159
160
|
- spec/k3s_spec.rb
|
|
160
161
|
- spec/spec_helper.rb
|