porkadot 0.1.0 → 0.18.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/porkadot/assets.rb +24 -0
- data/lib/porkadot/assets/bootstrap.rb +2 -2
- data/lib/porkadot/assets/etcd.rb +4 -1
- data/lib/porkadot/assets/etcd/etcd-server.yaml.erb +2 -0
- data/lib/porkadot/assets/kubelet.rb +3 -0
- data/lib/porkadot/assets/kubelet/config.yaml.erb +2 -0
- data/lib/porkadot/assets/kubelet/install-deps.sh.erb +21 -3
- data/lib/porkadot/assets/kubelet/install-pkgs.sh.erb +9 -1
- data/lib/porkadot/assets/kubernetes.rb +9 -0
- data/lib/porkadot/assets/kubernetes/manifests/coredns.yaml.erb +202 -0
- data/lib/porkadot/assets/kubernetes/manifests/dns-horizontal-autoscaler.yaml.erb +110 -0
- data/lib/porkadot/assets/kubernetes/manifests/flannel.yaml.erb +10 -10
- data/lib/porkadot/assets/kubernetes/manifests/kube-apiserver.secrets.yaml.erb +37 -0
- data/lib/porkadot/assets/kubernetes/manifests/kube-apiserver.yaml.erb +0 -36
- data/lib/porkadot/assets/kubernetes/manifests/kube-controller-manager.secrets.yaml.erb +16 -0
- data/lib/porkadot/assets/kubernetes/manifests/kube-controller-manager.yaml.erb +0 -15
- data/lib/porkadot/assets/kubernetes/manifests/kube-scheduler.yaml.erb +1 -1
- data/lib/porkadot/assets/kubernetes/manifests/kubelet-rubber-stamp.yaml.erb +11 -2
- data/lib/porkadot/assets/kubernetes/manifests/metallb.secrets.yaml.erb +13 -0
- data/lib/porkadot/assets/kubernetes/manifests/metallb.yaml.erb +116 -26
- data/lib/porkadot/assets/kubernetes/manifests/pod-checkpointer.yaml.erb +1 -1
- data/lib/porkadot/assets/kubernetes/manifests/porkadot.yaml.erb +23 -1
- data/lib/porkadot/assets/kubernetes/manifests/storage-version-migrator.yaml.erb +327 -0
- data/lib/porkadot/config.rb +8 -0
- data/lib/porkadot/configs/bootstrap.rb +13 -1
- data/lib/porkadot/configs/certs.rb +1 -1
- data/lib/porkadot/configs/certs/k8s.rb +6 -0
- data/lib/porkadot/configs/etcd.rb +7 -3
- data/lib/porkadot/configs/kubelet.rb +9 -1
- data/lib/porkadot/configs/kubernetes.rb +9 -0
- data/lib/porkadot/default.yaml +3 -2
- data/lib/porkadot/install/bootstrap.rb +7 -0
- data/lib/porkadot/install/kubelet.rb +4 -0
- data/lib/porkadot/install/kubernetes.rb +4 -0
- data/lib/porkadot/version.rb +1 -1
- metadata +8 -2
@@ -154,11 +154,11 @@ spec:
|
|
154
154
|
requiredDuringSchedulingIgnoredDuringExecution:
|
155
155
|
nodeSelectorTerms:
|
156
156
|
- matchExpressions:
|
157
|
-
- key:
|
157
|
+
- key: kubernetes.io/os
|
158
158
|
operator: In
|
159
159
|
values:
|
160
160
|
- linux
|
161
|
-
- key:
|
161
|
+
- key: kubernetes.io/arch
|
162
162
|
operator: In
|
163
163
|
values:
|
164
164
|
- amd64
|
@@ -248,11 +248,11 @@ spec:
|
|
248
248
|
requiredDuringSchedulingIgnoredDuringExecution:
|
249
249
|
nodeSelectorTerms:
|
250
250
|
- matchExpressions:
|
251
|
-
- key:
|
251
|
+
- key: kubernetes.io/os
|
252
252
|
operator: In
|
253
253
|
values:
|
254
254
|
- linux
|
255
|
-
- key:
|
255
|
+
- key: kubernetes.io/arch
|
256
256
|
operator: In
|
257
257
|
values:
|
258
258
|
- arm64
|
@@ -342,11 +342,11 @@ spec:
|
|
342
342
|
requiredDuringSchedulingIgnoredDuringExecution:
|
343
343
|
nodeSelectorTerms:
|
344
344
|
- matchExpressions:
|
345
|
-
- key:
|
345
|
+
- key: kubernetes.io/os
|
346
346
|
operator: In
|
347
347
|
values:
|
348
348
|
- linux
|
349
|
-
- key:
|
349
|
+
- key: kubernetes.io/arch
|
350
350
|
operator: In
|
351
351
|
values:
|
352
352
|
- arm
|
@@ -436,11 +436,11 @@ spec:
|
|
436
436
|
requiredDuringSchedulingIgnoredDuringExecution:
|
437
437
|
nodeSelectorTerms:
|
438
438
|
- matchExpressions:
|
439
|
-
- key:
|
439
|
+
- key: kubernetes.io/os
|
440
440
|
operator: In
|
441
441
|
values:
|
442
442
|
- linux
|
443
|
-
- key:
|
443
|
+
- key: kubernetes.io/arch
|
444
444
|
operator: In
|
445
445
|
values:
|
446
446
|
- ppc64le
|
@@ -530,11 +530,11 @@ spec:
|
|
530
530
|
requiredDuringSchedulingIgnoredDuringExecution:
|
531
531
|
nodeSelectorTerms:
|
532
532
|
- matchExpressions:
|
533
|
-
- key:
|
533
|
+
- key: kubernetes.io/os
|
534
534
|
operator: In
|
535
535
|
values:
|
536
536
|
- linux
|
537
|
-
- key:
|
537
|
+
- key: kubernetes.io/arch
|
538
538
|
operator: In
|
539
539
|
values:
|
540
540
|
- s390x
|
@@ -0,0 +1,37 @@
|
|
1
|
+
<% k8s = global_config.k8s -%>
|
2
|
+
---
|
3
|
+
apiVersion: v1
|
4
|
+
data:
|
5
|
+
apiserver.crt: <%= certs.kubernetes.to_base64(:apiserver_cert) %>
|
6
|
+
apiserver.key: <%= certs.kubernetes.to_base64(:apiserver_key) %>
|
7
|
+
ca.crt: <%= certs.kubernetes.to_base64(:ca_cert) %>
|
8
|
+
front-proxy-ca.crt: <%= certs.front_proxy.to_base64(:ca_cert) %>
|
9
|
+
front-proxy-client.crt: <%= certs.front_proxy.to_base64(:client_cert) %>
|
10
|
+
front-proxy-client.key: <%= certs.front_proxy.to_base64(:client_key) %>
|
11
|
+
kubelet-client.crt: <%= certs.kubernetes.to_base64(:kubelet_client_cert) %>
|
12
|
+
kubelet-client.key: <%= certs.kubernetes.to_base64(:kubelet_client_key) %>
|
13
|
+
sa.pub: <%= certs.kubernetes.to_base64(:sa_public_key) %>
|
14
|
+
kind: Secret
|
15
|
+
metadata:
|
16
|
+
name: kube-apiserver
|
17
|
+
namespace: kube-system
|
18
|
+
labels:
|
19
|
+
<%- k8s.apiserver.labels.each do |k, v| -%>
|
20
|
+
<%= k.to_s %>: <%= v %>
|
21
|
+
<%- end -%>
|
22
|
+
type: Opaque
|
23
|
+
---
|
24
|
+
apiVersion: v1
|
25
|
+
data:
|
26
|
+
ca.crt: <%= certs.etcd.to_base64(:ca_cert) %>
|
27
|
+
etcd-client.crt: <%= certs.etcd.to_base64(:client_cert) %>
|
28
|
+
etcd-client.key: <%= certs.etcd.to_base64(:client_key) %>
|
29
|
+
kind: Secret
|
30
|
+
metadata:
|
31
|
+
name: etcd-tls
|
32
|
+
namespace: kube-system
|
33
|
+
labels:
|
34
|
+
<%- k8s.apiserver.labels.each do |k, v| -%>
|
35
|
+
<%= k.to_s %>: <%= v %>
|
36
|
+
<%- end -%>
|
37
|
+
type: Opaque
|
@@ -1,41 +1,5 @@
|
|
1
1
|
<% k8s = global_config.k8s -%>
|
2
2
|
---
|
3
|
-
apiVersion: v1
|
4
|
-
data:
|
5
|
-
apiserver.crt: <%= certs.kubernetes.to_base64(:apiserver_cert) %>
|
6
|
-
apiserver.key: <%= certs.kubernetes.to_base64(:apiserver_key) %>
|
7
|
-
ca.crt: <%= certs.kubernetes.to_base64(:ca_cert) %>
|
8
|
-
front-proxy-ca.crt: <%= certs.front_proxy.to_base64(:ca_cert) %>
|
9
|
-
front-proxy-client.crt: <%= certs.front_proxy.to_base64(:client_cert) %>
|
10
|
-
front-proxy-client.key: <%= certs.front_proxy.to_base64(:client_key) %>
|
11
|
-
kubelet-client.crt: <%= certs.kubernetes.to_base64(:kubelet_client_cert) %>
|
12
|
-
kubelet-client.key: <%= certs.kubernetes.to_base64(:kubelet_client_key) %>
|
13
|
-
sa.pub: <%= certs.kubernetes.to_base64(:sa_public_key) %>
|
14
|
-
kind: Secret
|
15
|
-
metadata:
|
16
|
-
name: kube-apiserver
|
17
|
-
namespace: kube-system
|
18
|
-
labels:
|
19
|
-
<%- k8s.apiserver.labels.each do |k, v| -%>
|
20
|
-
<%= k.to_s %>: <%= v %>
|
21
|
-
<%- end -%>
|
22
|
-
type: Opaque
|
23
|
-
---
|
24
|
-
apiVersion: v1
|
25
|
-
data:
|
26
|
-
ca.crt: <%= certs.etcd.to_base64(:ca_cert) %>
|
27
|
-
etcd-client.crt: <%= certs.etcd.to_base64(:client_cert) %>
|
28
|
-
etcd-client.key: <%= certs.etcd.to_base64(:client_key) %>
|
29
|
-
kind: Secret
|
30
|
-
metadata:
|
31
|
-
name: etcd-tls
|
32
|
-
namespace: kube-system
|
33
|
-
labels:
|
34
|
-
<%- k8s.apiserver.labels.each do |k, v| -%>
|
35
|
-
<%= k.to_s %>: <%= v %>
|
36
|
-
<%- end -%>
|
37
|
-
type: Opaque
|
38
|
-
---
|
39
3
|
apiVersion: "apps/v1"
|
40
4
|
kind: DaemonSet
|
41
5
|
metadata:
|
@@ -0,0 +1,16 @@
|
|
1
|
+
<% k8s = global_config.k8s -%>
|
2
|
+
---
|
3
|
+
apiVersion: v1
|
4
|
+
data:
|
5
|
+
ca.crt: <%= certs.kubernetes.to_base64(:ca_cert) %>
|
6
|
+
ca.key: <%= certs.kubernetes.to_base64(:ca_key) %>
|
7
|
+
sa.key: <%= certs.kubernetes.to_base64(:sa_private_key) %>
|
8
|
+
kind: Secret
|
9
|
+
metadata:
|
10
|
+
name: kube-controller-manager
|
11
|
+
namespace: kube-system
|
12
|
+
labels:
|
13
|
+
<%- k8s.controller_manager.labels.each do |k, v| -%>
|
14
|
+
<%= k.to_s %>: <%= v %>
|
15
|
+
<%- end -%>
|
16
|
+
type: Opaque
|
@@ -44,21 +44,6 @@ metadata:
|
|
44
44
|
<%= k.to_s %>: <%= v %>
|
45
45
|
<%- end -%>
|
46
46
|
---
|
47
|
-
apiVersion: v1
|
48
|
-
data:
|
49
|
-
ca.crt: <%= certs.kubernetes.to_base64(:ca_cert) %>
|
50
|
-
ca.key: <%= certs.kubernetes.to_base64(:ca_key) %>
|
51
|
-
sa.key: <%= certs.kubernetes.to_base64(:sa_private_key) %>
|
52
|
-
kind: Secret
|
53
|
-
metadata:
|
54
|
-
name: kube-controller-manager
|
55
|
-
namespace: kube-system
|
56
|
-
labels:
|
57
|
-
<%- k8s.controller_manager.labels.each do |k, v| -%>
|
58
|
-
<%= k.to_s %>: <%= v %>
|
59
|
-
<%- end -%>
|
60
|
-
type: Opaque
|
61
|
-
---
|
62
47
|
apiVersion: apps/v1
|
63
48
|
kind: Deployment
|
64
49
|
metadata:
|
@@ -62,7 +62,7 @@ rules:
|
|
62
62
|
verbs: ["get", "watch", "list"]
|
63
63
|
- apiGroups: [""] # "" indicates the core API group
|
64
64
|
resources: ["secrets", "configmaps"]
|
65
|
-
verbs: ["get"]
|
65
|
+
verbs: ["get", "watch", "list"]
|
66
66
|
---
|
67
67
|
apiVersion: rbac.authorization.k8s.io/v1
|
68
68
|
kind: RoleBinding
|
@@ -24,7 +24,7 @@ spec:
|
|
24
24
|
- name: kubelet-rubber-stamp
|
25
25
|
# image: quay.io/kontena/kubelet-rubber-stamp-amd64:0.2
|
26
26
|
# Use following image until issue is fixed
|
27
|
-
image: yuanying/kubelet-rubber-stamp:0.
|
27
|
+
image: yuanying/kubelet-rubber-stamp:0.3.0.y01
|
28
28
|
args:
|
29
29
|
- "--v=2"
|
30
30
|
imagePullPolicy: Always
|
@@ -56,12 +56,21 @@ kind: ClusterRole
|
|
56
56
|
metadata:
|
57
57
|
name: kubelet-rubber-stamp
|
58
58
|
rules:
|
59
|
+
- apiGroups:
|
60
|
+
- certificates.k8s.io
|
61
|
+
resources:
|
62
|
+
- signers
|
63
|
+
# legacy-unknown: support before kubernetes-1.18.0
|
64
|
+
resourceNames:
|
65
|
+
- "kubernetes.io/legacy-unknown"
|
66
|
+
- "kubernetes.io/kubelet-serving"
|
67
|
+
verbs:
|
68
|
+
- approve
|
59
69
|
- apiGroups:
|
60
70
|
- certificates.k8s.io
|
61
71
|
resources:
|
62
72
|
- certificatesigningrequests
|
63
73
|
verbs:
|
64
|
-
- delete
|
65
74
|
- get
|
66
75
|
- list
|
67
76
|
- watch
|
@@ -0,0 +1,13 @@
|
|
1
|
+
<% require 'securerandom' -%>
|
2
|
+
<% k8s = global_config.k8s -%>
|
3
|
+
---
|
4
|
+
apiVersion: v1
|
5
|
+
stringData:
|
6
|
+
secretkey: <%= SecureRandom.base64(128) %>
|
7
|
+
kind: Secret
|
8
|
+
metadata:
|
9
|
+
name: memberlist
|
10
|
+
namespace: metallb-system
|
11
|
+
labels:
|
12
|
+
app: metallb
|
13
|
+
type: Opaque
|
@@ -8,6 +8,48 @@ metadata:
|
|
8
8
|
---
|
9
9
|
apiVersion: policy/v1beta1
|
10
10
|
kind: PodSecurityPolicy
|
11
|
+
metadata:
|
12
|
+
labels:
|
13
|
+
app: metallb
|
14
|
+
name: controller
|
15
|
+
namespace: metallb-system
|
16
|
+
spec:
|
17
|
+
allowPrivilegeEscalation: false
|
18
|
+
allowedCapabilities: []
|
19
|
+
allowedHostPaths: []
|
20
|
+
defaultAddCapabilities: []
|
21
|
+
defaultAllowPrivilegeEscalation: false
|
22
|
+
fsGroup:
|
23
|
+
ranges:
|
24
|
+
- max: 65535
|
25
|
+
min: 1
|
26
|
+
rule: MustRunAs
|
27
|
+
hostIPC: false
|
28
|
+
hostNetwork: false
|
29
|
+
hostPID: false
|
30
|
+
privileged: false
|
31
|
+
readOnlyRootFilesystem: true
|
32
|
+
requiredDropCapabilities:
|
33
|
+
- ALL
|
34
|
+
runAsUser:
|
35
|
+
ranges:
|
36
|
+
- max: 65535
|
37
|
+
min: 1
|
38
|
+
rule: MustRunAs
|
39
|
+
seLinux:
|
40
|
+
rule: RunAsAny
|
41
|
+
supplementalGroups:
|
42
|
+
ranges:
|
43
|
+
- max: 65535
|
44
|
+
min: 1
|
45
|
+
rule: MustRunAs
|
46
|
+
volumes:
|
47
|
+
- configMap
|
48
|
+
- secret
|
49
|
+
- emptyDir
|
50
|
+
---
|
51
|
+
apiVersion: policy/v1beta1
|
52
|
+
kind: PodSecurityPolicy
|
11
53
|
metadata:
|
12
54
|
labels:
|
13
55
|
app: metallb
|
@@ -19,13 +61,21 @@ spec:
|
|
19
61
|
- NET_ADMIN
|
20
62
|
- NET_RAW
|
21
63
|
- SYS_ADMIN
|
64
|
+
allowedHostPaths: []
|
65
|
+
defaultAddCapabilities: []
|
66
|
+
defaultAllowPrivilegeEscalation: false
|
22
67
|
fsGroup:
|
23
68
|
rule: RunAsAny
|
69
|
+
hostIPC: false
|
24
70
|
hostNetwork: true
|
71
|
+
hostPID: false
|
25
72
|
hostPorts:
|
26
73
|
- max: 7472
|
27
74
|
min: 7472
|
28
75
|
privileged: true
|
76
|
+
readOnlyRootFilesystem: true
|
77
|
+
requiredDropCapabilities:
|
78
|
+
- ALL
|
29
79
|
runAsUser:
|
30
80
|
rule: RunAsAny
|
31
81
|
seLinux:
|
@@ -33,7 +83,9 @@ spec:
|
|
33
83
|
supplementalGroups:
|
34
84
|
rule: RunAsAny
|
35
85
|
volumes:
|
36
|
-
-
|
86
|
+
- configMap
|
87
|
+
- secret
|
88
|
+
- emptyDir
|
37
89
|
---
|
38
90
|
apiVersion: v1
|
39
91
|
kind: ServiceAccount
|
@@ -80,6 +132,14 @@ rules:
|
|
80
132
|
verbs:
|
81
133
|
- create
|
82
134
|
- patch
|
135
|
+
- apiGroups:
|
136
|
+
- policy
|
137
|
+
resourceNames:
|
138
|
+
- controller
|
139
|
+
resources:
|
140
|
+
- podsecuritypolicies
|
141
|
+
verbs:
|
142
|
+
- use
|
83
143
|
---
|
84
144
|
apiVersion: rbac.authorization.k8s.io/v1
|
85
145
|
kind: ClusterRole
|
@@ -106,7 +166,7 @@ rules:
|
|
106
166
|
- create
|
107
167
|
- patch
|
108
168
|
- apiGroups:
|
109
|
-
-
|
169
|
+
- policy
|
110
170
|
resourceNames:
|
111
171
|
- speaker
|
112
172
|
resources:
|
@@ -132,6 +192,21 @@ rules:
|
|
132
192
|
- watch
|
133
193
|
---
|
134
194
|
apiVersion: rbac.authorization.k8s.io/v1
|
195
|
+
kind: Role
|
196
|
+
metadata:
|
197
|
+
labels:
|
198
|
+
app: metallb
|
199
|
+
name: pod-lister
|
200
|
+
namespace: metallb-system
|
201
|
+
rules:
|
202
|
+
- apiGroups:
|
203
|
+
- ''
|
204
|
+
resources:
|
205
|
+
- pods
|
206
|
+
verbs:
|
207
|
+
- list
|
208
|
+
---
|
209
|
+
apiVersion: rbac.authorization.k8s.io/v1
|
135
210
|
kind: ClusterRoleBinding
|
136
211
|
metadata:
|
137
212
|
labels:
|
@@ -178,6 +253,21 @@ subjects:
|
|
178
253
|
- kind: ServiceAccount
|
179
254
|
name: speaker
|
180
255
|
---
|
256
|
+
apiVersion: rbac.authorization.k8s.io/v1
|
257
|
+
kind: RoleBinding
|
258
|
+
metadata:
|
259
|
+
labels:
|
260
|
+
app: metallb
|
261
|
+
name: pod-lister
|
262
|
+
namespace: metallb-system
|
263
|
+
roleRef:
|
264
|
+
apiGroup: rbac.authorization.k8s.io
|
265
|
+
kind: Role
|
266
|
+
name: pod-lister
|
267
|
+
subjects:
|
268
|
+
- kind: ServiceAccount
|
269
|
+
name: speaker
|
270
|
+
---
|
181
271
|
apiVersion: apps/v1
|
182
272
|
kind: DaemonSet
|
183
273
|
metadata:
|
@@ -200,24 +290,6 @@ spec:
|
|
200
290
|
app: metallb
|
201
291
|
component: speaker
|
202
292
|
spec:
|
203
|
-
initContainers:
|
204
|
-
- command:
|
205
|
-
- "iptables"
|
206
|
-
- "-P"
|
207
|
-
- "FORWARD"
|
208
|
-
- "ACCEPT"
|
209
|
-
image: <%= k8s.image_repository %>/hyperkube:<%= k8s.kubernetes_version %>
|
210
|
-
imagePullPolicy: IfNotPresent
|
211
|
-
name: default-iptables
|
212
|
-
securityContext:
|
213
|
-
allowPrivilegeEscalation: false
|
214
|
-
capabilities:
|
215
|
-
add:
|
216
|
-
- NET_ADMIN
|
217
|
-
- NET_RAW
|
218
|
-
drop:
|
219
|
-
- ALL
|
220
|
-
readOnlyRootFilesystem: true
|
221
293
|
containers:
|
222
294
|
- args:
|
223
295
|
- --port=7472
|
@@ -231,8 +303,26 @@ spec:
|
|
231
303
|
valueFrom:
|
232
304
|
fieldRef:
|
233
305
|
fieldPath: status.hostIP
|
234
|
-
|
235
|
-
|
306
|
+
- name: METALLB_ML_BIND_ADDR
|
307
|
+
valueFrom:
|
308
|
+
fieldRef:
|
309
|
+
fieldPath: status.podIP
|
310
|
+
# needed when another software is also using memberlist / port 7946
|
311
|
+
#- name: METALLB_ML_BIND_PORT
|
312
|
+
# value: "7946"
|
313
|
+
- name: METALLB_ML_LABELS
|
314
|
+
value: "app=metallb,component=speaker"
|
315
|
+
- name: METALLB_ML_NAMESPACE
|
316
|
+
valueFrom:
|
317
|
+
fieldRef:
|
318
|
+
fieldPath: metadata.namespace
|
319
|
+
- name: METALLB_ML_SECRET_KEY
|
320
|
+
valueFrom:
|
321
|
+
secretKeyRef:
|
322
|
+
name: memberlist
|
323
|
+
key: secretkey
|
324
|
+
image: metallb/speaker:v0.9.4
|
325
|
+
imagePullPolicy: Always
|
236
326
|
name: speaker
|
237
327
|
ports:
|
238
328
|
- containerPort: 7472
|
@@ -255,7 +345,7 @@ spec:
|
|
255
345
|
nodeSelector:
|
256
346
|
beta.kubernetes.io/os: linux
|
257
347
|
serviceAccountName: speaker
|
258
|
-
terminationGracePeriodSeconds:
|
348
|
+
terminationGracePeriodSeconds: 2
|
259
349
|
tolerations:
|
260
350
|
- effect: NoSchedule
|
261
351
|
key: node-role.kubernetes.io/master
|
@@ -287,8 +377,8 @@ spec:
|
|
287
377
|
- args:
|
288
378
|
- --port=7472
|
289
379
|
- --config=config
|
290
|
-
image: metallb/controller:v0.
|
291
|
-
imagePullPolicy:
|
380
|
+
image: metallb/controller:v0.9.4
|
381
|
+
imagePullPolicy: Always
|
292
382
|
name: controller
|
293
383
|
ports:
|
294
384
|
- containerPort: 7472
|
@@ -304,7 +394,7 @@ spec:
|
|
304
394
|
- all
|
305
395
|
readOnlyRootFilesystem: true
|
306
396
|
nodeSelector:
|
307
|
-
|
397
|
+
kubernetes.io/os: linux
|
308
398
|
securityContext:
|
309
399
|
runAsNonRoot: true
|
310
400
|
runAsUser: 65534
|