@leverege/build-tools 2.43.11 → 2.44.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -3
- package/src/Utils.mjs +2 -2
- package/src/bash-funcs +6 -0
- package/src/helm-charts/cnpg-operator/.nohelm +0 -0
- package/src/helm-charts/cnpg-operator/.nohelmdn +0 -0
- package/src/helm-charts/cnpg-operator/helmdn.plugin +12 -0
- package/src/helm-charts/cnpg-operator/helmup.plugin +10 -0
- package/src/helm-charts/db-timescale-dense/.nohelm +0 -0
- package/src/helm-charts/db-timescale-dense/.nohelmdn +0 -0
- package/src/helm-charts/db-timescale-dense/db-timescale-dense.yaml.ovh +120 -0
- package/src/helm-charts/db-timescale-dense/gitignore +1 -0
- package/src/helm-charts/db-timescale-dense/helmdn.plugin +3 -0
- package/src/helm-charts/db-timescale-dense/helmup.plugin +5 -0
- package/src/helm-charts/transponder-dh/values-local.yaml +62 -0
- package/src/helmdn.sh +1 -1
- package/src/helmup.sh +224 -15
- package/src/overwhelm.mjs +1 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@leverege/build-tools",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.44.0",
|
|
4
4
|
"description": "A collection of build / support tools for Leverege developers",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"repository": {
|
|
@@ -67,11 +67,11 @@
|
|
|
67
67
|
"read-pkg": "^9.0.1",
|
|
68
68
|
"readline-sync": "^1.4.10",
|
|
69
69
|
"semver": "^7.5.4",
|
|
70
|
-
"simple-git": "^3.
|
|
70
|
+
"simple-git": "^3.22.0",
|
|
71
71
|
"zx": "^7.2.3"
|
|
72
72
|
},
|
|
73
73
|
"devDependencies": {
|
|
74
74
|
"@leverege/eslint-config-leverege": "^4.2.0",
|
|
75
|
-
"npm": "^10.
|
|
75
|
+
"npm": "^10.3.0"
|
|
76
76
|
}
|
|
77
77
|
}
|
package/src/Utils.mjs
CHANGED
|
@@ -126,8 +126,8 @@ export const parsePackageJson = async ( packageFileName ) => {
|
|
|
126
126
|
export const parseHelmChart = async ( helmroot ) => {
|
|
127
127
|
/* eslint-disable security/detect-non-literal-fs-filename */
|
|
128
128
|
const chartFiles = readdirSync( helmroot, { encoding : 'utf8', recursive : true } )
|
|
129
|
-
const chartYaml = YAML.load( readFileSync( `${helmroot}/Chart.yaml`, '
|
|
130
|
-
const valuesYaml = YAML.load( readFileSync( `${helmroot}/values.yaml`, '
|
|
129
|
+
const chartYaml = YAML.load( readFileSync( `${helmroot}/Chart.yaml`, 'utf8' ) )
|
|
130
|
+
const valuesYaml = YAML.load( readFileSync( `${helmroot}/values.yaml`, 'utf8' ) )
|
|
131
131
|
/* eslint-enable security/detect-non-literal-fs-filename */
|
|
132
132
|
return ( {
|
|
133
133
|
files : chartFiles,
|
package/src/bash-funcs
CHANGED
|
@@ -79,6 +79,12 @@ function setOrUnset() {
|
|
|
79
79
|
echo "$setting"
|
|
80
80
|
}
|
|
81
81
|
|
|
82
|
+
function generatePassword() {
|
|
83
|
+
local length=${1:-48}
|
|
84
|
+
local randompw=`openssl rand -base64 $length`
|
|
85
|
+
echo $randompw
|
|
86
|
+
}
|
|
87
|
+
|
|
82
88
|
function getManagedSecret() {
|
|
83
89
|
local SECNAME=$1
|
|
84
90
|
local PROJECT="${2:-leverege-registry}" # was leverege-docker-images
|
|
File without changes
|
|
File without changes
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
#
|
|
3
|
+
clearlyWarn DO_NOT_CLEAR
|
|
4
|
+
|
|
5
|
+
cat<<OPERATOR_SHUTDOWN
|
|
6
|
+
You are about to shutdown the `color r cnpg-operator` - are you certain that is what
|
|
7
|
+
you really intended to do?
|
|
8
|
+
OPERATOR_SHUTDOWN
|
|
9
|
+
|
|
10
|
+
yesToContinue "to delete the CNPG Operator"
|
|
11
|
+
|
|
12
|
+
helm uninstall --namespace cnpg-system cnpg # cnpg-cloudnative-pg
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
#
|
|
3
|
+
# See => https://github.com/cloudnative-pg/charts
|
|
4
|
+
addHelmRepo cnpg https://cloudnative-pg.github.io/charts
|
|
5
|
+
|
|
6
|
+
helm upgrade --install cnpg cnpg/cloudnative-pg \
|
|
7
|
+
--namespace cnpg-system --create-namespace \
|
|
8
|
+
--set webhook.port="10250"
|
|
9
|
+
|
|
10
|
+
removeHelmRepo cnpg
|
|
File without changes
|
|
File without changes
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
# Artifact Chart => https://artifacthub.io/packages/helm/cloudnative-pg/cloudnative-pg
|
|
2
|
+
# Examples Charts => https://cloudnative-pg.io/documentation/current/samples/
|
|
3
|
+
apiVersion: postgresql.cnpg.io/v1
|
|
4
|
+
kind: Cluster
|
|
5
|
+
metadata:
|
|
6
|
+
name: db-timescale-dense
|
|
7
|
+
namespace: cnpg-operands
|
|
8
|
+
spec:
|
|
9
|
+
instances: 1
|
|
10
|
+
logLevel: debug
|
|
11
|
+
|
|
12
|
+
description: "Leverege Dense Timescale DB"
|
|
13
|
+
imageName: us-docker.pkg.dev/leverege-registry/system/images/leverege-pgsql:16.1-cnpg-tsdb2-beta.6
|
|
14
|
+
bootstrap:
|
|
15
|
+
initdb:
|
|
16
|
+
postInitTemplateSQL:
|
|
17
|
+
- CREATE EXTENSION timescaledb;
|
|
18
|
+
- CREATE EXTENSION jsquery;
|
|
19
|
+
database: imagine
|
|
20
|
+
owner: imagine
|
|
21
|
+
secret:
|
|
22
|
+
name: db-timescale-dense-imagine-pw # imagine db pw
|
|
23
|
+
|
|
24
|
+
enableSuperuserAccess: true
|
|
25
|
+
superuserSecret:
|
|
26
|
+
name: db-timescale-dense-postgres-pw # postgres db pw
|
|
27
|
+
|
|
28
|
+
affinity:
|
|
29
|
+
# enablePodAntiAffinity: true
|
|
30
|
+
# topologyKey: failure-domain.beta.kubernetes.io/zone
|
|
31
|
+
tolerations:
|
|
32
|
+
- key: "database"
|
|
33
|
+
operator: "Equal"
|
|
34
|
+
value: "true"
|
|
35
|
+
effect: "NoSchedule"
|
|
36
|
+
|
|
37
|
+
# Parameters and pg_hba configuration will be append
|
|
38
|
+
# to the default ones to make the cluster work
|
|
39
|
+
postgresql:
|
|
40
|
+
shared_preload_libraries:
|
|
41
|
+
- timescaledb
|
|
42
|
+
- jsquery
|
|
43
|
+
parameters:
|
|
44
|
+
# max_worker_processes: "60"
|
|
45
|
+
max_worker_processes: "8"
|
|
46
|
+
max_parallel_workers: "8"
|
|
47
|
+
shared_buffers: 2GB
|
|
48
|
+
effective_cache_size: 6GB
|
|
49
|
+
maintenance_work_mem: 1GB
|
|
50
|
+
work_mem: 20MB
|
|
51
|
+
pg_hba:
|
|
52
|
+
# To access through TCP/IP you will need to get username
|
|
53
|
+
# and password from the secret cluster-example-custom-app
|
|
54
|
+
- host all all all md5
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
# Example of rolling update strategy:
|
|
58
|
+
# - unsupervised: automated update of the primary once all
|
|
59
|
+
# replicas have been upgraded (default)
|
|
60
|
+
# - supervised: requires manual supervision to perform
|
|
61
|
+
# the switchover of the primary
|
|
62
|
+
primaryUpdateStrategy: unsupervised
|
|
63
|
+
|
|
64
|
+
serviceAccountTemplate:
|
|
65
|
+
metadata:
|
|
66
|
+
annotations:
|
|
67
|
+
iam.gke.io/gcp-service-account: cnpg-operands-sa@OVH:<PROJECT_ID>.iam.gserviceaccount.com
|
|
68
|
+
|
|
69
|
+
# Require 50Gi of space per instance using ssd storage class
|
|
70
|
+
storage:
|
|
71
|
+
storageClass: premium-rwo
|
|
72
|
+
size: 50Gi
|
|
73
|
+
walStorage:
|
|
74
|
+
storageClass: premium-rwo
|
|
75
|
+
size: 1Gi
|
|
76
|
+
|
|
77
|
+
backup:
|
|
78
|
+
volumeSnapshot:
|
|
79
|
+
labels:
|
|
80
|
+
snapshotOf: db-timescale-dense
|
|
81
|
+
className: cnpg-snapshotclass
|
|
82
|
+
barmanObjectStore:
|
|
83
|
+
destinationPath: gs://OVH:<PROJECT_ID>-barman
|
|
84
|
+
googleCredentials:
|
|
85
|
+
gkeEnvironment: true
|
|
86
|
+
tags:
|
|
87
|
+
cnpgCluster: db-timescale-dense
|
|
88
|
+
wal:
|
|
89
|
+
compression: gzip
|
|
90
|
+
---
|
|
91
|
+
apiVersion: postgresql.cnpg.io/v1
|
|
92
|
+
kind: Pooler
|
|
93
|
+
metadata:
|
|
94
|
+
name: db-timescale-dense-pool-rw
|
|
95
|
+
namespace: cnpg-operands
|
|
96
|
+
spec:
|
|
97
|
+
cluster:
|
|
98
|
+
name: db-timescale-dense
|
|
99
|
+
|
|
100
|
+
instances: 1
|
|
101
|
+
type: rw
|
|
102
|
+
pgbouncer:
|
|
103
|
+
poolMode: session
|
|
104
|
+
parameters:
|
|
105
|
+
max_client_conn: "500"
|
|
106
|
+
default_pool_size: "10"
|
|
107
|
+
---
|
|
108
|
+
apiVersion: postgresql.cnpg.io/v1
|
|
109
|
+
kind: ScheduledBackup
|
|
110
|
+
metadata:
|
|
111
|
+
name: db-timescale-dense-backup
|
|
112
|
+
namespace: cnpg-operands
|
|
113
|
+
spec:
|
|
114
|
+
# https://pkg.go.dev/github.com/robfig/cron#hdr-CRON_Expression_Format
|
|
115
|
+
schedule: "0 0 * * * *"
|
|
116
|
+
cluster:
|
|
117
|
+
name: db-timescale-dense
|
|
118
|
+
method: volumeSnapshot
|
|
119
|
+
backupOwnerReference: self
|
|
120
|
+
immediate: true
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
db-timescale-dense.yaml
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
nameOverride: transponder-dh
|
|
2
|
+
|
|
3
|
+
image:
|
|
4
|
+
tag: # v4.3.3
|
|
5
|
+
|
|
6
|
+
config:
|
|
7
|
+
TRANSPONDER_RUN_MODE: "denseHistoryCombined"
|
|
8
|
+
LOG_CONFIG: '{"type":"pino","level":"info"}'
|
|
9
|
+
|
|
10
|
+
DENSE_HISTORY_TDB_CACHE_KEY_PREFIX: "timescale-dh"
|
|
11
|
+
DENSE_HISTORY_MIGRATE_PAGE_SIZE_LIMIT: "50"
|
|
12
|
+
|
|
13
|
+
# PG_HOST: "db-timescale-dense-rw"
|
|
14
|
+
PG_HOST: "db-timescale-dense-pool-rw.cnpg-operands"
|
|
15
|
+
|
|
16
|
+
# password from secret
|
|
17
|
+
PG_PASSWORD_SECRET: "db-timescale-dense-postgres-pw"
|
|
18
|
+
PG_PASSWORD_KEY: "password"
|
|
19
|
+
|
|
20
|
+
TRANSPORT_CONFIG: '{
|
|
21
|
+
"type": "pubsub",
|
|
22
|
+
"projectId": "OVH:<PROJECT_NAME>",
|
|
23
|
+
"publisher": {
|
|
24
|
+
"batching": {
|
|
25
|
+
"maxMilliseconds": 0,
|
|
26
|
+
"maxMessages": 999
|
|
27
|
+
}
|
|
28
|
+
},
|
|
29
|
+
"subscriptionOptions": {
|
|
30
|
+
"flowControl": {
|
|
31
|
+
"allowExcessMessages": false,
|
|
32
|
+
"maxMessages": 25
|
|
33
|
+
},
|
|
34
|
+
"streamingOptions": {
|
|
35
|
+
"maxStreams": 5
|
|
36
|
+
}
|
|
37
|
+
},
|
|
38
|
+
"channel": "writer-dh"
|
|
39
|
+
}'
|
|
40
|
+
|
|
41
|
+
# mode: "denseHistoryCombined"
|
|
42
|
+
# autoAck: "afterProcess"
|
|
43
|
+
# ackOnError: "false"
|
|
44
|
+
# allowForwarding: "true"
|
|
45
|
+
# denseHistoryMigratePageSizeLimit: "50"
|
|
46
|
+
|
|
47
|
+
autoscaling:
|
|
48
|
+
minReplicas: 3
|
|
49
|
+
maxReplicas: 16
|
|
50
|
+
maxCPU: 80
|
|
51
|
+
|
|
52
|
+
resources:
|
|
53
|
+
# We usually recommend not to specify default resources and to leave this as a conscious
|
|
54
|
+
# choice for the user. This also increases chances charts run on environments with little
|
|
55
|
+
# resources, such as Minikube. If you do want to specify resources, uncomment the following
|
|
56
|
+
# lines, adjust them as necessary, and remove the curly braces after 'resources:'.
|
|
57
|
+
limits:
|
|
58
|
+
cpu: 500m
|
|
59
|
+
memory: 1200Mi
|
|
60
|
+
requests:
|
|
61
|
+
cpu: 250m
|
|
62
|
+
memory: 600Mi
|
package/src/helmdn.sh
CHANGED
|
@@ -89,7 +89,7 @@ function helmDown() {
|
|
|
89
89
|
if [ -f "$1/.nohelmdn" ];
|
|
90
90
|
then
|
|
91
91
|
cat<<NOHELMDN
|
|
92
|
-
`color r "***ERROR: $1/.nohelmdn blocker
|
|
92
|
+
`color r "***ERROR: $1/.nohelmdn blocker present - helmdn blocked"`
|
|
93
93
|
|
|
94
94
|
Somebody really doesn't want you to helmdn $1
|
|
95
95
|
|
package/src/helmup.sh
CHANGED
|
@@ -33,6 +33,7 @@ PLATFORM=(
|
|
|
33
33
|
rule-engine
|
|
34
34
|
scheduler
|
|
35
35
|
transponder-bq
|
|
36
|
+
transponder-dh
|
|
36
37
|
transponder-rt
|
|
37
38
|
transponder-tsdb
|
|
38
39
|
)
|
|
@@ -46,13 +47,14 @@ AUXILIARY=(
|
|
|
46
47
|
vin-decoder-server
|
|
47
48
|
)
|
|
48
49
|
|
|
50
|
+
# zombie-killer disabled 20231228
|
|
49
51
|
SYSTEM=(
|
|
50
52
|
elasticsearch8
|
|
51
53
|
redis
|
|
52
54
|
postgres
|
|
53
55
|
timescale-db
|
|
54
56
|
preemptible-killer
|
|
55
|
-
|
|
57
|
+
reflector
|
|
56
58
|
reloader
|
|
57
59
|
traefik
|
|
58
60
|
velero
|
|
@@ -84,14 +86,31 @@ CATNAMI
|
|
|
84
86
|
addHelmRepo bitnami $BITNAMI_URL
|
|
85
87
|
}
|
|
86
88
|
|
|
89
|
+
# Accepts 3 paramaters - 1 mandatory.
|
|
90
|
+
# bindWorkloadIdentity <Service Account> <Namespace> <Target Service Account>
|
|
91
|
+
# where:
|
|
92
|
+
# Service Account = SA to receive the binding
|
|
93
|
+
# Namespace = optional - uses the default namespace if unspecified
|
|
94
|
+
# Target Service Account = SA to receive the workload binding
|
|
95
|
+
#
|
|
96
|
+
# We typically have a one to one binding for SA and target SA (and target seems
|
|
97
|
+
# like a poor choice of word), so for api-server running in the default namespace:
|
|
98
|
+
# bindWorkloadIdentity api-server
|
|
99
|
+
#
|
|
100
|
+
# but in the case of services like transponder which has various flavors but
|
|
101
|
+
# only the single trandponder services account we woul do:
|
|
102
|
+
# bindWorkloadIdentity transponder default tranponder-rt
|
|
103
|
+
#
|
|
87
104
|
function bindWorkloadIdentity() {
|
|
88
105
|
SVC_ACCT=$1
|
|
89
106
|
NAMESPACE=${2:-default}
|
|
107
|
+
BOUND_SVC_ACCT=${3:-$SVC_ACCT}
|
|
90
108
|
SVC_ACCT_EMAIL="${SVC_ACCT}@${GCP_PROJECT_ID}.iam.gserviceaccount.com"
|
|
91
109
|
|
|
92
110
|
cat<<WORKLOAD_ID
|
|
93
111
|
Binding workload identity IAM policy
|
|
94
|
-
|
|
112
|
+
Target Svc Acct : `color g $SVC_ACCT`
|
|
113
|
+
Bound Svc Acct : `color g $BOUND_SVC_ACCT`
|
|
95
114
|
Project : `color g $GCP_PROJECT_ID`
|
|
96
115
|
Namespace : `color g $NAMESPACE`
|
|
97
116
|
Service Email : `color g $SVC_ACCT_EMAIL`
|
|
@@ -100,38 +119,40 @@ WORKLOAD_ID
|
|
|
100
119
|
|
|
101
120
|
gcloud iam service-accounts add-iam-policy-binding \
|
|
102
121
|
--project $GCP_PROJECT_ID \
|
|
103
|
-
--member "serviceAccount:$GCP_PROJECT_ID.svc.id.goog[${NAMESPACE}/${
|
|
122
|
+
--member "serviceAccount:$GCP_PROJECT_ID.svc.id.goog[${NAMESPACE}/${BOUND_SVC_ACCT}]" \
|
|
104
123
|
--role roles/iam.workloadIdentityUser $SVC_ACCT_EMAIL
|
|
105
124
|
warnOnError $? "gcloud workload identity binding may have failed"
|
|
125
|
+
|
|
126
|
+
sleep 2 # give the policy a chance to propagate
|
|
106
127
|
|
|
107
128
|
# Verify the Kubernetes SA and create it if necessary
|
|
108
|
-
printf "\nChecking for the $NAMESPACE/$SVC_ACCT k8s SA\n"
|
|
129
|
+
printf "\nChecking for the `color g $NAMESPACE/$SVC_ACCT` k8s SA\n"
|
|
109
130
|
kubectl get sa -n $NAMESPACE $SVC_ACCT &> $DEVNULL
|
|
110
131
|
if [[ $? -eq 1 ]];
|
|
111
132
|
then
|
|
112
|
-
printf "Creating the k8s $SVC_ACCT SA in $NAMESPACE
|
|
133
|
+
printf "Creating the k8s `color g $SVC_ACCT` SA in `color g $NAMESPACE`\n"
|
|
113
134
|
kubectl create sa $SVC_ACCT -n $NAMESPACE
|
|
114
135
|
fi
|
|
115
136
|
|
|
116
|
-
printf "\nAnnotating the k8s SA $NAMESPACE/@$SVC_ACCT
|
|
137
|
+
printf "\nAnnotating the k8s SA `color g $NAMESPACE/@$SVC_ACCT`\n"
|
|
117
138
|
kubectl annotate serviceaccount $SVC_ACCT \
|
|
118
139
|
--namespace $NAMESPACE --overwrite \
|
|
119
140
|
"iam.gke.io/gcp-service-account=$SVC_ACCT_EMAIL"
|
|
120
141
|
warnOnError $? "Failed to annotate service account $NAMESPACE/$SVC_ACCT - things may fail"
|
|
121
|
-
}
|
|
122
142
|
|
|
123
|
-
|
|
124
|
-
RANDOMPW=`openssl rand -base64 12`
|
|
125
|
-
echo $RANDOMPW
|
|
143
|
+
sleep 2 # give the annotation a chance to propagate before exiting
|
|
126
144
|
}
|
|
127
145
|
|
|
128
146
|
function createNamespaceIfNeeded() {
|
|
129
147
|
# Check the namespace exists in kubernetes
|
|
130
|
-
|
|
148
|
+
local NS=$1
|
|
149
|
+
kubectl get namespace $NS &> $DEVNULL
|
|
131
150
|
if [[ $? -eq 1 ]];
|
|
132
151
|
then
|
|
133
|
-
printf "\n*** Creating $
|
|
134
|
-
kubectl create namespace $
|
|
152
|
+
printf "\n*** Creating $NS namespace ***\n\n"
|
|
153
|
+
kubectl create namespace $NS
|
|
154
|
+
# annotate to tell the network policy to allow redis access from NS
|
|
155
|
+
kubectl label namespaces $NS redis=external
|
|
135
156
|
fi
|
|
136
157
|
}
|
|
137
158
|
|
|
@@ -456,6 +477,170 @@ NEED_MANAGED_SECRET
|
|
|
456
477
|
fi
|
|
457
478
|
}
|
|
458
479
|
|
|
480
|
+
# See => https://github.com/emberstack/kubernetes-reflector
|
|
481
|
+
function reflectSecretIntoNamespaces() {
|
|
482
|
+
local SECRET="$1"
|
|
483
|
+
local SOURCE_NS="$2"
|
|
484
|
+
local REFLECT_INTO_NS="$3"
|
|
485
|
+
|
|
486
|
+
printf "\nReflecting secret `color y "$SOURCE_NS/$SECRET"` into `color g "$REFLECT_INTO_NS"` namespace\n"
|
|
487
|
+
|
|
488
|
+
kubectl annotate secrets $SECRET -n $SOURCE_NS \
|
|
489
|
+
reflector.v1.k8s.emberstack.com/reflection-allowed="true" &> $DEVNULL
|
|
490
|
+
kubectl annotate secrets $SECRET -n $SOURCE_NS \
|
|
491
|
+
reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces="$REFLECT_INTO_NS" &> $DEVNULL
|
|
492
|
+
|
|
493
|
+
cat<<REFLECTED_SECRET | kubectl apply -f -
|
|
494
|
+
apiVersion: v1
|
|
495
|
+
kind: Secret
|
|
496
|
+
metadata:
|
|
497
|
+
name: $SECRET
|
|
498
|
+
namespace: "$REFLECT_INTO_NS"
|
|
499
|
+
annotations:
|
|
500
|
+
reflector.v1.k8s.emberstack.com/reflects: "$SOURCE_NS/$SECRET"
|
|
501
|
+
REFLECTED_SECRET
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
# This function is used to randomly generate the postgresql database passwords
|
|
505
|
+
# for the postgres and imagine users in a cnpg operator configuration. Since
|
|
506
|
+
# the resultant passwords will live in the cnpg-operands namespace they will
|
|
507
|
+
# be mirrored to the default namespace via the reflector service. Lastly, the
|
|
508
|
+
# service account will have workload identity bound to it in the cnpg-operands
|
|
509
|
+
# namespace.
|
|
510
|
+
function initializeCnpgOperand() {
|
|
511
|
+
local CNPG_SERVICE="$1"
|
|
512
|
+
local CNPG_NAMESPACE="cnpg-operands"
|
|
513
|
+
local CNPG_SVC_ACCT="$CNPG_NAMESPACE-sa"
|
|
514
|
+
local IMAGINE_PW_NAME="$CNPG_SERVICE-imagine-pw"
|
|
515
|
+
local POSTGRES_PW_NAME="$CNPG_SERVICE-postgres-pw"
|
|
516
|
+
|
|
517
|
+
printf "\nCreating imagine db password for `color g $CNPG_SERVICE`\n"
|
|
518
|
+
kubectl create secret generic -n $CNPG_NAMESPACE "$IMAGINE_PW_NAME" \
|
|
519
|
+
--from-literal=username="imagine" \
|
|
520
|
+
--from-literal=password="$(generatePassword)" &> $DEVNULL
|
|
521
|
+
warnOnError $? "the `color y $IMAGINE_PW_NAME` secret may already exist"
|
|
522
|
+
|
|
523
|
+
# reflect the secret to default namespace
|
|
524
|
+
reflectSecretIntoNamespaces $IMAGINE_PW_NAME $CNPG_NAMESPACE "default"
|
|
525
|
+
|
|
526
|
+
printf "\nCreating postgres db password for `color g $CNPG_SERVICE`\n"
|
|
527
|
+
kubectl create secret generic -n $CNPG_NAMESPACE "$POSTGRES_PW_NAME" \
|
|
528
|
+
--from-literal=username="postgres" \
|
|
529
|
+
--from-literal=password="$(generatePassword)" &> $DEVNULL
|
|
530
|
+
warnOnError $? "the `color y $POSTGRES_PW_NAME` secret may already exist"
|
|
531
|
+
|
|
532
|
+
# reflect the secret to default namespace
|
|
533
|
+
reflectSecretIntoNamespaces $POSTGRES_PW_NAME $CNPG_NAMESPACE "default"
|
|
534
|
+
|
|
535
|
+
printf "\nBinding `color g $CNPG_SERVICE` to the `color y "$CNPG_SVC_ACCT"` SA\n"
|
|
536
|
+
bindWorkloadIdentity $CNPG_SVC_ACCT $CNPG_NAMESPACE $CNPG_SERVICE
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
function installCnpgOperatorEnvironment() {
|
|
540
|
+
# See => https://github.com/cloudnative-pg/charts
|
|
541
|
+
|
|
542
|
+
gcloud config set project $GCP_PROJECT_ID
|
|
543
|
+
|
|
544
|
+
# create an SA in the cnpg namespace for the operands to bind to
|
|
545
|
+
CNPG_NAMESPACE="cnpg-operands"
|
|
546
|
+
CNPG_SVC_ACCT="$CNPG_NAMESPACE-sa"
|
|
547
|
+
CNPG_SVC_EMAIL="$CNPG_SVC_ACCT@$GCP_PROJECT_ID.iam.gserviceaccount.com"
|
|
548
|
+
|
|
549
|
+
kubectl delete serviceaccounts -n $CNPG_NAMESPACE $CNPG_SVC_ACCT &> $DEVNULL
|
|
550
|
+
gcloud --quiet iam service-accounts delete $CNPG_SVC_EMAIL --project $GCP_PROJECT_ID&> $DEVNULL
|
|
551
|
+
|
|
552
|
+
# Create the operands namespace and annotate to allow redis access
|
|
553
|
+
createNamespaceIfNeeded $CNPG_NAMESPACE
|
|
554
|
+
|
|
555
|
+
# Create GCP SA and bind policies
|
|
556
|
+
printf "\nCreating the gcloud `color g $CNPG_SVC_ACCT` service account (SA)\n"
|
|
557
|
+
gcloud iam service-accounts create $CNPG_SVC_ACCT \
|
|
558
|
+
--project "$GCP_PROJECT_ID" \
|
|
559
|
+
--description "CNPG Operands SA" \
|
|
560
|
+
--display-name "CNPG Operands SA" &> $DEVNULL
|
|
561
|
+
warnOnError $? "the GCP SA `color y $CNPG_SVC_ACCT` may already exist"
|
|
562
|
+
|
|
563
|
+
CNPG_OPS_ROLE="cnpg.operands"
|
|
564
|
+
printf "\nCreating the `color g $CNPG_OPS_ROLE` IAM role\n"
|
|
565
|
+
## Attach roles
|
|
566
|
+
ROLE_PERMISSIONS=(
|
|
567
|
+
iam.serviceAccounts.actAs
|
|
568
|
+
iam.serviceAccounts.get
|
|
569
|
+
iam.serviceAccounts.getAccessToken
|
|
570
|
+
iam.serviceAccounts.getOpenIdToken
|
|
571
|
+
iam.serviceAccounts.implicitDelegation
|
|
572
|
+
iam.serviceAccounts.list
|
|
573
|
+
iam.serviceAccounts.signBlob
|
|
574
|
+
iam.serviceAccounts.signJwt
|
|
575
|
+
resourcemanager.projects.get
|
|
576
|
+
storage.buckets.create
|
|
577
|
+
storage.buckets.get
|
|
578
|
+
storage.buckets.getIamPolicy
|
|
579
|
+
storage.buckets.list
|
|
580
|
+
storage.buckets.update
|
|
581
|
+
storage.objects.create
|
|
582
|
+
storage.objects.delete
|
|
583
|
+
storage.objects.get
|
|
584
|
+
storage.objects.getIamPolicy
|
|
585
|
+
storage.objects.list
|
|
586
|
+
)
|
|
587
|
+
gcloud iam roles create $CNPG_OPS_ROLE \
|
|
588
|
+
--project $GCP_PROJECT_ID \
|
|
589
|
+
--title "CNPG Operands" \
|
|
590
|
+
--description "CNPG Operands" \
|
|
591
|
+
--permissions "$(IFS=","; echo "${ROLE_PERMISSIONS[*]}")" &> $DEVNULL
|
|
592
|
+
if [ $? -ne 0 ];
|
|
593
|
+
then
|
|
594
|
+
# assume error is that role exists so try updating instead
|
|
595
|
+
printf "\nUpdating the permissions on the `color g $CNPG_OPS_ROLE` IAM role\n"
|
|
596
|
+
gcloud iam roles update $CNPG_OPS_ROLE \
|
|
597
|
+
--project $GCP_PROJECT_ID \
|
|
598
|
+
--title "CNPG Operands" \
|
|
599
|
+
--permissions "$(IFS=","; echo "${ROLE_PERMISSIONS[*]}")" &> $DEVNULL
|
|
600
|
+
fi
|
|
601
|
+
warnOnError $? "Failed to add roles to $CNPG_OPS_ROLE - perhaps they already exist?"
|
|
602
|
+
|
|
603
|
+
printf "\nBinding IAM role to the SA `color g $CNPG_SVC_EMAIL`\n"
|
|
604
|
+
gcloud projects add-iam-policy-binding $GCP_PROJECT_ID \
|
|
605
|
+
--member="serviceAccount:$CNPG_SVC_EMAIL" \
|
|
606
|
+
--role "roles/storage.admin" &> $DEVNULL
|
|
607
|
+
# --role projects/$GCP_PROJECT_ID/roles/$CNPG_OPS_ROLE &> $DEVNULL
|
|
608
|
+
# --role "roles/storage.admin" &> $DEVNULL
|
|
609
|
+
warnOnError $? "gcloud problem binding IAM policy"
|
|
610
|
+
|
|
611
|
+
printf "\nBinding IAM the Token Creator role to the SA `color g $CNPG_SVC_EMAIL`\n"
|
|
612
|
+
gcloud projects add-iam-policy-binding $GCP_PROJECT_ID \
|
|
613
|
+
--member="serviceAccount:$CNPG_SVC_EMAIL" \
|
|
614
|
+
--role="roles/iam.serviceAccountTokenCreator"
|
|
615
|
+
|
|
616
|
+
# workload binding will be performed by each cnpg operand on startup
|
|
617
|
+
|
|
618
|
+
BARMAN_BUCKET="gs://$GCP_PROJECT_ID-barman"
|
|
619
|
+
gsutil ls -p $GCP_PROJECT_ID -b $BARMAN_BUCKET &> $DEVNULL
|
|
620
|
+
if [ "$?" -eq 0 ];
|
|
621
|
+
then
|
|
622
|
+
printf "\nBarman bucket `color g "$BARMAN_BUCKET"` already exists - skipping recreation\n"
|
|
623
|
+
else
|
|
624
|
+
printf "\nCreating a bucket for barman backups: `color g $BARMAN_BUCKET`\n"
|
|
625
|
+
gsutil mb -l $GCE_REGION $BARMAN_BUCKET &> $DEVNULL
|
|
626
|
+
gsutil iam ch serviceAccount:$CNPG_SVC_EMAIL:objectAdmin $BARMAN_BUCKET
|
|
627
|
+
gsutil ubla set on $BARMAN_BUCKET # uniform bucket access
|
|
628
|
+
fi
|
|
629
|
+
|
|
630
|
+
# define the snapshot storage class
|
|
631
|
+
printf "\nApplying the CNPG VolumeSnapshotClass `color g cnpg-snapshotclass` definition\n"
|
|
632
|
+
cat<<EOSNAPSC | kubectl apply -f -
|
|
633
|
+
apiVersion: snapshot.storage.k8s.io/v1
|
|
634
|
+
kind: VolumeSnapshotClass
|
|
635
|
+
metadata:
|
|
636
|
+
name: cnpg-snapshotclass
|
|
637
|
+
driver: pd.csi.storage.gke.io
|
|
638
|
+
deletionPolicy: Delete
|
|
639
|
+
EOSNAPSC
|
|
640
|
+
|
|
641
|
+
sleep 2 # hold up processing for a moment to allow IAM mods to propagate
|
|
642
|
+
}
|
|
643
|
+
|
|
459
644
|
function installVeleroEnvironment() {
|
|
460
645
|
# Follows instructions => https://github.com/vmware-tanzu/velero-plugin-for-gcp
|
|
461
646
|
showInstalling "Velero Backup Management"
|
|
@@ -469,10 +654,10 @@ function installVeleroEnvironment() {
|
|
|
469
654
|
|
|
470
655
|
## Create a bucket
|
|
471
656
|
BUCKET="$GCP_PROJECT_ID-velero-$VELERO_HELM_CHART"
|
|
472
|
-
GCE_REGION=`overwhelm -k GCE_REGION`
|
|
473
657
|
printf "\nCreating storage bucket `color g $BUCKET` in `color g $GCE_REGION`\n"
|
|
474
658
|
gsutil mb -l $GCE_REGION gs://$BUCKET/ &> $DEVNULL
|
|
475
659
|
warnOnError $? "Bucket failed to create or already exists - go check it\n"
|
|
660
|
+
gsutil ubla set on gs://$BUCKET &> $DEVNULL
|
|
476
661
|
|
|
477
662
|
## Create a service account
|
|
478
663
|
printf "\nCreating the service account\n"
|
|
@@ -615,7 +800,7 @@ function doInstall() {
|
|
|
615
800
|
|
|
616
801
|
local INVOKED_AS=`basename $0`
|
|
617
802
|
# set the dry-run flag on what
|
|
618
|
-
[ "$INVOKED_AS" == "helmwhat" ] && HELM_WHAT="--dry-run --debug"
|
|
803
|
+
[ "$INVOKED_AS" == "helmwhat" ] && HELM_WHAT="--dry-run --debug" && K8S_WHAT="--dry-run=server -o yaml"
|
|
619
804
|
|
|
620
805
|
# helminit is used to redeploy an existing service
|
|
621
806
|
if [ "$INVOKED_AS" == "helminit" ] && [[ $(isLeveregeService $SERVICE) -eq 1 ]];
|
|
@@ -660,6 +845,11 @@ CATBACKUP
|
|
|
660
845
|
bootstrapLocalSetup $SERVICE
|
|
661
846
|
;;
|
|
662
847
|
|
|
848
|
+
"cnpg-operator")
|
|
849
|
+
installCnpgOperatorEnvironment
|
|
850
|
+
bootstrapLocalSetup $SERVICE
|
|
851
|
+
;;
|
|
852
|
+
|
|
663
853
|
"cronZombieKiller"|"zombie-killer")
|
|
664
854
|
installCronZombieKiller # eradicate the gke 1.20+ zombies
|
|
665
855
|
;;
|
|
@@ -672,6 +862,16 @@ CATBACKUP
|
|
|
672
862
|
installDefaultConfigs
|
|
673
863
|
;;
|
|
674
864
|
|
|
865
|
+
db-postgres-*|db-timescale-*)
|
|
866
|
+
if [ ! -d $SERVICE ];
|
|
867
|
+
then
|
|
868
|
+
# this code only executes on the first deployment
|
|
869
|
+
initializeCnpgOperand $SERVICE
|
|
870
|
+
bootstrapLocalSetup $SERVICE
|
|
871
|
+
kubectl apply -f $SERVICE/$SERVICE.yaml $K8S_WHAT
|
|
872
|
+
fi
|
|
873
|
+
;;
|
|
874
|
+
|
|
675
875
|
db-curator-dh)
|
|
676
876
|
# db-curator-dh is a variant of db-curator with diff controlled in values-local.yaml
|
|
677
877
|
HELMUP="helm upgrade --install $SERVICE leverege/db-curator --values $SERVICE/values.yaml $CHARTVER $HELM_WHAT"
|
|
@@ -724,6 +924,13 @@ CATBACKUP
|
|
|
724
924
|
bootstrapLocalSetup $SERVICE
|
|
725
925
|
;;
|
|
726
926
|
|
|
927
|
+
"reflector")
|
|
928
|
+
showInstalling "Secrets/ConfigMap Namespace Reflector"
|
|
929
|
+
addHelmRepo emberstack https://emberstack.github.io/helm-charts
|
|
930
|
+
helm upgrade --install reflector emberstack/reflector
|
|
931
|
+
removeHelmRepo emberstack
|
|
932
|
+
;;
|
|
933
|
+
|
|
727
934
|
"reloader")
|
|
728
935
|
showInstalling "Stakater ConfigMap Reloader"
|
|
729
936
|
addHelmRepo stakater https://stakater.github.io/stakater-charts
|
|
@@ -842,6 +1049,8 @@ overwhelm
|
|
|
842
1049
|
|
|
843
1050
|
# The project id is used in a lot of places so just set it here
|
|
844
1051
|
GCP_PROJECT_ID="$(getProjectIdFromK8sContext)"
|
|
1052
|
+
# Same for the region, but not as prevalent
|
|
1053
|
+
GCE_REGION=`overwhelm -k GCE_REGION`
|
|
845
1054
|
|
|
846
1055
|
doInstall $@
|
|
847
1056
|
cd - &> $DEVNULL
|
package/src/overwhelm.mjs
CHANGED
|
@@ -82,7 +82,7 @@ debug( { over, yaml, }, '<==Contents of the over and yaml structs' )
|
|
|
82
82
|
let cfgs = yaml[context]
|
|
83
83
|
|
|
84
84
|
// If we just need to check context return 0 if cfgs exists, else error.
|
|
85
|
-
if ( args.checkctx ) { process.exit( !cfgs ) }
|
|
85
|
+
if ( args.checkctx ) { process.exit( !cfgs ? 1 : 0 ) }
|
|
86
86
|
|
|
87
87
|
if ( !cfgs ) {
|
|
88
88
|
log( `***ERROR: Unknown context => [${context}]\n` )
|