@colisweb/rescript-toolkit 4.14.14 → 4.14.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,2596 +0,0 @@
1
- #!/usr/bin/env bash
2
-
3
- #VARIABLES
4
- export SCRIPT_FULL_PATH=$(dirname "$0")
5
-
6
- ##FUNCTIONS
7
- # https://stackoverflow.com/questions/1527049/how-can-i-join-elements-of-an-array-in-bash
8
- join_by() {
9
- local d=${1-} f=${2-}
10
- if shift 2; then
11
- printf %s "$f" "${@/#/$d}"
12
- fi
13
- }
14
-
15
- mkstring() {
16
- local start=$1
17
- local separator=$2
18
- local end=$3
19
- shift 3
20
-
21
- if [ $# -gt 0 ]; then
22
- printf $start
23
- join_by $separator $*
24
- printf $end
25
- fi
26
- }
27
-
28
- md5all() {
29
- all_hash=$(mktemp)
30
- for name in $*; do
31
- find $name -type f -exec cat {} \; | md5sum | cut -f1 -d ' ' >> $all_hash
32
- done;
33
- cat $all_hash | md5sum | cut -f1 -d ' '
34
- }
35
-
36
- log() {
37
- echo "$*" >&2
38
- }
39
- #!/usr/bin/env bash
40
-
41
- check_args() {
42
- if [ -z $2 ] || [ "$1" != "$2" ]; then
43
- echo >&2 "missing argument $1"
44
- return 1
45
- fi
46
- }
47
-
48
- check_env_vars() {
49
- ArgsCount=$1 && shift
50
- for ((i = 0; i < $ArgsCount; i++)); do
51
- if [[ -z "${!1}" ]]; then
52
- echo >&2 "missing ENV $1"
53
- return 1
54
- fi
55
- shift
56
- done
57
- }
58
-
59
- extract_arg() {
60
- name=$1
61
- passed=$2
62
- value=$3
63
- if [ "--$name" != "$passed" ]; then
64
- echo "missing argument $name"
65
- exit 1
66
- fi
67
- eval $name='$value'
68
- }
69
-
70
- extract_args() {
71
- declare -a Array_Args
72
- ArgsCount=$1 && shift
73
- for ((i = 0; i < $ArgsCount; i++)); do
74
- Array_Args[i]=$1 && shift
75
- done
76
- for ArgName in "${Array_Args[@]}"; do
77
- extract_arg "$ArgName" $* && shift 2
78
- done
79
- }
80
-
81
- #!/usr/bin/env bash
82
-
83
- aws_ecr_login() {
84
- PATH=/root/.local/bin:$PATH
85
-
86
- aws ecr get-login-password \
87
- | docker login --username AWS --password-stdin 949316342391.dkr.ecr.eu-west-1.amazonaws.com \
88
- || (echo "you should update to AWS CLI version 2 https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2-mac.html " $(aws ecr get-login --region=eu-west-1 --no-include-email) )
89
- }
90
-
91
- aws_ecr_token() {
92
- aws ecr get-authorization-token --region=eu-west-1 --output text --query 'authorizationData[].authorizationToken'
93
- }
94
-
95
- # you will need jq to use these commands. You can install it using "brew install jq"
96
- # delete_images colisweb_api 8
97
- # will delete images older than 8 weeks
98
- delete_images() {
99
-
100
- REPO=$1
101
- WEEKS=${2:-16}
102
-
103
- WEEKS_AGO=$(date -v-${WEEKS}w +%F)
104
-
105
- #Get all ecr images
106
- IMAGES=$(aws ecr describe-images --repository-name $REPO --output json)
107
-
108
- #Filter unnecessary values and map `imagePushedAt` to EPOCH
109
- NON_LATEST_IMAGES=$(echo $IMAGES | jq '[.imageDetails[] | select(.imageTags | any(endswith("latest")) | not)]')
110
-
111
- #Filter on EPOCH
112
- OLD_IMAGES=$(echo $NON_LATEST_IMAGES | jq --arg date $WEEKS_AGO '.[] | select(.imagePushedAt[0:10] < $date).imageDigest')
113
- while IFS= read -r IMAGE; do
114
- if [ "$IMAGE" != "" ]; then
115
- echo "Deleting $IMAGE from $REPO"
116
- AWS_PAGER="" aws ecr batch-delete-image --repository-name $REPO --image-ids imageDigest=$IMAGE
117
- fi
118
- done <<< "$OLD_IMAGES"
119
- }
120
-
121
- # delete_images_all_repos 12
122
- # will delete images in all repositories older than 12 weeks
123
- delete_images_all_repos() {
124
- REPOSITORIES=$(aws ecr describe-repositories --output json | jq -r '.[]|.[].repositoryName')
125
-
126
- while IFS= read -r REPO; do
127
- echo "processing ECR repository $REPO"
128
- delete_images $REPO $1
129
- done <<< "$REPOSITORIES"
130
- }
131
-
132
- delete_old_cache() {
133
- DATE=${1:-$(date -v-1m +%F)}
134
- CACHE_BUCKET=${2:-"s3://gitlab-colisweb-distributed-cache/project/"}
135
-
136
- echo "deleting from cache $CACHE_BUCKET all older than $DATE"
137
-
138
- aws_ecr_login
139
-
140
- while read -r line; do
141
- datum=$(echo $line | cut -c1-10)
142
- if [[ "$datum" < "$DATE" ]] ; then
143
- # Shell Parameter Expansion: ${parameter##word}
144
- # Allow to return the result from "word" to the end of "parameters"
145
- # Here we need the end of the string after "project/" (corresponding to the S3 gitlab project id and filename)
146
- TO_DELETE="$CACHE_BUCKET${line##* project/}"
147
- echo $TO_DELETE
148
- aws s3 rm $TO_DELETE
149
- fi
150
- done < <(aws s3 ls $CACHE_BUCKET --recursive)
151
- }
152
-
153
- #!/usr/bin/env bash
154
-
155
- # If gitlab is down or pipeline are stuck, hotfixes need to be available
156
- # This script will publish docker images to ECR using your current git HEAD, then deploy them to a given environment.
157
- # Some local files (git-commit.conf and sentry.properties) will be updated, take caution.
158
- # No trace of this will appear on Gitlab (no releases, no pipelines, no tags).
159
- # create_hotfix_scala $ENVIRONMENT $CHART_NAME [ $MODULE_NAME $MODULE_PATH $DEPLOYMENT ]
160
- # create_hotfix_scala testing crm main modules/3-executables/main crm
161
- # create_hotfix_scala testing notification \
162
- # main-http modules/3-executables/main-http notification-http \
163
- # main-consumer modules/3-executables/main-consumer notification-consumer
164
-
165
- create_hotfix_scala() {
166
-
167
- ENVIRONMENT=$1
168
- CHART_NAME=$2
169
- shift 2
170
-
171
- SHORT_SHA=$(git rev-parse --short HEAD)
172
- HOTFIX_TAG="hotfix-$SHORT_SHA"
173
-
174
- gum confirm "Preparing $HOTFIX_TAG for $CHART_NAME ?" || exit
175
- prepare_hotfix_scala $HOTFIX_TAG
176
-
177
- gum confirm "Building $HOTFIX_TAG for $CHART_NAME ?" || exit
178
- while [[ $# -gt 2 ]] ; do
179
- build_hotfix_scala $HOTFIX_TAG "$1" "$2" "$3"
180
- shift 3
181
- done
182
-
183
- gum confirm "Deploying $HOTFIX_TAG for $CHART_NAME ?" || exit
184
- deploy_hotfix $CHART_NAME $ENVIRONMENT $HOTFIX_TAG
185
- }
186
-
187
- # Update local git-commit.conf and sentry.properties files using git short sha
188
- prepare_hotfix_scala() {
189
- HOTFIX_TAG=$1
190
-
191
- git secret reveal -f
192
- aws_ecr_login
193
-
194
- COMMIT_CONF_FILES=$(find . -name "git-commit.conf")
195
- SENTRY_PROPERTIES_FILES=$(find . -name "sentry.properties")
196
-
197
- for file in $(echo "$COMMIT_CONF_FILES\n$SENTRY_PROPERTIES_FILES"); do
198
- sed -i '' -e 's&GIT_COMMIT&'"$HOTFIX_TAG&" $file
199
- done
200
-
201
- }
202
-
203
- # Build docker images locally and publish them to AWS ECR.
204
- build_hotfix_scala() {
205
-
206
- HOTFIX_TAG=$1
207
- SBT_MODULE=$2
208
- DOCKER_PATH=$3
209
- DEPLOYMENT=$4
210
-
211
- DOCKER_REGISTRY_ID="949316342391"
212
- DOCKER_REGISTRY="$DOCKER_REGISTRY_ID.dkr.ecr.eu-west-1.amazonaws.com"
213
- DOCKER_IMAGE=$DOCKER_REGISTRY/$DEPLOYMENT
214
- HOTFIX_IMAGE=$DOCKER_IMAGE:$HOTFIX_TAG
215
-
216
- #Build
217
- sbt "project $SBT_MODULE" "Docker / stage"
218
-
219
- #Publish
220
- docker build --platform "linux/amd64" -t $HOTFIX_IMAGE --cache-from $DOCKER_IMAGE "$DOCKER_PATH/target/docker/stage"
221
- docker push $HOTFIX_IMAGE
222
-
223
- echo "Created hotfix $HOTFIX_IMAGE"
224
- }
225
-
226
- # Deploy the project in the given environment
227
- deploy_hotfix() {
228
- source $colisweb_scripts/ci/helm.sh
229
-
230
- CHART_NAME=$1
231
- ENVIRONMENT=$2
232
- HOTFIX_TAG=$3
233
-
234
- CONFIG_PATH=deploy
235
- CHART_PATH=$CONFIG_PATH/$CHART_NAME
236
- ROOT_PATH=$(pwd)
237
-
238
- # Unset Kubectl configuration made via the KUBECONFIG env variable
239
- # it would override the config made by configure_kubectl_for
240
- # for example, using Gitlab runners in Kubernetes sets this variable and causes conflict
241
- unset KUBECONFIG
242
-
243
- # Configure Kubectl
244
- configure_kubectl_for $ENVIRONMENT
245
-
246
- # Avoiding "no local-index.yaml" or "empty local-index.yaml" error
247
- cat > $HOME/Library/Caches/helm/repository/local-index.yaml <<EOT
248
- apiVersion: v1
249
- entries:
250
- cronjob:
251
- EOT
252
-
253
- # helm3 stable repo have changed and must be updated manually, in versions < v2.17.0
254
- helm3 repo add colisweb s3://colisweb-helm-charts/colisweb --force-update
255
- helm3 repo add stable https://charts.helm.sh/stable --force-update
256
- helm3 repo update
257
- helm3 dependency update ${ROOT_PATH}/${CHART_PATH}
258
-
259
- # Gather values/*.yaml files
260
- VALUES_PATH="${ROOT_PATH}/${CHART_NAME}/values"
261
- VALUES_FILES=''
262
- [ -d $VALUES_PATH ] && VALUES_FILES=$(find $VALUES_PATH -type f -maxdepth 1 -name "*.yaml" | sed 's/^/ -f /' | tr -d \\n | sed 's/%//')
263
-
264
- # Deploy
265
- helm3 upgrade --install \
266
- --namespace ${ENVIRONMENT} \
267
- ${VALUES_FILES} \
268
- -f ${ROOT_PATH}/${CONFIG_PATH}/common.yaml \
269
- -f ${ROOT_PATH}/${CONFIG_PATH}/${ENVIRONMENT}.yaml \
270
- -f ${ROOT_PATH}/${CONFIG_PATH}/${ENVIRONMENT}-secrets.yaml \
271
- --set global.version=$HOTFIX_TAG \
272
- ${CHART_NAME} ${ROOT_PATH}/${CHART_PATH}
273
-
274
-
275
- verify_deployments_v3 -t 10m $ENVIRONMENT $CHART_NAME
276
-
277
- }
278
-
279
- #!/usr/bin/env bash
280
-
281
- image_exists() {
282
- set -e
283
-
284
- REGISTRY=$1
285
- REPOSITORY=$2
286
- IMAGE=$3
287
-
288
- TAGGED_IMAGE="$REGISTRY/$REPOSITORY:$IMAGE"
289
-
290
- aws ecr describe-images --registry-id $REGISTRY --repository-name $REPOSITORY --image-ids "imageTag=$IMAGE"
291
-
292
- if [ $? -eq 0 ]
293
- then
294
- echo "Image $TAGGED_IMAGE already present in distant repo"
295
- return 0
296
- else
297
- echo "Image $TAGGED_IMAGE NOT present in distant repo"
298
- return 1
299
- fi
300
- }
301
- #!/usr/bin/env bash
302
-
303
- gmm() {
304
- git checkout $1
305
- git pull
306
- git checkout $2
307
- git pull
308
- git merge $1
309
- git push
310
- }
311
-
312
- git_damn_merge() {
313
- git checkout $1
314
- git pull
315
- git checkout $2
316
- git dammit
317
- git merge $1
318
- git push
319
- }
320
-
321
- git_prune_local_branches() {
322
- git branch -r |
323
- awk '{print $1}' |
324
- egrep -v -f /dev/fd/0 <(git branch -vv | grep origin) |
325
- awk '{print $1}' |
326
- xargs git branch -d
327
- }
328
-
329
- gum_checkout() {
330
- git branch -a | cut -f3- -d "/" | gum filter | xargs git checkout
331
- }
332
-
333
- # useful option :
334
- # export GIT_SUBLINE_MERGE_NON_INTERACTIVE_MODE=TRUE
335
- # see https://github.com/paulaltin/git-subline-merge
336
- setup_subline_merge() {
337
- location=${1:-"--local"}
338
-
339
- case $location in
340
- --local)
341
- if [ -d ".git" ]; then
342
- echo "* merge=subline" >>.git/info/attributes
343
- else
344
- echo "Cannot use local option, not in a git repository"
345
- return 1
346
- fi
347
- ;;
348
- --global)
349
- echo "* merge=subline" >>~/.gitattributes
350
- ;;
351
- *)
352
- echo "unknown argument $location"
353
- return 2
354
- ;;
355
- esac
356
-
357
- git config $location merge.conflictStyle diff3
358
- git config $location merge.subline.driver "$colisweb_scripts/shell-session/shell/dev/git-subline-merge %O %A %B %L %P"
359
- git config $location merge.subline.recursive binary
360
- }
361
-
362
- rebase_from_ancestor() {
363
- set -x
364
- branch=$1
365
- tip=$(git rev-parse HEAD)
366
- ancestor=$(git merge-base $branch $tip)
367
- commits=$(git log $ancestor..$tip)
368
- git reset --hard $ancestor
369
- git merge --squash $tip
370
- git commit -m "squashed commmits $commits" || echo "nothing committed"
371
- git rebase $branch -Xtheirs
372
- }
373
-
374
- #!/usr/bin/env bash
375
-
376
- import_all_pgp_keys() {
377
- echo "importing all PGP keys"
378
- gpg --import $SCRIPT_FULL_PATH/pgp_keys/*.key
379
- }
380
-
381
- remove_all_persons_from_secrets() {
382
- echo "cleanup git secret"
383
- WHO_KNOWS=($(git secret whoknows))
384
- git secret removeperson $WHO_KNOWS
385
- echo "Removed secrets access for $WHO_KNOWS"
386
- }
387
-
388
- all_pgp_emails() {
389
- gpg --show-key $SCRIPT_FULL_PATH/pgp_keys/*.key | sed -rn "s/.*<(.*)>/\1/p"
390
- }
391
-
392
- set_all_secret_keys() {
393
-
394
- import_all_pgp_keys
395
-
396
- git secret reveal -f
397
-
398
- remove_all_persons_from_secrets
399
-
400
- if [ $# -eq 0 ]; then
401
- echo "No emails supplied, using dev-tools pgp keys as source"
402
- IN_THE_KNOW=($(gum choose --no-limit $(all_pgp_emails)))
403
- else
404
- IN_THE_KNOW=($*)
405
- fi
406
-
407
- git secret tell $IN_THE_KNOW
408
- git secret hide
409
- git secret whoknows
410
-
411
- echo "all secrets updated, you'll need to commit the changes"
412
- }
413
-
414
- #!/usr/bin/env bash
415
-
416
- start_ssh_bastion() {
417
- ENV=$1
418
- SSH_LOCAL_PORT=$2
419
- POD_NAME=ssh-bastion-$USERNAME
420
- CONFIG_MAP_NAME=ssh-bastion-$USERNAME
421
- configure_kubectl_for $ENV
422
- kubectl get pods -o name | grep pod/$POD_NAME
423
- if [ $? -eq 0 ]; then
424
- echo "$POD_NAME is already running"
425
- else
426
- #configmap
427
- kubectl get configmap $CONFIG_MAP_NAME && kubectl delete configmap $CONFIG_MAP_NAME
428
- tempdir=$(mktemp -d)
429
- cat <<EOF > $tempdir/sshd_config
430
- AllowTcpForwarding yes
431
- Port 2222
432
- PermitRootLogin yes
433
- AuthorizedKeysFile /etc/ssh/authorized_keys
434
- EOF
435
- cp ~/.ssh/id_rsa.pub $tempdir/authorized_keys
436
- kubectl create configmap $CONFIG_MAP_NAME --from-file=$tempdir
437
-
438
- #pod
439
- kubectl get pod $POD_NAME && kubectl delete pod $POD_NAME
440
- cat <<EOF | kubectl create -f -
441
-
442
- apiVersion: v1
443
- kind: Pod
444
- metadata:
445
- name: $POD_NAME
446
- spec:
447
- containers:
448
- - name: $POD_NAME
449
- image: sickp/alpine-sshd:7.4
450
- ports:
451
- - containerPort: 2222
452
- volumeMounts:
453
- - mountPath: /etc/ssh/sshd_config
454
- name: ssh-config
455
- subPath: sshd_config
456
- - mountPath: /etc/ssh/authorized_keys
457
- name: ssh-config
458
- subPath: authorized_keys
459
- volumes:
460
- - name: ssh-config
461
- configMap:
462
- name: $CONFIG_MAP_NAME
463
- EOF
464
-
465
- fi
466
-
467
- # You need a recent kubectl for wait to work (1.15 works), install or upgrade
468
- # with brew :
469
- # brew install kubernetes-cli
470
- # brew upgrade kubernetes-cli
471
- kubectl wait --for=condition=Ready pod/$POD_NAME
472
-
473
- # kube port-forward
474
- lsof -ti tcp:$SSH_LOCAL_PORT | xargs kill
475
- kubectl port-forward $POD_NAME $SSH_LOCAL_PORT:2222 &
476
- while ! nc -z 127.0.0.1 $SSH_LOCAL_PORT; do
477
- sleep 1
478
- done
479
- echo "forwarding ssh via local port $SSH_LOCAL_PORT"
480
- echo "remember to terminate the bastion with 'stop_ssh_bastion'"
481
- }
482
-
483
- stop_ssh_bastion() {
484
- POD_NAME=ssh-bastion-$USERNAME
485
- kubectl delete pod $POD_NAME
486
- }
487
-
488
- #!/usr/bin/env bash
489
-
490
- configure_kubectl_for() {
491
- local infra_env="$1"
492
- local valid_envs="[testing][staging][production][performance][tests][recette]"
493
- echo "$valid_envs" | grep -q "\[$infra_env\]"
494
-
495
- if [ $? -ne 0 ]; then
496
- echo "Cannot configure kubectl for invalid env : $infra_env"
497
- echo "choose one of $valid_envs"
498
- return 1
499
- fi
500
-
501
- aws eks update-kubeconfig --name "toutatis-$infra_env-eks" >&2
502
- }
503
-
504
- #!/usr/bin/env bash
505
-
506
- # WARNING : never try to do a dump directly from the database_production_ca
507
- # this could cause lot of lock database issues.
508
- # always use database_production_read_replica_ca instead
509
- database_k8s() {
510
- MODE=$1
511
- case $MODE in
512
- "tests") SSH_LOCAL_PORT=2224;PG_LOCAL_PORT=24440;CA_LOCAL_PORT=25430;ENV="tests";;
513
- "testing") SSH_LOCAL_PORT=2225;PG_LOCAL_PORT=24441;CA_LOCAL_PORT=25431;ENV="testing";;
514
- "staging") SSH_LOCAL_PORT=2226;PG_LOCAL_PORT=24442;CA_LOCAL_PORT=25432;ENV="staging";;
515
- "production") SSH_LOCAL_PORT=2227;PG_LOCAL_PORT=24443;CA_LOCAL_PORT=25433;ENV="production";;
516
- "production_rw") SSH_LOCAL_PORT=2227;PG_LOCAL_PORT=24444;CA_LOCAL_PORT=25434;ENV="production";;
517
- "recette") SSH_LOCAL_PORT=2228;PG_LOCAL_PORT=24446;CA_LOCAL_PORT=25436;ENV="recette";;
518
- *) echo "Unsupported ENV : $MODE"; return 1 ;;
519
- esac
520
-
521
- start_ssh_bastion $ENV $SSH_LOCAL_PORT
522
-
523
- lsof -ti tcp:$PG_LOCAL_PORT | xargs kill
524
-
525
- bastion_config=$(mktemp)
526
- cat > "$bastion_config" <<EOF
527
- UserKnownHostsFile /dev/null
528
- StrictHostKeyChecking no
529
- User root
530
- Host bastion_tests
531
- HostName 127.0.0.1
532
- Port 2224
533
- LocalForward 24440 toutatis-tests-db.ca0rjdmnxf1x.eu-west-1.rds.amazonaws.com:5432
534
- Host bastion_testing
535
- HostName 127.0.0.1
536
- Port 2225
537
- LocalForward 24441 toutatis-testing-db.ca0rjdmnxf1x.eu-west-1.rds.amazonaws.com:5432
538
- LocalForward 25431 toutatis-testing-mysql-db.ca0rjdmnxf1x.eu-west-1.rds.amazonaws.com:3306
539
- LocalForward 25531 testapirds.ca0rjdmnxf1x.eu-west-1.rds.amazonaws.com:3306
540
- Host bastion_staging
541
- HostName 127.0.0.1
542
- Port 2226
543
- LocalForward 24442 toutatis-staging-db.ca0rjdmnxf1x.eu-west-1.rds.amazonaws.com:5432
544
- LocalForward 25432 testapirds.ca0rjdmnxf1x.eu-west-1.rds.amazonaws.com:3306
545
- Host bastion_recette
546
- HostName 127.0.0.1
547
- Port 2228
548
- LocalForward 24446 toutatis-recette-db.ca0rjdmnxf1x.eu-west-1.rds.amazonaws.com:5432
549
- LocalForward 25436 toutatis-recette-mysql-db.ca0rjdmnxf1x.eu-west-1.rds.amazonaws.com:3306
550
- LocalForward 25536 testapirds.ca0rjdmnxf1x.eu-west-1.rds.amazonaws.com:3306
551
- Host bastion_production
552
- HostName 127.0.0.1
553
- Port 2227
554
- LocalForward 24443 toutatis-production-db-replica.ca0rjdmnxf1x.eu-west-1.rds.amazonaws.com:5432
555
- LocalForward 25433 api-production-rds-read-replica.ca0rjdmnxf1x.eu-west-1.rds.amazonaws.com:3306
556
- LocalForward 25435 archive-ca.ca0rjdmnxf1x.eu-west-1.rds.amazonaws.com:3306
557
- EOF
558
- if [ "$MODE" = "production_rw" ] ; then
559
- cat >> "$bastion_config" <<EOF
560
- LocalForward 24444 toutatis-production-db.ca0rjdmnxf1x.eu-west-1.rds.amazonaws.com:5432
561
- LocalForward 25434 api-production-rds.ca0rjdmnxf1x.eu-west-1.rds.amazonaws.com:3306
562
- EOF
563
- fi
564
-
565
- ssh -f -N \
566
- -F "$bastion_config" \
567
- "bastion_$ENV"
568
-
569
- echo "sample command : 'psql postgres://postgres@127.0.0.1:$PG_LOCAL_PORT'"
570
- echo "sample command : 'mysql -u colisweb -h 127.0.0.1 -P $CA_LOCAL_PORT -p db_name'"
571
-
572
- echo "run 'kubectl delete pod $POD_NAME' when you have finished"
573
- }
574
-
575
- psql_on_k8() {
576
- NAMESPACE=$1
577
- SERVICE=$2
578
- CONNECTION=$3
579
- shift 3
580
-
581
- kubectl -n $NAMESPACE run ${SERVICE}-database-init \
582
- --image jbergknoff/postgresql-client \
583
- --restart=Never \
584
- --attach --rm \
585
- -- \
586
- postgresql://${CONNECTION} \
587
- "$*"
588
- }
589
-
590
- mysql_on_k8() {
591
- local namespace=$1
592
- local db_host=$2
593
- local db_port=$3
594
- local db_init_username=$4
595
- local db_init_password=$5
596
- local query=$6
597
-
598
- kubectl -n ${namespace} run datadog-database-init \
599
- --image widdpim/mysql-client \
600
- --restart=Never \
601
- --attach --rm \
602
- -- \
603
- mysql --host=$db_host --user=$db_init_username --password=$db_init_password --port=$db_port --execute="$query"
604
- }
605
- #!/usr/bin/env bash
606
-
607
- kube_init_database_once() {
608
-
609
- extract_args 8 namespace db_host db_port db_init_username db_init_password db_database db_username db_password $*
610
-
611
- echo "======================="
612
- echo " Initializing Database '$db_database' for namespace $namespace"
613
- echo "======================="
614
-
615
- set -x
616
-
617
- echo "Checking if Database '$db_database' exists"
618
- set +e
619
- psql_on_k8 $namespace once "$db_init_username:$db_init_password@$db_host:$db_port" -lqtA | cut -d\| -f1 | grep "^$db_database$"
620
- return_code=$?
621
- set -e
622
-
623
- if [ ${return_code} -eq 0 ]; then
624
- echo "Database $db_database already exists - nothing to do"
625
- else
626
- echo "Database $db_database does not exist - initializing"
627
-
628
- psql_on_k8 $namespace once "$db_init_username:$db_init_password@$db_host:$db_port" -c 'CREATE DATABASE '"$db_database"';'
629
- echo "DB created $db_database"
630
-
631
- psql_on_k8 $namespace once "$db_init_username:$db_init_password@$db_host:$db_port" -c 'CREATE USER '"$db_username"' WITH ENCRYPTED PASSWORD '"'$db_password'"';'
632
- echo "USER created $db_username"
633
-
634
- psql_on_k8 $namespace once "$db_init_username:$db_init_password@$db_host:$db_port" -c 'GRANT ALL PRIVILEGES ON DATABASE '"$db_database"' TO '"$db_username"';'
635
- echo "Granted all privileges for $db_username on $db_database"
636
- fi
637
-
638
- echo "======================="
639
- echo " Database '$db_database' Initialization complete for namespace $namespace"
640
- echo "======================="
641
- }
642
-
643
- kube_init_database_readonly_account() {
644
-
645
- extract_args 6 namespace service db_connection db_database db_readonly_username db_readonly_password $*
646
-
647
- echo "======================="
648
- echo " Initializing Readonly Account '$db_readonly_username' for '$db_database' for namespace $namespace"
649
- echo "======================="
650
-
651
- # Print commands before execution, except echo
652
- trap '[[ $BASH_COMMAND != echo* ]] && echo $BASH_COMMAND' DEBUG
653
-
654
- echo "Checking if Readonly account '$db_readonly_username' for '$db_database' exists"
655
- set +e
656
- psql_on_k8 $namespace $service $db_connection -qtAc 'SELECT rolname FROM pg_roles;' | grep "^$db_readonly_username$"
657
- return_code=$?
658
- set -e
659
-
660
- if [ ${return_code} -eq 0 ]; then
661
- echo "Account $db_readonly_username already exists - nothing to do"
662
- else
663
- echo "Account $db_readonly_username does not exist - creating"
664
-
665
- psql_on_k8 $namespace $service $db_connection -c 'CREATE USER '"$db_readonly_username"' WITH ENCRYPTED PASSWORD '"'$db_readonly_password'"';'
666
- psql_on_k8 $namespace $service $db_connection -c 'GRANT CONNECT ON DATABASE '"$db_database"' TO '"$db_readonly_username"';'
667
- psql_on_k8 $namespace $service $db_connection -c 'GRANT USAGE ON SCHEMA public TO '"$db_readonly_username"';'
668
- psql_on_k8 $namespace $service $db_connection -c 'GRANT SELECT ON ALL TABLES IN SCHEMA public TO '"$db_readonly_username"';'
669
- psql_on_k8 $namespace $service $db_connection -c 'ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO '"$db_readonly_username"';'
670
-
671
- echo "Created user with read-only permissions for $db_readonly_username on $db_database (schema public)"
672
- fi
673
- }
674
-
675
- kube_init_datadog_in_database() {
676
- extract_args 8 namespace db_host db_port db_init_username db_init_password db_datadog_username db_datadog_password db_datadog_schema $*
677
-
678
- echo "======================="
679
- echo " Initializing Datadog Agent Requiement for namespace $namespace"
680
- echo "======================="
681
-
682
- echo "Checking if User '$db_datadog_username' exists"
683
- set +e
684
- mysql_on_k8 $namespace $db_host $db_port $db_init_username $db_init_password 'SELECT user FROM mysql.user;' | grep "^$db_datadog_username$"
685
- return_code=$?
686
- set -e
687
-
688
- if [ ${return_code} -eq 0 ]; then
689
- echo "User $db_datadog_username already exists - nothing to do"
690
- else
691
- echo "User $db_datadog_username does not exist - initializing"
692
-
693
- # All the query come from this docs : https://docs.datadoghq.com/fr/database_monitoring/setup_mysql/selfhosted/?tab=mysql56
694
-
695
- mysql_on_k8 $namespace $db_host $db_port $db_init_username $db_init_password 'CREATE USER '"$db_datadog_username"'@"%" IDENTIFIED BY '"'$db_datadog_password'"';'
696
- echo "USER created $db_datadog_username"
697
-
698
- mysql_on_k8 $namespace $db_host $db_port $db_init_username $db_init_password 'GRANT REPLICATION CLIENT ON *.* TO datadog@"%" WITH MAX_USER_CONNECTIONS 5;'
699
- echo "ALTER USER $db_datadog_username"
700
-
701
- mysql_on_k8 $namespace $db_host $db_port $db_init_username $db_init_password 'GRANT PROCESS ON *.* TO '"$db_datadog_username"'@"%";'
702
- echo "Granted PROCESS for $db_datadog_username"
703
-
704
- mysql_on_k8 $namespace $db_host $db_port $db_init_username $db_init_password 'GRANT SELECT ON performance_schema.* TO '"$db_datadog_username"'@"%";'
705
- echo "Granted SELECT on performance_schema for $db_datadog_username"
706
-
707
- mysql_on_k8 $namespace $db_host $db_port $db_init_username $db_init_password 'CREATE SCHEMA IF NOT EXISTS datadog;'
708
- echo "CREATE SCHEMA datadog"
709
-
710
- mysql_on_k8 $namespace $db_host $db_port $db_init_username $db_init_password 'GRANT EXECUTE ON datadog.* to '"$db_datadog_username"'@"%";'
711
- echo "Granted 'GRANT EXECUTE for $db_datadog_username on datadog"
712
-
713
- mysql_on_k8 $namespace $db_host $db_port $db_init_username $db_init_password 'GRANT CREATE TEMPORARY TABLES ON datadog.* TO '"$db_datadog_username"'@"%";'
714
- echo "Granted CREATE TEMPORARY TABLES for $db_datadog_username"
715
-
716
-
717
- mysql_on_k8 $namespace $db_host $db_port $db_init_username $db_init_password 'DROP PROCEDURE IF EXISTS datadog.explain_statement;
718
- DELIMITER $$
719
- CREATE PROCEDURE datadog.explain_statement(IN query TEXT)
720
- SQL SECURITY DEFINER
721
- BEGIN
722
- SET @explain := CONCAT("EXPLAIN FORMAT=json ", query);
723
- PREPARE stmt FROM @explain;
724
- EXECUTE stmt;
725
- DEALLOCATE PREPARE stmt;
726
- END $$
727
- DELIMITER ;'
728
- echo "CREATE PROCEDURE PROCEDURE datadog.explain_statement"
729
-
730
- mysql_on_k8 $namespace $db_host $db_port $db_init_username $db_init_password 'DROP PROCEDURE IF EXISTS '"$db_datadog_username"'.explain_statement;
731
- DELIMITER $$
732
- CREATE PROCEDURE '"$db_datadog_username"'.explain_statement(IN query TEXT)
733
- SQL SECURITY DEFINER
734
- BEGIN
735
- SET @explain := CONCAT("EXPLAIN FORMAT=json ", query);
736
- PREPARE stmt FROM @explain;
737
- EXECUTE stmt;
738
- DEALLOCATE PREPARE stmt;
739
- END $$
740
- DELIMITER ;
741
- GRANT EXECUTE ON PROCEDURE '"$db_datadog_username"'.explain_statement TO datadog@"%";'
742
- echo "CREATE PROCEDURE on SCHEMA $db_datadog_schema for $db_datadog_username"
743
-
744
- mysql_on_k8 $namespace $db_host $db_port $db_init_username $db_init_password 'DROP PROCEDURE IF EXISTS datadog.enable_events_statements_consumers;
745
- DELIMITER $$
746
- CREATE PROCEDURE datadog.enable_events_statements_consumers()
747
- SQL SECURITY DEFINER
748
- BEGIN
749
- UPDATE performance_schema.setup_consumers SET enabled="YES" WHERE name LIKE "events_statements_%";
750
- END $$
751
- DELIMITER ;
752
- GRANT EXECUTE ON PROCEDURE datadog.enable_events_statements_consumers TO datadog@"%";'
753
-
754
- echo "CREATE PROCEDURE on datadog.enable_events_statements_consumers"
755
- fi
756
-
757
- echo "======================="
758
- echo " Database '$db_datadog_schema' Initialization complete for namespace $namespace"
759
- echo "======================="
760
- }
761
-
762
- kube_init_datadog_in_postgres_database() {
763
- extract_args 7 namespace db_host db_port db_init_username db_init_password db_datadog_username db_datadog_password $*
764
-
765
- local service="datadog"
766
- local db_connection="$db_init_username:$db_init_password@$db_host:$db_port"
767
-
768
- echo "======================="
769
- echo " Initializing $service Agent On PostgresSQL Database Requirement for namespace $namespace"
770
- echo "======================="
771
-
772
- echo "Checking if User '$db_datadog_username' exists"
773
-
774
- set +e
775
- if psql_on_k8 $namespace $service $db_connection -qtAc 'SELECT usename FROM pg_catalog.pg_user;' | grep "^$db_datadog_username$";
776
- then
777
- echo "User $db_datadog_username already exists - nothing to do"
778
- else
779
- echo "User $db_datadog_username does not exist - initializing"
780
-
781
- set -e
782
- psql_on_k8 $namespace $service $db_connection -qc 'CREATE USER '"$db_datadog_username"' WITH password '"'$db_datadog_password'"';'
783
- echo "User created $db_datadog_username"
784
-
785
- psql_on_k8 $namespace $service $db_connection -qc 'CREATE SCHEMA datadog;'
786
- echo "Schema datadog created"
787
-
788
- psql_on_k8 $namespace $service $db_connection -qc 'GRANT USAGE ON SCHEMA datadog TO datadog;'
789
- echo "Granted usage for datadog schema to datadog"
790
-
791
- psql_on_k8 $namespace $service $db_connection -qc 'GRANT USAGE ON SCHEMA public TO datadog;'
792
- echo "Granted usage for public schema to datadog"
793
-
794
- psql_on_k8 $namespace $service $db_connection -qc 'GRANT pg_monitor TO datadog;'
795
- echo "Granted pg_monitor to datadog"
796
-
797
- psql_on_k8 $namespace $service $db_connection -qc 'CREATE EXTENSION IF NOT EXISTS pg_stat_statements schema public;'
798
- echo "Extension pg_stat_statements created"
799
-
800
- local datadog_function_path="/tmp/datatog-explain-statement-function.sql"
801
- local datadog_function="CREATE OR REPLACE FUNCTION datadog.explain_statement(
802
- l_query TEXT,
803
- OUT explain JSON
804
- )
805
- RETURNS SETOF JSON AS
806
- \\$\\$
807
- DECLARE
808
- curs REFCURSOR;
809
- plan JSON;
810
-
811
- BEGIN
812
- OPEN curs FOR EXECUTE pg_catalog.concat('EXPLAIN (FORMAT JSON) ', l_query);
813
- FETCH curs INTO plan;
814
- CLOSE curs;
815
- RETURN QUERY SELECT plan;
816
- END;
817
- \\$\\$
818
- LANGUAGE 'plpgsql'
819
- RETURNS NULL ON NULL INPUT
820
- SECURITY DEFINER;"
821
-
822
- kubectl -n $namespace run $service-database-init \
823
- --image jbergknoff/postgresql-client \
824
- --restart=Never \
825
- --attach --rm \
826
- --command \
827
- -- \
828
- /bin/sh -c "echo -e \"$datadog_function\" > $datadog_function_path; psql postgresql://$db_connection -qf $datadog_function_path"
829
-
830
- echo "Function datadog.explain_statement created"
831
- fi
832
-
833
- echo "======================="
834
- echo " Database $service Initialization complete for namespace $namespace"
835
- echo "======================="
836
- }
837
-
838
- kube_init_service_database() {
839
-
840
- extract_args 9 namespace service db_host db_port db_init_username db_init_password db_database db_username db_password $*
841
-
842
- local db_connection="$db_init_username:$db_init_password@$db_host:$db_port"
843
-
844
- set -x
845
-
846
- echo "Checking if Database '$db_database' exists"
847
- set +e
848
- psql_on_k8 $namespace $service $db_connection -lqtA | cut -d\| -f1 | grep "^$db_database$"
849
- return_code=$?
850
- set -e
851
-
852
- if [ ${return_code} -eq 0 ]; then
853
- echo "Database $db_database already exists - nothing to do"
854
- else
855
- echo "Database $db_database does not exist - initializing"
856
-
857
- psql_on_k8 $namespace $service $db_connection -c 'CREATE DATABASE '"$db_database"';'
858
- echo "DB created $db_database"
859
-
860
- psql_on_k8 $namespace $service $db_connection -c 'CREATE USER '"$db_username"' WITH ENCRYPTED PASSWORD '"'$db_password'"';'
861
- echo "USER created $db_username"
862
-
863
- psql_on_k8 $namespace $service $db_connection -c 'GRANT ALL PRIVILEGES ON DATABASE '"$db_database"' TO '"$db_username"';'
864
- echo "Granted all privileges for $db_username on $db_database"
865
- fi
866
-
867
- echo "======================="
868
- echo " Database '$db_database' Initialization complete for namespace $namespace"
869
- echo "======================="
870
- }
871
-
872
- #!/usr/bin/env bash
873
-
874
- # Port forward on the first matching pod
875
- # Ex :
876
- # pod_forward testing notification-http
877
- # pod_forward testing colisweb-api-web 3333 3000
878
- pod_forward() {
879
- ENV=$1
880
- POD_FILTER=$2
881
- LOCAL_PORT=${3:-8080}
882
- POD_PORT=${4:-8080}
883
-
884
- if PID=$(lsof -ti tcp:$LOCAL_PORT); then
885
- echo "killing process $PID which uses port $LOCAL_PORT"
886
- kill $PID
887
- fi
888
-
889
- configure_kubectl_for $ENV
890
-
891
- POD=`pick_pod $ENV $POD_FILTER`
892
-
893
- echo "setting up forwarding to $POD"
894
- kubectl -n $ENV port-forward $POD $LOCAL_PORT:$POD_PORT &
895
- PID=$!
896
-
897
- while ! echo exit | nc localhost $LOCAL_PORT > /dev/null; do
898
- sleep 1
899
- echo "waiting for port $LOCAL_PORT to be open locally"
900
- done
901
- echo "port $LOCAL_PORT is now available on localhost, forwarding to $ENV $POD:$POD_PORT"
902
- echo 'you can terminate it with "kill '$PID'" or "kill $(lsof -ti tcp:'$LOCAL_PORT')"'
903
- }
904
-
905
- # prompts to pick a pod and run a command like bash inside
906
- # pod_exec testing
907
- # pod_exec testing bash
908
- # pod_exec testing bash colisweb-api
909
- pod_exec() {
910
- ENV=$1
911
- COMMAND=${2:-bash}
912
- configure_kubectl_for $ENV
913
- POD_FILTER=$3
914
- POD=`pick_pod $ENV $POD_FILTER`
915
- echo "running $COMMAND inside $POD"
916
- kubectl -n $ENV exec -ti $POD -- $COMMAND
917
- }
918
-
919
- # prompts to pick a pod and copy from a local file to the pod
920
- # pod_copy_to testing localfile remotefile
921
- # pod_copy_to testing localfile remotefile colisweb-api
922
- pod_copy_to() {
923
- ENV=$1
924
- LOCAL_FILE=$2
925
- REMOTE_FILE=$3
926
- configure_kubectl_for $ENV
927
- POD_FILTER=$4
928
- POD=`pick_pod $ENV $POD_FILTER`
929
- kubectl cp $LOCAL_FILE $ENV/$POD:$REMOTE_FILE
930
- }
931
-
932
-
933
- pick_pod() {
934
- ENV=$1
935
- POD_FILTER="pod/$2"
936
- configure_kubectl_for $ENV
937
-
938
- if [ -z "$2" ] ; then
939
- kubectl -n $ENV get pods | gum filter | cut -f1 -d" "
940
- else
941
- if PODS=$(kubectl -n $ENV get pods -o=name | grep "$POD_FILTER"); then
942
- echo $PODS | head -1 | sed -e 's/pod\///'
943
- else
944
- echo "no pods found on $ENV matching $POD_FILTER" >&2
945
- fi
946
- fi
947
- }
948
-
949
- #!/usr/bin/env bash
950
-
951
- redis_k8s() {
952
- MODE=$1
953
- case $MODE in
954
- "testing") SSH_LOCAL_PORT=2225;REDIS_LOCAL_PORT=63791;ENV="testing";;
955
- "staging") SSH_LOCAL_PORT=2226;REDIS_LOCAL_PORT=63792;ENV="staging";;
956
- "production") SSH_LOCAL_PORT=2227;REDIS_LOCAL_PORT=63793;ENV="production";;
957
- *) echo "Unsupported ENV : $MODE"; return 1 ;;
958
- esac
959
-
960
- start_ssh_bastion $ENV $SSH_LOCAL_PORT
961
-
962
- lsof -ti tcp:$REDIS_LOCAL_PORT | xargs kill
963
-
964
- bastion_config=$(mktemp)
965
- cat > "$bastion_config" <<EOF
966
- UserKnownHostsFile /dev/null
967
- StrictHostKeyChecking no
968
- User root
969
- Host bastion_testing
970
- HostName 127.0.0.1
971
- Port 2225
972
- LocalForward 63791 redis-testing.xufte6.0001.euw1.cache.amazonaws.com:6379
973
- Host bastion_staging
974
- HostName 127.0.0.1
975
- Port 2226
976
- LocalForward 63792 redis-sandbox.xufte6.0001.euw1.cache.amazonaws.com:6379
977
- Host bastion_production
978
- HostName 127.0.0.1
979
- Port 2227
980
- LocalForward 63793 redis-prod.xufte6.0001.euw1.cache.amazonaws.com:6379
981
- EOF
982
-
983
- ssh -f -N \
984
- -F "$bastion_config" \
985
- "bastion_$ENV"
986
-
987
- echo "sample command : 'redis-cli -p $REDIS_LOCAL_PORT'"
988
- echo "run 'kubectl delete pod $POD_NAME' when you have finished"
989
-
990
- redis-cli -p $REDIS_LOCAL_PORT
991
- }
992
-
993
- #!/usr/bin/env bash
994
-
995
- #Create a k8s cron jobs that will be run regularly
996
- #See run_cron_job_k8s -h for more details
997
-
998
- run_cron_job_k8s() {
999
-
1000
- #default values
1001
- local namespace="testing"
1002
- local name="$USERNAME"
1003
- local SCHEDULE="00 05 * * *"
1004
- local secret=""
1005
- local amm_folder=""
1006
- local amm_script=""
1007
-
1008
- while getopts ":e:c:p:f:s:t:h" opt; do
1009
- case $opt in
1010
- e)
1011
- namespace="$OPTARG" >&2
1012
- ;;
1013
- t)
1014
- SCHEDULE="$OPTARG" >&2
1015
- ;;
1016
- p)
1017
- name="$OPTARG" >&2
1018
- ;;
1019
- c)
1020
- secret="$OPTARG" >&2
1021
- ;;
1022
- f)
1023
- amm_folder="$OPTARG" >&2
1024
- ;;
1025
- s)
1026
- amm_script="$OPTARG" >&2
1027
- ;;
1028
- h)
1029
- show_help_cron_job
1030
- return 0
1031
- ;;
1032
- :)
1033
- echo "Option -$OPTARG requires an argument. Run run_cron_job_k8s -h for help" >&2
1034
- return 0
1035
- ;;
1036
- \?)
1037
- echo "Invalid option: -$OPTARG. Run run_cron_job_k8s -h for help" >&2
1038
- return 0
1039
- ;;
1040
- esac
1041
- done
1042
-
1043
- if [ -z "$amm_script" ]; then
1044
- echo 'Missing -s. Run run_cron_job_k8s -h for help' >&2
1045
- return 0
1046
- fi
1047
-
1048
- shift "$((OPTIND-1))"
1049
-
1050
- local script_args=$(
1051
- if [ "$#" -gt 0 ] ; then
1052
- printf '"'
1053
- join_by '", "' $*
1054
- printf '"'
1055
- fi
1056
- )
1057
-
1058
- local IMAGE="lolhens/ammonite:2.5.4"
1059
- local CRONJOB_NAME="cronjob-ammonite-$name"
1060
-
1061
-
1062
- configure_kubectl_for $namespace
1063
-
1064
- if [[ ! -r "$amm_script" ]]; then
1065
- echo "ammonite script not found $amm_script"
1066
- return 2
1067
- else
1068
- local CONFIG_MAP="config-$CRONJOB_NAME"
1069
- local SECRET_MAP="secret-$CRONJOB_NAME"
1070
- local CONFIG_MAP_DIR="$(mktemp -d)"
1071
-
1072
- if [[ ! -z $amm_folder && -d $amm_folder ]] ; then
1073
- cp -r "$amm_folder/" "$CONFIG_MAP_DIR"
1074
- fi
1075
- cp "$amm_script" "$CONFIG_MAP_DIR/script.sc"
1076
-
1077
- kubectl -n $namespace get configmap $CONFIG_MAP && kubectl -n $namespace delete configmap $CONFIG_MAP
1078
- kubectl -n $namespace create configmap $CONFIG_MAP --from-file="$CONFIG_MAP_DIR"
1079
-
1080
- kubectl -n $namespace get secret $SECRET_MAP && kubectl -n $namespace delete secret $SECRET_MAP
1081
- kubectl -n $namespace create secret generic $SECRET_MAP --from-file="$secret"
1082
-
1083
- kubectl -n $namespace get cronjob $CRONJOB_NAME && kubectl -n $namespace delete cronjob $CRONJOB_NAME
1084
-
1085
- echo "starting $CRONJOB_NAME with $IMAGE"
1086
-
1087
- JOB_DEFINITION='
1088
- apiVersion: batch/v1
1089
- kind: CronJob
1090
- metadata:
1091
- name: '$CRONJOB_NAME'
1092
- namespace: '$namespace'
1093
- spec:
1094
- schedule: "'$SCHEDULE'"
1095
- concurrencyPolicy: Forbid
1096
- jobTemplate:
1097
- spec:
1098
- backoffLimit: 0
1099
- template:
1100
- spec:
1101
- nodeSelector:
1102
- workType: "workers"
1103
- restartPolicy: Never
1104
- volumes:
1105
- - name: config
1106
- configMap:
1107
- name: '$CONFIG_MAP'
1108
- - name: secret
1109
- secret:
1110
- secretName: '$SECRET_MAP'
1111
- containers:
1112
- - name: '$CRONJOB_NAME'
1113
- command: ["amm", "/code/script.sc"]
1114
- image: '$IMAGE'
1115
- imagePullPolicy: IfNotPresent
1116
- args: ['$script_args']
1117
- env:
1118
- - name: POD_NAME
1119
- valueFrom:
1120
- fieldRef:
1121
- apiVersion: v1
1122
- fieldPath: metadata.name
1123
- - name: POD_NAMESPACE
1124
- valueFrom:
1125
- fieldRef:
1126
- apiVersion: v1
1127
- fieldPath: metadata.namespace
1128
- - name: HOST_IP
1129
- valueFrom:
1130
- fieldRef:
1131
- apiVersion: v1
1132
- fieldPath: status.hostIP
1133
- volumeMounts:
1134
- - name: config
1135
- mountPath: /code
1136
- - name: secret
1137
- mountPath: /conf
1138
- readOnly: true
1139
- resources:
1140
- requests:
1141
- cpu: 500m
1142
- memory: 256Mi
1143
- limits:
1144
- cpu: 4000m
1145
- memory: 512Mi
1146
- envFrom:
1147
- - configMapRef:
1148
- name: '$CONFIG_MAP'
1149
- - secretRef:
1150
- name: '$SECRET_MAP'
1151
- '
1152
-
1153
- echo $JOB_DEFINITION > /tmp/job.yaml
1154
-
1155
- kubectl -n $namespace apply -f /tmp/job.yaml
1156
-
1157
- fi
1158
- }
1159
-
1160
- # Usage info
1161
- show_help_cron_job() {
1162
- #p:f:s
1163
- local help="""Usage: run_cron_job_k8s -s SCRIPT [-t TIME] [-e ENV] [-c CONFIG] [-p POD] [-f FOLDER] [ARGS]
1164
- Create a k8s cron job that will be run a script regularly
1165
-
1166
- -h display this help and exit
1167
- -s SCRIPT run script SCRIPT on a pod (SCRIPT must be a .sc file)
1168
- -t TIME opt. time when the job will be launched. TIME should be in CRON syntax (default to 00 05 * * *, ie 5AM UTC)
1169
- -e ENV opt. set execution environment (default to testing)
1170
- -c CONFIG opt. secret file needed for the script (must be a .sc file, not a .secret file)
1171
- -p POD opt. name of the pod to create (default to $USERNAME)
1172
- -f FOLDER opt. name of the folder containing the scripts to execute (if SCRIPT needs other files)
1173
- ARGS opt. additional arguments for SCRIPT
1174
- """
1175
- echo "$help"
1176
- }
1177
-
1178
- #!/usr/bin/env bash
1179
-
1180
- # Usage info
1181
- show_help_job() {
1182
- local help="""Usage: run_job_k8s -s SCRIPT [-e ENV] [-c CONFIG] [-p POD] [-f FOLDER] [ARGS]
1183
- Create a k8s job executing a script
1184
-
1185
- -h display this help and exit
1186
- -s SCRIPT run script SCRIPT on a pod (SCRIPT must be a .sc file)
1187
- -e ENV opt. set execution environment (default to testing)
1188
- -c CONFIG opt. secret file needed for the script (must be a .sc file, not a .secret file)
1189
- -p POD opt. name of the pod to create (default to $USERNAME)
1190
- -f FOLDER opt. name of the folder containing the scripts to execute (if SCRIPT needs other files)
1191
- ARGS opt. additional arguments for SCRIPT
1192
-
1193
- The organisation of the files must be the same locally as on the pod :
1194
- - /code containing the script to execute (arg -s) and the other needed files (if the arg -f is used, it must reference this directory)
1195
- - /conf containing the secret file (arg -c if used)
1196
- E.g. in the script \"/code/script.sc\", to use a secret file \"/conf/secret.sc\", the import should look like \"import \$file.^.conf.secret.sc\"
1197
- """
1198
- echo "$help"
1199
- }
1200
-
1201
- run_job_k8s() {
1202
-
1203
- #default values
1204
- local namespace="testing"
1205
- local name="$USERNAME"
1206
- local secret=""
1207
- local amm_folder=""
1208
- local amm_script=""
1209
-
1210
- while getopts ":e:c:p:f:s:h" opt; do
1211
- case $opt in
1212
- e)
1213
- namespace="$OPTARG" >&2
1214
- ;;
1215
- p)
1216
- name="$OPTARG" >&2
1217
- ;;
1218
- c)
1219
- secret="$OPTARG" >&2
1220
- ;;
1221
- f)
1222
- amm_folder="$OPTARG" >&2
1223
- ;;
1224
- s)
1225
- amm_script="$OPTARG" >&2
1226
- ;;
1227
- h)
1228
- show_help_job
1229
- return 0
1230
- ;;
1231
- :)
1232
- echo "Option -$OPTARG requires an argument. Run run_cron_job_k8s -h for help" >&2
1233
- return 0
1234
- ;;
1235
- \?)
1236
- echo "Invalid option: -$OPTARG. Run run_cron_job_k8s -h for help" >&2
1237
- return 0
1238
- ;;
1239
- esac
1240
- done
1241
-
1242
- if [ -z "$amm_script" ]; then
1243
- echo 'Missing -s. Run run_job_k8s -h for help' >&2
1244
- return 0
1245
- fi
1246
-
1247
- shift "$((OPTIND-1))"
1248
-
1249
- local script_args=$(
1250
- if [ "$#" -gt 0 ] ; then
1251
- printf '"'
1252
- join_by '", "' $*
1253
- printf '"'
1254
- fi
1255
- )
1256
-
1257
- local IMAGE="lolhens/ammonite:2.5.4"
1258
- local JOB_NAME="job-ammonite-$name"
1259
-
1260
- if [[ ! -r "$amm_script" ]]; then
1261
- echo "ammonite script not found $amm_script"
1262
- return 2
1263
- else
1264
- local CONFIG_MAP="config-$JOB_NAME"
1265
- local CONFIG_MAP_DIR="$(mktemp -d)"
1266
- local SECRET_MAP="secret-$JOB_NAME"
1267
-
1268
- configure_kubectl_for $namespace
1269
-
1270
- if [[ ! -z $amm_folder && -d $amm_folder ]] ; then
1271
- cp -r "$amm_folder/" "$CONFIG_MAP_DIR"
1272
- fi
1273
- cp "$amm_script" "$CONFIG_MAP_DIR/script.sc"
1274
-
1275
- kubectl -n $namespace get configmap $CONFIG_MAP && kubectl -n $namespace delete configmap $CONFIG_MAP
1276
- kubectl -n $namespace create configmap $CONFIG_MAP --from-file="$CONFIG_MAP_DIR"
1277
-
1278
- kubectl -n $namespace get secret $SECRET_MAP && kubectl -n $namespace delete secret $SECRET_MAP
1279
- kubectl -n $namespace create secret generic $SECRET_MAP --from-file="$secret"
1280
-
1281
- kubectl -n $namespace get job $JOB_NAME && kubectl -n $namespace delete job $JOB_NAME
1282
-
1283
- echo "starting $JOB_NAME with $IMAGE"
1284
- fi
1285
-
1286
- JOB_DEFINITION='
1287
- apiVersion: batch/v1
1288
- kind: Job
1289
- metadata:
1290
- name: '$JOB_NAME'
1291
- namespace: '$namespace'
1292
- spec:
1293
- template:
1294
- spec:
1295
- containers:
1296
- - name: '$JOB_NAME'
1297
- command: ["amm", "/code/script.sc"]
1298
- image: '$IMAGE'
1299
- args: ['$script_args']
1300
- env:
1301
- - name: POD_NAME
1302
- valueFrom:
1303
- fieldRef:
1304
- apiVersion: v1
1305
- fieldPath: metadata.name
1306
- - name: POD_NAMESPACE
1307
- valueFrom:
1308
- fieldRef:
1309
- apiVersion: v1
1310
- fieldPath: metadata.namespace
1311
- - name: HOST_IP
1312
- valueFrom:
1313
- fieldRef:
1314
- apiVersion: v1
1315
- fieldPath: status.hostIP
1316
- volumeMounts:
1317
- - name: config
1318
- mountPath: /code
1319
- - name: secret
1320
- mountPath: /conf
1321
- readOnly: true
1322
- resources:
1323
- requests:
1324
- cpu: 500m
1325
- memory: 256Mi
1326
- limits:
1327
- cpu: 4000m
1328
- memory: 1Gi
1329
- nodeSelector:
1330
- workType: workers
1331
- restartPolicy: Never
1332
- volumes:
1333
- - name: config
1334
- configMap:
1335
- name: '$CONFIG_MAP'
1336
- - name: secret
1337
- secret:
1338
- secretName: '$SECRET_MAP'
1339
- '
1340
-
1341
-
1342
- echo $JOB_DEFINITION > /tmp/job.yaml
1343
-
1344
- kubectl -n $namespace apply -f /tmp/job.yaml
1345
-
1346
- }
1347
-
1348
-
1349
- #!/usr/bin/env bash
1350
-
1351
- run_task() {
1352
- set -e
1353
-
1354
- check_args "--namespace" $1
1355
- shift
1356
- NAMESPACE=$1
1357
- shift
1358
- check_args "--image" $1
1359
- shift
1360
- IMAGE=$1
1361
- shift
1362
- check_args "--name" $1
1363
- shift
1364
- NAME=$1
1365
- shift
1366
-
1367
- set -x
1368
-
1369
- kubectl -n ${NAMESPACE} run ${NAME} \
1370
- --image ${IMAGE} \
1371
- --restart=Never \
1372
- --attach --rm \
1373
- $*
1374
- }
1375
- geocode_address() {
1376
- ADDRESS=$(sed -e 's: :%20:g' <(echo "$*"))
1377
- URL="https://maps.googleapis.com/maps/api/geocode/json?address=${ADDRESS}&key=${GOOGLE_API_KEY}"
1378
- curl $URL
1379
- }
1380
-
1381
- search_business() {
1382
- SIREN=$1
1383
- shift
1384
- QUERY=$(sed -e 's: :+:g' <(echo "$*"))
1385
- URL="https://data.opendatasoft.com/api/records/1.0/search/?dataset=sirene_v3%40public&q=${QUERY}&sort=datederniertraitementetablissement&facet=trancheeffectifsetablissement&facet=libellecommuneetablissement&facet=departementetablissementi&refine.siren=${SIREN}"
1386
- curl $URL
1387
- }
1388
-
1389
- #!/bin/bash
1390
-
1391
- # source tolls.sh ; tolls antoine.thomas@colisweb.com
1392
- function tolls() {
1393
- USER=${1:-first.last@colisweb.com}
1394
- FROM_DATE=${2:-"2023-02-01"}
1395
- TO_DATE=${3:-"2023-02-28"}
1396
-
1397
- USER=$(gum input --prompt "username : " --value $USER)
1398
- TOKEN=$(./tour_details.sc login --user $USER --password $(gum input --password --placeholder password))
1399
- [ "$TOKEN" != "" ] && echo "connected" || return 1
1400
-
1401
- FROM_DATE=$(gum input --prompt "Date start : " --value $FROM_DATE)
1402
- TO_DATE=$(gum input --prompt "Date end : " --value $TO_DATE)
1403
- FILENAME="tours-${FROM_DATE}-TO-${TO_DATE}.json"
1404
- curl --cookie "session=$TOKEN" "https://api.production.colisweb.com/api/v6/routes-plans/external?from=${FROM_DATE}&to=${TO_DATE}" > ~/Downloads/$FILENAME
1405
- echo "Tournées téléchargées"
1406
-
1407
- projectIds=$(./tour_details.sc allProjects --file ~/Downloads/$FILENAME | gum choose --no-limit | cut -d "," -f 2)
1408
- echo "projets sélectionnés : $projectIds"
1409
- tourIds=$(./tour_details.sc allTours --file ~/Downloads/$FILENAME --projectIds "$projectIds")
1410
- echo "tournées sélectionnées : $tourIds"
1411
-
1412
- TARGET="${FROM_DATE}-TO-${TO_DATE}.csv"
1413
- echo "appels à HERE, écriture dans $TARGET"
1414
- ./tour_details.sc allToursDetails --token $TOKEN --hereApiKey $HERE_API_KEY --routeIds "$tourIds" > "$TARGET"
1415
-
1416
- echo "terminé"
1417
- }
1418
-
1419
- #!/usr/bin/env bash
1420
-
1421
- # possible syntax:
1422
- # login
1423
- # login testing
1424
- # login testing userid
1425
- login() {
1426
- ENV=${1:-`gum choose testing staging production recette`} && \
1427
- USER=${2:-`gum input --placeholder username`} && \
1428
- PASSWORD=`gum input --password --placeholder password` && \
1429
- TOKEN=`$SCRIPT_FULL_PATH/scala/auth.sc login --env $ENV --user $USER --password $PASSWORD` && \
1430
- export TOKEN_$ENV=$TOKEN && \
1431
- echo "login success for $USER on $ENV" >&2
1432
- }
1433
-
1434
- # you need to call login first (see above)
1435
- # possible syntax:
1436
- # recompute_tour
1437
- # recompute_tour testing
1438
- # recompute_tour testing draft
1439
- # recompute_tour testing draft 28bf9967-b5f3-4294-8855-cfd2fa36ec09
1440
- # recompute_tour testing draft 28bf9967-b5f3-4294-8855-cfd2fa36ec09 TODAY
1441
- # recompute_tour testing draft 28bf9967-b5f3-4294-8855-cfd2fa36ec09 FRIDAY
1442
- recompute_tour() {
1443
- ENV=${1:-`gum choose testing staging production recette`}
1444
- MODE=${2:-`gum choose draft definitive`}
1445
- PROJECT_ID=${3:-`pick_project $ENV`}
1446
- DAY=${4:-`gum choose TODAY MONDAY TUESDAY WEDNESDAY THURSDAY FRIDAY SATURDAY SUNDAY`}
1447
- jwt_token $ENV
1448
- scala/tour_config.sc $MODE -t $TOKEN -p $PROJECT_ID -d $DAY
1449
- }
1450
-
1451
- pick_project() {
1452
- ENV=${1:-`gum choose testing staging production recette`}
1453
- jwt_token $ENV
1454
- scala/tour_config.sc list -t $TOKEN -e $ENV | gum filter | cut -f1
1455
- }
1456
-
1457
- jwt_token() {
1458
- ENV=${1:-`gum choose testing staging production recette`}
1459
- eval 'TOKEN=$TOKEN_'$ENV
1460
- if ! $SCRIPT_FULL_PATH/scala/auth.sc check -t $TOKEN -e $ENV ; then
1461
- login $ENV
1462
- fi
1463
- }
1464
-
1465
- #!/usr/bin/env bash
1466
-
1467
- ftp_ikea_k8s() {
1468
- SSH_LOCAL_PORT=2230
1469
- FTP_LOCAL_PORT=25500
1470
- start_ssh_bastion testing $SSH_LOCAL_PORT
1471
-
1472
- lsof -ti tcp:$FTP_LOCAL_PORT | xargs kill
1473
-
1474
- bastion_config=$(mktemp)
1475
- cat > "$bastion_config" <<EOF
1476
- UserKnownHostsFile /dev/null
1477
- StrictHostKeyChecking no
1478
- User root
1479
- Host bastion_ftp
1480
- HostName 127.0.0.1
1481
- Port 2230
1482
- LocalForward 25500 ft.centiro.ikea.com:22
1483
- EOF
1484
-
1485
- ssh -f -N \
1486
- -F "$bastion_config" \
1487
- "bastion_ftp"
1488
-
1489
- sftp -P $FTP_LOCAL_PORT colisweb.fr@127.0.0.1
1490
- }
1491
-
1492
- #!/usr/bin/env bash
1493
-
1494
- # usage:
1495
- # jconsole_k8s testing colisweb-api-web
1496
-
1497
- jconsole_k8s() {
1498
- ENV=$1
1499
- NAME=$2
1500
-
1501
- start_ssh_bastion $ENV 2242
1502
- POD_IP=$( \
1503
- kubectl -n $ENV get pods -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.status.podIP}{"\n"}{end}' \
1504
- | grep "$NAME" | cut -d' ' -f2 | head -1 \
1505
- )
1506
- echo "selected POD with ip $POD_IP"
1507
- echo "use 'root' as password"
1508
- ssh -f -N -oUserKnownHostsFile=/dev/null -oStrictHostKeyChecking=no -D 7777 root@127.0.0.1 -p 2242
1509
-
1510
- jconsole \
1511
- -J-DsocksProxyHost=localhost \
1512
- -J-DsocksProxyPort=7777 \
1513
- -J-DsocksNonProxyHosts= \
1514
- service:jmx:rmi:///jndi/rmi://$POD_IP:7199/jmxrmi \
1515
- &
1516
-
1517
- echo "remember to stop with 'stop_ssh_bastion'"
1518
-
1519
- }
1520
-
1521
- #!/usr/bin/env bash
1522
-
1523
- # Interactive console on an new pod. See also run_ruby_k8s
1524
- # Ex :
1525
- # railsc_k8s production
1526
- # railsc_k8s production "User.where(email:'toni@colisweb.com')"
1527
- railsc_k8s() {
1528
- ENV=$1
1529
- COMMAND=$2
1530
- [[ $ENV = "production" || $ENV = "staging" ]] && default_tag="master-latest" || default_tag="${ENV}-latest"
1531
- local image_tag=${5:-$default_tag}
1532
- local IMAGE="949316342391.dkr.ecr.eu-west-1.amazonaws.com/colisweb-api:$image_tag"
1533
- local POD_NAME="colisweb-api-rails-console-$image_tag-$USERNAME"
1534
-
1535
- kubectl -n $ENV get pod $POD_NAME && kubectl -n $ENV delete pod $POD_NAME
1536
-
1537
- configure_kubectl_for $ENV
1538
- echo "starting with $IMAGE"
1539
-
1540
- kubectl -n $ENV run $POD_NAME \
1541
- --image $IMAGE \
1542
- --restart=Never \
1543
- --overrides='{
1544
- "spec":{
1545
- "nodeSelector":{
1546
- "workType": "workers"
1547
- },
1548
- "containers":[
1549
- {
1550
- "name":"'$POD_NAME'",
1551
- "image":"'$IMAGE'",
1552
- "imagePullPolicy":"Always",
1553
- "command":[
1554
- "sleep",
1555
- "infinity"
1556
- ],
1557
- "resources":{
1558
- "limits":{
1559
- "memory": "2048Mi"
1560
- }
1561
- },
1562
- "envFrom": [ {
1563
- "configMapRef": {
1564
- "name": "colisweb-api"
1565
- }
1566
- }, {
1567
- "secretRef": {
1568
- "name": "colisweb-api"
1569
- }
1570
- }
1571
- ]
1572
- }
1573
- ]
1574
- }
1575
- }
1576
- '
1577
-
1578
- sleep 5
1579
- KUBERAILS="kubectl -n $ENV exec -ti $POD_NAME -- /usr/src/app/bin/rails c"
1580
- [ -z "$COMMAND" ] && eval $KUBERAILS || echo $COMMAND | eval $KUBERAILS
1581
-
1582
- print "End of $POD_NAME "
1583
- kubectl -n $ENV delete pods $POD_NAME
1584
- }
1585
-
1586
- # Ex :
1587
- # create_user testing claire.lien@colisweb.com super_admin clairemdp
1588
- create_user() {
1589
- ENV=$1
1590
- EMAIL=$2
1591
- ROLE=$3
1592
- PASSWORD=$4
1593
- railsc_k8s $ENV "User.where(email:'$EMAIL', role:'$ROLE').first_or_create.update_attributes!(password: '$PASSWORD')"
1594
- }
1595
-
1596
- # Ex :
1597
- # delete_user testing claire.lien@colisweb.com
1598
- delete_user() {
1599
- ENV=$1
1600
- EMAIL=$2
1601
- railsc_k8s $ENV "User.find_by(email:'$EMAIL').destroy"
1602
- }
1603
-
1604
- # NON Interactive console on an new pod, for long-running tasks (a few minutes)
1605
- # See also railsc_k8s
1606
- # file.txt will be available from /conf/data.txt in the ruby code
1607
- # examples :
1608
- # run_ruby_k8s testing demo <(echo "pp JSON.parse(File.read('/conf/data.txt'))") <(echo '{ "content": 123 }')
1609
- # run_ruby_k8s testing demo ~/.oh-my-zsh/custom/dev-tools/shell-session/ruby/demo.rb <(echo '{ "content": 123 }')
1610
- run_ruby_k8s() {
1611
- if [ $# -lt 4 ]; then
1612
- echo "usage : run_ruby_k8s production name-for-pod script.rb file.txt"
1613
- return 1
1614
- fi
1615
- local namespace=$1
1616
- local name=$2
1617
- local ruby_script=$3
1618
- local input_data=$4
1619
- [[ $namespace = "production" || $namespace = "staging" ]] && default_tag="master-latest" || default_tag="${namespace}-latest"
1620
- local image_tag=${5:-$default_tag}
1621
-
1622
- if [ ! -r "$ruby_script" ]; then
1623
- echo "ruby script not found $ruby_script"
1624
- return 2
1625
- fi
1626
-
1627
- if [ ! -r "$input_data" ]; then
1628
- echo "data not found $input_data"
1629
- return 3
1630
- fi
1631
-
1632
-
1633
- local IMAGE="949316342391.dkr.ecr.eu-west-1.amazonaws.com/colisweb-api:$image_tag"
1634
- local POD_NAME="colisweb-api-script-$name"
1635
- local CONFIG_MAP="config-$POD_NAME"
1636
- local CONFIG_MAP_DIR="$(mktemp -d)"
1637
-
1638
-
1639
- configure_kubectl_for $namespace
1640
-
1641
-
1642
- cp "$ruby_script" "$CONFIG_MAP_DIR/script.rb"
1643
- cp "$input_data" "$CONFIG_MAP_DIR/data.txt"
1644
-
1645
- kubectl -n $namespace get configmap $CONFIG_MAP && kubectl -n $namespace delete configmap $CONFIG_MAP
1646
- kubectl -n $namespace create configmap $CONFIG_MAP --from-file="$CONFIG_MAP_DIR"
1647
-
1648
- kubectl -n $namespace get pod $POD_NAME && kubectl -n $namespace delete pod $POD_NAME
1649
-
1650
- echo "starting with $IMAGE"
1651
- kubectl -n $namespace run $POD_NAME \
1652
- --image $IMAGE \
1653
- -ti \
1654
- --restart=Never \
1655
- --attach \
1656
- --rm \
1657
- --overrides='{
1658
- "spec":{
1659
- "nodeSelector":{
1660
- "workType": "workers"
1661
- },
1662
- "containers":[
1663
- {
1664
- "name":"'$POD_NAME'",
1665
- "image":"'$IMAGE'",
1666
- "imagePullPolicy":"Always",
1667
- "command":[
1668
- "/usr/src/app/bin/rails",
1669
- "r",
1670
- "/conf/script.rb"
1671
- ],
1672
- "resources":{
1673
- "limits":{
1674
- "memory": "4096Mi"
1675
- }
1676
- },
1677
- "volumeMounts":[
1678
- {
1679
- "name":"conf",
1680
- "mountPath":"/conf"
1681
- }
1682
- ],
1683
- "envFrom": [ {
1684
- "configMapRef": {
1685
- "name": "colisweb-api"
1686
- }
1687
- }, {
1688
- "secretRef": {
1689
- "name": "colisweb-api"
1690
- }
1691
- }
1692
- ]
1693
- }
1694
- ],
1695
- "volumes":[
1696
- {
1697
- "name":"conf",
1698
- "configMap":{ "name":"'$CONFIG_MAP'" }
1699
- }
1700
- ]
1701
- }
1702
- }
1703
- '
1704
-
1705
- kubectl -n $namespace delete configmap $CONFIG_MAP
1706
- }
1707
-
1708
- # example:
1709
- # update_pickup_cp testing <( echo '{"wrong_cp": "59123", "corrected_cp": "59223", "delivery_ids": ["4192421", "4192425"]}' )
1710
- update_pickup_cp() {
1711
- run_ruby_k8s $1 update-pickup-cp "$SCRIPT_FULL_PATH/ruby/update_pickup_cp.rb" $2
1712
- }
1713
-
1714
-
1715
-
1716
- update_all_prices() {
1717
- local namespace=$1
1718
- local json_prices=$2
1719
-
1720
- local json_size=$(wc -c < "$json_prices")
1721
-
1722
- if ((json_size > 940000)); then
1723
- command -v jq || (echo "jq not found (use brew install jq)" && return 1)
1724
- local max_lines=3000
1725
- local total_lines=$(jq '. | length' $json_prices)
1726
- local iterations=$((total_lines / max_lines + 1))
1727
- echo "$json_prices is too big, I'll split it for you in blocks of $max_lines lines. It will take $iterations runs"
1728
- for (( i = 0 ; i < iterations ; i++ )) ; do
1729
- local start=$((i * max_lines))
1730
- local end=$(( (i + 1) * max_lines))
1731
- local split_file=$(mktemp)
1732
- jq -c ".[$start:$end]" $json_prices > $split_file
1733
- local split_lines=$(jq '. | length' $split_file)
1734
- echo "starting iteration $i from $start to $end with $split_file command -v has $split_lines lines"
1735
- run_ruby_k8s $namespace "update-prices-$i" "$SCRIPT_FULL_PATH/ruby/update_prices.rb" $split_file
1736
- done
1737
- else
1738
- run_ruby_k8s $namespace "update-prices" "$SCRIPT_FULL_PATH/ruby/update_prices.rb" $json_prices
1739
- fi
1740
- }
1741
-
1742
-
1743
- update_surveys() {
1744
- local namespace=$1
1745
- local csv_surveys=$2
1746
-
1747
- local csv_size=$(wc -c < "$csv_surveys")
1748
-
1749
-
1750
- if ((csv_size > 940000)); then
1751
- local max_lines=400
1752
- local total_lines=$(wc -l < $csv_surveys)
1753
- local iterations=$((total_lines / max_lines + 1))
1754
- echo "$csv_surveys is too big, I'll split it for you in blocks of $max_lines lines. It will take $iterations runs"
1755
- for (( i = 0 ; i < iterations ; i++ )) ; do
1756
- local start=$((i * max_lines + 2))
1757
- local end=$(( (i + 1) * max_lines + 1))
1758
- local split_file=$(mktemp)
1759
- head -1 $csv_surveys > $split_file
1760
- sed -n ''"$start,${end}p" $csv_surveys >> $split_file
1761
-
1762
-
1763
- local split_lines=$(wc -l < $split_file)
1764
- echo "starting iteration $i from $start to $end with $split_file command -v has $split_lines lines"
1765
- run_ruby_k8s $namespace "reimport-surveys-$i" "$SCRIPT_FULL_PATH/ruby/feedback_kpi_reuploader.rb" $split_file
1766
- done
1767
- else
1768
- run_ruby_k8s $namespace "reimport-surveys" "$SCRIPT_FULL_PATH/ruby/feedback_kpi_reuploader.rb" $csv_surveys
1769
- fi
1770
- }
1771
-
1772
- #!/usr/bin/env bash
1773
-
1774
- configure_gitlab_ssh() {
1775
- tmp_dir=$(mktemp -d)
1776
- ssh-keyscan gitlab.com > $tmp_dir/known_hosts
1777
- echo "$SSH_PRIVATE_KEY" > $tmp_dir/id_rsa
1778
- chmod 600 $tmp_dir/id_rsa
1779
- ssh -i $tmp_dir/id_rsa -T git@gitlab.com
1780
- rm -Rf $tmp_dir
1781
- }
1782
-
1783
-
1784
- configure_gitlab_ssh_home() {
1785
- mkdir ~/.ssh
1786
- ssh-keyscan gitlab.com >> ~/.ssh/known_hosts
1787
- echo "$SSH_PRIVATE_KEY" > ~/.ssh/id_rsa
1788
- chmod 600 ~/.ssh/id_rsa
1789
- ssh -T git@gitlab.com
1790
- }
1791
- #!/usr/bin/env bash
1792
-
1793
- datadog_schedule_downtime() {
1794
- SERVICES=$1
1795
- DOWNTIME_MINUTES=${2:-30}
1796
-
1797
- if [[ "$ENVIRONMENT" == "production" ]] ; then
1798
- log "scheduling downtime for $SERVICES in $ENVIRONMENT"
1799
- else
1800
- return 0
1801
- fi
1802
-
1803
- for SERVICE in $SERVICES ; do
1804
- datadog_schedule_downtime_single $SERVICE $DOWNTIME_MINUTES
1805
- done
1806
- }
1807
-
1808
- datadog_schedule_downtime_single() {
1809
- local SERVICE=$1
1810
- local DOWNTIME_MINUTES=$2
1811
-
1812
- START=$(date +%s)
1813
- END=$((START + 60 * DOWNTIME_MINUTES))
1814
-
1815
- log "scheduling a downtime on datadog for $SERVICE ($DOWNTIME_MINUTES minutes)"
1816
- curl -X POST "https://api.datadoghq.com/api/v1/downtime" \
1817
- -H "Content-Type: application/json" \
1818
- -H "DD-API-KEY: ${DD_API_KEY}" \
1819
- -H "DD-APPLICATION-KEY: ${DD_APP_KEY}" \
1820
- -d '
1821
- {
1822
- "active": true,
1823
- "downtime_type": 0,
1824
- "start": $START,
1825
- "end": $END,
1826
- "message": "CA Deployment - performance for $SERVICE may be lower for next $DOWNTIME_MINUTES min",
1827
- "monitor_tags": [
1828
- "service:$SERVICE",
1829
- "performance"
1830
- ],
1831
- "scope": [
1832
- "env:production"
1833
- ],
1834
- "timezone": "Europe/Paris"
1835
- }
1836
- '
1837
- }
1838
- #!/usr/bin/env bash
1839
-
1840
- docker_build_push() {
1841
- read -r -a BUILD_ARGS <<< "$1"
1842
- DOCKER_BUILD_ARGS="--build-arg VCS_REF=$(git rev-parse --short HEAD)"
1843
- for ARG_NAME in "${BUILD_ARGS[@]}"
1844
- do
1845
- DOCKER_BUILD_ARGS="$DOCKER_BUILD_ARGS --build-arg $ARG_NAME=${!ARG_NAME}"
1846
- done
1847
-
1848
- if ! image_exists $DOCKER_REGISTRY_ID $APPLICATION $CI_COMMIT_SHORT_SHA ; then
1849
- docker pull $DOCKER_IMAGE || true
1850
- docker build $DOCKER_BUILD_ARGS -t $DOCKER_IMAGE_SHA --cache-from $DOCKER_IMAGE $DOCKER_STAGE_PATH
1851
- docker push $DOCKER_IMAGE_SHA
1852
- fi
1853
- }
1854
-
1855
- docker_promote() {
1856
- # inspired by https://dille.name/blog/2018/09/20/how-to-tag-docker-images-without-pulling-them/
1857
- OLD_TAG=${1//[^0-9a-zA-Z-.]/_}
1858
- NEW_TAG=${2//[^0-9a-zA-Z-.]/_}
1859
- echo "promoting from $OLD_TAG to $NEW_TAG"
1860
- TOKEN=$(aws_ecr_token)
1861
- CONTENT_TYPE="application/vnd.docker.distribution.manifest.v2+json"
1862
- MANIFESTS_API="https://${DOCKER_REGISTRY}/v2/${APPLICATION}/manifests"
1863
-
1864
- if MANIFEST=$(curl --fail -H "Authorization: Basic $TOKEN" -H "Accept: ${CONTENT_TYPE}" "$MANIFESTS_API/${OLD_TAG}"); then
1865
- echo "authenticated on $MANIFESTS_API"
1866
- else
1867
- return 1
1868
- fi
1869
- if curl --fail -H "Authorization: Basic $TOKEN" -X PUT -H "Content-Type: ${CONTENT_TYPE}" -d "${MANIFEST}" "$MANIFESTS_API/$NEW_TAG" ; then
1870
- echo "promoted ${APPLICATION} from $OLD_TAG to $NEW_TAG"
1871
- else
1872
- return 2
1873
- fi
1874
- }
1875
-
1876
- ensure_images_exists() {
1877
- for IMAGE_TO_CHECK in $(echo $1 | tr "," "\n"); do
1878
- image_exists ${DOCKER_REGISTRY_ID} ${IMAGE_TO_CHECK} ${VERSION} || return 1
1879
- done
1880
- }
1881
- #!/usr/bin/env bash
1882
-
1883
- extract_yaml_config_variable() {
1884
- set +e
1885
- set +x
1886
-
1887
- check_args "--environment" $1
1888
- shift
1889
- ENVIRONMENT=$1
1890
- shift
1891
-
1892
- check_args "--configs-path" $1
1893
- shift
1894
- CONFIGS_PATH=$1
1895
- shift
1896
-
1897
- check_args "--variable" $1
1898
- shift
1899
- VARIABLE=$1
1900
- shift
1901
-
1902
- [[ "$1" == "--optional" ]] && OPTIONAL=true || OPTIONAL=false
1903
-
1904
- if [ ! -f ${CONFIGS_PATH}/common.yaml ]; then
1905
- echo >&2 "Missing $CONFIGS_PATH/common.yaml configuration file"
1906
- exit 1
1907
- fi
1908
- if [ ! -f ${CONFIGS_PATH}/${ENVIRONMENT}.yaml ]; then
1909
- echo >&2 "Missing $CONFIGS_PATH/$ENVIRONMENT.yaml configuration file"
1910
- exit 1
1911
- fi
1912
- if [ ! -f ${CONFIGS_PATH}/${ENVIRONMENT}-secrets.yaml ]; then
1913
- echo >&2 "Missing $CONFIGS_PATH/$ENVIRONMENT-secrets.yaml configuration file"
1914
- exit 1
1915
- fi
1916
-
1917
- result=$(yq -r ${VARIABLE} "$CONFIGS_PATH/$ENVIRONMENT-secrets.yaml")
1918
- if [ $? -ne 0 ] || [ "$result" = "null" ]; then
1919
- result=$(yq -r ${VARIABLE} "$CONFIGS_PATH/$ENVIRONMENT.yaml")
1920
- if [ $? -ne 0 ] || [ "$result" = "null" ]; then
1921
- result=$(yq -r ${VARIABLE} "$CONFIGS_PATH/common.yaml")
1922
- if [ $? -ne 0 ] || [ "$result" = "null" ]; then
1923
- if [ $OPTIONAL = true ]; then
1924
- echo ""
1925
- exit 0
1926
- else
1927
- echo >&2 "Missing path $VARIABLE in $CONFIGS_PATH/$ENVIRONMENT-secrets.yaml, $CONFIGS_PATH/$ENVIRONMENT.yaml or $CONFIGS_PATH/common.yaml"
1928
- exit 1
1929
- fi
1930
- fi
1931
- fi
1932
- fi
1933
- echo ${result}
1934
- }
1935
- #!/usr/bin/env bash
1936
-
1937
- flyway_clean() {
1938
- HOST="$1"
1939
- PORT="$2"
1940
- DATABASE="$3"
1941
- USER="$4"
1942
- PASSWORD="$5"
1943
-
1944
- kubectl run -it --rm flywayclean \
1945
- --image=flyway/flyway \
1946
- --restart=Never \
1947
- -- \
1948
- -cleanDisabled=false \
1949
- -url="jdbc:postgresql://$HOST:$PORT/$DATABASE" \
1950
- -user="$USER" \
1951
- -password="$PASSWORD" \
1952
- clean
1953
- }
1954
-
1955
- #!/usr/bin/env bash
1956
-
1957
- FLYWAY_VERSION="7.4.0"
1958
-
1959
-
1960
- get_yaml_variable() {
1961
- extract_yaml_config_variable --environment ${ENVIRONMENT} --configs-path $(pwd)/deploy --variable $@
1962
- }
1963
-
1964
- init_migrate_db() {
1965
- set -e
1966
-
1967
- check_env_vars 4 "APPLICATION" "ENVIRONMENT" "FLYWAY_VERSION" "MIGRATION_SQL_PATH"
1968
-
1969
- PG_YAML_PATH=".${APPLICATION}config.postgres"
1970
-
1971
- DB_PORT="5432"
1972
- DB_HOST=$(get_yaml_variable "${PG_YAML_PATH}.host")
1973
- DB_INIT_USERNAME=$(get_yaml_variable "${PG_YAML_PATH}.initUsername")
1974
- DB_INIT_PASSWORD=$(get_yaml_variable "${PG_YAML_PATH}.initPassword")
1975
- DB_DATABASE=$(get_yaml_variable "${PG_YAML_PATH}.database")
1976
- DB_USER=$(get_yaml_variable "${PG_YAML_PATH}.user")
1977
- DB_PASSWORD=$(get_yaml_variable "${PG_YAML_PATH}.password")
1978
- DB_URL="jdbc:postgresql://${DB_HOST}:${DB_PORT}/${DB_DATABASE}"
1979
-
1980
- DB_RO_USER=$(get_yaml_variable "${PG_YAML_PATH}.readOnlyUser" --optional)
1981
- DB_RO_PASSWORD=$(get_yaml_variable "${PG_YAML_PATH}.readOnlyPassword" --optional)
1982
-
1983
- unset KUBECONFIG
1984
-
1985
- configure_kubectl_for_ci ${ENVIRONMENT}
1986
-
1987
- kube_init_service_database \
1988
- --namespace ${ENVIRONMENT} \
1989
- --service ${APPLICATION} \
1990
- --db_host ${DB_HOST} \
1991
- --db_port ${DB_PORT} \
1992
- --db_init_username ${DB_INIT_USERNAME} \
1993
- --db_init_password ${DB_INIT_PASSWORD} \
1994
- --db_database ${DB_DATABASE} \
1995
- --db_username ${DB_USER} \
1996
- --db_password ${DB_PASSWORD}
1997
-
1998
- if [[ ! -z "$DB_RO_USER" ]] && [[ ! -z "$DB_RO_USER" ]]; then
1999
- kube_init_database_readonly_account \
2000
- --namespace ${ENVIRONMENT} \
2001
- --service ${APPLICATION} \
2002
- --db_connection "$DB_INIT_USERNAME:$DB_INIT_PASSWORD@$DB_HOST:$DB_PORT" \
2003
- --db_database ${DB_DATABASE} \
2004
- --db_readonly_username ${DB_RO_USER} \
2005
- --db_readonly_password ${DB_RO_PASSWORD}
2006
- fi
2007
-
2008
- flyway_migrate \
2009
- --environment ${ENVIRONMENT} \
2010
- --namespace ${ENVIRONMENT} \
2011
- --service ${APPLICATION} \
2012
- --db_url ${DB_URL} \
2013
- --db_user ${DB_USER} \
2014
- --db_password ${DB_PASSWORD} \
2015
- --flyway_version ${FLYWAY_VERSION} \
2016
- --flyway_sql_folder $(pwd)/${MIGRATION_SQL_PATH}
2017
- }
2018
-
2019
- flyway_migrate() {
2020
- set -e
2021
-
2022
- extract_args 8 \
2023
- environment namespace service db_url db_user db_password flyway_version flyway_sql_folder $*
2024
-
2025
- echo "running flyway migrations for service $service in environment $environment namespace $namespace for db_url $db_url with user $db_user"
2026
- echo "migration files expected in $flyway_sql_folder"
2027
-
2028
- CONFIGMAP_NAME="$service-flyway-migration-sql"
2029
- POD_NAME="$service-flyway-migration"
2030
-
2031
- configure_kubectl_for_ci $environment
2032
-
2033
- kubectl -n $namespace delete configmap $CONFIGMAP_NAME --ignore-not-found
2034
- kubectl -n $namespace delete pod $POD_NAME --ignore-not-found
2035
- kubectl -n $namespace create configmap $CONFIGMAP_NAME --from-file=$flyway_sql_folder
2036
-
2037
- kubectl -n $namespace run $POD_NAME --image ignored -ti --restart=Never --attach --rm --overrides='
2038
- {
2039
- "spec":{
2040
- "containers":[
2041
- {
2042
- "name":"'$POD_NAME'",
2043
- "image":"flyway/flyway:'$flyway_version'",
2044
- "command":["flyway", "-url='$db_url'", "-user='$db_user'", "-password='$db_password'", "migrate"],
2045
- "volumeMounts":[
2046
- {
2047
- "name":"sql",
2048
- "mountPath":"/flyway/sql"
2049
- }
2050
- ]
2051
- }
2052
- ],
2053
- "volumes":[
2054
- {
2055
- "name":"sql",
2056
- "configMap":{
2057
- "name":"'$CONFIGMAP_NAME'"
2058
- }
2059
- }
2060
- ]
2061
- }
2062
- }
2063
- '
2064
-
2065
- kubectl -n $namespace delete configmap $CONFIGMAP_NAME
2066
- }
2067
-
2068
- #!/usr/bin/env bash
2069
- flyway_repair() {
2070
- set -e
2071
- check_env_vars 4 "APPLICATION" "ENVIRONMENT" "FLYWAY_VERSION" "MIGRATION_SQL_PATH"
2072
-
2073
- PG_YAML_PATH=".${APPLICATION}config.postgres"
2074
-
2075
- DB_PORT="5432"
2076
- DB_HOST=$(get_yaml_variable "${PG_YAML_PATH}.host")
2077
- DB_DATABASE=$(get_yaml_variable "${PG_YAML_PATH}.database")
2078
- DB_USER=$(get_yaml_variable "${PG_YAML_PATH}.user")
2079
- DB_PASSWORD=$(get_yaml_variable "${PG_YAML_PATH}.password")
2080
- DB_URL="jdbc:postgresql://${DB_HOST}:${DB_PORT}/${DB_DATABASE}"
2081
-
2082
- flyway_sql_folder=$(pwd)/${MIGRATION_SQL_PATH}
2083
-
2084
- configure_kubectl_for_ci "${ENVIRONMENT}"
2085
- POD_NAME="${APPLICATION}-flyway-repair"
2086
- CONFIGMAP_NAME="${APPLICATION}-flyway-repair-sql"
2087
-
2088
- kubectl -n "${ENVIRONMENT}" delete configmap $CONFIGMAP_NAME --ignore-not-found
2089
- kubectl -n "${ENVIRONMENT}" delete pod $POD_NAME --ignore-not-found
2090
- kubectl -n "${ENVIRONMENT}" create configmap $CONFIGMAP_NAME --from-file="${flyway_sql_folder}"
2091
-
2092
- kubectl -n "${ENVIRONMENT}" run --rm -it "${POD_NAME}" \
2093
- --image=flyway/flyway \
2094
- --restart=Never \
2095
- --overrides='
2096
- {
2097
- "spec":{
2098
- "containers":[
2099
- {
2100
- "name":"'$POD_NAME'",
2101
- "image":"flyway/flyway:'${FLYWAY_VERSION}'",
2102
- "command":["flyway", "-url='$DB_URL'", "-user='$DB_USER'", "-password='$DB_PASSWORD'", "repair"],
2103
- "volumeMounts":[
2104
- {
2105
- "name":"sql",
2106
- "mountPath":"/flyway/sql"
2107
- }
2108
- ]
2109
- }
2110
- ],
2111
- "volumes":[
2112
- {
2113
- "name":"sql",
2114
- "configMap":{
2115
- "name":"'$CONFIGMAP_NAME'"
2116
- }
2117
- }
2118
- ]
2119
- }
2120
- }
2121
- '
2122
- kubectl -n "${ENVIRONMENT}" delete configmap $CONFIGMAP_NAME
2123
- }
2124
-
2125
- #!/usr/bin/env bash
2126
-
2127
- record_git_commit() {
2128
- for file in $GIT_COMMIT_FILES; do
2129
- sed -i 's&GIT_COMMIT&'"${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}&" "$file"
2130
- done
2131
- }
2132
-
2133
- gitlab_import_pgp_key() {
2134
- if [ "$GITLAB_PGP_PRIVATE_KEY" != "" ]
2135
- then
2136
- KEY_FOLDER=<(echo "$GITLAB_PGP_PRIVATE_KEY")
2137
- gpg --import $KEY_FOLDER > /dev/null
2138
- else
2139
- echo '$GITLAB_PGP_PRIVATE_KEY is not set'
2140
- return 1
2141
- fi
2142
- }
2143
-
2144
- git_reveal() {
2145
- gitlab_import_pgp_key
2146
- gpg --decrypt $1
2147
- }
2148
- #!/usr/bin/env bash
2149
-
2150
- helm_deploy_v3() {
2151
- APPLICATION=$1
2152
- ENVIRONMENT=$2
2153
- VERSION=$3
2154
- deploy_chart_v3 \
2155
- --path_configs deploy \
2156
- --path_chart deploy/$APPLICATION \
2157
- --application $APPLICATION \
2158
- --environment $ENVIRONMENT \
2159
- --namespace $ENVIRONMENT \
2160
- --helm_extra_args --set global.version=$VERSION
2161
- }
2162
-
2163
- deploy_chart_v3() {
2164
- set -e
2165
- set -x
2166
-
2167
- # Rigid parsing, but all args are mandatory (expect last) and flexible order is unnecessary
2168
- check_args "--path_configs" $1; shift
2169
- path_configs=$1; shift
2170
- check_args "--path_chart" $1; shift
2171
- path_chart=$1; shift
2172
- check_args "--application" $1; shift
2173
- application=$1; shift
2174
- check_args "--environment" $1; shift
2175
- environment=$1; shift
2176
- check_args "--namespace" $1; shift
2177
- namespace=$1; shift
2178
- if [ $# -ne 0 ]; then
2179
- check_args "--helm_extra_args" $1; shift
2180
- helm_extra_args=$*
2181
- fi
2182
-
2183
- echo "================================"
2184
- echo " Deploying $application"
2185
- echo " - Environment: $environment"
2186
- echo " - Namespace: $namespace"
2187
- echo "================================"
2188
-
2189
- root_path=$(pwd)
2190
-
2191
- # Check the configs exists
2192
-
2193
- check_config_file ${root_path}/${path_configs}/common.yaml
2194
- check_config_file ${root_path}/${path_configs}/${namespace}.yaml
2195
- check_config_file ${root_path}/${path_configs}/${namespace}-secrets.yaml
2196
-
2197
- # Check the chart exists
2198
- if [ ! -d ${root_path}/${path_chart} ] || [ ! -f ${root_path}/${path_chart}/Chart.yaml ]; then
2199
- echo "Bad Chart $root_path/$path_chart : does not exists or missing Chart.yaml"
2200
- print_usage
2201
- exit 1
2202
- fi
2203
-
2204
- # Unset Kubectl configuration made via the KUBECONFIG env variable
2205
- # it would override the config made by configure_kubectl_for
2206
- # for example, using Gitlab runners in Kubernetes sets this variable and causes conflict
2207
- unset KUBECONFIG
2208
-
2209
- # Configure Kubectl
2210
- configure_kubectl_for_ci ${environment}
2211
-
2212
- # Configure helm3
2213
- helm3 version --namespace ${namespace} || true
2214
- # helm3 stable repo have changed and must be updated manually, in versions < v2.17.0
2215
- helm3 repo add colisweb s3://colisweb-helm-charts/colisweb
2216
- helm3 repo add stable https://charts.helm.sh/stable
2217
- helm3 repo update
2218
- helm3 dependency update ${root_path}/${path_chart}
2219
-
2220
- # Gather values/*.yaml files
2221
- values_path="${root_path}/${path_chart}/values"
2222
- values_files=''
2223
- [ -d $values_path ] && values_files=$(find $values_path -type f -maxdepth 1 -name "*.yaml" | sed 's/^/ -f /' | tr -d \\n | sed 's/%//')
2224
-
2225
- # Deploy
2226
- helm3 upgrade --install \
2227
- --namespace ${namespace} \
2228
- ${values_files} \
2229
- -f ${root_path}/${path_configs}/common.yaml \
2230
- -f ${root_path}/${path_configs}/${namespace}.yaml \
2231
- -f ${root_path}/${path_configs}/${namespace}-secrets.yaml \
2232
- ${helm_extra_args} \
2233
- ${application} ${root_path}/${path_chart}
2234
-
2235
- #send event to dd
2236
- PUBLISHED_VERSION="$CI_COMMIT_REF_NAME-$CI_COMMIT_SHA"
2237
- emit_datadog_deploy_event --environment $environment --service $application --version $PUBLISHED_VERSION
2238
-
2239
- echo "================================"
2240
- echo " Deployed $application"
2241
- echo " - Environment: $environment"
2242
- echo " - Namespace: $namespace"
2243
- echo "================================"
2244
-
2245
- set +x
2246
- }
2247
-
2248
- verify_deployments_v3() {
2249
- set -e
2250
-
2251
- # usage :
2252
- # verify_deployments staging price
2253
- # verify_deployments -t 15m testing price
2254
-
2255
- if [ "$1" = "-t" ] ; then
2256
- TIMEOUT=$2
2257
- shift
2258
- shift
2259
- else
2260
- TIMEOUT=5m
2261
- fi
2262
-
2263
- NAMESPACE=$1
2264
- RELEASE=$2
2265
-
2266
- # Get all Deployments names from the deployed chart
2267
- DEPLOYMENTS=(
2268
- $(helm3 get manifest --namespace $NAMESPACE $RELEASE | yq -rs '.[] | select(.kind=="Deployment") | .metadata.name')
2269
- )
2270
-
2271
- echo "verifying on $NAMESPACE deployments ${DEPLOYMENTS[@]} with a timeout of $TIMEOUT"
2272
-
2273
- PIDS=()
2274
- for D in "${DEPLOYMENTS[@]}"; do
2275
- kubectl -n ${NAMESPACE} rollout status deployment ${D} --timeout=${TIMEOUT} &
2276
- PIDS+=($!)
2277
- done
2278
-
2279
- for P in ${PIDS[@]}; do
2280
- wait $P
2281
-
2282
- if [ $? -ne 0 ]; then
2283
- echo "at least one deployment failed or timed out (after $TIMEOUT)"
2284
- exit 1
2285
- fi
2286
- done
2287
-
2288
- }
2289
-
2290
- print_usage() {
2291
- echo "Usage:"
2292
- echo "deploy_chart \\"
2293
- echo " --path_configs <path to .yaml namespaces and secret config files>"
2294
- echo " --path_chart <path to Helm Chart>"
2295
- echo " --application <application name used by Helm>"
2296
- echo " --environment <infrastructure environment>"
2297
- echo " --namespace <namespace>"
2298
- echo " --helm-extra-args <extra args to pass to helm, ex: --set my.value=42 --set your.setting=on>"
2299
- echo ""
2300
- }
2301
-
2302
- check_config_file() {
2303
- local filename=$1
2304
- if [ ! -f ${filename} ]; then
2305
- echo "Missing $filename configuration file"
2306
- print_usage
2307
- exit 1
2308
- fi
2309
- }
2310
-
2311
- #!/usr/bin/env bash
2312
-
2313
- configure_kubectl_for_ci() {
2314
- if [ -z ${GITLAB_PAT} ]; then
2315
- echo "Cannot configure kubectl: no GITLAB_PAT configured"
2316
- exit 1
2317
- fi
2318
-
2319
- infra_env="$1"
2320
- valid_envs="[testing][staging][production][performance][tests][recette]"
2321
- echo "$valid_envs" | grep -q "\[$infra_env\]"
2322
-
2323
- if [ $? -ne 0 ]; then
2324
- echo "Cannot configure kubectl for invalid env : $infra_env"
2325
- echo "choose one of $valid_envs"
2326
- exit 1
2327
- fi
2328
-
2329
- mkdir -p ~/.kube
2330
- curl -fsS \
2331
- --header "PRIVATE-TOKEN: $GITLAB_PAT" \
2332
- "https://gitlab.com/api/v4/projects/8141053/jobs/artifacts/$infra_env/raw/$infra_env.kubeconfig?job=4_kubernetes_config_output" \
2333
- > ~/.kube/$infra_env.kubeconfig
2334
-
2335
- curl_return_code=$?
2336
- if [ ${curl_return_code} -ne 0 ]; then
2337
- echo "Cannot configure kubectl for $infra_env, get configuration failed with code $curl_return_code"
2338
- exit ${curl_return_code}
2339
- fi
2340
-
2341
- rm -f ~/.kube/config
2342
- ln -s ~/.kube/$infra_env.kubeconfig ~/.kube/config
2343
- echo "Configured kubectl for env : $infra_env"
2344
- }
2345
- notify_new_deployment() {
2346
- jq --version || (apt update && apt install -y jq)
2347
-
2348
- CHAT_URL=${1:-$DEFAULT_CHAT_URL}
2349
-
2350
- STATUS=$(echo $CI_JOB_STATUS | tr '[:lower:]' '[:upper:]' )
2351
- ENV_NAME=$(echo $ENVIRONMENT | tr '[:lower:]' '[:upper:]' )
2352
-
2353
- JOB_LINK="<$CI_JOB_URL| $CI_JOB_NAME $CI_JOB_ID>"
2354
-
2355
- DESCRIPTION="
2356
- $STATUS : Deployment for $CI_PROJECT_NAME on $ENV_NAME
2357
- $JOB_LINK
2358
- $CI_COMMIT_TITLE
2359
- "
2360
-
2361
- JSON_MESSAGE=$(jq -n --arg text "$DESCRIPTION" '{text: $text }')
2362
- curl -X POST $CHAT_URL \
2363
- --header "Content-Type: application/json" \
2364
- --data "$JSON_MESSAGE"
2365
- }
2366
- notify_new_version() {
2367
-
2368
- ! test -z $CI_COMMIT_TAG || exit 0
2369
-
2370
- jq --version || (apt update && apt install -y jq)
2371
-
2372
- KIND=$1
2373
- CHAT_URL=${2:-$DEFAULT_CHAT_URL}
2374
-
2375
- STATUS=$(echo $CI_JOB_STATUS | tr '[:lower:]' '[:upper:]' )
2376
- ENV_NAME=$(echo $ENVIRONMENT | tr '[:lower:]' '[:upper:]' )
2377
- TITLE="$ENV_NAME *$STATUS* $KIND for version *$CI_COMMIT_TAG* of *$CI_PROJECT_NAME* "
2378
-
2379
- RELEASE_URL="https://gitlab.com/api/v4/projects/$CI_PROJECT_ID/releases/$CI_COMMIT_TAG"
2380
-
2381
- NOTES=$(curl --header "PRIVATE-TOKEN: $GITLAB_TOKEN" $RELEASE_URL |
2382
- jq .description |
2383
- sed -e 's/^"//' -e 's/"$//' |
2384
- sed -E 's/\[([^]]+)\]\(([^)]+)\)/<\2|\1>/g' |
2385
- sed -E 's/\\n/\'$'\n/g')
2386
-
2387
- JOB_LINK="<$CI_JOB_URL| $CI_JOB_NAME $CI_JOB_ID>"
2388
-
2389
- DESCRIPTION="
2390
- $TITLE
2391
- $JOB_LINK
2392
- $NOTES
2393
- "
2394
-
2395
- JSON_MESSAGE=$(jq -n --arg text "$DESCRIPTION" '{text: $text }')
2396
- curl -X POST $CHAT_URL \
2397
- --header "Content-Type: application/json" \
2398
- --data "$JSON_MESSAGE"
2399
- }
2400
- #!/usr/bin/env bash
2401
-
2402
- skip_sbt_compile_cache() {
2403
- COMPARED_BRANCH="${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:-$CI_DEFAULT_BRANCH}"
2404
- echo "branch to compare to: $COMPARED_BRANCH"
2405
- git fetch origin $COMPARED_BRANCH
2406
- echo "fetched $COMPARED_BRANCH"
2407
- [[ "$CI_COMMIT_REF_NAME" =~ ^(master|develop)$ || $(git diff origin/$COMPARED_BRANCH --exit-code -- project) ]]
2408
- }
2409
- #!/usr/bin/env bash
2410
-
2411
- # in case of trouble with functions for update history during import
2412
- # https://stackoverflow.com/questions/56729192/pg-restore-fails-when-trying-to-create-function-referencing-table-that-does-not
2413
-
2414
- # example: clone_databases --source_env testing --destination_env recette --services "order,notification,parcel,ikea"
2415
- clone_databases() {
2416
- export USERNAME="database-cloner"
2417
-
2418
- set -e
2419
-
2420
- extract_args 3 source_env destination_env services $*
2421
-
2422
- dump_databases "$source_env" "$services"
2423
- import_databases "$destination_env" "$services"
2424
- }
2425
-
2426
- dump_databases() {
2427
- local env="$1"
2428
- local services=$(echo -n "$2" | tr ',' '\n')
2429
-
2430
- database_k8s_output_dump_path="/tmp/database_k8s_output_dump"
2431
-
2432
- configure_kubectl_for "$env"
2433
- set +e
2434
- database_k8s "$env" > "$database_k8s_output_dump_path"
2435
- set -e
2436
-
2437
- source_pg_local_port=$(extract_pg_local_port "$database_k8s_output_dump_path")
2438
-
2439
- for service in $services
2440
- do
2441
- service_path="/tmp/$service"
2442
-
2443
- set +e
2444
- git clone "git@gitlab.com:colisweb/back/$service.git" "$service_path"
2445
- set -e
2446
-
2447
- if cd "$service_path"; then
2448
- echo "dump the database for service $service.."
2449
-
2450
- git secret reveal -f
2451
-
2452
- PG_YAML_PATH=".${service}config.postgres"
2453
-
2454
- SOURCE_DB_DATABASE=$(extract_yaml_config_variable --environment "$env" --configsPath ./deploy --variable "${PG_YAML_PATH}.database")
2455
- SOURCE_DB_USER=$(extract_yaml_config_variable --environment "$env" --configsPath ./deploy --variable "${PG_YAML_PATH}.user")
2456
- SOURCE_DB_PASSWORD=$(extract_yaml_config_variable --environment "$env" --configsPath ./deploy --variable "${PG_YAML_PATH}.password")
2457
-
2458
- export PGPASSWORD="$SOURCE_DB_PASSWORD"
2459
-
2460
- DUMP_PATH="/tmp/db_dump_${service}.sql"
2461
- pg_dump --no-owner -h localhost -p "$source_pg_local_port" -U "$SOURCE_DB_USER" "$SOURCE_DB_DATABASE" > "$DUMP_PATH"
2462
-
2463
- cd ..
2464
- rm -rf "$service_path"
2465
- else
2466
- echo "WARN: failed to clone $service - skipping"
2467
- fi
2468
- done
2469
- }
2470
-
2471
- import_databases() {
2472
- local env="$1"
2473
- local services=$(echo -n "$2" | tr ',' '\n')
2474
-
2475
- database_k8s_output_import_path="/tmp/database_k8s_output_import"
2476
-
2477
- configure_kubectl_for "$env"
2478
- set +e
2479
- database_k8s "$env" > "$database_k8s_output_import_path"
2480
- set -e
2481
-
2482
- destination_pg_local_port=$(extract_pg_local_port "$database_k8s_output_import_path")
2483
-
2484
- for service in $services
2485
- do
2486
- service_path="/tmp/$service"
2487
-
2488
- set +e
2489
- git clone "git@gitlab.com:colisweb/back/$service.git" "$service_path"
2490
- set -e
2491
-
2492
- if cd "$service_path"; then
2493
- echo "create and import database for $service.."
2494
-
2495
- git secret reveal -f
2496
-
2497
- PG_YAML_PATH=".${service}config.postgres"
2498
-
2499
- DB_PORT="5432"
2500
- DB_HOST=$(extract_yaml_config_variable --environment "$env" --configsPath ./deploy --variable "${PG_YAML_PATH}.host")
2501
- DB_INIT_USERNAME=$(extract_yaml_config_variable --environment "$env" --configsPath ./deploy --variable "${PG_YAML_PATH}.initUsername")
2502
- DB_INIT_PASSWORD=$(extract_yaml_config_variable --environment "$env" --configsPath ./deploy --variable "${PG_YAML_PATH}.initPassword")
2503
- DB_DATABASE=$(extract_yaml_config_variable --environment "$env" --configsPath ./deploy --variable "${PG_YAML_PATH}.database")
2504
- DB_USER=$(extract_yaml_config_variable --environment "$env" --configsPath ./deploy --variable "${PG_YAML_PATH}.user")
2505
- DB_PASSWORD=$(extract_yaml_config_variable --environment "$env" --configsPath ./deploy --variable "${PG_YAML_PATH}.password")
2506
-
2507
- kube_init_service_database \
2508
- --namespace ${env} \
2509
- --service ${service} \
2510
- --db_host ${DB_HOST} \
2511
- --db_port ${DB_PORT} \
2512
- --db_init_username ${DB_INIT_USERNAME} \
2513
- --db_init_password ${DB_INIT_PASSWORD} \
2514
- --db_database ${DB_DATABASE} \
2515
- --db_username ${DB_USER} \
2516
- --db_password ${DB_PASSWORD}
2517
-
2518
- echo "WARN: A complete clean of $DB_DATABASE on $DB_HOST will be operated"
2519
- read -rsn1 -p"Press any key to continue";echo
2520
- flyway_clean "$DB_HOST" "$DB_PORT" "$DB_DATABASE" "$DB_USER" "$DB_PASSWORD"
2521
-
2522
- DUMP_PATH="/tmp/db_dump_${service}.sql"
2523
- export PGPASSWORD="$DB_PASSWORD"
2524
- set +e
2525
- psql "postgres://$DB_USER@127.0.0.1:$destination_pg_local_port" -p "$DB_DATABASE" -f "$DUMP_PATH"
2526
- set -e
2527
-
2528
- cd ..
2529
- rm -rf "$service_path"
2530
- else
2531
- echo "WARN: failed to clone $service - skipping"
2532
- fi
2533
- done
2534
- }
2535
-
2536
- extract_pg_local_port() {
2537
- cat "$1" | grep 'postgres@127.0.0.1:' | sed 's/.*postgres@127.0.0.1:\(.*[0-9]\).*/\1/g'
2538
- }
2539
- #!/usr/bin/env bash
2540
-
2541
- emit_datadog_deploy_event() {
2542
- extract_args 3 environment service version $*
2543
- check_env_vars 1 "DD_API_KEY"
2544
-
2545
- response=$(
2546
- curl -X POST -H "Content-type: application/json" \
2547
- -d '{
2548
- "title": "deploying '"$service"' to '"$environment"'",
2549
- "text": "deploying '"$service"' version '"$version"' to '"$environment"'",
2550
- "priority": "normal",
2551
- "tags": ["service:'"$service"' ", "env:'"$environment"'" ,"action:'"deployment"'"] ,
2552
-
2553
- "alert_type": "Info"
2554
- }' \
2555
- "https://api.datadoghq.com/api/v1/events?api_key=$DD_API_KEY"
2556
- )
2557
-
2558
- #echo $response
2559
- EventID=$(echo $response | jq ".event.id")
2560
- url=$(echo $response | jq ".event.url")
2561
-
2562
- if [[ $EventID -ne 0 ]]; then
2563
- echo "event successfully created check in datadog UI : $url"
2564
- else
2565
- echo " failed to create event "
2566
- exit 1
2567
- fi
2568
- }
2569
-
2570
- #!/usr/bin/env bash
2571
-
2572
- # DEPRECATED
2573
- emit_datadog_error_events() {
2574
- set -e
2575
- extract_args 4 title text priority environment $*
2576
- check_env_vars 1 "DD_API_KEY"
2577
-
2578
- curl -X POST -H "Content-type: application/json" \
2579
- -d '{
2580
- "title": "'"$title"'",
2581
- "text": "'"$text"'",
2582
- "priority": "'"$priority"'",
2583
- "tags": ["environment:'"$environment"'"],
2584
- "alert_type": "Error"
2585
- }' \
2586
- "https://api.datadoghq.com/api/v1/events?api_key=$DD_API_KEY"
2587
- }
2588
-
2589
- #!/usr/bin/env bash
2590
- terraform_init() {
2591
- SECTION=$1
2592
- ENV=$2
2593
- cd $SECTION
2594
- terraform init -input=false
2595
- terraform workspace select $ENV || terraform workspace new $ENV
2596
- }