@colisweb/rescript-toolkit 5.8.0 → 5.9.0

package/.secure_files/{ci-functions-v17.7.1 → ci-functions-v17.12.0-feat-add-mysql-service-1.0.2beta}

@@ -577,7 +577,7 @@ psql_on_k8() {
   CONNECTION=$3
   shift 3
 
-  kubectl -n $NAMESPACE run ${SERVICE}-database-init \
+  kubectl -n $NAMESPACE run ${SERVICE}-postgres-init \
     --image jbergknoff/postgresql-client \
     --restart=Never \
     --attach --rm \
@@ -588,14 +588,15 @@ psql_on_k8() {
 
 mysql_on_k8() {
   local namespace=$1
-  local db_host=$2
-  local db_port=$3
-  local db_init_username=$4
-  local db_init_password=$5
-  local query=$6
-
-  kubectl -n ${namespace} run datadog-database-init \
-    --image widdpim/mysql-client \
+  local service=$2
+  local db_host=$3
+  local db_port=$4
+  local db_init_username=$5
+  local db_init_password=$6
+  local query=$7
+
+  kubectl -n ${namespace} run ${service}-mysql-init \
+    --image arey/mysql-client \
     --restart=Never \
     --attach --rm \
     -- \
@@ -675,12 +676,14 @@ kube_init_datadog_in_database() {
   extract_args 8 namespace db_host db_port db_init_username db_init_password db_datadog_username db_datadog_password db_datadog_schema $*
 
   echo "======================="
-  echo " Initializing Datadog Agent Requiement for namespace $namespace"
+  echo " Initializing Datadog Agent Requirement for namespace $namespace"
   echo "======================="
 
   echo "Checking if User  '$db_datadog_username' exists"
+  local service="datadog"
+  found_db_users=$(mysql_on_k8 $namespace $service $db_host $db_port $db_init_username $db_init_password 'SELECT user FROM mysql.user;')
   set +e
-  mysql_on_k8 $namespace $db_host $db_port $db_init_username $db_init_password 'SELECT user FROM mysql.user;' | grep "^$db_datadog_username$"
+  echo "$found_db_users" | grep "^$db_datadog_username$"
   return_code=$?
   set -e
 
@@ -691,29 +694,29 @@ kube_init_datadog_in_database() {
 
     # All the query come from this docs : https://docs.datadoghq.com/fr/database_monitoring/setup_mysql/selfhosted/?tab=mysql56
 
-    mysql_on_k8 $namespace $db_host $db_port $db_init_username $db_init_password 'CREATE USER '"$db_datadog_username"'@"%" IDENTIFIED BY '"'$db_datadog_password'"';'
+    mysql_on_k8 $namespace $service $db_host $db_port $db_init_username $db_init_password 'CREATE USER '"$db_datadog_username"'@"%" IDENTIFIED BY '"'$db_datadog_password'"';'
     echo "USER created $db_datadog_username"
 
-    mysql_on_k8 $namespace $db_host $db_port $db_init_username $db_init_password 'GRANT REPLICATION CLIENT ON *.* TO datadog@"%" WITH MAX_USER_CONNECTIONS 5;'
+    mysql_on_k8 $namespace $service $db_host $db_port $db_init_username $db_init_password 'GRANT REPLICATION CLIENT ON *.* TO datadog@"%" WITH MAX_USER_CONNECTIONS 5;'
     echo "ALTER USER $db_datadog_username"
 
-    mysql_on_k8 $namespace $db_host $db_port $db_init_username $db_init_password 'GRANT PROCESS ON *.* TO '"$db_datadog_username"'@"%";'
+    mysql_on_k8 $namespace $service $db_host $db_port $db_init_username $db_init_password 'GRANT PROCESS ON *.* TO '"$db_datadog_username"'@"%";'
     echo "Granted PROCESS for $db_datadog_username"
 
-    mysql_on_k8 $namespace $db_host $db_port $db_init_username $db_init_password 'GRANT SELECT ON performance_schema.* TO '"$db_datadog_username"'@"%";'
+    mysql_on_k8 $namespace $service $db_host $db_port $db_init_username $db_init_password 'GRANT SELECT ON performance_schema.* TO '"$db_datadog_username"'@"%";'
     echo "Granted SELECT on performance_schema for $db_datadog_username"
 
-    mysql_on_k8 $namespace $db_host $db_port $db_init_username $db_init_password 'CREATE SCHEMA IF NOT EXISTS datadog;'
+    mysql_on_k8 $namespace $service $db_host $db_port $db_init_username $db_init_password 'CREATE SCHEMA IF NOT EXISTS datadog;'
     echo "CREATE SCHEMA datadog"
 
-    mysql_on_k8 $namespace $db_host $db_port $db_init_username $db_init_password 'GRANT EXECUTE ON datadog.* to '"$db_datadog_username"'@"%";'
+    mysql_on_k8 $namespace $service $db_host $db_port $db_init_username $db_init_password 'GRANT EXECUTE ON datadog.* to '"$db_datadog_username"'@"%";'
     echo "Granted 'GRANT EXECUTE for $db_datadog_username on datadog"
 
-    mysql_on_k8 $namespace $db_host $db_port $db_init_username $db_init_password 'GRANT CREATE TEMPORARY TABLES ON datadog.* TO '"$db_datadog_username"'@"%";'
+    mysql_on_k8 $namespace $service $db_host $db_port $db_init_username $db_init_password 'GRANT CREATE TEMPORARY TABLES ON datadog.* TO '"$db_datadog_username"'@"%";'
     echo "Granted  CREATE TEMPORARY TABLES for $db_datadog_username"
 
 
-    mysql_on_k8 $namespace $db_host $db_port $db_init_username $db_init_password 'DROP PROCEDURE IF EXISTS datadog.explain_statement;
+    mysql_on_k8 $namespace $service $db_host $db_port $db_init_username $db_init_password 'DROP PROCEDURE IF EXISTS datadog.explain_statement;
                   DELIMITER $$
                   CREATE PROCEDURE datadog.explain_statement(IN query TEXT)
                       SQL SECURITY DEFINER
@@ -726,7 +729,7 @@ kube_init_datadog_in_database() {
                   DELIMITER ;'
    echo "CREATE PROCEDURE PROCEDURE datadog.explain_statement"
 
-    mysql_on_k8 $namespace $db_host $db_port $db_init_username $db_init_password 'DROP PROCEDURE IF EXISTS '"$db_datadog_username"'.explain_statement;
+    mysql_on_k8 $namespace $service $db_host $db_port $db_init_username $db_init_password 'DROP PROCEDURE IF EXISTS '"$db_datadog_username"'.explain_statement;
                     DELIMITER $$
                     CREATE PROCEDURE '"$db_datadog_username"'.explain_statement(IN query TEXT)
                         SQL SECURITY DEFINER
@@ -740,7 +743,7 @@ kube_init_datadog_in_database() {
                     GRANT EXECUTE ON PROCEDURE '"$db_datadog_username"'.explain_statement TO datadog@"%";'
     echo "CREATE PROCEDURE on SCHEMA $db_datadog_schema for $db_datadog_username"
 
-    mysql_on_k8 $namespace $db_host $db_port $db_init_username $db_init_password 'DROP PROCEDURE IF EXISTS datadog.enable_events_statements_consumers;
+    mysql_on_k8 $namespace $service $db_host $db_port $db_init_username $db_init_password 'DROP PROCEDURE IF EXISTS datadog.enable_events_statements_consumers;
                   DELIMITER $$
                   CREATE PROCEDURE datadog.enable_events_statements_consumers()
                       SQL SECURITY DEFINER
@@ -818,7 +821,7 @@ kube_init_datadog_in_postgres_database() {
       RETURNS NULL ON NULL INPUT
       SECURITY DEFINER;"
 
-    kubectl -n $namespace run $service-database-init \
+    kubectl -n $namespace run $service-postgres-init \
       --image jbergknoff/postgresql-client \
       --restart=Never \
       --attach --rm \
@@ -870,6 +873,57 @@ kube_init_service_database() {
 
 #!/usr/bin/env bash
 
+# Allow to use JMX connection to retrieve data and metrics from the pods within kubernetes
+# You will need visualVM to use this tool https://visualvm.github.io/
+# ex: bind_jmx testing notification
+bind_jmx() {
+
+  local ENV=$1
+  local SERVICE_NAME=$2
+  local PORT=2242
+
+  start_ssh_bastion $ENV $PORT
+
+  echo "root" | ssh -f -N -oUserKnownHostsFile=/dev/null -oStrictHostKeyChecking=no -D 7777 root@127.0.0.1 -p 2242
+  local PODS=$(kubectl -n $ENV get pods -o wide | grep $SERVICE_NAME | grep -Eo '^[^ ]+')
+
+  echo "Choose one of the following pod to get metrics from..."
+  local POD_NAME=$(gum choose $PODS)
+  local POD_IP=$(
+    kubectl -n $ENV get pods -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.status.podIP}{"\n"}{end}' |
+    grep $POD_NAME |
+    cut -d' ' -f2 |
+    head -1
+  )
+
+  jconsole -J-DsocksProxyHost=localhost \
+    -J-DsocksProxyPort=7777 \
+    service:jmx:rmi:///jndi/rmi://$POD_IP:7199/jmxrmi \
+    -J-DsocksNonProxyHosts= &
+
+  cat << EOF
+  Now start VisualVM
+  Preferences > Network > Manual Proxy Settings
+  SOCKS Proxy Line: Set 'localhost' and Port '7777'
+  File > Add JMX Connection
+  Set $POD_IP:7199, check 'do not require an SSL connection'
+  Remember to kill you bastion afterward using 'stop_ssh_bastion'
+EOF
+}
+#!/usr/bin/env bash
+
+k8_nodes_stats() {
+kubectl get nodes -o name  |
+  xargs kubectl describe |
+  grep "^Name\|workType\|cpu  \|memory  "  |
+  sed -r 's/[ :=]+/\t/g' |
+  sed 's/\tworkType\t//g' |
+  sed -r 's/^Name/---\nName/g' |
+  grep --color "Name\|web\|workers\|cpu\|memory\|---"
+}
+
+#!/usr/bin/env bash
+
 # Port forward on the first matching pod
 # Ex :
 # pod_forward testing notification-http
@@ -947,12 +1001,58 @@ pick_pod() {
 
 #!/usr/bin/env bash
 
+bastion_config_for_redis_ca() {
+  ssh_config xufte6.0001.euw1.cache.amazonaws.com redis 2223 63789 tests testing recette-001 sandbox prod > $1
+}
+
+bastion_config_for_redis_toutatis() {
+  ssh_config xufte6.0001.euw1.cache.amazonaws.com toutatis 2223 63789 tests testing recette staging production > $1
+}
+
+ssh_config() {
+  host=$1
+  host_prefix=$2
+  port0=$3
+  forward0=$4
+  shift 4
+  instance_names=("$@") # /!\ indices start at 1 with zsh
+  ssh_header
+
+  environments=(tests testing recette staging production)
+
+  length=${#environments[@]}
+  for (( i=1; i<=${length}; i++ ));
+  do
+    bastion_block bastion_${environments[$i]} $(($port0 + $i)) $(($forward0 + $i)) ${host_prefix}-${instance_names[$i]}.$host
+  done
+}
+
+ssh_header() {
+  cat  <<EOF
+    UserKnownHostsFile /dev/null
+    StrictHostKeyChecking no
+    User root
+EOF
+}
+
+bastion_block() {
+  cat <<EOF
+    Host $1
+      HostName 127.0.0.1
+      Port $2
+      LocalForward $3 $4:6379
+EOF
+}
+
 redis_k8s() {
   MODE=$1
+  REDIS_INSTANCE=${2:-ca}
   case $MODE in
+	   "tests") SSH_LOCAL_PORT=2224;REDIS_LOCAL_PORT=63790;ENV="tests";;
 	   "testing") SSH_LOCAL_PORT=2225;REDIS_LOCAL_PORT=63791;ENV="testing";;
-	   "staging") SSH_LOCAL_PORT=2226;REDIS_LOCAL_PORT=63792;ENV="staging";;
-	   "production") SSH_LOCAL_PORT=2227;REDIS_LOCAL_PORT=63793;ENV="production";;
+	   "recette") SSH_LOCAL_PORT=2226;REDIS_LOCAL_PORT=63792;ENV="recette";;
+	   "staging") SSH_LOCAL_PORT=2227;REDIS_LOCAL_PORT=63793;ENV="staging";;
+	   "production") SSH_LOCAL_PORT=2228;REDIS_LOCAL_PORT=63794;ENV="production";;
 	   *) echo "Unsupported ENV : $MODE"; return 1 ;;
   esac
 
@@ -961,23 +1061,11 @@ redis_k8s() {
   lsof -ti  tcp:$REDIS_LOCAL_PORT | xargs kill
 
   bastion_config=$(mktemp)
-  cat > "$bastion_config" <<EOF
-	  UserKnownHostsFile /dev/null
-	  StrictHostKeyChecking no
-	  User root
-	  Host bastion_testing
-		HostName 127.0.0.1
-		Port 2225
-		LocalForward 63791 redis-testing.xufte6.0001.euw1.cache.amazonaws.com:6379
-	  Host bastion_staging
-		HostName 127.0.0.1
-		Port 2226
-		LocalForward 63792 redis-sandbox.xufte6.0001.euw1.cache.amazonaws.com:6379
-	  Host bastion_production
-		HostName 127.0.0.1
-		Port 2227
-    LocalForward 63793 redis-prod.xufte6.0001.euw1.cache.amazonaws.com:6379
-EOF
+  case $REDIS_INSTANCE in
+    "ca") bastion_config_for_redis_ca "$bastion_config";;
+    "toutatis") bastion_config_for_redis_toutatis "$bastion_config";;
+    *) echo "Unsupported redis instance (ca or toutatis available) : $REDIS_INSTANCE"; return 1;;
+  esac
 
   ssh -f -N \
   	-F "$bastion_config" \
@@ -1820,11 +1908,11 @@ datadog_schedule_downtime_single() {
       {
         "active": true,
           "downtime_type": 0,
-          "start": $START,
-          "end": $END,
-          "message": "CA Deployment - performance for $SERVICE may be lower for next $DOWNTIME_MINUTES min",
+          "start": '$START',
+          "end": '$END',
+          "message": "CA Deployment - performance for '$SERVICE' may be lower for next '$DOWNTIME_MINUTES' min",
           "monitor_tags": [
-            "service:$SERVICE",
+            "service:'$SERVICE'",
             "performance"
           ],
           "scope": [
@@ -1834,6 +1922,7 @@ datadog_schedule_downtime_single() {
       }
       '
 }
+
 #!/usr/bin/env bash
 
 docker_build_push() {
@@ -1989,7 +2078,7 @@ init_migrate_db() {
 
   unset KUBECONFIG
 
-  configure_kubectl_for_ci ${ENVIRONMENT}
+  configure_kubectl_for ${ENVIRONMENT}
 
   kube_init_service_database \
     --namespace ${ENVIRONMENT} \
@@ -2035,7 +2124,7 @@ flyway_migrate() {
   CONFIGMAP_NAME="$service-flyway-migration-sql"
   POD_NAME="$service-flyway-migration"
 
-  configure_kubectl_for_ci $environment
+  configure_kubectl_for $environment
 
   kubectl -n $namespace delete configmap $CONFIGMAP_NAME --ignore-not-found
   kubectl -n $namespace delete pod $POD_NAME --ignore-not-found
@@ -2088,7 +2177,7 @@ flyway_migrate() {
 
   flyway_sql_folder=$(pwd)/${MIGRATION_SQL_PATH}
 
-  configure_kubectl_for_ci "${ENVIRONMENT}"
+  configure_kubectl_for "${ENVIRONMENT}"
   POD_NAME="${APPLICATION}-flyway-repair"
   CONFIGMAP_NAME="${APPLICATION}-flyway-repair-sql"
 
@@ -2214,7 +2303,7 @@ deploy_chart_v3() {
   unset KUBECONFIG
 
   # Configure Kubectl
-  configure_kubectl_for_ci ${environment}
+  configure_kubectl_for ${environment}
 
   # Configure helm3
   helm3 version --namespace ${namespace} || true
@@ -2315,40 +2404,6 @@ check_config_file() {
     fi
 }
 
-#!/usr/bin/env bash
-
-configure_kubectl_for_ci() {
-  if [ -z ${GITLAB_PAT} ]; then
-    echo "Cannot configure kubectl: no GITLAB_PAT configured"
-    exit 1
-  fi
-
-  infra_env="$1"
-  valid_envs="[testing][staging][production][performance][tests][recette]"
-  echo "$valid_envs" | grep -q "\[$infra_env\]"
-
-  if [ $? -ne 0 ]; then
-    echo "Cannot configure kubectl for invalid env : $infra_env"
-    echo "choose one of $valid_envs"
-    exit 1
-  fi
-
-  mkdir -p ~/.kube
-  curl -fsS \
-  --header "PRIVATE-TOKEN: $GITLAB_PAT" \
-  "https://gitlab.com/api/v4/projects/8141053/jobs/artifacts/$infra_env/raw/$infra_env.kubeconfig?job=4_kubernetes_config_output" \
-  > ~/.kube/$infra_env.kubeconfig
-
-  curl_return_code=$?
-  if [ ${curl_return_code} -ne 0 ]; then
-    echo "Cannot configure kubectl for $infra_env, get configuration failed with code $curl_return_code"
-    exit ${curl_return_code}
-  fi
-
-  rm -f ~/.kube/config
-  ln -s ~/.kube/$infra_env.kubeconfig ~/.kube/config
-  echo "Configured kubectl for env : $infra_env"
-}
 notify_new_deployment() {
   jq --version || (apt update && apt install -y jq)