@colisweb/rescript-toolkit 4.14.16 → 4.15.1

package/.secure_files/{ci-functions-v16.6.0 → ci-functions-v17.7.2}

@@ -97,22 +97,19 @@ aws_ecr_token() {
 # will delete images older than 8 weeks
 delete_images() {
 
-  aws --version | grep -q "^aws-cli/2" || (echo "You must have aws-cli v2 installed to use this script") ; return 1
-
   REPO=$1
   WEEKS=${2:-16}
 
-  WEEKS_AGO=$(date -j -v-${WEEKS}w +%s)
+  WEEKS_AGO=$(date -v-${WEEKS}w +%F)
 
   #Get all ecr images
   IMAGES=$(aws ecr describe-images --repository-name $REPO --output json)
 
   #Filter unnecessary values and map `imagePushedAt` to EPOCH
-  TIMED_IMAGES=$(echo $IMAGES | jq .'[]' | jq "map({imagePushedAt: (.imagePushedAt[0:19]+\"Z\" | fromdateiso8601), imageDigest: .imageDigest}) | sort_by(.imagePushedAt) | .[:-1]")
+  NON_LATEST_IMAGES=$(echo $IMAGES | jq '[.imageDetails[] | select(.imageTags | any(endswith("latest")) | not)]')
 
   #Filter on EPOCH
-  OLD_IMAGES=$(echo $TIMED_IMAGES | jq "map(select (.imagePushedAt < $WEEKS_AGO)) | .[] " | jq -r '.imageDigest')
-
+  OLD_IMAGES=$(echo $NON_LATEST_IMAGES | jq --arg date $WEEKS_AGO '.[] | select(.imagePushedAt[0:10] < $date).imageDigest')
   while IFS= read -r IMAGE; do
     if [ "$IMAGE" != "" ]; then
       echo "Deleting $IMAGE from $REPO"
@@ -132,6 +129,153 @@ delete_images_all_repos() {
   done <<< "$REPOSITORIES"
 }
 
+delete_old_cache() {
+  DATE=${1:-$(date -v-1m +%F)}
+  CACHE_BUCKET=${2:-"s3://gitlab-colisweb-distributed-cache/project/"}
+
+  echo "deleting from cache $CACHE_BUCKET all older than $DATE"
+
+  aws_ecr_login
+
+  while read -r line; do
+    datum=$(echo $line | cut -c1-10)
+    if [[ "$datum" < "$DATE" ]] ; then
+      # Shell Parameter Expansion: ${parameter##word}
+      # Allow to return the result from "word" to the end of "parameters"
+      # Here we need the end of the string after "project/" (corresponding to the S3 gitlab project id and filename)
+      TO_DELETE="$CACHE_BUCKET${line##* project/}"
+      echo $TO_DELETE
+      aws s3 rm $TO_DELETE
+    fi
+  done < <(aws s3 ls $CACHE_BUCKET  --recursive)
+}
+
+#!/usr/bin/env bash
+
+# If gitlab is down or pipeline are stuck, hotfixes need to be available
+# This script will publish docker images to ECR using your current git HEAD, then deploy them to a given environment.
+# Some local files (git-commit.conf and sentry.properties) will be updated, take caution.
+# No trace of this will appear on Gitlab (no releases, no pipelines, no tags).
+# create_hotfix_scala $ENVIRONMENT $CHART_NAME [ $MODULE_NAME $MODULE_PATH $DEPLOYMENT ]
+# create_hotfix_scala testing crm main modules/3-executables/main crm
+# create_hotfix_scala testing notification \
+#   main-http modules/3-executables/main-http notification-http \
+#   main-consumer modules/3-executables/main-consumer notification-consumer
+
+create_hotfix_scala() {
+
+  ENVIRONMENT=$1
+  CHART_NAME=$2
+  shift 2
+
+  SHORT_SHA=$(git rev-parse --short HEAD)
+  HOTFIX_TAG="hotfix-$SHORT_SHA"
+
+  gum confirm "Preparing $HOTFIX_TAG for $CHART_NAME ?" || exit
+  prepare_hotfix_scala $HOTFIX_TAG
+
+  gum confirm "Building $HOTFIX_TAG for $CHART_NAME ?" || exit
+  while [[ $# -gt 2 ]] ; do
+    build_hotfix_scala $HOTFIX_TAG "$1" "$2" "$3"
+    shift 3
+  done
+
+  gum confirm "Deploying $HOTFIX_TAG for $CHART_NAME ?" || exit
+  deploy_hotfix $CHART_NAME $ENVIRONMENT $HOTFIX_TAG
+}
+
+# Update local git-commit.conf and sentry.properties files using git short sha
+prepare_hotfix_scala() {
+  HOTFIX_TAG=$1
+
+  git secret reveal -f
+  aws_ecr_login
+
+  COMMIT_CONF_FILES=$(find . -name "git-commit.conf")
+  SENTRY_PROPERTIES_FILES=$(find . -name "sentry.properties")
+
+  for file in $(echo "$COMMIT_CONF_FILES\n$SENTRY_PROPERTIES_FILES"); do
+      sed -i '' -e 's&GIT_COMMIT&'"$HOTFIX_TAG&" $file
+  done
+
+}
+
+# Build docker images locally and publish them to AWS ECR.
+build_hotfix_scala() {
+
+  HOTFIX_TAG=$1
+  SBT_MODULE=$2
+  DOCKER_PATH=$3
+  DEPLOYMENT=$4
+
+  DOCKER_REGISTRY_ID="949316342391"
+  DOCKER_REGISTRY="$DOCKER_REGISTRY_ID.dkr.ecr.eu-west-1.amazonaws.com"
+  DOCKER_IMAGE=$DOCKER_REGISTRY/$DEPLOYMENT
+  HOTFIX_IMAGE=$DOCKER_IMAGE:$HOTFIX_TAG
+
+  #Build
+  sbt "project $SBT_MODULE" "Docker / stage"
+
+  #Publish
+  docker build --platform "linux/amd64" -t $HOTFIX_IMAGE --cache-from $DOCKER_IMAGE "$DOCKER_PATH/target/docker/stage"
+  docker push $HOTFIX_IMAGE
+
+  echo "Created hotfix $HOTFIX_IMAGE"
+}
+
+# Deploy the project in the given environment
+deploy_hotfix() {
+  source $colisweb_scripts/ci/helm.sh
+
+  CHART_NAME=$1
+  ENVIRONMENT=$2
+  HOTFIX_TAG=$3
+
+  CONFIG_PATH=deploy
+  CHART_PATH=$CONFIG_PATH/$CHART_NAME
+  ROOT_PATH=$(pwd)
+
+  # Unset Kubectl configuration made via the KUBECONFIG env variable
+  #  it would override the config made by configure_kubectl_for
+  #  for example, using Gitlab runners in Kubernetes sets this variable and causes conflict
+  unset KUBECONFIG
+
+  # Configure Kubectl
+  configure_kubectl_for $ENVIRONMENT
+
+  # Avoiding "no local-index.yaml" or "empty local-index.yaml" error
+  cat > $HOME/Library/Caches/helm/repository/local-index.yaml <<EOT
+apiVersion: v1
+entries:
+  cronjob:
+EOT
+
+  # helm3 stable repo have changed and must be updated manually, in versions < v2.17.0
+  helm3 repo add colisweb s3://colisweb-helm-charts/colisweb  --force-update
+  helm3 repo add stable https://charts.helm.sh/stable --force-update
+  helm3 repo update
+  helm3 dependency update ${ROOT_PATH}/${CHART_PATH}
+
+  # Gather values/*.yaml files
+  VALUES_PATH="${ROOT_PATH}/${CHART_NAME}/values"
+  VALUES_FILES=''
+  [ -d $VALUES_PATH ] && VALUES_FILES=$(find $VALUES_PATH -type f -maxdepth 1 -name "*.yaml" | sed 's/^/ -f /' | tr -d \\n | sed 's/%//')
+
+  # Deploy
+  helm3 upgrade --install \
+    --namespace ${ENVIRONMENT} \
+    ${VALUES_FILES} \
+    -f ${ROOT_PATH}/${CONFIG_PATH}/common.yaml \
+    -f ${ROOT_PATH}/${CONFIG_PATH}/${ENVIRONMENT}.yaml \
+    -f ${ROOT_PATH}/${CONFIG_PATH}/${ENVIRONMENT}-secrets.yaml \
+    --set global.version=$HOTFIX_TAG \
+    ${CHART_NAME} ${ROOT_PATH}/${CHART_PATH}
+
+
+  verify_deployments_v3 -t 10m $ENVIRONMENT $CHART_NAME
+
+}
+
 #!/usr/bin/env bash
 
 image_exists() {
@@ -391,28 +535,29 @@ database_k8s() {
 		HostName 127.0.0.1
 		Port 2225
 		LocalForward 24441 toutatis-testing-db.ca0rjdmnxf1x.eu-west-1.rds.amazonaws.com:5432
-		LocalForward 25431 testapirds.ca0rjdmnxf1x.eu-west-1.rds.amazonaws.com:3306
+		LocalForward 25431 toutatis-testing-mysql-db.ca0rjdmnxf1x.eu-west-1.rds.amazonaws.com:3306
+		LocalForward 25531 testapirds.ca0rjdmnxf1x.eu-west-1.rds.amazonaws.com:3306
 	  Host bastion_staging
 		HostName 127.0.0.1
 		Port 2226
 		LocalForward 24442 toutatis-staging-db.ca0rjdmnxf1x.eu-west-1.rds.amazonaws.com:5432
-		LocalForward 25432 testapirds.ca0rjdmnxf1x.eu-west-1.rds.amazonaws.com:3306
+		LocalForward 25432 toutatis-staging-mysql-db.ca0rjdmnxf1x.eu-west-1.rds.amazonaws.com:3306
     Host bastion_recette
 		HostName 127.0.0.1
 		Port 2228
 		LocalForward 24446 toutatis-recette-db.ca0rjdmnxf1x.eu-west-1.rds.amazonaws.com:5432
-		LocalForward 25436 testapirds.ca0rjdmnxf1x.eu-west-1.rds.amazonaws.com:3306
+		LocalForward 25436 toutatis-recette-mysql-db.ca0rjdmnxf1x.eu-west-1.rds.amazonaws.com:3306
+		LocalForward 25536 testapirds.ca0rjdmnxf1x.eu-west-1.rds.amazonaws.com:3306
 	  Host bastion_production
 		HostName 127.0.0.1
 		Port 2227
     LocalForward 24443 toutatis-production-db-replica.ca0rjdmnxf1x.eu-west-1.rds.amazonaws.com:5432
-    LocalForward 25433 api-production-rds-read-replica.ca0rjdmnxf1x.eu-west-1.rds.amazonaws.com:3306
-    LocalForward 25435 archive-ca.ca0rjdmnxf1x.eu-west-1.rds.amazonaws.com:3306
+    LocalForward 25433 toutatis-production-mysql-db-replica.ca0rjdmnxf1x.eu-west-1.rds.amazonaws.com:3306
 EOF
   if [ "$MODE" = "production_rw" ] ; then
     cat >> "$bastion_config" <<EOF
       LocalForward 24444 toutatis-production-db.ca0rjdmnxf1x.eu-west-1.rds.amazonaws.com:5432
-      LocalForward 25434 api-production-rds.ca0rjdmnxf1x.eu-west-1.rds.amazonaws.com:3306
+      LocalForward 25434 toutatis-production-mysql-db.ca0rjdmnxf1x.eu-west-1.rds.amazonaws.com:3306
 EOF
   fi
 
@@ -432,7 +577,7 @@ psql_on_k8() {
   CONNECTION=$3
   shift 3
 
-  kubectl -n $NAMESPACE run ${SERVICE}-database-init \
+  kubectl -n $NAMESPACE run ${SERVICE}-postgres-init \
     --image jbergknoff/postgresql-client \
     --restart=Never \
     --attach --rm \
@@ -443,13 +588,14 @@ psql_on_k8() {
 
 mysql_on_k8() {
   local namespace=$1
-  local db_host=$2
-  local db_port=$3
-  local db_init_username=$4
-  local db_init_password=$5
-  local query=$6
-
-  kubectl -n ${namespace} run datadog-database-init \
+  local service=$2
+  local db_host=$3
+  local db_port=$4
+  local db_init_username=$5
+  local db_init_password=$6
+  local query=$7
+
+  kubectl -n ${namespace} run ${service}-mysql-init \
     --image widdpim/mysql-client \
     --restart=Never \
     --attach --rm \
@@ -533,11 +679,11 @@ kube_init_datadog_in_database() {
   echo " Initializing Datadog Agent Requiement for namespace $namespace"
   echo "======================="
 
-  set -x
-
-  echo "Checking if Database '$db_datadog_username' exists"
+  echo "Checking if User  '$db_datadog_username' exists"
+  local service="datadog"
+  found_db_users=$(mysql_on_k8 $namespace $service $db_host $db_port $db_init_username $db_init_password 'SELECT user FROM mysql.user;')
   set +e
-  mysql_on_k8 $namespace $db_host $db_port $db_init_username $db_init_password 'SELECT user FROM mysql.user;' | grep "^$db_datadog_username$"
+  echo $found_db_users | grep "^$db_datadog_username$"
   return_code=$?
   set -e
 
@@ -548,29 +694,29 @@ kube_init_datadog_in_database() {
 
     # All the query come from this docs : https://docs.datadoghq.com/fr/database_monitoring/setup_mysql/selfhosted/?tab=mysql56
 
-    mysql_on_k8 $namespace $db_host $db_port $db_init_username $db_init_password 'CREATE USER '"$db_datadog_username"'@"%" IDENTIFIED BY '"'$db_datadog_password'"';'
+    mysql_on_k8 $namespace $service $db_host $db_port $db_init_username $db_init_password 'CREATE USER '"$db_datadog_username"'@"%" IDENTIFIED BY '"'$db_datadog_password'"';'
     echo "USER created $db_datadog_username"
 
-    mysql_on_k8 $namespace $db_host $db_port $db_init_username $db_init_password 'GRANT REPLICATION CLIENT ON *.* TO datadog@"%" WITH MAX_USER_CONNECTIONS 5;'
+    mysql_on_k8 $namespace $service $db_host $db_port $db_init_username $db_init_password 'GRANT REPLICATION CLIENT ON *.* TO datadog@"%" WITH MAX_USER_CONNECTIONS 5;'
     echo "ALTER USER $db_datadog_username"
 
-    mysql_on_k8 $namespace $db_host $db_port $db_init_username $db_init_password 'GRANT PROCESS ON *.* TO '"$db_datadog_username"'@"%";'
+    mysql_on_k8 $namespace $service $db_host $db_port $db_init_username $db_init_password 'GRANT PROCESS ON *.* TO '"$db_datadog_username"'@"%";'
     echo "Granted PROCESS for $db_datadog_username"
 
-    mysql_on_k8 $namespace $db_host $db_port $db_init_username $db_init_password 'GRANT SELECT ON performance_schema.* TO '"$db_datadog_username"'@"%";'
+    mysql_on_k8 $namespace $service $db_host $db_port $db_init_username $db_init_password 'GRANT SELECT ON performance_schema.* TO '"$db_datadog_username"'@"%";'
     echo "Granted SELECT on performance_schema for $db_datadog_username"
 
-    mysql_on_k8 $namespace $db_host $db_port $db_init_username $db_init_password 'CREATE SCHEMA IF NOT EXISTS datadog;'
+    mysql_on_k8 $namespace $service $db_host $db_port $db_init_username $db_init_password 'CREATE SCHEMA IF NOT EXISTS datadog;'
     echo "CREATE SCHEMA datadog"
 
-    mysql_on_k8 $namespace $db_host $db_port $db_init_username $db_init_password 'GRANT EXECUTE ON datadog.* to '"$db_datadog_username"'@"%";'
+    mysql_on_k8 $namespace $service $db_host $db_port $db_init_username $db_init_password 'GRANT EXECUTE ON datadog.* to '"$db_datadog_username"'@"%";'
     echo "Granted 'GRANT EXECUTE for $db_datadog_username on datadog"
 
-    mysql_on_k8 $namespace $db_host $db_port $db_init_username $db_init_password 'GRANT CREATE TEMPORARY TABLES ON datadog.* TO '"$db_datadog_username"'@"%";'
+    mysql_on_k8 $namespace $service $db_host $db_port $db_init_username $db_init_password 'GRANT CREATE TEMPORARY TABLES ON datadog.* TO '"$db_datadog_username"'@"%";'
     echo "Granted  CREATE TEMPORARY TABLES for $db_datadog_username"
 
 
-    mysql_on_k8 $namespace $db_host $db_port $db_init_username $db_init_password 'DROP PROCEDURE IF EXISTS datadog.explain_statement;
+    mysql_on_k8 $namespace $service $db_host $db_port $db_init_username $db_init_password 'DROP PROCEDURE IF EXISTS datadog.explain_statement;
                   DELIMITER $$
                   CREATE PROCEDURE datadog.explain_statement(IN query TEXT)
                       SQL SECURITY DEFINER
@@ -583,7 +729,7 @@ kube_init_datadog_in_database() {
                   DELIMITER ;'
    echo "CREATE PROCEDURE PROCEDURE datadog.explain_statement"
 
-    mysql_on_k8 $namespace $db_host $db_port $db_init_username $db_init_password 'DROP PROCEDURE IF EXISTS '"$db_datadog_username"'.explain_statement;
+    mysql_on_k8 $namespace $service $db_host $db_port $db_init_username $db_init_password 'DROP PROCEDURE IF EXISTS '"$db_datadog_username"'.explain_statement;
                     DELIMITER $$
                     CREATE PROCEDURE '"$db_datadog_username"'.explain_statement(IN query TEXT)
                         SQL SECURITY DEFINER
@@ -597,7 +743,7 @@ kube_init_datadog_in_database() {
                     GRANT EXECUTE ON PROCEDURE '"$db_datadog_username"'.explain_statement TO datadog@"%";'
     echo "CREATE PROCEDURE on SCHEMA $db_datadog_schema for $db_datadog_username"
 
-    mysql_on_k8 $namespace $db_host $db_port $db_init_username $db_init_password 'DROP PROCEDURE IF EXISTS datadog.enable_events_statements_consumers;
+    mysql_on_k8 $namespace $service $db_host $db_port $db_init_username $db_init_password 'DROP PROCEDURE IF EXISTS datadog.enable_events_statements_consumers;
                   DELIMITER $$
                   CREATE PROCEDURE datadog.enable_events_statements_consumers()
                       SQL SECURITY DEFINER
@@ -615,6 +761,82 @@ kube_init_datadog_in_database() {
   echo "======================="
 }
 
+kube_init_datadog_in_postgres_database() {
+  extract_args 7 namespace db_host db_port db_init_username db_init_password db_datadog_username db_datadog_password $*
+
+  local service="datadog"
+  local db_connection="$db_init_username:$db_init_password@$db_host:$db_port"
+
+  echo "======================="
+  echo " Initializing $service Agent On PostgresSQL Database Requirement for namespace $namespace"
+  echo "======================="
+
+  echo "Checking if User  '$db_datadog_username' exists"
+
+  set +e
+  if psql_on_k8 $namespace $service $db_connection -qtAc 'SELECT usename FROM pg_catalog.pg_user;' | grep "^$db_datadog_username$";
+  then
+    echo "User $db_datadog_username already exists - nothing to do"
+  else
+    echo "User $db_datadog_username does not exist - initializing"
+
+    set -e
+    psql_on_k8 $namespace $service $db_connection -qc 'CREATE USER '"$db_datadog_username"' WITH password '"'$db_datadog_password'"';'
+    echo "User created $db_datadog_username"
+
+    psql_on_k8 $namespace $service $db_connection -qc 'CREATE SCHEMA datadog;'
+    echo "Schema datadog created"
+
+    psql_on_k8 $namespace $service $db_connection -qc 'GRANT USAGE ON SCHEMA datadog TO datadog;'
+    echo "Granted usage for datadog schema to datadog"
+
+    psql_on_k8 $namespace $service $db_connection -qc 'GRANT USAGE ON SCHEMA public TO datadog;'
+    echo "Granted usage for public schema to datadog"
+
+    psql_on_k8 $namespace $service $db_connection -qc 'GRANT pg_monitor TO datadog;'
+    echo "Granted pg_monitor to datadog"
+
+    psql_on_k8 $namespace $service $db_connection -qc 'CREATE EXTENSION IF NOT EXISTS pg_stat_statements schema public;'
+    echo "Extension pg_stat_statements created"
+
+    local datadog_function_path="/tmp/datatog-explain-statement-function.sql"
+    local datadog_function="CREATE OR REPLACE FUNCTION datadog.explain_statement(
+         l_query TEXT,
+         OUT explain JSON
+      )
+      RETURNS SETOF JSON AS
+      \\$\\$
+      DECLARE
+      curs REFCURSOR;
+      plan JSON;
+
+      BEGIN
+         OPEN curs FOR EXECUTE pg_catalog.concat('EXPLAIN (FORMAT JSON) ', l_query);
+         FETCH curs INTO plan;
+         CLOSE curs;
+         RETURN QUERY SELECT plan;
+      END;
+      \\$\\$
+      LANGUAGE 'plpgsql'
+      RETURNS NULL ON NULL INPUT
+      SECURITY DEFINER;"
+
+    kubectl -n $namespace run $service-postgres-init \
+      --image jbergknoff/postgresql-client \
+      --restart=Never \
+      --attach --rm \
+      --command \
+      -- \
+      /bin/sh -c "echo -e \"$datadog_function\" > $datadog_function_path; psql postgresql://$db_connection -qf $datadog_function_path"
+
+    echo "Function datadog.explain_statement created"
+  fi
+
+  echo "======================="
+  echo " Database $service Initialization complete for namespace $namespace"
+  echo "======================="
+}
+
 kube_init_service_database() {
 
   extract_args 9 namespace service db_host db_port db_init_username db_init_password db_database db_username db_password $*
@@ -637,17 +859,18 @@ kube_init_service_database() {
     psql_on_k8 $namespace $service $db_connection -c 'CREATE DATABASE '"$db_database"';'
     echo "DB created $db_database"
 
-    psql_on_k8 $namespace $service $db_connection -c 'CREATE USER '"$db_datadog_username"' WITH ENCRYPTED PASSWORD '"'$db_password'"';'
-    echo "USER created $db_datadog_username"
+    psql_on_k8 $namespace $service $db_connection -c 'CREATE USER '"$db_username"' WITH ENCRYPTED PASSWORD '"'$db_password'"';'
+    echo "USER created $db_username"
 
-    psql_on_k8 $namespace $service $db_connection -c 'GRANT ALL PRIVILEGES ON DATABASE '"$db_database"' TO '"$db_datadog_username"';'
-    echo "Granted all privileges for $db_datadog_username on $db_database"
+    psql_on_k8 $namespace $service $db_connection -c 'GRANT ALL PRIVILEGES ON DATABASE '"$db_database"' TO '"$db_username"';'
+    echo "Granted all privileges for $db_username on $db_database"
   fi
 
   echo "======================="
-  echo " Database '$db_database' Initialization complete for namespace $namespace"
+  echo " Database '$db_database' Initialization complete for  namespace $namespace"
   echo "======================="
 }
+
 #!/usr/bin/env bash
 
 # Port forward on the first matching pod
@@ -1165,6 +1388,36 @@ search_business() {
   curl $URL
 }
 
+#!/bin/bash
+
+# source tolls.sh ; tolls antoine.thomas@colisweb.com
+function tolls() {
+  USER=${1:-first.last@colisweb.com}
+  FROM_DATE=${2:-"2023-02-01"}
+  TO_DATE=${3:-"2023-02-28"}
+
+  USER=$(gum input --prompt "username : " --value $USER)
+  TOKEN=$(./tour_details.sc login --user $USER --password $(gum input --password --placeholder password))
+  [ "$TOKEN" != "" ] && echo "connected" || return 1
+
+  FROM_DATE=$(gum input --prompt "Date start : " --value $FROM_DATE)
+  TO_DATE=$(gum input --prompt "Date end : " --value $TO_DATE)
+  FILENAME="tours-${FROM_DATE}-TO-${TO_DATE}.json"
+  curl --cookie "session=$TOKEN" "https://api.production.colisweb.com/api/v6/routes-plans/external?from=${FROM_DATE}&to=${TO_DATE}" > ~/Downloads/$FILENAME
+  echo "Tournées téléchargées"
+
+  projectIds=$(./tour_details.sc allProjects --file ~/Downloads/$FILENAME | gum choose --no-limit | cut -d "," -f 2)
+  echo "projets sélectionnés : $projectIds"
+  tourIds=$(./tour_details.sc allTours --file ~/Downloads/$FILENAME --projectIds "$projectIds")
+  echo "tournées sélectionnées : $tourIds"
+
+  TARGET="${FROM_DATE}-TO-${TO_DATE}.csv"
+  echo "appels à HERE, écriture dans $TARGET"
+  ./tour_details.sc allToursDetails --token $TOKEN --hereApiKey $HERE_API_KEY --routeIds "$tourIds" > "$TARGET"
+
+  echo "terminé"
+}
+
 #!/usr/bin/env bash
 
 # possible syntax:
@@ -1269,17 +1522,67 @@ jconsole_k8s() {
 
 #!/usr/bin/env bash
 
-# Interactive console on an existing pod. See also run_ruby_k8s
+# Interactive console on an new pod. See also run_ruby_k8s
 # Ex :
 # railsc_k8s production
 # railsc_k8s production "User.where(email:'toni@colisweb.com')"
 railsc_k8s() {
   ENV=$1
   COMMAND=$2
+  [[ $ENV = "production" || $ENV = "staging" ]] && default_tag="master-latest" || default_tag="${ENV}-latest"
+  local image_tag=${5:-$default_tag}
+  local IMAGE="949316342391.dkr.ecr.eu-west-1.amazonaws.com/colisweb-api:$image_tag"
+  local POD_NAME="colisweb-api-rails-console-$image_tag-$USERNAME"
+
+  kubectl -n $ENV get pod $POD_NAME && kubectl -n $ENV delete pod $POD_NAME
+
   configure_kubectl_for $ENV
-  POD=$(kubectl -n $ENV get pods -o=name | grep colisweb-api-web | head -1 | sed -e 's/pod\///')
-  KUBERAILS="kubectl -n $ENV exec -ti $POD -- /usr/src/app/bin/rails c"
+  echo "starting with $IMAGE"
+
+  kubectl -n $ENV run $POD_NAME \
+       --image $IMAGE \
+       --restart=Never \
+       --overrides='{
+       "spec":{
+          "nodeSelector":{
+            "workType": "workers"
+          },
+          "containers":[
+             {
+                "name":"'$POD_NAME'",
+                "image":"'$IMAGE'",
+                "imagePullPolicy":"Always",
+                "command":[
+                  "sleep",
+                  "infinity"
+                ],
+                "resources":{
+                  "limits":{
+                    "memory": "2048Mi"
+                  }
+                },
+                "envFrom": [ {
+                     "configMapRef": {
+                         "name": "colisweb-api"
+                     }
+                   }, {
+                     "secretRef": {
+                         "name": "colisweb-api"
+                     }
+                   }
+                ]
+             }
+          ]
+       }
+    }
+    '
+
+  sleep 5
+  KUBERAILS="kubectl -n $ENV exec -ti $POD_NAME -- /usr/src/app/bin/rails c"
   [ -z "$COMMAND" ] && eval $KUBERAILS || echo $COMMAND | eval $KUBERAILS
+
+  print "End of $POD_NAME "
+  kubectl -n $ENV delete pods $POD_NAME
 }
 
 # Ex :
@@ -1315,7 +1618,7 @@ run_ruby_k8s() {
   local name=$2
   local ruby_script=$3
   local input_data=$4
-  [[ $namespace = "production" ]] && default_tag="master-latest" || default_tag="${namespace}-latest"
+  [[ $namespace = "production" || $namespace = "staging" ]] && default_tag="master-latest" || default_tag="${namespace}-latest"
   local image_tag=${5:-$default_tag}
 
   if [ ! -r "$ruby_script" ]; then
@@ -1546,11 +1849,18 @@ docker_build_push() {
 
   if ! image_exists $DOCKER_REGISTRY_ID $APPLICATION $CI_COMMIT_SHORT_SHA ; then
     docker pull $DOCKER_IMAGE || true
-    docker build $DOCKER_BUILD_ARGS -t $DOCKER_IMAGE_SHA --cache-from $DOCKER_IMAGE $DOCKER_STAGE_PATH
+    SOURCE_URL=${CI_PROJECT_URL:8} # without "https://" protocol, like gitlab.com/colisweb-idl/colisweb/back/packing
+    docker build $DOCKER_BUILD_ARGS \
+      -t $DOCKER_IMAGE_SHA \
+      --label org.opencontainers.image.revision=$(git rev-parse HEAD) \
+      --label org.opencontainers.image.source=$SOURCE_URL \
+      --cache-from $DOCKER_IMAGE \
+      $DOCKER_STAGE_PATH
     docker push $DOCKER_IMAGE_SHA
   fi
 }
 
+
 docker_promote() {
    # inspired by https://dille.name/blog/2018/09/20/how-to-tag-docker-images-without-pulling-them/
    OLD_TAG=${1//[^0-9a-zA-Z-.]/_}
@@ -1577,6 +1887,7 @@ docker_promote() {
      image_exists ${DOCKER_REGISTRY_ID} ${IMAGE_TO_CHECK} ${VERSION} || return 1
    done
  }
+
 #!/usr/bin/env bash
 
 extract_yaml_config_variable() {
@@ -1653,7 +1964,7 @@ flyway_clean() {
 
 #!/usr/bin/env bash
 
-FLYWAY_VERSION="5.2.4"
+FLYWAY_VERSION="7.4.0"
 
 
 get_yaml_variable() {
@@ -1739,7 +2050,7 @@ flyway_migrate() {
       "containers":[
          {
             "name":"'$POD_NAME'",
-            "image":"boxfuse/flyway:'$flyway_version'",
+            "image":"flyway/flyway:'$flyway_version'",
             "command":["flyway", "-url='$db_url'", "-user='$db_user'", "-password='$db_password'", "migrate"],
             "volumeMounts":[
                {
@@ -1764,6 +2075,63 @@ flyway_migrate() {
   kubectl -n $namespace delete configmap $CONFIGMAP_NAME
 }
 
+#!/usr/bin/env bash
+ flyway_repair() {
+  set -e
+  check_env_vars 4 "APPLICATION" "ENVIRONMENT" "FLYWAY_VERSION" "MIGRATION_SQL_PATH"
+
+  PG_YAML_PATH=".${APPLICATION}config.postgres"
+
+  DB_PORT="5432"
+  DB_HOST=$(get_yaml_variable "${PG_YAML_PATH}.host")
+  DB_DATABASE=$(get_yaml_variable "${PG_YAML_PATH}.database")
+  DB_USER=$(get_yaml_variable "${PG_YAML_PATH}.user")
+  DB_PASSWORD=$(get_yaml_variable "${PG_YAML_PATH}.password")
+  DB_URL="jdbc:postgresql://${DB_HOST}:${DB_PORT}/${DB_DATABASE}"
+
+  flyway_sql_folder=$(pwd)/${MIGRATION_SQL_PATH}
+
+  configure_kubectl_for_ci "${ENVIRONMENT}"
+  POD_NAME="${APPLICATION}-flyway-repair"
+  CONFIGMAP_NAME="${APPLICATION}-flyway-repair-sql"
+
+  kubectl -n "${ENVIRONMENT}" delete configmap $CONFIGMAP_NAME --ignore-not-found
+  kubectl -n "${ENVIRONMENT}" delete pod $POD_NAME --ignore-not-found
+  kubectl -n "${ENVIRONMENT}" create configmap $CONFIGMAP_NAME --from-file="${flyway_sql_folder}"
+
+  kubectl -n "${ENVIRONMENT}" run --rm  -it "${POD_NAME}" \
+  --image=flyway/flyway \
+  --restart=Never \
+  --overrides='
+  {
+     "spec":{
+        "containers":[
+           {
+              "name":"'$POD_NAME'",
+              "image":"flyway/flyway:'${FLYWAY_VERSION}'",
+              "command":["flyway", "-url='$DB_URL'", "-user='$DB_USER'", "-password='$DB_PASSWORD'", "repair"],
+              "volumeMounts":[
+                 {
+                    "name":"sql",
+                    "mountPath":"/flyway/sql"
+                 }
+              ]
+           }
+        ],
+        "volumes":[
+           {
+              "name":"sql",
+              "configMap":{
+                 "name":"'$CONFIGMAP_NAME'"
+              }
+           }
+        ]
+     }
+  }
+  '
+  kubectl -n "${ENVIRONMENT}" delete configmap $CONFIGMAP_NAME
+}
+
 #!/usr/bin/env bash
 
 record_git_commit() {
@@ -1789,152 +2157,6 @@ git_reveal() {
 }
 #!/usr/bin/env bash
 
-helm_deploy() {
-  APPLICATION=$1
-  ENVIRONMENT=$2
-  VERSION=$3
-  deploy_chart \
-    --path_configs deploy \
-    --path_chart deploy/$APPLICATION \
-    --application $APPLICATION \
-    --environment $ENVIRONMENT \
-    --namespace $ENVIRONMENT \
-    --helm_extra_args --set global.version=$VERSION
-}
-
-deploy_chart() {
-  set -e
-  set -x
-
-  # Rigid parsing, but all args are mandatory (expect last) and flexible order is unnecessary
-  check_args "--path_configs" $1; shift
-  path_configs=$1; shift
-
-  check_args "--path_chart" $1; shift
-  path_chart=$1; shift
-
-  check_args "--application" $1; shift
-  application=$1; shift
-
-  check_args "--environment" $1; shift
-  environment=$1; shift
-
-  check_args "--namespace" $1; shift
-  namespace=$1; shift
-
-  if [ $# -ne 0 ]; then
-    check_args "--helm_extra_args" $1; shift
-    helm_extra_args=$*
-  fi
-
-  echo "================================"
-  echo " Deploying $application"
-  echo " - Environment: $environment"
-  echo " - Namespace: $namespace"
-  echo "================================"
-
-  root_path=$(pwd)
-
-  # Check the configs exists
-
-  check_config_file ${root_path}/${path_configs}/common.yaml
-  check_config_file ${root_path}/${path_configs}/${namespace}.yaml
-  check_config_file ${root_path}/${path_configs}/${namespace}-secrets.yaml
-
-  # Check the chart exists
-  if [ ! -d ${root_path}/${path_chart} ] || [ ! -f ${root_path}/${path_chart}/Chart.yaml ]; then
-      echo "Bad Chart $root_path/$path_chart : does not exists or missing Chart.yaml"
-      print_usage
-      exit 1
-  fi
-
-  # Unset Kubectl configuration made via the KUBECONFIG env variable
-  #  it would override the config made by configure_kubectl_for
-  #  for example, using Gitlab runners in Kubernetes sets this variable and causes conflict
-  unset KUBECONFIG
-
-  # Configure Kubectl
-  configure_kubectl_for_ci ${environment}
-
-  # Configure Helm
-  helm version --tiller-namespace ${namespace} || true
-  # Helm stable repo have changed and must be updated manually, in versions < v2.17.0
-  helm init --tiller-namespace ${namespace} --client-only --stable-repo-url https://charts.helm.sh/stable
-  helm repo add colisweb s3://colisweb-helm-charts/colisweb
-  helm repo update --strict
-  helm dependency update --tiller-namespace ${namespace} ${root_path}/${path_chart}
-
-  # Gather values/*.yaml files
-  values_path="${root_path}/${path_chart}/values"
-  values_files=''
-  [ -d $values_path ] && values_files=$(find $values_path -type f -maxdepth 1 -name "*.yaml" | sed 's/^/ -f /' | tr -d \\n | sed 's/%//')
-
-  # Deploy
-  helm upgrade -i \
-    --namespace ${namespace} \
-    --tiller-namespace ${namespace} \
-    ${values_files} \
-    -f ${root_path}/${path_configs}/common.yaml \
-    -f ${root_path}/${path_configs}/${namespace}.yaml \
-    -f ${root_path}/${path_configs}/${namespace}-secrets.yaml \
-    ${helm_extra_args} \
-    ${application} ${root_path}/${path_chart}
-
-  #send event to dd
-  PUBLISHED_VERSION="$CI_COMMIT_REF_NAME-$CI_COMMIT_SHA"
-  emit_datadog_deploy_event  --environment $environment --service $application  --version $PUBLISHED_VERSION
-
-  echo "================================"
-  echo " Deployed $application"
-  echo " - Environment: $environment"
-  echo " - Namespace: $namespace"
-  echo "================================"
-
-  set +x
-}
-
-verify_deployments() {
-  set -e
-
-  # usage :
-  # verify_deployments staging price
-  # verify_deployments -t 15m testing price
-
-  if [ "$1" == "-t" ]; then
-    TIMEOUT=$2
-    shift
-    shift
-  else
-    TIMEOUT=5m
-  fi
-
-  NAMESPACE=$1
-  RELEASE=$2
-
-  # Get all Deployments names from the deployed chart
-  DEPLOYMENTS=(
-    $(helm get manifest --tiller-namespace $NAMESPACE $RELEASE | yq -rs '.[] | select(.kind=="Deployment") | .metadata.name')
-  )
-
-  echo "verifying on $NAMESPACE deployments ${DEPLOYMENTS[@]} with a timeout of $TIMEOUT"
-
-  PIDS=()
-  for D in "${DEPLOYMENTS[@]}"; do
-    kubectl -n ${NAMESPACE} rollout status deployment ${D} --timeout=${TIMEOUT} &
-    PIDS+=($!)
-  done
-
-  for P in ${PIDS[@]}; do
-    wait $P
-
-    if [ $? -ne 0 ]; then
-      echo "at least one deployment failed or timed out (after $TIMEOUT)"
-      exit 1
-    fi
-  done
-
-}
-
 helm_deploy_v3() {
   APPLICATION=$1
   ENVIRONMENT=$2
@@ -2040,7 +2262,7 @@ verify_deployments_v3() {
   # verify_deployments staging price
   # verify_deployments -t 15m testing price
 
-  if [ "$1" == "-t" ]; then
+  if [ "$1" = "-t" ] ; then
     TIMEOUT=$2
     shift
     shift