@leverege/build-tools 2.67.0 → 2.67.2-pedro.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.67.0",
+  "version": "2.67.2-pedro.1",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {
@@ -92,4 +92,4 @@
     "@leverege/eslint-config-leverege": "^5.0.1",
     "npm": "^11.4.1"
   }
-}
+}

package/src/DockerPy.mjs

@@ -221,9 +221,9 @@ async function buildContainerImage( {
   await shellCmd( `${cloudBuildCmd} ${cloudBuildLogs} ${cloudBuildCfg} ./build`, { stdio : 'inherit' } )
 }
 
-async function repoCleanup() {
+async function repoCleanup( doNotCleanupPaths = [] ) {
   // remove garbage from the repo
-  const garbage = [
+  let garbage = [
     'build',
     'dist',
     'docker/.keep',
@@ -233,6 +233,10 @@ async function repoCleanup() {
     'docker/Dockerfile',
     'docker/workspace',
   ]
+
+  // Remove any paths that are in the doNotCleanupPaths array from the garbage list
+  garbage = garbage.filter( path => !doNotCleanupPaths.includes( path ) ) 
+
   garbage.forEach( trash => fs.rmSync( trash, { recursive : true, force : true } ) )
 
   fs.writeFileSync( 'docker/.gitignore', '.previous\nDockerfile' )

package/src/Utils.mjs

@@ -537,7 +537,7 @@ export const analyzeRepository = async ( giveGuidance ) => {
   const containerName = closestPackageJson?.name.replace( /^@[^/]+\//, '' ) || rootPyProjectToml?.project.name
   const artifactProject = closestPackageJson?.artifactRegistry.project || 'unused'
   const artifactRegistry = closestPackageJson?.leverege.registry || rootPyProjectToml?.registry
-  const packageVersion = `v${closestPackageJson?.version}` || rootPyProjectToml?.project.version
+  const packageVersion = `v${closestPackageJson?.version || rootPyProjectToml?.project.version}`
 
   const currentDir = process.cwd()
   const closestPkg = await packageUp()

package/src/docker-to-registry-py.mjs

@@ -89,10 +89,10 @@ const pyproject = toml.parse( tomlStr )
 const repoDescr = {
   artifactProject : pyproject.leverege.project || 'leverege-registry',
   containerName : pyproject.project.name,
-  packageVersion : pyproject.project.version,
+  packageVersion : `v${pyproject.project.version}`,
   artifactRegistry : pyproject.leverege.registry || 'us-docker.pkg.dev/leverege-registry/stack',
-  registryFolder : `${pyproject.leverege.registryFolder}/images/${pyproject.project.name}`,
-  imageBase : pyproject.leverege.imageBase,
+  registryFolder : `${pyproject.leverege['registry-folder']}/images/${pyproject.project.name}`,
+  imageBase : pyproject.leverege['image-base'],
   helmChart : await parseHelmChart(),
 }
 
@@ -124,7 +124,11 @@ const registryFolder = `${artifactRegistry}/images/${containerName}`
 // await shellCmd( 'npm install', { stdio : 'inherit' } )
 
 if ( giveGuidance ) {
-  await Docker.repoCleanup() // adjusts gitignore files and removes cruft
+  const doNotCleanupPaths = [
+    'docker/Dockerfile',
+  ]
+
+  await Docker.repoCleanup( doNotCleanupPaths ) // adjusts gitignore files and removes cruft
   log( `
 
 

package/src/helm-charts/pgbouncer/helmup.plugin

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+showInstalling "PG Bouncer"
+
+OCI_CHART="oci://ghcr.io/icoretech/charts/pgbouncer"
+
+[ -z "$PGBOUNCER_CHART_VERSION" ] && PGBOUNCER_CHART_VERSION="2.7.0"
+
+helm upgrade --install pgbouncer $OCI_CHART        \
+             --values  pgbouncer/values-local.yaml \
+             --version $PGBOUNCER_CHART_VERSION $HELM_WHAT

package/src/helm-charts/pgbouncer/values-local.yaml

@@ -0,0 +1,56 @@
+# pgbouncer.values.yaml - https://artifacthub.io/packages/helm/icoretech/pgbouncer
+
+replicaCount: 1
+
+config:
+  existingAdminSecret: "pgbouncer-admin"
+  existingUserlistSecret: "pgbouncer-userlist-secret"
+  pgbouncer:
+    default_pool_size: "20"
+    idle_transaction_timeout: "60"
+    ignore_startup_parameters: "extra_float_digits"
+    max_client_conn: "1000"
+    pool_mode: "transaction"         # Options: session | transaction | statement
+    query_wait_timeout: "120"
+    reserve_pool_size: "5"
+    reserve_pool_timeout: "5"
+    server_connect_timeout: "15"
+    server_idle_timeout: "600"
+    server_lifetime: "3600"
+    server_login_retry: "15"
+    # https://www.pgbouncer.org/config.html#log-settings
+    log_connections: "1"
+    log_disconnections: "0"
+    verbose: "0"
+  databases:
+    scheduler:
+      host: postgres-postgresql
+      port: 5432
+      dbname: scheduler
+      user: postgres
+    models:
+      host: postgres-postgresql
+      port: 5432
+      dbname: models
+      user: postgres
+
+service:
+  port: 8432
+
+resources:
+  requests:
+    cpu: 100m
+    memory: 128Mi
+  limits:
+    cpu: 200m
+    memory: 256Mi
+
+# Optional: Configure a serviceAccount if needed
+serviceAccount:
+  create: true
+  name: ""
+
+# Optional: Add any needed nodeSelectors or tolerations
+nodeSelector: {}
+tolerations: []
+affinity: {}

package/src/helm-charts/prom-operator/helmup.plugin

@@ -6,7 +6,7 @@ addHelmRepo prometheus-community https://prometheus-community.github.io/helm-cha
 NS="--namespace prometheus"
 
 showInstalling "The Prometheus Operator (kube-prometheus-stack)"
-[ -z "$PROMETHEUS_STACK_CHART_VERSION" ] && PROMETHEUS_STACK_CHART_VERSION="73"
+[ -z "$PROMETHEUS_STACK_CHART_VERSION" ] && PROMETHEUS_STACK_CHART_VERSION="74"
 
 # https://artifacthub.io/packages/helm/prometheus-community/kube-prometheus-stack
 helm upgrade $NS --install prometheus-stack prometheus-community/kube-prometheus-stack \

package/src/helm-charts/prom-operator/prometheus-stack.yaml.ovh

@@ -109,7 +109,13 @@ alertmanager:
     operator: "Equal"
     value: "true"
     effect: "NoSchedule"
-
+  storage:
+    volumeClaimTemplate:
+      spec:
+        accessModes: ["ReadWriteOnce"]
+        resources:
+          requests:
+            storage: 10Gi  # or however much you need
   strategy:
     type: Recreate
 

package/src/helm-charts/prom-operator/rules/elasticsearch-rules.yaml

@@ -162,19 +162,19 @@ spec:
               Elasticsearch has pending tasks. Cluster works slowly.
                 VALUE = {{ $value }}
                 LABELS = {{ $labels }}
-        - alert: ElasticsearchNoNewDocuments
-          expr: increase(elasticsearch_indices_indexing_index_total{es_data_node="true"}[10m]) < 1
-          for: 0m
-          labels:
-            severity: warning
-          annotations:
-            summary: Elasticsearch no new documents (instance {{ $labels.instance }})
-            description: |-
-              No new documents for 10 min!
-                VALUE = {{ $value }}
-                LABELS = {{ $labels }}
+#       - alert: ElasticsearchNoNewDocuments
+#         expr: increase(elasticsearch_indices_indexing_index_total{es_data_node="true"}[10m]) < 1
+#         for: 0m
+#         labels:
+#           severity: warning
+#         annotations:
+#           summary: Elasticsearch no new documents (instance {{ $labels.instance }})
+#           description: |-
+#             No new documents for 10 min!
+#               VALUE = {{ $value }}
+#               LABELS = {{ $labels }}
         - alert: ElasticsearchHighIndexingLatency
-          expr: elasticsearch_indices_indexing_index_time_seconds_total / elasticsearch_indices_indexing_index_total > 0.005 # awesome was 0.0005
+          expr: elasticsearch_indices_indexing_index_time_seconds_total / elasticsearch_indices_indexing_index_total > 0.010 # awesome was 0.0005
           for: 10m
           labels:
             severity: warning

package/src/helm-charts/prom-operator/rules/kubestate-rules.yaml

@@ -154,7 +154,7 @@ spec:
                 LABELS = {{ $labels }}
         - alert: KubernetesStatefulsetDown
           expr: kube_statefulset_replicas != kube_statefulset_status_replicas_ready > 0
-          for: 1m
+          for: 3m # awesome was 1m
           labels:
             severity: critical
           annotations:
@@ -194,17 +194,17 @@ spec:
             summary: Kubernetes HPA scale maximum (instance {{ $labels.instance }})
             description: |-
               HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has hit max replica pods of {{ $value }}
-        - alert: KubernetesHpaUnderutilized
-          expr: max(quantile_over_time(0.5, kube_horizontalpodautoscaler_status_desired_replicas[1d]) == kube_horizontalpodautoscaler_spec_min_replicas) by (horizontalpodautoscaler) > 3
-          for: 0m
-          labels:
-            severity: info
-          annotations:
-            summary: Kubernetes HPA underutilized (instance {{ $labels.instance }})
-            description: |-
-              HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} is constantly at minimum replicas for 50% of the time. Potential cost saving here.
-                VALUE = {{ $value }}
-                LABELS = {{ $labels }}
+#       - alert: KubernetesHpaUnderutilized
+#         expr: max(quantile_over_time(0.5, kube_horizontalpodautoscaler_status_desired_replicas[1d]) == kube_horizontalpodautoscaler_spec_min_replicas) by (horizontalpodautoscaler) > 3
+#         for: 0m
+#         labels:
+#           severity: info
+#         annotations:
+#           summary: Kubernetes HPA underutilized (instance {{ $labels.instance }})
+#           description: |-
+#             HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} is constantly at minimum replicas for 50% of the time. Potential cost saving here.
+#               VALUE = {{ $value }}
+#               LABELS = {{ $labels }}
         - alert: KubernetesPodNotHealthy
           expr: sum by (namespace, pod) (kube_pod_status_phase{phase=~"Pending|Unknown|Failed"}) > 0
           for: 15m

package/src/helm-charts/traefik/values-local.yaml

@@ -2,10 +2,10 @@
 # reserved static IPs and for creating a secondary LB for UDP traffic.
 # See the *** UDP PORTS *** section below for more info.
 #
-#service:
-#  spec:
-#    loadBalancerIP: ${TRAEFIK_IP} # set in overwhelm.yaml
-#  single: true # true by default,  set to false for UDP support
+service:
+  spec:
+    loadBalancerIP: ${TRAEFIK_IP} # set in overwhelm.yaml
+  single: true # true by default,  set to false for UDP support
 
 deployment:
   replicas: 3
@@ -28,7 +28,7 @@ resources:
 metrics:
   prometheus:
     serviceMonitor:
-      enabled: false
+      enabled: true
       interval: 30s
       namespace: prometheus
 

package/src/helmup.sh

@@ -40,6 +40,7 @@ AUXILIARY=(
   fota-server
   geotile-server
   overdose
+  pgbouncer,
   pubsub-pulse
   push-notifier
   pusher
@@ -198,6 +199,27 @@ function setupStackBuckets() {
   makeUniformBucketWithCors geotile-server PUBLIC
 }
 
+function setupPgbouncerSecret() {
+  # pgbouncer uses a k8s secret for the user access list which is in the form
+  # of => "user" "password" - this function will create the pgbouncer secret
+  # if it does not exist by pulling the password from the PG_PASSWORD secret
+  # managed by the Google Secret Manager service.
+  USERLIST_SECRET='pgbouncer-userlist-secret'
+  kubectl get secret $USERLIST_SECRET &> $DEVNULL
+  [[ $? -eq 0 ]] && return # already exists - bail
+
+  cat<<PGB_SECRET
+
+  `color y "Creatng $USERLIST_SECRET k8s secret from secret manager PG_PASSWORD secret..."`
+
+PGB_SECRET
+  DECODED_PSQL_PASSWORD=$(gcloud secrets versions access latest --secret=PG_PASSWORD --project $GCP_PROJECT_ID)
+  exitOnError $? "Failed to fetch PG_PASSWORD from the secret manager"
+
+  kubectl create secret generic $USERLIST_SECRET --from-literal=userlist.txt="\"postgres\" \"$DECODED_PSQL_PASSWORD\""
+  exitOnError $? "Failed to create the $USERLIST_SECRET k8s secret"
+}
+
 function doPlatormPreInstallChecks() {
   # This is where we handle platform specific things that need to be setup,
   # usuall once, and then promptly forgotten about.
@@ -1126,6 +1148,11 @@ CATBACKUP
         setupStackBuckets && exit 0
         ;;
 
+      "pgbouncer")
+        setupPgbouncerSecret
+        bootstrapLocalSetup $SERVICE
+        ;;
+
       "postgres")
         bootstrapLocalSetup $SERVICE
         ;;

package/src/klog.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-[ -f overwhelm.yaml ] && [ -f package.json ] && npm run ahoy
+overwhelm
 
 DTS="`date +\"%Y%m%d-%H%M%S\"`"
 PRETTY="pino-pretty -c -t 'HH:MM:ss.l' -i 'severity'"

package/.vscode/settings.json

@@ -1,5 +0,0 @@
-{
-  "files.associations": {
-    "*.yaml.ovh": "yaml"
-  }
-}