@vaharoni/devops 1.0.47

package/src/target-templates/.devops/manifests/deployment-web.yaml.hb

@@ -0,0 +1,53 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: {{app_name}}
+    env: {{monorepo_env}}
+  name: {{app_name}}
+  namespace: {{namespace}}
+spec:
+  replicas: {{replicas}}
+  selector:
+    matchLabels:
+      app: {{app_name}}
+  template:
+    metadata:
+      labels:
+        app: {{app_name}}
+        env: {{monorepo_env}}
+    spec:
+      volumes:
+        - name: secret-injection-hook
+          secret:
+            secretName: {{env_secret_name}}
+      containers:
+        - image: {{image_path}}
+          name: {{app_name}}
+          args:
+            - "{{project_name}}"
+          env:
+            - name: MONOREPO_ENV
+              value: {{monorepo_env}}
+            - name: MONOREPO_NAMESPACE
+              value: {{namespace}}
+            - name: IS_KUBERNETES
+              value: 'true'
+            - name: MONOREPO_BASE_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: {{env_secret_name}}
+                  key: {{env_base_secret_key}}
+          volumeMounts:
+            - name: secret-injection-hook
+              mountPath: /etc/kubernetes/secrets
+              readOnly: true
+          startupProbe:
+            httpGet:
+              path: /healthz
+              port: {{port}}
+            failureThreshold: 50
+            periodSeconds: 2
+          resources:
+            requests:
+              memory: 250Mi

package/src/target-templates/.devops/manifests/ingress.yaml.hb

@@ -0,0 +1,21 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: {{app_name}}
+  namespace: {{namespace}}
+  labels:
+    app: {{app_name}}
+    env: {{monorepo_env}}
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: {{subdomain}}.{{domain_name}}
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: {{service_name}}
+                port:
+                  number: 80

package/src/target-templates/.devops/manifests/prefect.yaml.hb

@@ -0,0 +1,62 @@
+{{#each prefectFlows}}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: {{../app_name}}
+    env: {{../monorepo_env}}
+  name: {{../app_name}}-{{flow_name}}
+  namespace: {{../namespace}}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: {{../app_name}}
+  template:
+    metadata:
+      labels:
+        app: {{../app_name}}
+        env: {{../monorepo_env}}
+    spec:
+      volumes:
+        - name: secret-injection-hook
+          secret:
+            secretName: {{../env_secret_name}}
+      containers:
+        - image: {{../image_path}}
+          name: {{flow_name}}
+          command:
+            - ./python-exec.sh
+          args:
+            - ./devopspy
+            - exec
+            - --in
+            - {{../app_name}}
+            - python
+            - -m
+            - {{script_path}}
+          env:
+            - name: PREFECT_API_URL
+              value: http://prefect-server.prefect-{{../monorepo_env}}:4200/api
+            - name: MONOREPO_ENV
+              value: {{../monorepo_env}}
+            - name: MONOREPO_NAMESPACE
+              value: {{../namespace}}
+            - name: IS_KUBERNETES
+              value: 'true'
+            - name: MONOREPO_BASE_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: {{../env_secret_name}}
+                  key: {{../env_base_secret_key}}
+          volumeMounts:
+            - name: secret-injection-hook
+              mountPath: /etc/kubernetes/secrets
+              readOnly: true
+          resources:
+            requests:
+              memory: 250Mi
+{{#unless @last}}
+---
+{{/unless}}
+{{/each}}

package/src/target-templates/.devops/manifests/service.yaml.hb

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{service_name}}
+  labels:
+    app: {{app_name}}
+    env: {{monorepo_env}}
+  namespace: {{namespace}}
+spec:
+  selector:
+    app: {{app_name}}
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: {{port}}

package/src/target-templates/.devops/milvus/production/milvus-values.yaml

@@ -0,0 +1,2 @@
+cluster:
+  enabled: true

package/src/target-templates/.devops/milvus/staging/milvus-values.yaml

@@ -0,0 +1,2 @@
+cluster:
+  enabled: false

package/src/target-templates/.devops/postgres/DailyOperatorRestart.yaml

@@ -0,0 +1,54 @@
+# New service account for managing deployments in stackgres namespace
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: stackgres-deployment-manager
+  namespace: stackgres
+---
+# Role with permissions to restart deployments
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: deployment-manager
+  namespace: stackgres
+rules:
+- apiGroups: ["apps"]
+  resources: ["deployments"]
+  verbs: ["get", "list", "watch", "patch"]
+---
+# Bind the role to the service account
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: stackgres-deployment-manager-binding
+  namespace: stackgres
+subjects:
+- kind: ServiceAccount
+  name: stackgres-deployment-manager
+  namespace: stackgres
+roleRef:
+  kind: Role
+  name: deployment-manager
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: stackgres-operator-restart-all-deployments
+  namespace: stackgres
+spec:
+  schedule: "0 2 * * *"
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          serviceAccountName: stackgres-deployment-manager
+          containers:
+          - name: kubectl
+            image: bitnami/kubectl:latest
+            command:
+            - /bin/sh
+            - -c
+            # Restarts all deployments in the stackgres namespace, which include stackgres-collector, stackgres-operator, and stackgres-restapi
+            - kubectl rollout restart deployment -n stackgres
+          restartPolicy: OnFailure

package/src/target-templates/.devops/postgres/production/cluster/PodDisruptionBudget.yaml

@@ -0,0 +1,27 @@
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: db-production-replicas-pdb
+  namespace: db-production
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      app: StackGresCluster
+      role: replica
+      stackgres.io/cluster-name: db-production
+      stackgres.io/cluster: "true"
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: db-production-primary-pdb
+  namespace: db-production
+spec:
+  minAvailable: 1
+  selector:
+    matchLabels:
+      app: StackGresCluster
+      role: primary
+      stackgres.io/cluster-name: db-production
+      stackgres.io/cluster: "true"

package/src/target-templates/.devops/postgres/production/cluster/SGCluster.yaml

@@ -0,0 +1,47 @@
+apiVersion: stackgres.io/v1
+kind: SGCluster
+metadata:
+  namespace: db-production
+  name: db-production
+spec:
+  postgres:
+    version: '17.4'
+    extensions:
+    # - name: 'pg_repack'
+    - name: 'pg_trgm'  # required for prefect
+  instances: 2
+  # sgInstanceProfile: 'size-m'
+  profile: 'production'
+  pods:
+    persistentVolume:
+      size: '20Gi'
+      storageClass: 'hcloud-volumes-retain'
+    scheduling:
+      tolerations:
+        - key: "stackgres"
+          operator: "Exists"
+          effect: "NoSchedule"
+  configurations:
+    # sgPostgresConfig: 'pgconfig'
+    # sgPoolingConfig: 'poolconfig'
+    backups:
+    - sgObjectStorage: 'backupconfig'
+      cronSchedule: '30 02 * * *'
+      retention: 7
+      compression: lz4
+      performance:
+        maxDiskBandwidth: 26214400 #25 MB per seceod
+        maxNetworkBandwidth: 52428800 #50 MB per second
+        uploadDiskConcurrency: 2
+  # We do not use for now as it requires adding a few more nodes
+  # distributedLogs:
+  #   sgDistributedLogs: 'distributedlogs'
+  managedSql:
+    scripts:
+      - sgScript: create-db-script
+  postgresServices:
+    primary:
+      type: ClusterIP
+    replicas:
+      type: ClusterIP
+  prometheusAutobind: true

package/src/target-templates/.devops/postgres/production/cluster/StackGres-alerts.yaml

@@ -0,0 +1,191 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  labels:
+    app: kube-prometheus-stack
+    release: monitoring
+  name: stackgres-rules
+  namespace: monitoring
+spec:
+  groups:
+  - name: StackGres
+    rules:
+    - alert: PostgresInstanceDown
+      expr: pg_up == 0
+      for: 1m
+      labels:
+        severity: critical
+        service: "PostgreSQL"
+        cluster: "StackGres"
+      annotations:
+        summary: "Postgres server instance is down"
+        description: "Postgres has not been responding for the past 1 minutes on {{ $labels.instance }}"
+        title: "Postgres server instance {{ $labels.instance }} is down "
+    - alert: PostgresExporterErrors
+      expr: pg_exporter_last_scrape_error == 1
+      for: 10m
+      labels:
+        severity: critical
+        service: "PostgreSQL"
+        cluster: "StackGres"
+      annotations:
+        summary: "Postgres Exporter is down or is showing errors"
+        description: "postgres-exporter is not running or it is showing errors {{ $labels.instance }}"
+    - alert: PostgresReplicationLagSizeTooLarge
+      expr: pg_replication_status_lag_size > 1e+09
+      for: 5m
+      labels:
+        severity: critical
+        service: "PostgreSQL"
+        cluster: "StackGres"
+      annotations:
+        summary: "Postgres replication lag size is to large"
+        description: "Replication lag size on server {{$labels.instance}} ({{$labels.application_name}}) is currently {{ $value | humanize1024}}B behind the leader in cluster {{$labels.cluster_name}}"
+    - alert: PostgresTooManyDeadTuples
+      expr: ((pg_stat_user_tables_n_dead_tup > 1e+06) / (pg_stat_user_tables_n_live_tup + pg_stat_user_tables_n_dead_tup)) >= 0.1
+      for: 30m
+      labels:
+        severity: info
+        service: "PostgreSQL"
+        cluster: "StackGres"
+      annotations:
+        summary: "PostgreSQL dead tuples is too large"
+        description: "The dead tuple ratio of {{$labels.relname}} on database {{$labels.datname}} is greater than 10% in cluster {{$labels.cluster_name}}"
+    - alert: PostgresInactiveReplicationSlots
+      expr: pg_replication_slots_active == 0
+      for: 10m
+      labels:
+        severity: warning
+        service: "PostgreSQL"
+        cluster: "StackGres"
+      annotations:
+        summary: "There are inactive replications slots"
+        description: "The are some inactive replication slots on {{$labels.instance}} in cluster {{$labels.cluster_name}}"
+    - alert: PostgresSplitBrain
+      expr: count by(cluster_name) (pg_replication_is_replica == 0) > 1
+      for: 1m
+      labels:
+        severity: critical
+        service: "PostgreSQL"
+        cluster: "StackGres"
+      annotations:
+        summary: "There are more than one instance in read-write mode"
+        description: "Split Brain: too many postgres databases in cluster {{$labels.cluster_name}} in read-write mode"
+    - alert: PostgresTooManyConnections
+      expr: sum by (datname) (pg_stat_activity_count{datname!~"template.*|postgres"}) > pg_settings_max_connections * 0.9
+      for: 5m
+      labels:
+        severity: warning
+        service: "PostgreSQL"
+        cluster: "StackGres"
+      annotations:
+        summary: Postgresql too many connections (instance {{ $labels.instance }} in cluster {{$labels.cluster_name}})
+        description: "PostgreSQL instance has too many connections\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"
+    - alert: PostgresNotEnoughConnections
+      expr: sum by (datname) (pg_stat_activity_count{datname!~"template.*|postgres"}) < 5
+      for: 5m
+      labels:
+        severity: warning
+        service: "PostgreSQL"
+        cluster: "StackGres"
+      annotations:
+        summary: Postgresql not enough connections (instance {{ $labels.instance }} in cluster {{$labels.cluster_name}})
+        description: "PostgreSQL instance should have more connections (> 5)\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"
+    - alert: PostgresPromotedNode
+      expr: pg_replication_is_replica and changes(pg_replication_is_replica[1m]) > 0
+      for: 5m
+      labels:
+        severity: warning
+        service: "PostgreSQL"
+        cluster: "StackGres"
+      annotations:
+        summary: "Postgresql promoted node (instance {{ $labels.instance }}, cluster {{ $labels.cluster_name }})"
+        description: "Postgresql standby server has been promoted as primary node\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"
+    - alert: PostgresLongRunningTransactionLeaderNode
+      expr: pg_long_running_transactions_transactions>0 and on (instance)(pg_replication_is_replica == 0)
+      for: 10m
+      labels:
+        severity: warning
+        service: "PostgreSQL"
+        cluster: "StackGres"
+      annotations:
+        summary: "Long running transaction on the leader node, {{ $labels.instance }}
+          cluster {{ $labels.cluster_name }}"
+        description: |-
+          There are long running transactions on the leader node, {{ $labels.instance }}
+            COUNT = {{ $value }}
+            LABELS: {{ $labels }}
+    - alert: PostgresLongRunningTransactionReplicaNode
+      expr: pg_long_running_transactions_transactions>0 and on (instance)(pg_replication_is_replica == 1)
+      for: 10m
+      labels:
+        severity: warning
+        service: "PostgreSQL"
+        cluster: "StackGres"
+      annotations:
+        summary: "Long running transaction on a replica node, {{ $labels.instance }}, cluster {{ $labels.cluster_name }}"
+        description: |-
+          There are long running transactions on a replica node, {{ $labels.instance }}
+            COUNT = {{ $value }}
+            LABELS: {{ $labels }}
+    # Connection Pooling alerts
+    - alert: PgBouncerWaitingClients
+      expr: pgbouncer_show_pools_cl_waiting > 0
+      for: 10m
+      labels:
+        severity: warning
+        service: "PostgreSQL"
+        cluster: "StackGres"
+      annotations:
+        summary: PgBouncer has waiting clients on instance {{ $labels.instance }} in cluster {{$labels.cluster_name}})
+        description: "PgBouncer instance has waiting clients\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"
+    - alert: PgBouncerNotEnoughConnections
+      expr: (sum by (database,instance) (pgbouncer_show_pools_cl_active{database!~"template.*|postgres|pgbouncer"}) + sum by (database, instance) (pgbouncer_show_pools_cl_waiting{database!~"template.*|postgres|pgbouncer"}))  - on (database,instance) (pgbouncer_show_databases_pool_size{database!~"template.*|postgres|pgbouncer"}) > 0
+      for: 10m
+      labels:
+        severity: critical
+        service: "PostgreSQL"
+        cluster: "StackGres"
+      annotations:
+        summary: PgBouncer pool size is not enough for the current connections on {{ $labels.instance }} in cluster {{$labels.cluster_name}})
+        description: "PgBouncer is getting more connections than the pool size, extra connections = {{ $value }}"
+    - alert: PgBouncerPoolFillingUp
+      expr: (sum by (database,instance) (pgbouncer_show_databases_pool_size{database!~"template.*|postgres|pgbouncer"})  - on (database,instance) pgbouncer_show_databases_current_connections)  <= 15
+      for: 1m
+      labels:
+        severity: warning
+        service: "PostgreSQL"
+        cluster: "StackGres"
+      annotations:
+        summary: PgBouncer pool is filling up on {{ $labels.instance }} in cluster {{$labels.cluster_name}})
+        description: "PgBouncer pool is filling up, remaining connections = {{ $value }}"
+    - alert: PgBouncerAvgWaitTimeTooHigh
+      expr: pgbouncer_show_stats_avg_wait_time > 1e+6
+      for: 5m
+      labels:
+        severity: warning
+        service: "PostgreSQL"
+        cluster: "StackGres"
+      annotations:
+        summary: PgBouncer time spent by clients waiting for a connections is too high on {{ $labels.instance }} in cluster {{$labels.cluster_name}})
+        description: "PgBouncer wait for a server connections is too high = {{ $value }}"
+    - alert: PgBouncerQueryTimeTooHigh
+      expr: pgbouncer_show_stats_avg_query_time > 5e+6
+      for: 5m
+      labels:
+        severity: warning
+        service: "PostgreSQL"
+        cluster: "StackGres"
+      annotations:
+        summary: PgBouncer average query duration more than 5 seconds on {{ $labels.instance }} in cluster {{$labels.cluster_name}})
+        description: "PgBouncer average query duration more than 5 seconds = {{ $value }}"
+    - alert: DatabaseLowDiskAvailable
+      expr: (1.0 - node_filesystem_avail_bytes{mountpoint="/var/lib/postgresql",fstype!=""} / node_filesystem_size_bytes{mountpoint="/var/lib/postgresql",fstype!=""}) * 100 >= 80
+      for: 15m
+      labels:
+        severity: warning
+        service: "PostgreSQL"
+        cluster: "StackGres"
+      annotations:
+        summary: Database disk is filling up currently have less than 20% available on {{ $labels.instance }} in cluster {{$labels.cluster_name}})
+        description: "Database disk is filling up currently have less than 20%, currently occupied {{ $value }} %"

package/src/target-templates/.devops/postgres/production/configurations/06-SGDistributedLogs.yaml

@@ -0,0 +1,11 @@
+# We do not use for now as it requires adding a few more nodes
+
+# apiVersion: stackgres.io/v1
+# kind: SGDistributedLogs
+# metadata:
+#   name: distributedlogs
+#   namespace: db-production
+# spec:
+#   persistentVolume:
+#     size: 10Gi
+

package/src/target-templates/.devops/postgres/production/configurations/07-SGObjectStorage.yaml

@@ -0,0 +1,18 @@
+apiVersion: stackgres.io/v1beta1
+kind: SGObjectStorage
+metadata:
+  namespace: db-production
+  name: backupconfig
+spec:
+  type: s3Compatible
+  s3Compatible:
+    bucket: changeme-backups
+    endpoint: https://hel1.your-objectstorage.com
+    awsCredentials:
+      secretKeySelectors:
+        accessKeyId: 
+          name: s3-backup-bucket-secret
+          key: accessKey
+        secretAccessKey:
+          name: s3-backup-bucket-secret
+          key: secretKey

package/src/target-templates/.devops/postgres/production/configurations/08-SGScript.yaml

@@ -0,0 +1,12 @@
+apiVersion: stackgres.io/v1
+kind: SGScript
+metadata:
+  name: create-db-script
+  namespace: db-production
+spec:
+  managedVersions: true
+  continueOnError: false
+  scripts:
+  - name: create-database
+    script: |
+      CREATE DATABASE changeme WITH OWNER postgres;

package/src/target-templates/.devops/postgres/staging/cluster/SGCluster.yaml

@@ -0,0 +1,42 @@
+apiVersion: stackgres.io/v1
+kind: SGCluster
+metadata:
+  namespace: db-staging
+  name: db-staging
+spec:
+  postgres:
+    version: '17.4'
+    extensions:
+    # - name: 'pg_repack'
+    - name: 'pg_trgm'  # required for prefect
+  instances: 1
+  # sgInstanceProfile: 'size-m'
+  profile: 'testing'
+  pods:
+    persistentVolume:
+      size: '10Gi'
+      storageClass: 'hcloud-volumes-retain'
+    scheduling:
+      tolerations:
+        - key: "stackgres"
+          operator: "Exists"
+          effect: "NoSchedule"
+  configurations:
+    # sgPostgresConfig: 'pgconfig'
+    # sgPoolingConfig: 'poolconfig'
+    backups:
+    - sgObjectStorage: 'backupconfig'
+      # cronSchedule: '30 02 */4 * *'
+      cronSchedule: '30 02 * * *'
+      retention: 4
+  # distributedLogs:
+  #   sgDistributedLogs: 'distributedlogs'
+  managedSql:
+    scripts:
+      - sgScript: create-db-script
+  postgresServices:
+    primary:
+      type: ClusterIP
+    replicas:
+      type: ClusterIP
+  prometheusAutobind: true

package/src/target-templates/.devops/postgres/staging/configurations/07-SGObjectStorage.yaml

@@ -0,0 +1,18 @@
+apiVersion: stackgres.io/v1beta1
+kind: SGObjectStorage
+metadata:
+  namespace: db-staging
+  name: backupconfig
+spec:
+  type: s3Compatible
+  s3Compatible:
+    bucket: changeme-backups
+    endpoint: https://hel1.your-objectstorage.com
+    awsCredentials:
+      secretKeySelectors:
+        accessKeyId: 
+          name: s3-backup-bucket-secret
+          key: accessKey
+        secretAccessKey:
+          name: s3-backup-bucket-secret
+          key: secretKey

package/src/target-templates/.devops/postgres/staging/configurations/08-SGScript.yaml

@@ -0,0 +1,12 @@
+apiVersion: stackgres.io/v1
+kind: SGScript
+metadata:
+  name: create-db-script
+  namespace: db-staging
+spec:
+  managedVersions: true
+  continueOnError: false
+  scripts:
+  - name: create-database
+    script: |
+      CREATE DATABASE changeme WITH OWNER postgres;

package/src/target-templates/.devops/prefect/production/prefect-values.yaml

@@ -0,0 +1,14 @@
+ingress:
+  enabled: true
+  host:
+    hostname: "prefect-production.staging.com"
+server:
+  uiConfig:
+    prefectUiApiUrl: "https://prefect-production.staging.com/api"
+
+# We use the environment's stackgres postgres instance
+postgresql:
+  enabled: false
+secret:
+  create: false 
+  name: prefect-server-postgresql-connection

package/src/target-templates/.devops/prefect/staging/prefect-values.yaml

@@ -0,0 +1,14 @@
+ingress:
+  enabled: true
+  host:
+    hostname: "prefect-staging.staging.com"
+server:
+  uiConfig:
+    prefectUiApiUrl: "https://prefect-staging.staging.com/api"
+
+# We use the environment's stackgres postgres instance
+postgresql:
+  enabled: false
+secret:
+  create: false 
+  name: prefect-server-postgresql-connection

package/src/target-templates/.devops/redis/production/redis-values.yaml

@@ -0,0 +1,20 @@
+architecture: replication
+global:
+  defaultStorageClass: hcloud-volumes-retain
+master:
+  resources: 
+    requests:
+      cpu: 1
+      memory: 256Mi
+    limits:
+      cpu: 3
+      memory: 1024Mi
+replica:
+  replicaCount: 1
+  resources: 
+    requests:
+      cpu: 1
+      memory: 256Mi
+    limits:
+      cpu: 3
+      memory: 1024Mi  

package/src/target-templates/.devops/redis/staging/redis-values.yaml

@@ -0,0 +1,8 @@
+architecture: standalone
+global:
+  defaultStorageClass: hcloud-volumes-retain
+master:
+  resources: 
+    limits:
+      cpu: 1
+      memory: 256Mi

package/src/target-templates/.envrc

@@ -0,0 +1,5 @@
+if [ -f "$PWD/config/kubeconfig" ]; then
+  export KUBECONFIG="$PWD/config/kubeconfig"
+else
+  export KUBECONFIG="${HOME}/.kube/config"
+fi