@vaharoni/devops 1.2.13 → 1.2.14

package/dist/src/target-templates/infra-variants/hetzner/.devops/infra/hetzner/harbor-values.yaml

@@ -0,0 +1,76 @@
+expose:
+  type: ingress
+  ingress:
+    hosts:
+      core: $REGISTRY_BASE_URL
+  tls:
+    certSource: secret
+    secret:
+      secretName: tls-secret
+externalURL: https://$REGISTRY_BASE_URL
+persistence:
+  enabled: true
+  persistentVolumeClaim:
+    registry:
+      storageClass: "hcloud-volumes"
+      accessMode: ReadWriteOnce
+      size: 20Gi
+    jobservice:
+      storageClass: "hcloud-volumes"
+      accessMode: ReadWriteOnce
+      size: 5Gi
+    chartmuseum:
+      storageClass: "hcloud-volumes"
+      accessMode: ReadWriteOnce
+      size: 5Gi
+database:
+  internal:
+    livenessProbe:
+      timeoutSeconds: 5
+    readinessProbe:
+      timeoutSeconds: 5
+core:
+  livenessProbe:
+    initialDelaySeconds: 10
+    periodSeconds: 10
+    timeoutSeconds: 5
+    failureThreshold: 3
+  readinessProbe:
+    initialDelaySeconds: 5
+    periodSeconds: 10
+    timeoutSeconds: 5
+    failureThreshold: 3
+jobservice:
+  livenessProbe:
+    initialDelaySeconds: 10
+    periodSeconds: 10
+    timeoutSeconds: 5
+    failureThreshold: 3
+  readinessProbe:
+    initialDelaySeconds: 5
+    periodSeconds: 10
+    timeoutSeconds: 5
+    failureThreshold: 3
+registry:
+  registry:
+    livenessProbe:
+      initialDelaySeconds: 10
+      periodSeconds: 10
+      timeoutSeconds: 5
+      failureThreshold: 3
+    readinessProbe:
+      initialDelaySeconds: 5
+      periodSeconds: 10
+      timeoutSeconds: 5
+      failureThreshold: 3
+  controller:
+    livenessProbe:
+      initialDelaySeconds: 10
+      periodSeconds: 10
+      timeoutSeconds: 5
+      failureThreshold: 3
+    readinessProbe:
+      initialDelaySeconds: 5
+      periodSeconds: 10
+      timeoutSeconds: 5
+      failureThreshold: 3

package/dist/src/target-templates/infra-variants/hetzner/.devops/infra/hetzner/hcloud-config.yaml

@@ -0,0 +1,113 @@
+---
+cluster_name: $PROJECT_NAME
+kubeconfig_path: "./config/kubeconfig"
+k3s_version: v1.32.2+k3s1
+
+networking:
+  ssh:
+    port: 22
+    use_agent: false # set to true if your key has a passphrase
+    public_key_path: "~/.ssh/id_hcloud.pub"
+    private_key_path: "~/.ssh/id_hcloud"
+  allowed_networks:
+    ssh:
+      - 0.0.0.0/0
+    api: # this will firewall port 6443 on the nodes
+      - 0.0.0.0/0
+  public_network:
+    ipv4: true
+    ipv6: true
+  private_network:
+    enabled: true
+    subnet: 10.0.0.0/16
+    existing_network_name: ""
+  cni:
+    enabled: true
+    encryption: false
+    mode: flannel
+
+  # cluster_cidr: 10.244.0.0/16 # optional: a custom IPv4/IPv6 network CIDR to use for pod IPs
+  # service_cidr: 10.43.0.0/16 # optional: a custom IPv4/IPv6 network CIDR to use for service IPs. Warning, if you change this, you should also change cluster_dns!
+  # cluster_dns: 10.43.0.10 # optional: IPv4 Cluster IP for coredns service. Needs to be an address from the service_cidr range
+
+
+# manifests:
+#   cloud_controller_manager_manifest_url: "https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/v1.23.0/ccm-networks.yaml"
+#   csi_driver_manifest_url: "https://raw.githubusercontent.com/hetznercloud/csi-driver/v2.12.0/deploy/kubernetes/hcloud-csi.yml"
+#   system_upgrade_controller_deployment_manifest_url: "https://github.com/rancher/system-upgrade-controller/releases/download/v0.14.2/system-upgrade-controller.yaml"
+#   system_upgrade_controller_crd_manifest_url: "https://github.com/rancher/system-upgrade-controller/releases/download/v0.14.2/crd.yaml"
+#   cluster_autoscaler_manifest_url: "https://raw.githubusercontent.com/kubernetes/autoscaler/master/cluster-autoscaler/cloudprovider/hetzner/examples/cluster-autoscaler-run-on-master.yaml"
+#   cluster_autoscaler_container_image_tag: "v1.32.0"
+
+# datastore:
+#   mode: etcd # etcd (default) or external
+#   external_datastore_endpoint: postgres://....
+
+schedule_workloads_on_masters: false
+
+# image: rocky-9 # optional: default is ubuntu-24.04
+# autoscaling_image: 103908130 # optional, defaults to the `image` setting
+# snapshot_os: microos # optional: specified the os type when using a custom snapshot
+
+masters_pool:
+  instance_type: cpx21
+  instance_count: 1 # for HA; you can also create a single master cluster for dev and testing (not recommended for production)
+  locations: # You can choose a single location for single master clusters or if you prefer to have all masters in the same location. For regional clusters (which are only available in the eu-central network zone), each master needs to be placed in a separate location.
+    # - fsn1
+    # - hel1
+    - nbg1
+
+worker_node_pools:
+- name: small-static
+  instance_type: cx32
+  instance_count: 2
+  location: nbg1
+  # image: debian-11
+  # labels:
+  #   - key: purpose
+  #     value: blah
+  # taints:
+  #   - key: something
+  #     value: value1:NoSchedule
+# - name: medium-autoscaled
+#   instance_type: cpx31
+#   location: fsn1
+#   autoscaling:
+#     enabled: true
+#     min_instances: 0
+#     max_instances: 3
+
+embedded_registry_mirror:
+  enabled: false # Enables fast p2p distribution of container images between nodes for faster pod startup. Check if your k3s version is compatible before enabling this option. You can find more information at https://docs.k3s.io/installation/registry-mirror
+
+protect_against_deletion: true
+
+create_load_balancer_for_the_kubernetes_api: false # Just a heads up: right now, we can’t limit access to the load balancer by IP through the firewall. This feature hasn’t been added by Hetzner yet.
+
+# additional_packages:
+# - somepackage
+
+# post_create_commands:
+# - apt update
+# - apt upgrade -y
+# - apt autoremove -y
+
+# kube_api_server_args:
+# - arg1
+# - ...
+# kube_scheduler_args:
+# - arg1
+# - ...
+# kube_controller_manager_args:
+# - arg1
+# - ...
+# kube_cloud_controller_manager_args:
+# - arg1
+# - ...
+# kubelet_args:
+# - arg1
+# - ...
+# kube_proxy_args:
+# - arg1
+# - ...
+# api_server_hostname: k8s.example.com # optional: DNS for the k8s API LoadBalancer. After the script has run, create a DNS record with the address of the API LoadBalancer.

package/dist/src/target-templates/infra-variants/hetzner/.devops/infra/hetzner/ingress-nginx-annotations.yaml

@@ -0,0 +1,49 @@
+# INSTALLATION
+# 1. Install Helm: https://helm.sh/docs/intro/install/
+# 2. Add ingress-nginx Helm repo: helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx
+# 3. Update information of available charts: helm repo update
+# 4. Install ingress-nginx:
+# helm upgrade --install \
+# ingress-nginx ingress-nginx/ingress-nginx \
+# --set controller.ingressClassResource.default=true \ # Remove this line if you don’t want Nginx to be the default Ingress Controller
+# -f ./ingress-nginx-annotations.yaml \
+# --namespace ingress-nginx \
+# --create-namespace
+
+# LIST of all ANNOTATIONS: https://github.com/hetznercloud/hcloud-cloud-controller-manager/blob/master/internal/annotation/load_balancer.go
+
+controller:
+  kind: DaemonSet
+  service:
+    annotations:
+      # Germany:
+      # - nbg1 (Nuremberg)
+      # - fsn1 (Falkenstein)
+      # Finland:
+      # - hel1 (Helsinki)
+      # USA:
+      # - ash (Ashburn, Virginia)
+      # Without this, the load balancer won’t be provisioned and will stay in "pending" state.
+      # You can check this state using "kubectl get svc -n ingress-nginx"
+      load-balancer.hetzner.cloud/location: nbg1
+
+      # Name of the load balancer. This name will appear in your Hetzner cloud console under "Your project -> Load Balancers".
+      # NOTE: This is NOT the load balancer created automatically for HA clusters. You need to specify a different name here to create a separate load balancer for ingress Nginx.
+      load-balancer.hetzner.cloud/name: ingress-lb
+
+      # Ensures communication between the load balancer and cluster nodes happens through the private network.
+      load-balancer.hetzner.cloud/use-private-ip: "true"
+
+      # [ START: Use these annotations if you care about seeing the actual client IP ]
+      # "uses-proxyprotocol" enables the proxy protocol on the load balancer so that the ingress controller and applications can see the real client IP.
+      # "hostname" is needed if you use cert-manager (LetsEncrypt SSL certificates). It fixes HTTP01 challenges for cert-manager (https://cert-manager.io/docs/).
+      # Check this link for more details: https://github.com/compumike/hairpin-proxy
+      # In short: the easiest fix provided by some providers (including Hetzner) is to configure the load balancer to use a hostname instead of an IP.
+      load-balancer.hetzner.cloud/uses-proxyprotocol: 'true'
+
+      # 1. "yourDomain.com" must be correctly configured in DNS to point to the Nginx load balancer; otherwise, certificate provisioning won’t work.
+      # 2. If you use multiple domains, specify any one.
+      # load-balancer.hetzner.cloud/hostname: yourDomain.com
+      # [ END: Use these annotations if you care about seeing the actual client IP ]
+
+      load-balancer.hetzner.cloud/http-redirect-https: 'false'

package/dist/src/target-templates/infra-variants/hetzner/.devops/infra/hetzner/ingress-nginx-configmap.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  # Do not change the name - this is required by the Nginx Ingress Controller
+  name: ingress-nginx-controller
+  namespace: ingress-nginx
+data:
+  use-proxy-protocol: "true"

package/dist/src/target-templates/infra-variants/hetzner/.devops/infra/hetzner/retain-storage-class.yaml

@@ -0,0 +1,8 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: hcloud-volumes-retain
+provisioner: csi.hetzner.cloud
+reclaimPolicy: Retain
+volumeBindingMode: WaitForFirstConsumer
+allowVolumeExpansion: true

package/dist/src/target-templates/infra-variants/hetzner/.github/workflows/k8s-build.yaml

@@ -0,0 +1,93 @@
+name: "Monorepo Build and Deploy"
+
+on:
+  push:
+    branches:
+      - staging
+      - production
+
+permissions:
+  contents: read
+  packages: read
+  # For deploying images to Cloud Run
+  # id-token: write
+
+jobs:
+  build_images:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - image_name: main-node
+          - image_name: main-python
+            cache_path: /root/.cache/uv
+    steps:
+      # Fetch the last 50 commits so that devops affected works
+      - name: Checkout repo and history
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 50
+
+      - name: Setup prerequesites
+        uses: ./.github/actions/setup-prereq@v1
+
+      - name: Connect to Hetzner
+        uses: ./.github/actions/connect-to-hetzner@v1
+        with:
+          kubeconfig: ${{ secrets.HCLOUD_KUBECONFIG }}
+          harbor_user: ${{ secrets.HARBOR_USER }}
+          harbor_password: ${{ secrets.HARBOR_PASSWORD }}
+
+      # For deploying images to Cloud Run
+      # - name: Connect to Cloud Run
+      #   uses: ./.github/actions/connect-to-cloud-run@v1
+      #   with:
+      #     project_id: ${{ vars.GCP_PROJECT_ID }}
+      #     project_number: ${{ vars.GCP_PROJECT_NUMBER }}
+      #     region: ${{ vars.GCP_ARTIFACT_REGISTRY_REGION }}
+
+      - name: Build image
+        uses: ./.github/actions/build-image@v1
+        with:
+          image_name: ${{ matrix.image_name }}
+          cache_path: ${{ matrix.cache_path || '/root/.bun/install/cache' }}
+
+  db_migrate_and_deploy:
+    needs: [build_images]
+    runs-on: ubuntu-latest
+    steps:
+      # Fetch the last 50 commits so that devops affected works
+      - name: Checkout repo and history
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 50
+
+      - name: Setup prerequesites
+        uses: ./.github/actions/setup-prereq@v1
+
+      - name: Connect to Hetzner
+        uses: ./.github/actions/connect-to-hetzner@v1
+        with:
+          kubeconfig: ${{ secrets.HCLOUD_KUBECONFIG }}
+          harbor_user: ${{ secrets.HARBOR_USER }}
+          harbor_password: ${{ secrets.HARBOR_PASSWORD }}
+
+      # For deploying images to Cloud Run
+      # - name: Connect to Cloud Run
+      #   uses: ./.github/actions/connect-to-cloud-run@v1
+      #   with:
+      #     project_id: ${{ vars.GCP_PROJECT_ID }}
+      #     project_number: ${{ vars.GCP_PROJECT_NUMBER }}
+      #     region: ${{ vars.GCP_ARTIFACT_REGISTRY_REGION }}
+
+      - name: Run DB Migrate
+        uses: ./.github/actions/db-migrate@v1
+
+      # Repeat per image (it checks if the image is affected and deploys it if it is)
+      - name: Deploy main node
+        uses: ./.github/actions/deploy-image-k8s@v1
+        with: { "image_name": "main-node" }
+
+      - name: Deploy main python
+        uses: ./.github/actions/deploy-image-k8s@v1
+        with: { "image_name": "main-python" }

package/dist/src/target-templates/lang-variants-common/README.md

@@ -0,0 +1,4 @@
+All files under `typescript` are copied under the root project folder.
+
+During `./devops init`, the user is asked whether python-related files should be added to the repository.
+If the user answers positively, the files under the `python` folder are also copied, overriding files with the same path.

package/dist/src/target-templates/lang-variants-common/python/.cursor/rules/monorepo-python.mdc

@@ -0,0 +1,56 @@
+---
+alwaysApply: true
+---
+
+# Python Monorepo
+
+This project includes a Python workspace alongside the TypeScript workspace.
+
+- **`./devops`** is for TypeScript/Node packages (package.json tree)
+- **`./devopspy`** is for Python packages (pyproject.toml tree)
+
+The Python workspace is configured through a root `pyproject.toml` using uv workspaces.
+
+## Structure
+
+Packages under `applications` depend on packages under `libs` using workspace dependencies. It makes sense to extract code to packages under `libs` when it is used in more than one application. Applications never rely on other applications.
+
+All web services found under `applications` should respond to a `GET healthz` with status 200.
+
+## Using the devopspy CLI (Python only)
+
+We use `uv` as our package manager. The `./devopspy` script is able to run scripts defined in pyproject.toml inside a workspace, run a shell process inside a workspace, and execute the tests in a project:
+
+```bash
+# Most important commands
+./devopspy run <project-name>:<script-name>
+./devopspy exec --in <project-name> <command>
+./devopspy test
+./devopspy test <project-name>
+
+# Examples
+./devopspy run my-api:dev
+./devopspy exec --in my-api uv add some-package
+./devopspy test my-service
+
+# To sync dependencies on all workspaces in the monorepo, use uv normally
+uv sync
+```
+
+Scripts are defined in pyproject.toml under `[tool.devops.scripts]`:
+
+```toml
+[tool.devops.scripts]
+dev = "python -c 'from src.my_package.scripts import dev; dev()'"
+start = "python -c 'from src.my_package.scripts import start; start()'"
+```
+
+`./devopspy` automatically loads the right env variables, so don't use python-dotenv.
+
+There is no need to `cd` into package directories. Prefer to use `./devopspy run` and `./devopspy exec`.
+
+## Testing
+
+Use pytest.
+
+**Always run tests with `./devopspy test`** to ensure the test environment is used.

package/dist/src/target-templates/lang-variants-common/python/.devops/config/images.yaml

@@ -0,0 +1,89 @@
+# The file structure is as follows:
+#
+# Under `templates` key:
+#   <template-key>:
+#     copy-common: <boolean>
+#     extra-content:
+#       - <file-name>
+#       - <file-name>
+#
+# Where:
+# <template-key>       The identifier of the template. 
+#                      This is used to refer to the template in the images section.
+# copy-common          Whether to copy the common Docker files from .devops/docker-images/common to the image.
+# extra-content        Any files to include in the image that are not part of the build context. By default, all applications and their dependencies are copied, as well as the 
+#                      content of .devops/docker-images/<image-name>. If you need to include additional files, list them here. These are typically from the root of the project.
+#                      Files and full folders are supported. Globs (wildcards) are currently not supported.
+#
+#
+# Under `images` key:
+#   <image-key>:
+#     image-template: <image-template>
+#     language: <language>
+#     debug-template: <debug-template>
+#     can-db-migrate: <boolean>
+#     domains:
+#       production: <production-domain>
+#       staging: <staging-domain>
+#     image-extra-content:
+#       - <file-name>
+#       - <file-name>
+#     applications:
+#     - <application-name>
+#     - <application-name>
+#
+# Where:
+# <image-name>          The identifier of the image. Github action should refer to this name in a hard-coded manner when invoking devops scripts.
+#                       This is also the basis for the name of the image in the registry.
+# image-template        The build process copies .devops/docker-images/<image-template>.Dockerfile and .devops/docker-images/<image-template>/ to the image. 
+# language              The language of the image. Currently only "node" and "python" are supported.
+# debug-template        Each image comes with a debug pod that can be acccessed as a console. This is the name of the template to use. 
+#                       Only "debug-console" is currently supported. Can be left out if the image should not have a debug pod.
+# can-db-migrate        Whether this image can be used to run DB migrations. If set to true, the image could be used to run DB migrations if any project that uses this image depends 
+#                       on the db project and if the db project changed.
+# domains               The domains for the image. This is used to generate the ingress rules. 
+#                       This can be left out if the image does not have ingress rules, such as a worker image.
+# cloudrun              If this image should be pushed to cloudrun instead of the cluster registry, set to "true".
+# applications          List of applications that use this image. There is no need to specify dependencies - they will be derived as long as they are declared in package.json.
+#
+
+templates: 
+  node-services:
+    copy-common: true
+    extra-content:
+      - devops
+      - package.json
+      - bun.lock
+      - tsconfig.json
+
+  python-services:
+    copy-common: true
+    extra-content:
+      - devopspy
+      - pyproject.toml
+      - uv.lock    
+
+images:
+  main-node:
+    image-template: node-services
+    language: node
+    debug-template: debug-console
+    can-db-migrate: true
+    domains:
+      production: $PRODUCTION_DOMAIN
+      staging: $STAGING_DOMAIN
+    applications:
+    - example-node
+    - jobs
+
+  main-python:
+    image-template: python-services
+    language: python
+    debug-template: debug-console
+    can-db-migrate: false
+    domains:
+      production: $PRODUCTION_DOMAIN
+      staging: $STAGING_DOMAIN
+    applications:
+    - example-python
+    - example-data-pipeline

package/dist/src/target-templates/lang-variants-common/python/.devops/docker-images/python-services/python-exec.sh

@@ -0,0 +1,8 @@
+#! /usr/bin/env bash
+
+source "$(dirname "$0")/docker-common.sh"
+
+check_env
+setup_config
+pause_if_no_args "$#"
+"$@"

package/dist/src/target-templates/lang-variants-common/python/.devops/docker-images/python-services/python-run.sh

@@ -0,0 +1,8 @@
+#! /usr/bin/env bash
+
+source "$(dirname "$0")/docker-common.sh"
+
+check_env
+setup_config
+pause_if_no_args "$#"
+./devopspy run $1:${2:-start}

package/dist/src/target-templates/lang-variants-common/python/.devops/docker-images/python-services.Dockerfile

@@ -0,0 +1,29 @@
+FROM ghcr.io/astral-sh/uv:python3.12-bookworm-slim
+
+ENV UV_SYSTEM_PYTHON=1
+ENV PATH="/root/.local/bin:$PATH"
+
+WORKDIR /app
+
+ARG MONOREPO_ENV
+ENV MONOREPO_ENV=${MONOREPO_ENV}
+RUN echo "Building for environment: $MONOREPO_ENV"
+
+# Install dependencies
+RUN apt-get update && apt-get install -y jq libpq-dev curl git parallel --fix-missing
+
+# Copy project files
+COPY . .
+
+RUN \
+  --mount=type=cache,target=/root/.cache/uv \
+  uv sync --all-packages --all-extras
+
+# For prisma client, if used
+RUN ./devopspy run-many generate
+RUN ./devopspy run-many build
+
+# The config folder will be mounted when the pod starts with up-to-date env variables that are used in runtime by server-side code
+RUN rm -rf config/
+
+ENTRYPOINT [ "./python-run.sh" ]

package/dist/src/target-templates/lang-variants-common/python/.devops/manifests/_index.yaml

@@ -0,0 +1,21 @@
+# db-migrate is a required entry that is hard-coded in the devops-cli
+# The debug template used per image is dynamic and can be configured in images.yaml
+# The deployment template used per application is dynamic and can be configured in its package.json
+
+db-migrate: 
+  - db-migrate-job.yaml.hb
+debug-console:
+  - deployment-debug.yaml.hb
+cron-jobs:
+  - cron-jobs.yaml.hb
+backend-process:
+  - deployment-process.yaml.hb
+internal-service:
+  - deployment-web.yaml.hb
+  - service.yaml.hb
+external-service:
+  - deployment-web.yaml.hb
+  - service.yaml.hb
+  - ingress.yaml.hb
+prefect:
+  - prefect.yaml.hb

package/dist/src/target-templates/lang-variants-common/python/.devops/manifests/prefect.yaml.hb

@@ -0,0 +1,63 @@
+{{#each prefectFlows}}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: {{../app_name}}
+    env: {{../monorepo_env}}
+  name: {{../app_name}}-{{flow_name}}
+  namespace: {{../namespace}}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: {{../app_name}}
+  template:
+    metadata:
+      labels:
+        app: {{../app_name}}
+        env: {{../monorepo_env}}
+    spec:
+      volumes:
+        - name: secret-injection-hook
+          secret:
+            secretName: {{../env_secret_name}}
+      containers:
+        - image: {{../image_path}}
+          name: {{flow_name}}
+          command:
+            - ./python-exec.sh
+          args:
+            - ./devopspy
+            - exec
+            - --in
+            - {{../app_name}}
+            - python
+            - -m
+            - {{script_path}}
+          env:
+            - name: PREFECT_API_URL
+              value: http://prefect-server.prefect-{{../monorepo_env}}:4200/api
+            - name: MONOREPO_ENV
+              value: {{../monorepo_env}}
+            - name: MONOREPO_NAMESPACE
+              value: {{../namespace}}
+            - name: IS_KUBERNETES
+              value: 'true'
+            - name: MONOREPO_BASE_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: {{../env_secret_name}}
+                  key: {{../env_base_secret_key}}
+          volumeMounts:
+            - name: secret-injection-hook
+              mountPath: /etc/kubernetes/secrets
+              readOnly: true
+          resources:
+            requests:
+              memory: 250Mi
+              cpu: 250m
+{{#unless @last}}
+---
+{{/unless}}
+{{/each}}

package/dist/src/target-templates/lang-variants-common/python/applications/example-data-pipeline/pyproject.toml

@@ -0,0 +1,14 @@
+[project]
+name = "example-data-pipeline"
+version = "0.1.0"
+description = ""
+authors = []
+requires-python = ">=3.12"
+dependencies = ["prefect (>=3.2.14,<4.0.0)"]
+
+[tool.devops.deployment]
+template = "prefect"
+
+[[tool.devops.deployment.prefectFlows]]
+flow_name = "test-flow"
+script_path = "src.example_data_pipeline.main"