@vaharoni/devops 1.0.47

package/src/libs/validate-env.ts

@@ -0,0 +1,266 @@
+import fs from 'fs';
+import yaml from 'yaml';
+import { IGNORED_PATHS } from './discovery/process-common';
+
+type EnvRequirement = 'optional' | 'boolean' | 'required' | string[];
+type ParsedEnvYaml = Record<string, EnvRequirement>;
+type CombinedErrors = Record<string, string[]>;
+type KeysFromFiles = Record<string, string[]>;
+
+export class CombinedEnvValidator {
+  envYamlPaths: string[];
+  dotEnvPaths: string[];
+
+  yamlValidators: SingleEnvValidator[] = [];
+  dotEnvParsers: DotEnvParser[] = [];
+
+  keysFromYamlFiles: Set<string> = new Set();
+  keysFromDotEnvFiles: KeysFromFiles = {};
+
+  errors: CombinedErrors = {};
+  warnings: string[] = [];
+
+  constructor(envYamlPaths: string[], dotEnvPaths: string[] = []) {
+    this.envYamlPaths = envYamlPaths.filter(path => 
+      !IGNORED_PATHS.some((ignoredPath) => path.includes(ignoredPath))
+    )
+    this.dotEnvPaths = dotEnvPaths;
+  }
+
+  validate() {
+    this._handleYamlFiles();
+    this._handleDotEnvFiles();
+    this._finalize();
+  }
+
+  _handleYamlFiles() {
+    this._loadYamlFiles(this.envYamlPaths);
+    this._validateYamlFiles();
+    this._haltIfParsingErrors();
+    this._extractErrors();
+  }
+
+  _handleDotEnvFiles() {
+    this._loadDotEnvFiles(this.dotEnvPaths);
+    this._parseDotEnvFiles();
+    this._combineDotEnvFiles();
+    this._extractWarnings();
+  }
+
+  _loadYamlFiles(envYamlPaths: string[]) {
+    envYamlPaths.forEach((path) => {
+      const validator = new SingleEnvValidator(path);
+      this.yamlValidators.push(validator);
+    });
+  }
+
+  _validateYamlFiles() {
+    this.yamlValidators.forEach((x) => x.validate());
+  }
+
+  _haltIfParsingErrors() {
+    const filesWithParsingErrors = this.yamlValidators.filter((validator) =>
+      Boolean(validator.parsingError),
+    );
+    if (filesWithParsingErrors.length === 0) return;
+
+    console.error('The following env.yaml files have parsing errors:');
+    filesWithParsingErrors.forEach((validator: SingleEnvValidator) => {
+      console.error(`\t${validator.parsingError}`);
+    });
+    process.exit(1);
+  }
+
+  _extractErrors() {
+    this.yamlValidators.forEach((validator) => {
+      Object.keys(validator.parsedEnvYaml ?? {}).forEach((envVar) => {
+        this.keysFromYamlFiles.add(envVar);
+      });
+      Object.entries(validator.errors).forEach(([envVar, error]) => {
+        this.errors[envVar] ??= [];
+        this.errors[envVar].push(error);
+      });
+    });
+  }
+
+  _loadDotEnvFiles(dotEnvPaths: string[] = []) {
+    dotEnvPaths.forEach((path) => {
+      const parser = new DotEnvParser(path);
+      this.dotEnvParsers.push(parser);
+    });
+  }
+
+  _parseDotEnvFiles() {
+    this.dotEnvParsers.forEach((x) => x.parse());
+  }
+
+  _combineDotEnvFiles() {
+    this.dotEnvParsers.forEach((parser) => {
+      if (!parser.keys) return;
+      parser.keys.forEach((key) => {
+        this.keysFromDotEnvFiles[key] ??= [];
+        this.keysFromDotEnvFiles[key].push(parser.path);
+      });
+    });
+  }
+
+  _extractWarnings() {
+    const unusedKeys = Object.keys(this.keysFromDotEnvFiles).filter(
+      (x) => !this.keysFromYamlFiles.has(x),
+    );
+    unusedKeys.forEach((x) => {
+      this.warnings.push(`${x} in: ${this.keysFromDotEnvFiles[x].join(', ')}`);
+    });
+  }
+
+  _finalize() {
+    if (this.warnings.length > 0) {
+      console.error(
+        'WARNING: some env variables exist in .env but not in env.yaml:',
+      );
+      this.warnings.forEach((warning) => console.error(`\t${warning}`));
+      console.error();
+    }
+    if (Object.keys(this.errors).length > 0) {
+      Object.entries(this.errors).forEach(([key, errors]) => {
+        console.error(`Errors for ${key}:`);
+        errors.forEach((error) => console.error(`\t${error}`));
+        console.error();
+      });
+      console.error();
+      process.exit(1);
+    }
+  }
+}
+
+/**
+ * While we don't strictly need to parse .env files (we can simply use process.env), it is useful to give
+ * warnings to the user that there are unused entries in .env files compared to the stated requirements
+ * captured in env.yaml files.
+ */
+export class DotEnvParser {
+  path: string;
+  keys: string[] | undefined;
+
+  constructor(path: string) {
+    this.path = path;
+  }
+
+  parse() {
+    const text = this._readFile(this.path);
+    if (text) this.keys = this._parse(text);
+  }
+
+  _readFile(path: string) {
+    if (!fs.existsSync(path)) return;
+    return fs.readFileSync(path).toString();
+  }
+
+  _parse(text: string) {
+    const lines = text.split('\n');
+    const withoutComments = lines
+      .map((x) => x.replace(/#.*$/, '').trim())
+      .filter(Boolean);
+    const keys = withoutComments
+      .map((x) => x.split('=').map((y) => y.trim()))
+      .filter((x) => x.length > 1)
+      .map((x) => x[0]);
+    return keys;
+  }
+}
+
+export class SingleEnvValidator {
+  envYamlPath: string;
+  parsedEnvYaml: ParsedEnvYaml | undefined;
+  parsingError: string | undefined;
+  errors: Record<string, string> = {};
+
+  constructor(envYamlPath: string) {
+    this.envYamlPath = envYamlPath;
+  }
+
+  validate() {
+    this.parsedEnvYaml = this._parse();
+    if (!this.parsingError) this._addAllErrors();
+  }
+
+  _readFile() {
+    if (!fs.existsSync(this.envYamlPath)) {
+      console.error(`Skipping ${this.envYamlPath}: does not exist`);
+      return;
+    }
+    return yaml.parse(fs.readFileSync(this.envYamlPath).toString());
+  }
+
+  _generateError(message: string) {
+    return `Error in ${this.envYamlPath}: ${message}`;
+  }
+
+  _setParsingError(message: string) {
+    this.parsingError = this._generateError(message);
+  }
+
+  _addError(key: string, message: string) {
+    this.errors[key] = this._generateError(message);
+  }
+
+  _parse() {
+    const allEnv: ParsedEnvYaml = {};
+    const envManifest = this._readFile();
+    if (!envManifest) return;
+    if (!(envManifest instanceof Array)) {
+      this._setParsingError(`env.yaml file must resolve to an array`);
+      return;
+    }
+    envManifest.forEach((env: string | object) => {
+      if (env instanceof Object) {
+        const entries = Object.entries(env);
+        if (entries.length > 1) {
+          this._setParsingError(
+            `every object in env.yaml must have one key. Error near: ${entries[0][0]}`,
+          );
+          return;
+        }
+        const [name, value] = entries[0];
+        if (
+          !(value instanceof Array) &&
+          !['optional', 'boolean'].includes(value as string)
+        ) {
+          this._setParsingError(
+            `invalid value for ${name}: ${JSON.stringify(value)}`,
+          );
+          return;
+        }
+        allEnv[name] = value;
+      } else {
+        allEnv[env] = 'required';
+      }
+    });
+    return allEnv;
+  }
+
+  _addAllErrors() {
+    Object.entries(this.parsedEnvYaml!).forEach(([key, requirement]) => {
+      const value = process.env[key];
+      if (requirement !== 'optional' && !value) {
+        this._addError(key, `${key} is required but missing`);
+      } else if (
+        requirement === 'boolean' &&
+        !['true', 'false'].includes(String(value))
+      ) {
+        this._addError(
+          key,
+          `${key} must be either true or false. Value: ${value}`,
+        );
+      } else if (
+        requirement instanceof Array &&
+        !requirement.includes(value ?? '')
+      ) {
+        this._addError(
+          key,
+          `${key} must be one of ${requirement.join(', ')}. Value: ${value}`,
+        );
+      }
+    });
+  }
+}

package/src/target-templates/.devops/config/constants.yaml

@@ -0,0 +1,17 @@
+# These will be used when generating kubernetes entities
+project-name: changeme
+
+# Supported: hetzner or digitalocean
+infra: hetzner
+
+# Only relevant for Digital Ocean. Determines the number of versions to keep for each docker image.
+image-versions-to-keep: 5
+
+registry-base-url: registry.staging.com
+registry-name: changeme
+
+# production and staging are supported by default
+extra-remote-environments: []
+
+# development and test are supported by default
+extra-local-environments: []

package/src/target-templates/.devops/config/images.yaml

@@ -0,0 +1,88 @@
+# The file structure is as follows:
+#
+# Under `templates` key:
+#   <template-key>:
+#     copy-common: <boolean>
+#     extra-content:
+#       - <file-name>
+#       - <file-name>
+#
+# Where:
+# <template-key>       The identifier of the template. 
+#                      This is used to refer to the template in the images section.
+# copy-common          Whether to copy the common Docker files from .devops/docker-images/common to the image.
+# extra-content        Any files to include in the image that are not part of the build context. By default, all applications and their dependencies are copied, as well as the 
+#                      content of .devops/docker-images/<image-name>. If you need to include additional files, list them here. These are typically from the root of the project.
+#                      Files and full folders are supported. Globs (wildcards) are currently not supported.
+#
+#
+# Under `images` key:
+#   <image-key>:
+#     image-template: <image-template>
+#     language: <language>
+#     debug-template: <debug-template>
+#     can-db-migrate: <boolean>
+#     domains:
+#       production: <production-domain>
+#       staging: <staging-domain>
+#     image-extra-content:
+#       - <file-name>
+#       - <file-name>
+#     applications:
+#     - <application-name>
+#     - <application-name>
+#
+# Where:
+# <image-name>          The identifier of the image. Github action should refer to this name in a hard-coded manner when invoking devops scripts.
+#                       This is also the basis for the name of the image in the registry.
+# image-template        The build process copies .devops/docker-images/<image-template>.Dockerfile and .devops/docker-images/<image-template>/ to the image. 
+# language              The language of the image. Currently only "node" and "python" are supported.
+# debug-template        Each image comes with a debug pod that can be acccessed as a console. This is the name of the template to use. Only "debug-console" is currently supported.
+# can-db-migrate        Whether this image can be used to run DB migrations. If set to true, the image could be used to run DB migrations if any project that uses this image depends 
+#                       on the db project and if the db project changed.
+# domains               The domains for the image. This is used to generate the ingress rules. 
+#                       This can be left out if the image does not have ingress rules, such as a worker image.
+# applications          List of applications that use this image. There is no need to specify dependencies - they will be derived as long as they are declared in package.json.
+#
+
+templates: 
+  node-services:
+    copy-common: true
+    extra-content:
+      - devops
+      - package.json
+      - bun.lock
+      - tsconfig.json
+      - .npmrc
+
+  python-services:
+    copy-common: true
+    extra-content:
+      - devopspy
+      - pyproject.toml
+      - uv.lock    
+
+images:
+  main-node:
+    image-template: node-services
+    language: node
+    debug-template: debug-console
+    can-db-migrate: true
+    domains:
+      production: production.com
+      staging: staging.com
+    applications:
+    - example-node
+    - jobs
+
+  main-python:
+    image-template: python-services
+    language: python
+    debug-template: debug-console
+    can-db-migrate: false
+    domains:
+      production: production.com
+      staging: staging.com
+    applications:
+    - example-python
+    - example-data-pipeline

package/src/target-templates/.devops/docker-images/common/docker-common.sh

@@ -0,0 +1,23 @@
+check_env() {
+  if [ -z "${MONOREPO_ENV}" ]; then
+    echo "ERROR: MONOREPO_ENV does not exist"
+    exit 1
+  fi
+}
+
+setup_config() {
+  mkdir -p /app/config
+
+  if [ -f /etc/kubernetes/secrets/env_json ]; then
+    jq -r 'to_entries|map("\(.key)=\(.value|tostring)")|.[]' /etc/kubernetes/secrets/env_json > /app/config/.env.global
+  else
+    echo "WARNING: /etc/kubernetes/secrets/env_json does not exist"
+  fi
+}
+
+pause_if_no_args() {
+  if [ "$1" -eq 0 ]; then
+    echo "WARNING: No args provided. Pausing."
+    tail -f /dev/null
+  fi
+}

package/src/target-templates/.devops/docker-images/node-services/node-exec.sh

@@ -0,0 +1,8 @@
+#! /usr/bin/env bash
+
+source "$(dirname "$0")/docker-common.sh"
+
+check_env
+setup_config
+pause_if_no_args "$#"
+"$@"

package/src/target-templates/.devops/docker-images/node-services/node-run.sh

@@ -0,0 +1,8 @@
+#! /usr/bin/env bash
+
+source "$(dirname "$0")/docker-common.sh"
+
+check_env
+setup_config
+pause_if_no_args "$#"
+./devops run $1:${2:-start}

package/src/target-templates/.devops/docker-images/node-services.Dockerfile

@@ -0,0 +1,34 @@
+# FROM node:bookworm-slim
+FROM node:22.4.1-bookworm-slim
+
+RUN apt-get update && apt-get install -y jq curl
+
+WORKDIR /app
+
+ENV NODE_ENV=production
+
+ARG MONOREPO_ENV
+ENV MONOREPO_ENV=${MONOREPO_ENV}
+RUN echo "Building for environment: $MONOREPO_ENV"
+
+RUN npm install -g bun
+
+# This assumes devops prep-build was called by the host, which creates the config/ folder with necessary env variables
+# that are needed to be statitcally resolved by devops run-many build (e.g. NEXT_PUBLIC_*)
+COPY . .
+
+# Install dependencies using bun
+# Mount the GH_PAT_TOKEN secret and use it during bun install
+RUN --mount=type=secret,id=GH_PAT_TOKEN \
+  --mount=type=cache,target=/root/.bun/install/cache \
+  GH_PAT_TOKEN=$(cat /run/secrets/GH_PAT_TOKEN) bun install
+
+# For prisma client, if used
+RUN ./devops run-many generate
+RUN ./devops run-many build
+
+# The config folder will be mounted when the pod starts with up-to-date env variables that are used in runtime by server-side code
+RUN rm -rf config/
+
+# Pods may override this entrypoint to `node-exec.sh` using the `command` field in the pod spec.
+ENTRYPOINT [ "./node-run.sh" ]

package/src/target-templates/.devops/docker-images/python-services/python-exec.sh

@@ -0,0 +1,8 @@
+#! /usr/bin/env bash
+
+source "$(dirname "$0")/docker-common.sh"
+
+check_env
+setup_config
+pause_if_no_args "$#"
+"$@"

package/src/target-templates/.devops/docker-images/python-services/python-run.sh

@@ -0,0 +1,8 @@
+#! /usr/bin/env bash
+
+source "$(dirname "$0")/docker-common.sh"
+
+check_env
+setup_config
+pause_if_no_args "$#"
+./devopspy run $1:${2:-start}

package/src/target-templates/.devops/docker-images/python-services.Dockerfile

@@ -0,0 +1,29 @@
+FROM ghcr.io/astral-sh/uv:python3.12-bookworm-slim
+
+ENV UV_SYSTEM_PYTHON=1
+ENV PATH="/root/.local/bin:$PATH"
+
+WORKDIR /app
+
+ARG MONOREPO_ENV
+ENV MONOREPO_ENV=${MONOREPO_ENV}
+RUN echo "Building for environment: $MONOREPO_ENV"
+
+# Install dependencies
+RUN apt-get update && apt-get install -y jq libpq-dev curl git parallel --fix-missing
+
+# Copy project files
+COPY . .
+
+RUN \
+  --mount=type=cache,target=/root/.cache/uv \
+  uv sync --all-packages --all-extras
+
+# For prisma client, if used
+RUN ./devopspy run-many generate
+RUN ./devopspy run-many build
+
+# The config folder will be mounted when the pod starts with up-to-date env variables that are used in runtime by server-side code
+RUN rm -rf config/
+
+ENTRYPOINT [ "./python-run.sh" ]

package/src/target-templates/.devops/env.example.yaml

@@ -0,0 +1,23 @@
+# env.yaml files should be placed in workspace folders, at the same level of their 
+# package.json / pyproject.toml file.
+# The env.yaml file contains a list of environment variables the workspace depends on.
+# It is intended to be useful for both humans and machines.
+#
+# You are encouraged to use comments to structure the file and make it easier to read.
+#
+# The commands `./devops run` and `./devops exec` inject the content of the environment files in `config/` into the process.
+# Before executing, they validate the variables present in all applicable env.yaml files. If something is amiss, an error is raised.
+#
+# The file is structured as a YAML array of environment variables.
+# If an environment variable is stated by name only, it is a required string.
+# If it is stated as an object, it can have the following structure:
+# - ENV_NAME: optional              # => may be ommitted or empty
+# - ENV_NAME: boolean               # => must be either `true` or `false`
+# - ENV_NAME: ["value1", "value2"]  # => must be one of the listed values
+#
+# Example:
+# - NEXT_PUBLIC_REQUIRED_MESSAGE
+# - REDIS_PROTOCOL: ["redis", "rediss"]
+# - IS_SPECIAL_FEATURE_AVAILABLE: boolean
+# - MESSAGE_OVERRIDE: optional
+#

package/src/target-templates/.devops/infra/hetzner/abandoned/harbor-values.yaml

@@ -0,0 +1,30 @@
+# While this works when setting Cloudflare TLS encryption mode to "full", it fails when too big layers are attempted to be pushed
+expose:
+  type: ingress
+  ingress:
+    hosts:
+      core: $REGISTRY_DOMAIN
+    className: "nginx"
+    annotations:
+      nginx.ingress.kubernetes.io/ssl-redirect: "false"
+      nginx.ingress.kubernetes.io/proxy-body-size: "0"
+      ingress.kubernetes.io/ssl-redirect: "false"
+      ingress.kubernetes.io/proxy-body-size: "0"
+  tls:
+    enabled: false
+externalURL: https://$REGISTRY_DOMAIN
+persistence:
+  enabled: true
+  persistentVolumeClaim:
+    registry:
+      storageClass: "hcloud-volumes"
+      accessMode: ReadWriteOnce
+      size: 20Gi
+    jobservice:
+      storageClass: "hcloud-volumes"
+      accessMode: ReadWriteOnce
+      size: 5Gi
+    chartmuseum:
+      storageClass: "hcloud-volumes"
+      accessMode: ReadWriteOnce
+      size: 5Gi

package/src/target-templates/.devops/infra/hetzner/abandoned/hcloud-config.yaml

@@ -0,0 +1,134 @@
+---
+cluster_name: changeme
+kubeconfig_path: "./kubeconfig"
+k3s_version: v1.32.2+k3s1
+
+networking:
+  ssh:
+    port: 22
+    use_agent: false # set to true if your key has a passphrase
+    public_key_path: "~/.ssh/id_hcloud.pub"
+    private_key_path: "~/.ssh/id_hcloud"
+  allowed_networks:
+    ssh:
+      - 0.0.0.0/0
+    api: # this will firewall port 6443 on the nodes
+      - 0.0.0.0/0
+  public_network:
+    ipv4: false
+    ipv6: false
+  private_network:
+    enabled: true
+    subnet: 10.0.0.0/16
+    existing_network_name: "cluster-network"
+  cni:
+    enabled: true
+    encryption: false
+    mode: flannel
+
+  # cluster_cidr: 10.244.0.0/16 # optional: a custom IPv4/IPv6 network CIDR to use for pod IPs
+  # service_cidr: 10.43.0.0/16 # optional: a custom IPv4/IPv6 network CIDR to use for service IPs. Warning, if you change this, you should also change cluster_dns!
+  # cluster_dns: 10.43.0.10 # optional: IPv4 Cluster IP for coredns service. Needs to be an address from the service_cidr range
+
+
+# manifests:
+#   cloud_controller_manager_manifest_url: "https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/v1.23.0/ccm-networks.yaml"
+#   csi_driver_manifest_url: "https://raw.githubusercontent.com/hetznercloud/csi-driver/v2.12.0/deploy/kubernetes/hcloud-csi.yml"
+#   system_upgrade_controller_deployment_manifest_url: "https://github.com/rancher/system-upgrade-controller/releases/download/v0.14.2/system-upgrade-controller.yaml"
+#   system_upgrade_controller_crd_manifest_url: "https://github.com/rancher/system-upgrade-controller/releases/download/v0.14.2/crd.yaml"
+#   cluster_autoscaler_manifest_url: "https://raw.githubusercontent.com/kubernetes/autoscaler/master/cluster-autoscaler/cloudprovider/hetzner/examples/cluster-autoscaler-run-on-master.yaml"
+#   cluster_autoscaler_container_image_tag: "v1.32.0"
+
+# datastore:
+#   mode: etcd # etcd (default) or external
+#   external_datastore_endpoint: postgres://....
+
+schedule_workloads_on_masters: false
+
+# image: rocky-9 # optional: default is ubuntu-24.04
+# autoscaling_image: 103908130 # optional, defaults to the `image` setting
+# snapshot_os: microos # optional: specified the os type when using a custom snapshot
+
+masters_pool:
+  instance_type: cpx21
+  instance_count: 1 # for HA; you can also create a single master cluster for dev and testing (not recommended for production)
+  locations: # You can choose a single location for single master clusters or if you prefer to have all masters in the same location. For regional clusters (which are only available in the eu-central network zone), each master needs to be placed in a separate location.
+    # - fsn1
+    # - hel1
+    - nbg1
+
+worker_node_pools:
+- name: small-static
+  instance_type: cpx21
+  instance_count: 2
+  location: nbg1
+  # image: debian-11
+  # labels:
+  #   - key: purpose
+  #     value: blah
+  # taints:
+  #   - key: something
+  #     value: value1:NoSchedule
+# - name: medium-autoscaled
+#   instance_type: cpx31
+#   location: fsn1
+#   autoscaling:
+#     enabled: true
+#     min_instances: 0
+#     max_instances: 3
+
+embedded_registry_mirror:
+  enabled: false # Enables fast p2p distribution of container images between nodes for faster pod startup. Check if your k3s version is compatible before enabling this option. You can find more information at https://docs.k3s.io/installation/registry-mirror
+
+protect_against_deletion: true
+
+create_load_balancer_for_the_kubernetes_api: false # Just a heads up: right now, we can’t limit access to the load balancer by IP through the firewall. This feature hasn’t been added by Hetzner yet.
+
+# See https://github.com/vitobotta/hetzner-k3s/blob/main/docs/Private_clusters_with_public_network_interface_disabled.md
+post_create_commands:
+- apt update
+- apt upgrade -y
+- apt install ifupdown resolvconf -y
+- apt autoremove -y hc-utils
+- apt purge -y hc-utils
+- echo "auto enp7s0" > /etc/network/interfaces.d/60-private
+- echo "iface enp7s0 inet dhcp" >> /etc/network/interfaces.d/60-private
+- echo "    post-up ip route add default via 10.0.0.1"  >> /etc/network/interfaces.d/60-private
+- echo "[Resolve]" > /etc/systemd/resolved.conf
+- echo "DNS=1.1.1.1 1.0.0.1" >> /etc/systemd/resolved.conf
+- ifdown enp7s0
+- ifup enp7s0
+- systemctl start resolvconf
+- systemctl enable resolvconf
+- echo "nameserver 1.1.1.1" >> /etc/resolvconf/resolv.conf.d/head
+- echo "nameserver 1.0.0.1" >> /etc/resolvconf/resolv.conf.d/head
+- resolvconf --enable-updates
+- resolvconf -u
+
+# additional_packages:
+# - somepackage
+
+# post_create_commands:
+# - apt update
+# - apt upgrade -y
+# - apt autoremove -y
+
+# kube_api_server_args:
+# - arg1
+# - ...
+# kube_scheduler_args:
+# - arg1
+# - ...
+# kube_controller_manager_args:
+# - arg1
+# - ...
+# kube_cloud_controller_manager_args:
+# - arg1
+# - ...
+# kubelet_args:
+# - arg1
+# - ...
+# kube_proxy_args:
+# - arg1
+# - ...
+# api_server_hostname: k8s.example.com # optional: DNS for the k8s API LoadBalancer. After the script has run, create a DNS record with the address of the API LoadBalancer.