@vaharoni/devops 1.2.13 → 1.2.14

package/dist/src/target-templates/lang-variants-common/python/applications/example-data-pipeline/src/example_data_pipeline/main.py

@@ -0,0 +1,38 @@
+import httpx
+
+from prefect import flow, task # Prefect flow and task decorators
+
+
+@flow(log_prints=True)
+def show_stars(github_repos: list[str]):
+    """Flow: Show the number of stars that GitHub repos have"""
+    for repo in github_repos:
+        # Call Task 1
+        repo_stats = fetch_stats(repo)
+
+        # Call Task 2
+        stars = get_stars(repo_stats)
+
+        # Print the result
+        print(f"{repo}: {stars} stars")
+
+
+@task
+def fetch_stats(github_repo: str):
+    """Task 1: Fetch the statistics for a GitHub repo"""
+    return httpx.get(f"https://api.github.com/repos/{github_repo}").json()
+
+
+@task
+def get_stars(repo_stats: dict):
+    """Task 2: Get the number of stars from GitHub repo statistics"""
+    return repo_stats['stargazers_count']
+
+
+# Run the flow
+if __name__ == "__main__":
+    show_stars.serve(
+        name="Show Stars",
+        parameters={"github_repos": ["PrefectHQ/prefect", "pydantic/pydantic", "huggingface/transformers"]},
+        interval=60*60
+    )

package/dist/src/target-templates/lang-variants-common/python/applications/example-python/pyproject.toml

@@ -0,0 +1,20 @@
+[project]
+name = "example-python"
+version = "0.1.0"
+description = ""
+authors = []
+requires-python = ">=3.12"
+dependencies = ["example-python-lib", "fastapi[standard]"]
+
+[tool.devops.scripts]
+dev = "python -c 'from src.example_python.scripts import dev; dev()'"
+start = "python -c 'from src.example_python.scripts import start; start()'"
+
+[tool.uv.sources]
+example-python-lib = { workspace = true }
+
+
+[tool.devops.deployment]
+template = "external-service"
+service_name = "example-python"
+port = 3002

package/dist/src/target-templates/lang-variants-common/python/applications/example-python/src/example_python/main.py

@@ -0,0 +1,13 @@
+import os
+from fastapi import FastAPI
+
+app = FastAPI()
+
+
+@app.get("/")
+def read_root():
+    return "Hello from python"
+
+@app.get("/healthz")
+def healthz():
+    return {"status": "ok"}

package/dist/src/target-templates/lang-variants-common/python/applications/example-python/src/example_python/scripts.py

@@ -0,0 +1,17 @@
+import subprocess
+from devops_runner_python.pyproject import get_pyproject_data
+
+package_name = "example_python"
+data = get_pyproject_data()
+port = data.deployment["port"]
+workers = data.deployment.get("workers")
+
+def dev():
+    subprocess.run(["fastapi", "dev", f"src/{package_name}/main.py", "--port", str(port)], check=True)
+
+def start():
+    cmd = ["fastapi", "run", f"src/{package_name}/main.py", "--port", str(port)]
+    if (workers):
+        cmd.extend(["--workers", str(workers)])
+
+    subprocess.run(cmd, check=True)

package/dist/src/target-templates/lang-variants-common/python/devopspy

@@ -0,0 +1,3 @@
+#! /usr/bin/env bash
+
+uv run devopspy "$@"

package/dist/src/target-templates/lang-variants-common/python/libs/example-python-lib/pyproject.toml

@@ -0,0 +1,11 @@
+[project]
+name = "example-python-lib"
+version = "0.1.0"
+description = ""
+authors = []
+requires-python = ">=3.12"
+dependencies = []
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"

package/dist/src/target-templates/lang-variants-common/python/libs/example-python-lib/src/example_python_lib/__init__.py

@@ -0,0 +1,2 @@
+def test():
+    return "test"

package/dist/src/target-templates/lang-variants-common/python/pyproject.toml

@@ -0,0 +1,16 @@
+[project]
+name = "$PROJECT_NAME"
+version = "0.1.0"
+description = ""
+authors = []
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = ["devops-runner-python"]
+
+[tool.uv.workspace]
+members = [
+  "db",
+  "applications/example-data-pipeline",
+  "applications/example-python",
+  "libs/example-python-lib",
+]

package/dist/src/target-templates/lang-variants-common/typescript/.cursor/rules/monorepo-typescript.mdc

@@ -0,0 +1,51 @@
+---
+alwaysApply: true
+---
+
+# Monorepo Structure
+
+We have a monorepo setup configured through a global package.json. There is a `./devops` script to execute commands inside typescript packages.
+
+## Structure
+
+Packages under `applications` depend on packages under `libs` using `workspace:*` notation from package.json. It makes sense to extract code to packages under `libs` when it is used in more than one application. Applications never rely on other applications.
+
+Applications are encouraged to extract private concerns to "local libs" modules. A typical use-case are persistence-related operations (getters and mutations).
+
+All web services found under `applications` should respond to a `GET healthz` with status 200.
+
+## Using the devops CLI
+
+We use `bun` as our package manager. The `./devops` script is able to run package.json scripts inside a workspace, run a shell process inside a workspace, and execute the tests in a project:
+
+```bash
+# Most important commands
+./devops run <project-name>:<script-name>
+./devops exec --in <project-name> <command>
+./devops test
+./devops test <project-name>
+
+# Examples
+./devops run my-api:dev
+./devops exec --in my-api bun add some-package
+./devops test my-service
+
+# To install dependencies on all workspaces in the monorepo, use bun normally
+bun install
+```
+
+`./devops` automatically loads the right env variables, so don't use dotenv.
+
+There is no need to `cd` into package directories. Prefer to use `./devops run` and `./devops exec`.
+
+## Testing
+
+Use vitest.
+
+**Always run tests with `./devops test`** to ensure the test environment is used.
+
+## Inter-service Communication
+
+This repo runs in kubernetes and localhost contexts, which means the endpoint URLs will be different depending on the context. To communicate between services, the package `@vaharoni/devops` exports `function getServiceEndpoint(serviceName: string): string`. This function does not return a trailing slash. The `serviceName` can be found in the `deployment.service_name` custom key of the `package.json` of the application you wish to send requests to. If this key does not exist, halt and alert the user.
+
+However, in general, importing a shared lib is preferred over crossing the wire.

package/dist/src/target-templates/lang-variants-common/typescript/.devops/config/images.yaml

@@ -0,0 +1,69 @@
+# The file structure is as follows:
+#
+# Under `templates` key:
+#   <template-key>:
+#     copy-common: <boolean>
+#     extra-content:
+#       - <file-name>
+#       - <file-name>
+#
+# Where:
+# <template-key>       The identifier of the template. 
+#                      This is used to refer to the template in the images section.
+# copy-common          Whether to copy the common Docker files from .devops/docker-images/common to the image.
+# extra-content        Any files to include in the image that are not part of the build context. By default, all applications and their dependencies are copied, as well as the 
+#                      content of .devops/docker-images/<image-name>. If you need to include additional files, list them here. These are typically from the root of the project.
+#                      Files and full folders are supported. Globs (wildcards) are currently not supported.
+#
+#
+# Under `images` key:
+#   <image-key>:
+#     image-template: <image-template>
+#     language: <language>
+#     debug-template: <debug-template>
+#     can-db-migrate: <boolean>
+#     domains:
+#       production: <production-domain>
+#       staging: <staging-domain>
+#     image-extra-content:
+#       - <file-name>
+#       - <file-name>
+#     applications:
+#     - <application-name>
+#     - <application-name>
+#
+# Where:
+# <image-name>          The identifier of the image. Github action should refer to this name in a hard-coded manner when invoking devops scripts.
+#                       This is also the basis for the name of the image in the registry.
+# image-template        The build process copies .devops/docker-images/<image-template>.Dockerfile and .devops/docker-images/<image-template>/ to the image. 
+# language              The language of the image. Currently only "node" and "python" are supported.
+# debug-template        Each image comes with a debug pod that can be acccessed as a console. This is the name of the template to use. Only "debug-console" is currently supported.
+# can-db-migrate        Whether this image can be used to run DB migrations. If set to true, the image could be used to run DB migrations if any project that uses this image depends 
+#                       on the db project and if the db project changed.
+# domains               The domains for the image. This is used to generate the ingress rules. 
+#                       This can be left out if the image does not have ingress rules, such as a worker image.
+# applications          List of applications that use this image. There is no need to specify dependencies - they will be derived as long as they are declared in package.json.
+#
+
+templates: 
+  node-services:
+    copy-common: true
+    extra-content:
+      - devops
+      - package.json
+      - bun.lock
+      - tsconfig.json
+
+images:
+  main-node:
+    image-template: node-services
+    language: node
+    debug-template: debug-console
+    can-db-migrate: true
+    domains:
+      production: $PRODUCTION_DOMAIN
+      staging: $STAGING_DOMAIN
+    applications:
+    - example-node
+    - jobs
+

package/dist/src/target-templates/lang-variants-common/typescript/.devops/docker-images/cloudrun.Dockerfile

@@ -0,0 +1,31 @@
+FROM node:24-bookworm-slim AS builder
+
+RUN apt-get update && apt-get install -y jq curl
+
+WORKDIR /app
+
+ENV NODE_ENV=production
+
+ARG MONOREPO_ENV
+ENV MONOREPO_ENV=${MONOREPO_ENV}
+RUN echo "Building for environment: $MONOREPO_ENV"
+
+RUN npm install -g bun
+
+# This assumes devops prep-build was called by the host, which creates the config/ folder with necessary env variables
+# that are needed to be statitcally resolved by devops run-many build (e.g. NEXT_PUBLIC_*)
+COPY . .
+
+# Install dependencies using bun
+RUN --mount=type=cache,target=/root/.bun/install/cache bun install
+
+# This assumes the image has only one application that has a build script which outputs to dist/
+RUN ./devops run-many build
+
+
+FROM gcr.io/distroless/nodejs24-debian12 AS runner
+WORKDIR /app
+ENV NODE_ENV=production
+COPY --from=builder /app/dist ./dist
+EXPOSE 8080
+CMD ["dist/index.js"]

package/dist/src/target-templates/lang-variants-common/typescript/.devops/docker-images/common/docker-common.sh

@@ -0,0 +1,23 @@
+check_env() {
+  if [ -z "${MONOREPO_ENV}" ]; then
+    echo "ERROR: MONOREPO_ENV does not exist"
+    exit 1
+  fi
+}
+
+setup_config() {
+  mkdir -p /app/config
+
+  if [ -f /etc/kubernetes/secrets/env_json ]; then
+    jq -r 'to_entries|map("\(.key)=\(.value|tostring)")|.[]' /etc/kubernetes/secrets/env_json > /app/config/.env.global
+  else
+    echo "WARNING: /etc/kubernetes/secrets/env_json does not exist"
+  fi
+}
+
+pause_if_no_args() {
+  if [ "$1" -eq 0 ]; then
+    echo "WARNING: No args provided. Pausing."
+    tail -f /dev/null
+  fi
+}

package/dist/src/target-templates/lang-variants-common/typescript/.devops/docker-images/node-services/node-exec.sh

@@ -0,0 +1,8 @@
+#! /usr/bin/env bash
+
+source "$(dirname "$0")/docker-common.sh"
+
+check_env
+setup_config
+pause_if_no_args "$#"
+"$@"

package/dist/src/target-templates/lang-variants-common/typescript/.devops/docker-images/node-services/node-run.sh

@@ -0,0 +1,8 @@
+#! /usr/bin/env bash
+
+source "$(dirname "$0")/docker-common.sh"
+
+check_env
+setup_config
+pause_if_no_args "$#"
+./devops run $1:${2:-start}

package/dist/src/target-templates/lang-variants-common/typescript/.devops/docker-images/node-services.Dockerfile

@@ -0,0 +1,31 @@
+# FROM node:bookworm-slim
+FROM node:22.4.1-bookworm-slim
+
+RUN apt-get update && apt-get install -y jq curl
+
+WORKDIR /app
+
+ENV NODE_ENV=production
+
+ARG MONOREPO_ENV
+ENV MONOREPO_ENV=${MONOREPO_ENV}
+RUN echo "Building for environment: $MONOREPO_ENV"
+
+RUN npm install -g bun
+
+# This assumes devops prep-build was called by the host, which creates the config/ folder with necessary env variables
+# that are needed to be statitcally resolved by devops run-many build (e.g. NEXT_PUBLIC_*)
+COPY . .
+
+# Install dependencies using bun
+RUN --mount=type=cache,target=/root/.bun/install/cache bun install
+
+# For prisma client, if used
+RUN ./devops run-many generate
+RUN ./devops run-many build
+
+# The config folder will be mounted when the pod starts with up-to-date env variables that are used in runtime by server-side code
+RUN rm -rf config/
+
+# Pods may override this entrypoint to `node-exec.sh` using the `command` field in the pod spec.
+ENTRYPOINT [ "./node-run.sh" ]

package/dist/src/target-templates/lang-variants-common/typescript/.devops/env.example.yaml

@@ -0,0 +1,23 @@
+# env.yaml files should be placed in workspace folders, at the same level of their 
+# package.json / pyproject.toml file.
+# The env.yaml file contains a list of environment variables the workspace depends on.
+# It is intended to be useful for both humans and machines.
+#
+# You are encouraged to use comments to structure the file and make it easier to read.
+#
+# The commands `./devops run` and `./devops exec` inject the content of the environment files in `config/` into the process.
+# Before executing, they validate the variables present in all applicable env.yaml files. If something is amiss, an error is raised.
+#
+# The file is structured as a YAML array of environment variables.
+# If an environment variable is stated by name only, it is a required string.
+# If it is stated as an object, it can have the following structure:
+# - ENV_NAME: optional              # => may be ommitted or empty
+# - ENV_NAME: boolean               # => must be either `true` or `false`
+# - ENV_NAME: ["value1", "value2"]  # => must be one of the listed values
+#
+# Example:
+# - NEXT_PUBLIC_REQUIRED_MESSAGE
+# - REDIS_PROTOCOL: ["redis", "rediss"]
+# - IS_SPECIAL_FEATURE_AVAILABLE: boolean
+# - MESSAGE_OVERRIDE: optional
+#

package/dist/src/target-templates/lang-variants-common/typescript/.devops/manifests/_index.yaml

@@ -0,0 +1,19 @@
+# db-migrate is a required entry that is hard-coded in the devops-cli
+# The debug template used per image is dynamic and can be configured in images.yaml
+# The deployment template used per application is dynamic and can be configured in its package.json
+
+db-migrate: 
+  - db-migrate-job.yaml.hb
+debug-console:
+  - deployment-debug.yaml.hb
+cron-jobs:
+  - cron-jobs.yaml.hb
+backend-process:
+  - deployment-process.yaml.hb
+internal-service:
+  - deployment-web.yaml.hb
+  - service.yaml.hb
+external-service:
+  - deployment-web.yaml.hb
+  - service.yaml.hb
+  - ingress.yaml.hb

package/dist/src/target-templates/lang-variants-common/typescript/.devops/manifests/cron-jobs.yaml.hb

@@ -0,0 +1,56 @@
+{{#each cronJobs}}
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  labels:
+    app: {{../app_name}}
+    env: {{../monorepo_env}}
+  name: {{name}}
+  namespace: {{../namespace}}
+spec:
+  schedule: "{{cron}}"
+  jobTemplate:
+    spec:
+      ttlSecondsAfterFinished: 86400
+      template:
+        spec:
+          volumes:
+            - name: secret-injection-hook
+              secret:
+                secretName: {{../env_secret_name}}
+          containers:
+            - image: {{../image_path}}
+              name: {{name}}
+              command:
+                - ./node-exec.sh
+              args:
+                - ./devops
+                - internal-curl
+                {{#each curl}}
+                - {{this}}
+                {{/each}}
+              env:
+                - name: MONOREPO_ENV
+                  value: {{../monorepo_env}}
+                - name: MONOREPO_NAMESPACE
+                  value: {{../namespace}}
+                - name: IS_KUBERNETES
+                  value: 'true'
+                - name: MONOREPO_BASE_SECRET
+                  valueFrom:
+                    secretKeyRef:
+                      name: {{../env_secret_name}}
+                      key: {{../env_base_secret_key}}
+              volumeMounts:
+                - name: secret-injection-hook
+                  mountPath: /etc/kubernetes/secrets
+                  readOnly: true
+              resources:
+                requests:
+                  memory: 250Mi
+                  cpu: 100m
+          restartPolicy: Never
+{{#unless @last}}
+---
+{{/unless}}
+{{/each}}

package/dist/src/target-templates/lang-variants-common/typescript/.devops/manifests/db-migrate-job.yaml.hb

@@ -0,0 +1,43 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{db_migrate_job_name}}
+  namespace: {{namespace}}
+  labels:
+    env: {{monorepo_env}}
+spec:
+  template:
+    spec:
+      volumes:
+        - name: secret-injection-hook
+          secret:
+            secretName: {{env_secret_name}}
+      containers:
+        - name: db-migrate-job
+          image: {{image_path}}
+          args:
+            - db
+            - migrate-deploy
+          env:
+            - name: MONOREPO_ENV
+              value: {{monorepo_env}}
+            - name: MONOREPO_NAMESPACE
+              value: {{namespace}}
+            - name: IS_KUBERNETES
+              value: 'true'
+            - name: MONOREPO_BASE_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: {{env_secret_name}}
+                  key: {{env_base_secret_key}}
+          volumeMounts:
+            - name: secret-injection-hook
+              mountPath: /etc/kubernetes/secrets
+              readOnly: true
+          resources:
+            requests:
+              memory: 100Mi
+              cpu: 250m
+      restartPolicy: Never
+  backoffLimit: 0
+  ttlSecondsAfterFinished: 3600

package/dist/src/target-templates/lang-variants-common/typescript/.devops/manifests/deployment-debug.yaml.hb

@@ -0,0 +1,42 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  # Name and namespace should not be stated - they are provided by the runner
+  labels:
+    app: {{debug_pod_name}}
+    env: {{monorepo_env}}
+    ephemeral-shell: "true"
+spec:
+  volumes:
+    - name: secret-injection-hook
+      secret:
+        secretName: {{env_secret_name}}
+  containers:
+    - image: {{image_path}}
+      name: {{debug_pod_name}}
+      stdin: true
+      tty: true
+      command:
+      - ./node-exec.sh
+      args:
+      - /bin/bash
+      env:
+        - name: MONOREPO_ENV
+          value: {{monorepo_env}}
+        - name: MONOREPO_NAMESPACE
+          value: {{namespace}}
+        - name: IS_KUBERNETES
+          value: 'true'
+        - name: MONOREPO_BASE_SECRET
+          valueFrom:
+            secretKeyRef:
+              name: {{env_secret_name}}
+              key: {{env_base_secret_key}}
+      volumeMounts:
+        - name: secret-injection-hook
+          mountPath: /etc/kubernetes/secrets
+          readOnly: true
+      resources:
+        requests:
+          memory: 250Mi
+          cpu: 100m

package/dist/src/target-templates/lang-variants-common/typescript/.devops/manifests/deployment-process.yaml.hb

@@ -0,0 +1,48 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: {{app_name}}
+    env: {{monorepo_env}}
+  name: {{app_name}}
+  namespace: {{namespace}}
+spec:
+  replicas: {{replicas}}
+  selector:
+    matchLabels:
+      app: {{app_name}}
+  template:
+    metadata:
+      labels:
+        app: {{app_name}}
+        env: {{monorepo_env}}
+    spec:
+      volumes:
+        - name: secret-injection-hook
+          secret:
+            secretName: {{env_secret_name}}
+      containers:
+        - image: {{image_path}}
+          name: {{app_name}}
+          args:
+            - "{{project_name}}"
+          env:
+            - name: MONOREPO_ENV
+              value: {{monorepo_env}}
+            - name: MONOREPO_NAMESPACE
+              value: {{namespace}}
+            - name: IS_KUBERNETES
+              value: 'true'
+            - name: MONOREPO_BASE_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: {{env_secret_name}}
+                  key: {{env_base_secret_key}}
+          volumeMounts:
+            - name: secret-injection-hook
+              mountPath: /etc/kubernetes/secrets
+              readOnly: true
+          resources:
+            requests:
+              memory: 250Mi
+              cpu: 250m

package/dist/src/target-templates/lang-variants-common/typescript/.devops/manifests/deployment-web.yaml.hb

@@ -0,0 +1,54 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: {{app_name}}
+    env: {{monorepo_env}}
+  name: {{app_name}}
+  namespace: {{namespace}}
+spec:
+  replicas: {{replicas}}
+  selector:
+    matchLabels:
+      app: {{app_name}}
+  template:
+    metadata:
+      labels:
+        app: {{app_name}}
+        env: {{monorepo_env}}
+    spec:
+      volumes:
+        - name: secret-injection-hook
+          secret:
+            secretName: {{env_secret_name}}
+      containers:
+        - image: {{image_path}}
+          name: {{app_name}}
+          args:
+            - "{{project_name}}"
+          env:
+            - name: MONOREPO_ENV
+              value: {{monorepo_env}}
+            - name: MONOREPO_NAMESPACE
+              value: {{namespace}}
+            - name: IS_KUBERNETES
+              value: 'true'
+            - name: MONOREPO_BASE_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: {{env_secret_name}}
+                  key: {{env_base_secret_key}}
+          volumeMounts:
+            - name: secret-injection-hook
+              mountPath: /etc/kubernetes/secrets
+              readOnly: true
+          startupProbe:
+            httpGet:
+              path: /healthz
+              port: {{port}}
+            failureThreshold: 50
+            periodSeconds: 2
+          resources:
+            requests:
+              memory: 250Mi
+              cpu: 250m