@vaharoni/devops 1.0.47

package/src/target-templates/.github/actions/build-image@v1/action.yaml

@@ -0,0 +1,86 @@
+name: "Build image"
+description: "Build the specified image if it's affected"
+inputs:
+  image_name:
+    description: 'The image key in images.yaml'
+    required: true
+  gh_pat_token:
+    description: "GitHub personal access token"
+    required: true
+  cache_path:
+    description: "The path to cache inside the container"
+    required: true
+outputs:
+  affected:
+    description: "Whether the image is affected"
+    value: ${{ steps.check_affected.outputs.affected }}    
+runs:
+  using: "composite"
+  steps:
+    - name: Setup basic vars
+      shell: bash
+      run: |
+        echo "IMAGE_NAME=${{ inputs.image_name }}" >> $GITHUB_ENV
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - name: Check if affected
+      id: check_affected
+      shell: bash
+      run: |
+        AFFECTED=$(devops affected image $IMAGE_NAME --from-live-version)
+        echo "affected=$AFFECTED" >> $GITHUB_OUTPUT
+        if [[ "$AFFECTED" == "true" ]]; then
+          echo "${{ env.IMAGE_NAME }} is affected. Proceeding with build."
+        else
+          echo "${{ env.IMAGE_NAME }} is not affected. Skipping."
+        fi
+
+    - name: Prepare build
+      shell: bash
+      if: steps.check_affected.outputs.affected == 'true'
+      run: |
+        echo "CACHE_PREFIX=v1-${{ runner.os }}-${{ env.IMAGE_NAME }}-${{ github.ref_name }}" >> $GITHUB_ENV
+        echo "ECR_URL=$(devops registry repo-url ${{ env.IMAGE_NAME }} ${{ github.sha }})" >> $GITHUB_ENV
+        folder=$(devops prep-build ${{ env.IMAGE_NAME }})
+        mv $folder /home/runner/build
+        echo MONOREPO_ENV=${{ env.MONOREPO_ENV }}
+
+    - name: Setup cache
+      if: steps.check_affected.outputs.affected == 'true'
+      id: setup-cache
+      uses: actions/cache@v4
+      with:
+        path: cache-path
+        key: ${{ env.CACHE_PREFIX }}
+
+    - name: Inject cache into docker
+      if: steps.check_affected.outputs.affected == 'true'
+      uses: reproducible-containers/buildkit-cache-dance@v3.1.0
+      with:
+        cache-map: |
+          {
+            "cache-path": "${{ inputs.cache_path }}"
+          }
+        skip-extraction: ${{ steps.setup-cache.outputs.cache-hit }}          
+
+    - name: Build and push Docker image
+      if: steps.check_affected.outputs.affected == 'true'
+      uses: docker/build-push-action@v6
+      with:
+        context: /home/runner/build
+        push: true
+        tags: ${{ env.ECR_URL }}
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
+        secrets: |
+          GH_PAT_TOKEN=${{ inputs.gh_pat_token }}
+        build-args: |
+          MONOREPO_ENV=${{ env.MONOREPO_ENV }}
+
+    - name: Prune registry
+      if: steps.check_affected.outputs.affected == 'true'
+      shell: bash
+      run: |
+        devops registry prune ${{ env.IMAGE_NAME }}

package/src/target-templates/.github/actions/connect-to-digital-ocean@v1/action.yaml

@@ -0,0 +1,29 @@
+name: "Connect to Digital Ocean"
+description: "Sets up kubernetes connection to Digital Ocean and ensures connection"
+inputs:
+  access_token:
+    description: "DigitalOcean access token"
+    required: true
+  cluster_name:
+    description: "DigitalOcean cluster name"
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - name: Install doctl
+      uses: digitalocean/action-doctl@v2
+      with:
+        token: ${{ inputs.access_token }}
+
+    - name: Log in to DigitalOcean Container Registry with short-lived credentials
+      run: doctl registry login --expiry-seconds 1200
+      shell: bash
+
+    - name: Save DigitalOcean kubeconfig with short-lived credentials
+      run: | 
+        doctl kubernetes cluster kubeconfig save --expiry-seconds 1200 ${{ inputs.cluster_name }}
+      shell: bash
+
+    - name: verify namepsace exists
+      run: devops namespace check --env ${{ github.ref_name }}
+      shell: bash

package/src/target-templates/.github/actions/connect-to-hetzner@v1/action.yaml

@@ -0,0 +1,31 @@
+name: "Connect to Hetzner"
+description: "Sets up kubernetes connection to Hetzner and ensures connection"
+inputs:
+  kubeconfig:
+    description: "The Hetzner kubeconfig file"
+    required: true
+  harbor_user:
+    description: "The user name for the harbor registry"
+    required: true
+  harbor_password:
+    description: "The password for the harbor registry"
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - name: Create a kubeconfig file
+      run: | 
+        mkdir -p ~/.kube
+        echo "${{ inputs.kubeconfig }}" > ~/.kube/config
+        chmod 600 ~/.kube/config
+      shell: bash
+
+    - name: Verify cluster connection and that namepsace exists
+      run: devops namespace check --env ${{ github.ref_name }}
+      shell: bash
+
+    - name: Connect to the registry
+      run: | 
+        server_url=$(devops registry server-url)
+        docker login $server_url -u '${{ inputs.harbor_user }}' -p ${{ inputs.harbor_password }}
+      shell: bash

package/src/target-templates/.github/actions/connect-to-infra@v1/action.yaml

@@ -0,0 +1,46 @@
+name: "Connect to the infrastructure"
+description: "Convenience action to connect either to Digital Ocean or Hetzner based on the inputs provided"
+inputs:
+  do_access_token:
+    description: "DigitalOcean access token"
+    required: false
+  do_cluster_name:
+    description: "DigitalOcean cluster name"
+    required: false
+  hetzner_kubeconfig:
+    description: "The Hetzner kubeconfig file"
+    required: false
+  harbor_user:
+    description: "The user name for the harbor registry"
+    required: false
+  harbor_password:
+    description: "The password for the harbor registry"
+    required: false
+outputs:
+  infra:
+    description: "Which infrastructure is being used"
+    value: steps.determine_infrastructure.outputs.infra
+runs:
+  using: "composite"
+  steps:
+    - name: Determine the infrastructure
+      id: determine_infrastructure
+      shell: bash
+      run: |
+        INFRA=$(devops constant infra)
+        echo "infra=$INFRA" >> $GITHUB_OUTPUT
+
+    - name: Connect to Digital Ocean
+      if: steps.determine_infrastructure.outputs.infra == 'digitalocean'
+      uses: ./.github/actions/connect-to-digital-ocean@v1
+      with:
+        access_token: ${{ inputs.do_access_token }}
+        cluster_name: ${{ inputs.do_cluster_name }}
+
+    - name: Connect to Hetzner
+      if: steps.determine_infrastructure.outputs.infra == 'hetzner'
+      uses: ./.github/actions/connect-to-hetzner@v1
+      with:
+        kubeconfig: ${{ inputs.hetzner_kubeconfig }}
+        harbor_user: ${{ inputs.harbor_user }}
+        harbor_password: ${{ inputs.harbor_password }}

package/src/target-templates/.github/actions/db-migrate@v1/action.yaml

@@ -0,0 +1,23 @@
+name: "DB Migrate"
+description: "Run DB migrate on an affected image, if applicable"
+runs:
+  using: "composite"
+  steps:
+    - name: run-db-migrate
+      shell: bash
+      run: |
+        set +e
+        IMAGE=$(devops affected find-migrator --from-live-version)
+        status=$?
+        set -e
+        if [[ $status -eq 13 ]]; then
+          echo "db project missing, skipping DB migration"
+        elif [[ $status -ne 0 ]]; then
+          exit $status
+        elif [[ "$IMAGE" != "" ]]; then
+          devops job db-migrate create $IMAGE ${{ github.sha }} --timeout 240
+        else
+          echo "No image requires a DB migration"
+        fi
+
+

package/src/target-templates/.github/actions/deploy-image@v1/action.yaml

@@ -0,0 +1,33 @@
+name: "Deploy image"
+description: "Deploy the specified image if it's affected and set its version"
+inputs:
+  image_name:
+    description: 'The image key in images.yaml'
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - name: Setup basic vars
+      shell: bash
+      run: |
+        echo "IMAGE_NAME=${{ inputs.image_name }}" >> $GITHUB_ENV
+
+    - name: Check if affected
+      id: check_affected
+      shell: bash
+      run: |
+        AFFECTED=$(devops affected image $IMAGE_NAME --from-live-version)
+        echo "affected=$AFFECTED" >> $GITHUB_OUTPUT
+        echo "affected=$AFFECTED"
+        if [[ "$AFFECTED" == "true" ]]; then
+          echo "${{ env.IMAGE_NAME }} is affected. Proceeding with deployment."
+        else
+          echo "${{ env.IMAGE_NAME }} is not affected. Skipping."
+        fi
+
+    - name: Deploy
+      shell: bash
+      if: steps.check_affected.outputs.affected == 'true'
+      run: |
+        devops image deployment create ${{ env.IMAGE_NAME }} ${{ github.sha }}
+        devops image version set ${{ env.IMAGE_NAME }} ${{ github.sha }}

package/src/target-templates/.github/actions/setup-prereq@v1/action.yaml

@@ -0,0 +1,29 @@
+name: "Install prerequesites"
+description: "Sets up Node, Bun, and devops"
+inputs:
+  gh_pat_token:
+    description: "The GitHub Personal Access Token"
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - name: Setup Node.js
+      uses: actions/setup-node@v4
+      with:
+        node-version: 23
+
+    - name: Setup bun
+      uses: oven-sh/setup-bun@v2
+
+    - name: set $MONOREPO_ENV
+      shell: bash
+      run: |
+        BRANCH_NAME=${{ github.ref_name }}
+        echo "MONOREPO_ENV=$BRANCH_NAME" >> $GITHUB_ENV
+        echo "GH_PAT_TOKEN=${{ inputs.gh_pat_token }}" >> $GITHUB_ENV
+
+    - name: Install the devops tool
+      shell: bash
+      run: | 
+        bun install @vaharoni/devops
+        echo "$(pwd)/node_modules/.bin" >> $GITHUB_PATH

package/src/target-templates/.github/workflows/k8s-build.yaml

@@ -0,0 +1,84 @@
+name: "Monorepo Build and Deploy"
+
+on:
+  push:
+    branches:
+      - staging
+      - production
+
+permissions:
+  contents: read
+  packages: read
+
+jobs:
+  build_images:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - image_name: main-node
+          - image_name: main-python
+            cache_path: /root/.cache/uv
+    steps:
+      # Fetch the last 50 commits so that devops affected works
+      - name: Checkout repo and history
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 50
+
+      - name: Setup prerequesites
+        uses: ./.github/actions/setup-prereq@v1
+        with:
+          gh_pat_token: ${{ secrets.GH_PAT_TOKEN }}
+
+      - name: Connect to infrastructure
+        uses: ./.github/actions/connect-to-infra@v1
+        with:
+          do_access_token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
+          do_cluster_name: ${{ secrets.DIGITALOCEAN_CLUSTER_NAME }}
+          hetzner_kubeconfig: ${{ secrets.HCLOUD_KUBECONFIG }}
+          harbor_user: ${{ secrets.HARBOR_USER }}
+          harbor_password: ${{ secrets.HARBOR_PASSWORD }}
+
+      - name: Build image
+        uses: ./.github/actions/build-image@v1
+        with:
+          image_name: ${{ matrix.image_name }}
+          gh_pat_token: ${{ secrets.GH_PAT_TOKEN }}
+          cache_path: ${{ matrix.cache_path || '/root/.bun/install/cache' }}
+
+  db_migrate_and_deploy:
+    needs: [build_images]
+    runs-on: ubuntu-latest
+    steps:
+      # Fetch the last 50 commits so that devops affected works
+      - name: Checkout repo and history
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 50
+
+      - name: Setup prerequesites
+        uses: ./.github/actions/setup-prereq@v1
+        with:
+          gh_pat_token: ${{ secrets.GH_PAT_TOKEN }}
+
+      - name: Connect to infrastructure
+        uses: ./.github/actions/connect-to-infra@v1
+        with:
+          do_access_token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
+          do_cluster_name: ${{ secrets.DIGITALOCEAN_CLUSTER_NAME }}
+          hetzner_kubeconfig: ${{ secrets.HCLOUD_KUBECONFIG }}
+          harbor_user: ${{ secrets.HARBOR_USER }}
+          harbor_password: ${{ secrets.HARBOR_PASSWORD }}
+
+      - name: Run DB Migrate
+        uses: ./.github/actions/db-migrate@v1
+
+      # Repeat per image (it checks if the image is affected and deploys it if it is)
+      - name: Deploy main node
+        uses: ./.github/actions/deploy-image@v1
+        with: { "image_name": "main-node" }
+
+      - name: Deploy main python
+        uses: ./.github/actions/deploy-image@v1
+        with: { "image_name": "main-python" }

package/src/target-templates/applications/example-data-pipeline/pyproject.toml

@@ -0,0 +1,14 @@
+[project]
+name = "example-data-pipeline"
+version = "0.1.0"
+description = ""
+authors = []
+requires-python = ">=3.12"
+dependencies = ["prefect (>=3.2.14,<4.0.0)"]
+
+[tool.devops.deployment]
+template = "prefect"
+
+[[tool.devops.deployment.prefectFlows]]
+flow_name = "test-flow"
+script_path = "src.example_data_pipeline.main"

package/src/target-templates/applications/example-data-pipeline/src/example_data_pipeline/main.py

@@ -0,0 +1,38 @@
+import httpx
+
+from prefect import flow, task # Prefect flow and task decorators
+
+
+@flow(log_prints=True)
+def show_stars(github_repos: list[str]):
+    """Flow: Show the number of stars that GitHub repos have"""
+    for repo in github_repos:
+        # Call Task 1
+        repo_stats = fetch_stats(repo)
+
+        # Call Task 2
+        stars = get_stars(repo_stats)
+
+        # Print the result
+        print(f"{repo}: {stars} stars")
+
+
+@task
+def fetch_stats(github_repo: str):
+    """Task 1: Fetch the statistics for a GitHub repo"""
+    return httpx.get(f"https://api.github.com/repos/{github_repo}").json()
+
+
+@task
+def get_stars(repo_stats: dict):
+    """Task 2: Get the number of stars from GitHub repo statistics"""
+    return repo_stats['stargazers_count']
+
+
+# Run the flow
+if __name__ == "__main__":
+    show_stars.serve(
+        name="Show Stars",
+        parameters={"github_repos": ["PrefectHQ/prefect", "pydantic/pydantic", "huggingface/transformers"]},
+        interval=60*60
+    )

package/src/target-templates/applications/example-node/index.ts

@@ -0,0 +1,30 @@
+import express from "express";
+import { InternalToken } from '@vaharoni/devops';
+
+const app = express();
+const port = 3001;
+
+app.get("/", (req, res) => {
+  res.send("Hello from node");
+});
+
+// See applications/jobs/README.md for more information
+app.post("/ping-from-jobs", (req, res) => {
+  const authorizationHeader = req.headers['authorization'];
+  try {
+    new InternalToken('jobs').verifyFromHeaderOrThrow(authorizationHeader);
+  } catch {
+    res.status(401).json({ status: 'unauthorized' });
+    return;
+  }
+  console.log('Pong');
+  res.json({ status: 'ok' });
+});
+
+app.get("/healthz", (req, res) => {
+  res.send("OK");
+});
+
+app.listen(port, () => {
+  console.log(`Server is listening on ${port}`);
+});

package/src/target-templates/applications/example-node/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "example-node",
+  "module": "index.ts",
+  "type": "module",
+  "private": true,
+  "deployment": {
+    "service_name": "example-node",
+    "port": 3001,
+    "template": "external-service"
+  },
+  "scripts": {
+    "start": "tsx index.ts",
+    "verify": "tsc -p ./tsconfig.json --noEmit"
+  },
+  "dependencies": {
+    "express": "^4.21.2",
+    "example-node-lib": "workspace:*"
+  },
+  "devDependencies": {
+    "@types/bun": "latest",
+    "@types/express": "^5.0.0"
+  },
+  "peerDependencies": {
+    "typescript": "^5"
+  }
+}

package/src/target-templates/applications/example-node/tsconfig.json

@@ -0,0 +1,3 @@
+{
+  "extends": "../../tsconfig.json"
+}

package/src/target-templates/applications/example-python/pyproject.toml

@@ -0,0 +1,20 @@
+[project]
+name = "example-python"
+version = "0.1.0"
+description = ""
+authors = []
+requires-python = ">=3.12"
+dependencies = ["example-python-lib", "fastapi[standard]"]
+
+[tool.devops.scripts]
+dev = "python -c 'from src.example_python.scripts import dev; dev()'"
+start = "python -c 'from src.example_python.scripts import start; start()'"
+
+[tool.uv.sources]
+example-python-lib = { workspace = true }
+
+
+[tool.devops.deployment]
+template = "external-service"
+service_name = "example-python"
+port = 3002

package/src/target-templates/applications/example-python/src/example_python/main.py

@@ -0,0 +1,13 @@
+import os
+from fastapi import FastAPI
+
+app = FastAPI()
+
+
+@app.get("/")
+def read_root():
+    return "Hello from python"
+
+@app.get("/healthz")
+def healthz():
+    return {"status": "ok"}

package/src/target-templates/applications/example-python/src/example_python/scripts.py

@@ -0,0 +1,17 @@
+import subprocess
+from devops_python.pyproject import get_pyproject_data
+
+package_name = "example_python"
+data = get_pyproject_data()
+port = data.deployment["port"]
+workers = data.deployment.get("workers")
+
+def dev():
+    subprocess.run(["fastapi", "dev", f"src/{package_name}/main.py", "--port", str(port)], check=True)
+
+def start():
+    cmd = ["fastapi", "run", f"src/{package_name}/main.py", "--port", str(port)]
+    if (workers):
+        cmd.extend(["--workers", str(workers)])
+
+    subprocess.run(cmd, check=True)

package/src/target-templates/applications/jobs/README.md

@@ -0,0 +1,68 @@
+This application allows running commands at a certain schedule via k8s cron jobs.
+Currently only curl is supported. A special auth token is injected to the Authorization Bearer which can
+be verified by the receiver.
+
+## Adding a cron job
+
+To add a cron job, add an entry in `package.json` under `deployments.cronJobs`:
+```json
+  "deployment": {
+    "template": "cron-job",
+    "cronJobs": [
+      {
+        "name": "call-check-something-in-example",
+        "cron": "*/15 * * * *",
+        "curl": ["jobs", "-X", "POST", "http://example/jobs/check-something"]
+      }
+    ]
+  }  
+```
+
+- `name` - must be unique across jobs and a valid k8s name. No spaces are allowed. 
+- `cron` - must be a valid Cron format per [here](https://en.wikipedia.org/wiki/Cron).
+- `curl` - must be an array of strings, where each arg is an element in the array. 
+
+A few things to note about the `curl` argument:
+- We use an array here, since curl relies heavily on args with spaces, e.g. `Content-Type: application/json` is a single arg. 
+- The first element must be the subject of the token sent to the endpoint. The endpoint should verify the request header contains a bearer token with the expected subject.
+- The endpoint can be a DNS internal to the cluster, i.e. simply use the service name if it is in the same namepsace.
+
+Downsides of this approach:
+- the domain name used is a duplication of the `deployment.service_name` of the target application. This can be addressed in the future, e.g. by adding another arg to the cronjob with the `appName` that performs service discovery.
+
+## Securing an API endpoint
+
+Next.js example:
+
+```typescript
+// app/someroute/route.ts
+
+import { NextResponse } from 'next/server';
+import { InternalToken } from '@vaharoni/devops';
+
+export async function POST(request: Request) {
+  const authorizationHeader = request.headers.get('Authorization');
+  try {
+    new InternalToken('jobs').verifyFromHeaderOrThrow(authorizationHeader);
+  } catch {
+    return NextResponse.json({ status: 'unauthorized' }, { status: 401 });
+  }
+
+  // Do something
+
+  return NextResponse.json({ status: 'ok' });
+}
+```
+
+## Testing a secure endpoint in local development
+
+Make sure your `config/.env.global` has something like:
+```text
+MONOREPO_BASE_SECRET=123456789abcdef
+```
+
+Then run:
+
+```bash
+devops internal-curl jobs -v -X POST localhost:3001/jobs/someroute
+```

package/src/target-templates/applications/jobs/index.ts

@@ -0,0 +1 @@
+// no-op

package/src/target-templates/applications/jobs/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "jobs",
+  "type": "module",
+  "version": "0.0.1",
+  "main": "index.ts",
+  "nx": {
+    "projectType": "application"
+  },
+  "scripts": {
+    "print-token": "bunx tsx print-token.ts",
+    "verify": "tsc -p ./tsconfig.json --noEmit"
+  },
+  "author": "",
+  "license": "ISC",
+  "deployment": {
+    "template": "cron-jobs",
+    "cronJobs": [
+      {
+        "name": "test",
+        "cron": "*/15 * * * *",
+        "curl": [
+          "jobs",
+          "-X",
+          "POST",
+          "http://example-node/ping-from-jobs"
+        ]
+      }
+    ]
+  }
+}

package/src/target-templates/applications/jobs/tsconfig.json

@@ -0,0 +1,3 @@
+{
+  "extends": "../../tsconfig.json"
+}

package/src/target-templates/config/.env.development

@@ -0,0 +1 @@
+# Add here env variables that are for local development environment

package/src/target-templates/config/.env.global

@@ -0,0 +1,4 @@
+# Add here env variables that are global to all environments
+
+# This should remain locally on your machine. There is no need to set this in the remote environment.
+MONOREPO_BASE_SECRET=123456789abcdef

package/src/target-templates/config/.env.test

@@ -0,0 +1 @@
+# Add here env variables that are for local test environment