@vaharoni/devops 1.1.0 → 1.1.2

package/src/cli/init.ts

@@ -1,13 +1,9 @@
-import url from "url";
-import path from "path";
-import fs from "fs-extra";
-
-const __file__ = url.fileURLToPath(import.meta.url);
-const __root__ = path.join(path.dirname(__file__), "../..");
-const templatesDir = path.join(__root__, "src/target-templates");
-const targetDir = process.cwd(); // User's current working directory
-
+import inquirer from "inquirer";
+import { InitGenerator, type InitGeneratorFileInfo } from "../libs/init-generator";
 import { CLICommandParser, printUsageAndExit } from "./common";
+import chalk from "chalk";
+import fs from 'fs-extra';
+import type { ConstFileSchema } from "../types";
 
 const oneLiner =
   "Initializes the devops utility by copying template files to the current folder";
@@ -25,56 +21,233 @@ EXAMPLES
 
 async function run(cmdObj: CLICommandParser) {
   if (cmdObj.help) printUsageAndExit(usage);
-  copyTemplates();
+  createFiles();
 }
 
 export default {
   init: { oneLiner, keyExamples, run },
 };
 
-async function copyTemplates() {
-  console.log(`Initializing devops files from ${templatesDir} to ${targetDir}`);
-
-  try {
-    // Copy files without overriding existing ones
-    await fs.copy(templatesDir, targetDir, {
-      overwrite: false,
-      errorOnExist: false,
-      dereference: false,
-    });
-
-    console.log(successMessage);
-  } catch (error) {
-    console.error("❌ Failed to initialize devops files:", error);
-    process.exit(1);
+async function createFiles() {
+  const tc = new InitGenerator();
+  const userChoices = await getUserChoices(tc.projectName);
+
+  // Language variants
+  tc.addCopiedFolder("lang-variants-common/typescript", ".");
+  if (userChoices.usePython) {
+    tc.addCopiedFolder("lang-variants-common/python", ".");
+    tc.enableSubtitution("pyproject.toml");
+  }
+  tc.enableSubtitution(".devops/config/images.yaml");
+  tc.setMessageGenerator(".envrc", envrcMessage);
+
+  // gitignore
+  const gitIgnore = gitIgnoreContent(userChoices.infraVariant, userChoices.usePython)
+  tc.addGeneratedFile(".gitignore", gitIgnore);
+  tc.setMessageGenerator(".gitignore", gitignoreMessageGen(gitIgnore));
+
+  // Infra variants
+  tc.addCopiedFolder(`infra-variants/${userChoices.infraVariant}`, ".");
+  tc.enableSubtitution(".devops/config/constants.yaml");
+  if (userChoices.infraVariant === "hetzner") {
+    tc.enableSubtitution(".devops/infra/hetzner/harbor-cert.yaml");
+    tc.enableSubtitution(".devops/infra/hetzner/harbor-values.yaml");
+    tc.enableSubtitution(".devops/infra/hetzner/hcloud-config.yaml");
   }
-}
 
-const successMessage = `
-✅ Devops files initialized successfully!
+  // Prisma
+  if (userChoices.usePrisma) {
+    tc.addCopiedFolder("lang-variants-prisma/typescript", ".");
+    if (userChoices.usePython) {
+      tc.addCopiedFolder("lang-variants-prisma/python", ".");
+    }
+  }
 
-To finish the setup: 
+  // Cluster resources
+  const clusterResources = new Set(userChoices.clusterResources);
+  if (clusterResources.has("dns-test")) {
+    tc.addCopiedFolder("cluster-resource-options/dns-test", ".devops/infra/dns-test");
+  }
+  if (clusterResources.has("monitoring-ingress")) {
+    tc.addCopiedFolder("cluster-resource-options/monitoring-ingress", ".devops/infra/monitoring-ingress");
+  }
+  if (clusterResources.has("postgres")) {
+    tc.addCopiedFolder("cluster-resource-options/postgres", ".devops/infra/postgres");
+    // prettier-ignore
+    tc.enableSubtitution(".devops/infra/postgres/staging/configurations/07-SGObjectStorage.yaml");
+    // prettier-ignore
+    tc.enableSubtitution(".devops/infra/postgres/staging/configurations/08-SGScript.yaml");
+    // prettier-ignore
+    tc.enableSubtitution(".devops/infra/postgres/production/configurations/07-SGObjectStorage.yaml");
+    // prettier-ignore
+    tc.enableSubtitution(".devops/infra/postgres/production/configurations/08-SGScript.yaml");
+  }
+  if (clusterResources.has("redis")) {
+    tc.addCopiedFolder("cluster-resource-options/redis", ".devops/infra/redis");
+  }
+  if (clusterResources.has("milvus")) {
+    tc.addCopiedFolder("cluster-resource-options/milvus", ".devops/infra/milvus");
+  }
+  if (clusterResources.has("prefect") && userChoices.usePython) {
+    tc.addCopiedFolder("cluster-resource-options/prefect", ".devops/infra/prefect");
+  }
 
-1. add the following entry to the main package.json:
-  "workspaces": [
-    "libs/**",
-    "applications/**",
+  tc.run({
+    substitution: {
+      'PROJECT_NAME': userChoices.projectName,
+      'STAGING_DOMAIN': userChoices.stagingDomain,
+      'PRODUCTION_DOMAIN': userChoices.productionDomain,
+      'GCLOUD_PROJECT_ID': userChoices.gcloudProjectId,
+      'REGISTRY_IMAGE_PATH_PREFIX': userChoices.registryImagePathPrefix,
+      'REGISTRY_BASE_URL': userChoices.registryBaseUrl,
+    },
+    messages: [
+      packageJsonMessage(userChoices.usePrisma)
+    ]
+  })
+}
+
+function packageJsonMessage(usePrisma: boolean) {
+  const prismaMessage = usePrisma 
+    ? `,
     "db/**",
-    "dml/**"
-  ],
+    "dml/**"` 
+    : "";
 
-2. add the following to your .gitignore:
-**/.env*
+  return `add the following entry to the main ${chalk.blue("package.json")}:
+  ${chalk.yellow(`"workspaces": [
+    "libs/**",
+    "applications/**"${prismaMessage}
+  ],`)}`
+}
+
+function gitIgnoreContent(infraVariant: UserChoices["infraVariant"], usePython: boolean) {
+  const common = `**/.env*
 config/kubeconfig
 tmp/**
-!tmp/**/.gitkeep
-venv/
-**/__pycache__
-
-3. optionally create an .envrc file with the following content and run direnv allow: 
-if [ -f "$PWD/config/kubeconfig" ]; then
-  export KUBECONFIG="$PWD/config/kubeconfig"
-else
-  export KUBECONFIG="$HOME/.kube/config"
-fi
-`;
+!tmp/**/.gitkeep`;
+
+  const gcloud = infraVariant === 'gcloud' 
+    ? 'config/gke_gcloud_auth_plugin_cache'
+    : null;
+
+  const python = usePython
+    ? `venv/
+**/__pycache__`
+    : null;
+
+  return [common, gcloud, python].filter(Boolean).join('\n');
+}
+
+function gitignoreMessageGen(content: string) {
+  return (exists: boolean) => {
+    if (!exists) return;
+    return `add the following to your ${chalk.blue(".gitignore")}:
+${chalk.yellow(content)}`;   
+  }
+}
+
+function envrcMessage(targetExists: boolean, fileInfo: InitGeneratorFileInfo) {
+  if (fileInfo.type !== 'copied') throw new Error(`envrcMessage() expects a copied file, got ${fileInfo.type}`);
+  if (targetExists) {
+    const content = fs.readFileSync(fileInfo.sourceAbs, 'utf-8');
+    return `add the following to your ${chalk.blue(".envrc")} and run ${chalk.yellow("direnv allow")}:
+${chalk.yellow(content)}`;
+  } else {
+    return `Enable ${chalk.blue(".envrc")} by installing ${chalk.blue('direnv')} and running ${chalk.yellow("direnv allow")}`;
+  }
+}
+
+type UserChoices = {
+  projectName: string;
+  stagingDomain: string;
+  productionDomain: string;
+  infraVariant: ConstFileSchema["infra"];
+  gcloudProjectId?: string;
+  registryImagePathPrefix?: string;
+  registryBaseUrl?: string;
+  usePython: boolean;
+  usePrisma: boolean;
+  clusterResources: string[];
+};
+
+function getUserChoices(projectName: string | undefined): Promise<UserChoices> {
+  const defaultProjectName = projectName || "changeme";
+
+  return inquirer.prompt([
+    {
+      type: "input",
+      name: "projectName",
+      message: `Enter the project name (default: '${defaultProjectName}')`,
+      default: defaultProjectName,
+    },
+    {
+      type: "input",
+      name: "stagingDomain",
+      message: "Enter the staging domain (default: 'staging.com')",
+      default: "staging.com",
+    },
+    {
+      type: "input",
+      name: "productionDomain",
+      message: "Enter the production domain (default: 'production.com')",
+      default: "production.com",
+    },
+    {
+      type: "list",
+      name: "infraVariant",
+      message: "Where does your cluster run?",
+      choices: [
+        { name: "Google Cloud", value: "gcloud" },
+        { name: "Digital Ocean", value: "digitalocean" },
+        { name: "Hetzner", value: "hetzner" },
+      ],
+    },
+    {
+      type: "input",
+      name: "gcloudProjectId",
+      message: "Enter the GCP project ID (default: 'changeme')",
+      default: "changeme",
+      when: (answers) => answers.infraVariant === "gcloud",
+    },
+    {
+      type: "input",
+      name: "registryImagePathPrefix",
+      message: (answers) => `Enter your Digital Ocean container registry name (default: '${answers.projectName}')`,
+      default: (answers) => answers.projectName,
+      when: (answers) => answers.infraVariant === "digitalocean",
+    },
+    {
+      type: "input",
+      name: "registryBaseUrl",
+      message: (answers) => `Enter your registry base URL (default: 'registry.${answers.stagingDomain}')`,
+      default: (answers) => `registry.${answers.stagingDomain}`,
+      when: (answers) => answers.infraVariant === "hetzner",
+    },
+    {
+      type: "confirm",
+      name: "usePython",
+      message: "Add support for Python?",
+      default: true,
+    },
+    {
+      type: "confirm",
+      name: "usePrisma",
+      message: "Add support for Prisma?",
+      default: true,
+    },
+    {
+      type: "checkbox",
+      name: "clusterResources",
+      message: "Optional manifests and helm charts to add",
+      choices: (answers) => [
+        { name: "Manifest to test DNS setup", value: "dns-test" },
+        { name: "Manifest to setup ingress for graphana and prometheus", value: "monitoring-ingress" },
+        { name: "Stackgres CRDs and manifests for Postgres", value: "postgres" },
+        { name: "Redis Helm chart values", value: "redis" },
+        { name: "Milvus helm chart values", value: "milvus" },
+        ...(answers.usePython ? [{ name: "Prefect Helm chart values", value: "prefect" }] : [])
+      ]
+    }
+  ])
+}

package/src/libs/init-generator.ts

@@ -0,0 +1,165 @@
+import url from "url";
+import path from "path";
+import fs from "fs-extra";
+import chalk from "chalk";
+import fg from 'fast-glob';
+
+const __file__ = url.fileURLToPath(import.meta.url);
+const __root__ = path.join(path.dirname(__file__), "../..");
+const templatesDir = path.join(__root__, "src/target-templates");
+const targetDir = process.cwd(); // User's current working directory
+
+type MessageGeneratorFn = (targetExists: boolean, fileInfo: InitGeneratorFileInfo) => string | null | undefined;
+
+type CommonFileInfo = {
+  targetRel: string;
+  targetAbs: string;
+  targetFolderAbs: string;
+  targetExists: boolean;
+  messageGenerator?: MessageGeneratorFn;
+}
+
+export type InitGeneratorCopiedFileInfo = CommonFileInfo &{
+  type: "copied";
+  sourceRel: string;
+  sourceAbs: string;
+  enableSubstitution?: boolean;
+}
+
+type InitGeneratorGeneratedFileInfo = CommonFileInfo & {
+  type: "generated";
+  content: string;
+}
+
+export type InitGeneratorFileInfo = InitGeneratorCopiedFileInfo | InitGeneratorGeneratedFileInfo;
+
+export class InitGenerator {
+  projectName?: string;
+
+  /** The key is targetRel */
+  files: Record<string, InitGeneratorFileInfo> = {};
+
+  constructor() {
+    if (fs.existsSync("package.json")) {
+      const packageJson = fs.readJSONSync("package.json");
+      this.projectName = packageJson.name;
+    }
+  }
+
+  _ensureFileExists(targetRel: string) {
+    if (!this.files[targetRel]) {
+      throw new Error(`File for target "${targetRel}" not found.`);
+    }
+  }
+
+  enableSubtitution(targetRel: string) {
+    this._ensureFileExists(targetRel);
+    if (this.files[targetRel].type !== "copied") {
+      throw new Error(`File for target "${targetRel}" is not a copied file.`);
+    }
+    this.files[targetRel].enableSubstitution = true;
+  }
+
+  setMessageGenerator(targetRel: string, messageGen: MessageGeneratorFn) {
+    this._ensureFileExists(targetRel);
+    this.files[targetRel].messageGenerator = messageGen;
+  }
+
+  addGeneratedFile(targetRel: string, content: string) {
+    const targetAbs = path.join(targetDir, targetRel);
+    const targetFolderAbs = path.dirname(targetAbs);
+    const exists = fs.existsSync(targetAbs);
+    this.files[targetRel] = {
+      type: "generated",
+      targetRel,
+      targetAbs,
+      targetFolderAbs,
+      targetExists: exists,
+      content,
+    };
+  }
+
+  /**
+   * @param source relative path under the templates folder. All files and folders under `source` are copied directly under `target`.
+   * @param target relative path under the project root folder.
+   * If the target file exists already in `files`, it will be overridden.
+   */
+  addCopiedFolder(source: string, target: string) {
+    const pathPrefix = path.join(templatesDir, source);
+    const glob = path.join(pathPrefix, '**/*');
+    fg.globSync(glob, { dot: true }).forEach((sourceAbs) => {
+      const sourceRel = path.relative(templatesDir, sourceAbs);
+      const pathUnderSource = path.relative(pathPrefix, sourceAbs);
+      const targetRel = path.join(target, pathUnderSource);
+      const targetAbs = path.join(targetDir, targetRel);
+      const targetFolderAbs = path.dirname(targetAbs);
+      const exists = fs.existsSync(targetAbs);
+      this.files[targetRel] = {
+        type: "copied",
+        sourceRel,
+        targetRel,
+        sourceAbs,
+        targetAbs,
+        targetFolderAbs,
+        targetExists: exists,
+      };
+    })
+  }
+
+  run({ 
+    substitution = {}, 
+    messages = [],
+  } : { 
+    substitution?: Record<string, string | undefined>; 
+    messages?: string[];
+  }) {
+    const fileMessages: string[] = [];
+    Object.values(this.files).forEach((fileInfo) => {
+      if (fileInfo.messageGenerator) {
+        const message = fileInfo.messageGenerator(fileInfo.targetExists, fileInfo);
+        if (message) {
+          fileMessages.push(message);
+        }
+      }
+
+      if (fileInfo.targetExists) {
+        console.log(`Skipped ${chalk.yellow(fileInfo.targetRel)} (exists)`);
+        return;
+      } 
+
+      // Create or copy
+      if (!fs.existsSync(fileInfo.targetFolderAbs)) {
+        fs.mkdirSync(fileInfo.targetFolderAbs, { recursive: true });
+      }
+
+      if (fileInfo.type === 'generated') {
+        fs.writeFileSync(fileInfo.targetAbs, fileInfo.content, 'utf8');
+      } else if (fileInfo.enableSubstitution) {
+        const content = fs.readFileSync(fileInfo.sourceAbs, 'utf8');
+        const substitutedContent = content.replace(/\$([A-Z_]+)/g, (_, varName) => {
+          const value = substitution[varName];
+          if (!value) {
+            throw new Error(`${chalk.blue("TemplateCopier.run()")}: Variable ${chalk.yellow(varName)} is needed by ${chalk.yellow(fileInfo.targetRel)} but is undefined.`);
+          }
+          return value;
+        });
+        fs.writeFileSync(fileInfo.targetAbs, substitutedContent);
+      } else {
+        fs.copySync(fileInfo.sourceAbs, fileInfo.targetAbs, {
+          overwrite: false,
+          errorOnExist: false,
+          dereference: false,
+        });
+      }
+      console.log(`Created ${chalk.green(fileInfo.targetRel)}`);
+    });
+
+    const allMessages = [...messages, ...fileMessages];
+    if (!allMessages.length) return;
+
+    console.log(chalk.blue("\nNext steps:"));
+    allMessages.forEach((msg, i) => {
+      console.log(`${i + 1}. ${msg}\n`);
+    });
+  }
+}

package/src/target-templates/README.md

@@ -0,0 +1 @@
+This folder contains the templates that are copied to the target project's folder based on answers the user provides during `./devops init`. Some of these answers may change the content of files, which is done by substituting variables prefixed by `$`. To avoid potential issues, such substitution is not pursued globally for all files in this folder. Files requiring substitution must be referenced by name in `./devops init` code.

package/src/target-templates/cluster-resource-options/README.md

@@ -0,0 +1,2 @@
+During `./devops init`, the user is asked to select options for generating manifests of various cluster resources.
+The files of each selected option `$OPTION` is copied under `.devops/infra/$OPTION`.

package/src/target-templates/{.devops → cluster-resource-options}/postgres/production/configurations/07-SGObjectStorage.yaml

@@ -6,7 +6,7 @@ metadata:
 spec:
   type: s3Compatible
   s3Compatible:
-    bucket: changeme-backups
+    bucket: $PROJECT_NAME-backups
     endpoint: https://hel1.your-objectstorage.com
     awsCredentials:
       secretKeySelectors:

package/src/target-templates/{.devops → cluster-resource-options}/postgres/production/configurations/08-SGScript.yaml

@@ -9,4 +9,4 @@ spec:
   scripts:
   - name: create-database
     script: |
-      CREATE DATABASE changeme WITH OWNER postgres;
+      CREATE DATABASE $PROJECT_NAME WITH OWNER postgres;

package/src/target-templates/{.devops → cluster-resource-options}/postgres/staging/configurations/07-SGObjectStorage.yaml

@@ -6,7 +6,7 @@ metadata:
 spec:
   type: s3Compatible
   s3Compatible:
-    bucket: changeme-backups
+    bucket: $PROJECT_NAME-backups
     endpoint: https://hel1.your-objectstorage.com
     awsCredentials:
       secretKeySelectors:

package/src/target-templates/{.devops → cluster-resource-options}/postgres/staging/configurations/08-SGScript.yaml

@@ -9,4 +9,4 @@ spec:
   scripts:
   - name: create-database
     script: |
-      CREATE DATABASE changeme WITH OWNER postgres;
+      CREATE DATABASE $PROJECT_NAME WITH OWNER postgres;

package/src/target-templates/infra-variants/README.md

@@ -0,0 +1,2 @@
+During `./devops init`, the user is asked to select an infra variant.
+The files under the selected `$OPTION` are copied under the root project folder.

package/src/target-templates/infra-variants/digitalocean/.devops/config/constants.yaml

@@ -0,0 +1,18 @@
+# These will be used when generating kubernetes entities
+project-name: $PROJECT_NAME
+
+# Supported: hetzner, digitalocean, or gcloud
+infra: digitalocean
+
+# Only relevant for Digital Ocean. Determines the number of versions to keep for each docker image.
+image-versions-to-keep: 5
+
+registry-base-url: registry.digitalocean.com
+# What comes before <image-name>:<tag>. Can be empty.
+registry-image-path-prefix: $REGISTRY_IMAGE_PATH_PREFIX
+
+# production and staging are supported by default
+extra-remote-environments: []
+
+# development and test are supported by default
+extra-local-environments: []

package/src/target-templates/infra-variants/digitalocean/.github/workflows/k8s-build.yaml

@@ -0,0 +1,73 @@
+name: "Monorepo Build and Deploy"
+
+on:
+  push:
+    branches:
+      - staging
+      - production
+
+permissions:
+  contents: read
+  packages: read
+
+jobs:
+  build_images:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - image_name: main-node
+          - image_name: main-python
+            cache_path: /root/.cache/uv
+    steps:
+      # Fetch the last 50 commits so that devops affected works
+      - name: Checkout repo and history
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 50
+
+      - name: Setup prerequesites
+        uses: ./.github/actions/setup-prereq@v1
+
+      - name: Connect to infrastructure
+        uses: ./.github/actions/connect-to-infra@v1
+        with:
+          do_access_token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
+          do_cluster_name: ${{ secrets.DIGITALOCEAN_CLUSTER_NAME }}
+
+      - name: Build image
+        uses: ./.github/actions/build-image@v1
+        with:
+          image_name: ${{ matrix.image_name }}
+          cache_path: ${{ matrix.cache_path || '/root/.bun/install/cache' }}
+
+  db_migrate_and_deploy:
+    needs: [build_images]
+    runs-on: ubuntu-latest
+    steps:
+      # Fetch the last 50 commits so that devops affected works
+      - name: Checkout repo and history
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 50
+
+      - name: Setup prerequesites
+        uses: ./.github/actions/setup-prereq@v1
+
+      - name: Connect to infrastructure
+        uses: ./.github/actions/connect-to-infra@v1
+        with:
+          do_access_token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
+          do_cluster_name: ${{ secrets.DIGITALOCEAN_CLUSTER_NAME }}
+
+      - name: Run DB Migrate
+        uses: ./.github/actions/db-migrate@v1
+
+      # Repeat per image (it checks if the image is affected and deploys it if it is)
+      - name: Deploy main node
+        uses: ./.github/actions/deploy-image@v1
+        with: { "image_name": "main-node" }
+
+      - name: Deploy main python
+        uses: ./.github/actions/deploy-image@v1
+        with: { "image_name": "main-python" }

package/src/target-templates/infra-variants/gcloud/.devops/config/constants.yaml

@@ -0,0 +1,15 @@
+# These will be used when generating kubernetes entities
+project-name: $PROJECT_NAME
+
+# Supported: hetzner, digitalocean, or gcloud
+infra: gcloud
+
+registry-base-url: gcr.io
+# What comes before <image-name>:<tag>. Can be empty.
+registry-image-path-prefix: $GCLOUD_PROJECT_ID
+
+# production and staging are supported by default
+extra-remote-environments: []
+
+# development and test are supported by default
+extra-local-environments: []

package/src/target-templates/infra-variants/gcloud/.devops/manifests/ingress.yaml.hb

@@ -0,0 +1,22 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: {{app_name}}
+  namespace: {{namespace}}
+  labels:
+    app: {{app_name}}
+    env: {{monorepo_env}}
+  annotations:
+    kubernetes.io/ingress.class: "gce"    
+spec:
+  rules:
+    - host: {{subdomain}}.{{domain_name}}
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: {{service_name}}
+                port:
+                  number: 80