@vaharoni/devops 1.1.6 → 1.1.8

package/src/cli/prep-build.ts

@@ -3,8 +3,10 @@ import os from "os";
 import path from "path";
 import { CLICommandParser, printUsageAndExit } from "./common";
 import { getImageData, getTemplateData } from "../libs/config";
-import { getMonorepoSecret } from "../libs/k8s-secrets-manager";
+import { getMonorepoSecretStr } from "../libs/k8s-secrets-manager";
 import { getImageDescendentData } from "../libs/discovery/images";
+import { isLocalOrRemoteEnv } from "../libs/k8s-constants";
+import chalk from "chalk";
 
 const oneLiner =
   "Copies all dependencies of an image to a temporary folder in preparation for a Docker build";
@@ -16,7 +18,10 @@ const usage = `
 ${oneLiner}
 
 USAGE
-    devops prep-build <image>
+    devops prep-build <image> --env <env>
+
+    If <env> is a remote environment (e.g. staging, production), the environment variables are 
+    fetched from the cluster and injected in case they are needed during the build process.
 
 EXAMPLES
     ${keyExamples}
@@ -61,8 +66,10 @@ async function run(cmdObj: CLICommandParser) {
     fs.copySync(dockerCommonPayloadPath, destFolder);
   }
 
-  console.warn(`COPYING Docker image payload`);
-  fs.copySync(dockerImagePayloadPath, destFolder);
+  if (fs.existsSync(dockerImagePayloadPath)) {
+    console.warn(`COPYING Docker image payload`);
+    fs.copySync(dockerImagePayloadPath, destFolder);
+  }
 
   console.warn(`COPYING .devops/config`);
   fs.mkdirSync(path.join(destFolder, ".devops"));
@@ -71,8 +78,29 @@ async function run(cmdObj: CLICommandParser) {
   // Create config directory. It should be deleted by the docker image so that it can be mounted as a volume when the pod is run
   console.warn(`CREATING config for the build process`);
   fs.mkdirSync(path.join(destFolder, "config"));
-  const envFileData = getMonorepoSecret(cmdObj.env);
-  fs.writeFileSync(path.join(destFolder, `config/.env.global`), envFileData);
+  const destGlobalEnvPath = path.join(destFolder, "config/.env.global");
+  if (isLocalOrRemoteEnv(cmdObj.env) === "remote") {
+    const envFileData = getMonorepoSecretStr(cmdObj.env);
+    fs.writeFileSync(destGlobalEnvPath, envFileData);
+  } else {
+    let anyCopied = false;
+    const localGlobalEnvPath = "config/.env.global";
+    const localEnvPath = `config/.env.${cmdObj.env}`;
+    const destEnvPath = path.join(destFolder, `config/.env.${cmdObj.env}`);
+    if (fs.existsSync(localGlobalEnvPath)) {
+      console.warn(`COPYING ${localGlobalEnvPath} to ${destGlobalEnvPath}`);
+      fs.copyFileSync(localGlobalEnvPath, destGlobalEnvPath);
+      anyCopied = true;
+    }
+    if (fs.existsSync(localEnvPath)) {
+      console.warn(`COPYING ${localEnvPath} to ${destEnvPath}`);
+      fs.copyFileSync(localEnvPath, destEnvPath);
+      anyCopied = true;
+    }
+    if (!anyCopied) {
+      console.warn(chalk.red(`\nWarning: local environment ${cmdObj.env} has no .env files. Environment variables will not be injected.\n`));
+    }
+  }
 
   // Copy all dependencies
   getImageDescendentData(image).forEach((project) => {

package/src/cli/redis.ts

@@ -64,7 +64,7 @@ function run(cmdObj: CLICommandParser) {
   const parsed = cmdObj.parseOptions({ params: ["-p"] });
 
   const [command, namespace] = parsed.args;
-  const port = parsed.options["-p"] as string;
+  const port = parsed.options["-p"];
   // @ts-expect-error left as an exercise for the reader
   const handler = handlers[command];
   if (!handler) {

package/src/cli/registry.ts

@@ -11,21 +11,27 @@ const oneLiner = "Manage container repositories";
 const keyExamples = `
     $ devops registry server-url
     $ devops registry reg-url
-    $ devops registry repo-url my-image sha
-    $ devops registry prune    my-image
+    $ devops registry repo-url   my-image sha
+    $ devops registry image-name my-image 
+    $ devops registry prune      my-image
 `.trim();
 
 const usage = `
 ${oneLiner}
 
 USAGE
-  Get base URLs
+  Get base URLs for the container registry of the cluster:
       devops registry server-url
       devops registry reg-url
 
+      Note: for cloudrun images these URLs are not relevant. 
+
   Gets the URL of an image in the container registry:
     devops registry repo-url <image> <sha> --env <env>
 
+  Gets the image name in the container registry:
+    devops registry image-name <image> --env <env>
+
   Prunes the repository of old images to enforce the "image-versions-to-keep" constant in config/constants.yaml:
     devops registry prune <image> --env <env>
 
@@ -47,6 +53,9 @@ const handlers = {
       )
     );
   },
+  "image-name": (opts: StrongParams) => {
+    console.log(containerRegistryImageName(opts.required("image"), opts.required("env")));
+  },
   prune: (opts: StrongParams) => {
     const regName = containerRegistryPath();
     const repoName = containerRegistryImageName(

package/src/devops.ts

@@ -22,6 +22,7 @@ import namespace from "./cli/namespace";
 import image from "./cli/image";
 import template from "./cli/template";
 import job from "./cli/job";
+import cloudrun from "./cli/cloudrun";
 
 const [_node, _scriptPath, ...commandArgs] = process.argv;
 
@@ -45,11 +46,12 @@ const allImports = [
   job,
   //= Deployment
   prepBuild,
+  cloudrun,
   affected,
   constant,
   registry,
   internalCurl,
-  jwt
+  jwt,
 ];
 
 const commands: {

package/src/libs/cloudrun-helpers.ts

@@ -0,0 +1,118 @@
+import { randomBytes } from "crypto";
+import { CommandExecutor } from "../cli/common";
+import { containerRegistryRepoPath, isLocalOrRemoteEnv } from "./k8s-constants";
+import { getImageData } from "./config";
+import { getMonorepoSecretObject } from "./k8s-secrets-manager";
+import chalk from "chalk";
+
+function verifyCloudrunImage(image: string) {
+  const imageData = getImageData(image);
+  if (!imageData["cloudrun"]) {
+    console.error(`Image ${image} is not a cloudrun image. Add "cloudrun: true" in images.yaml`);
+    process.exit(1);
+  }
+}
+
+function getEnvValuesToForward(env: string, forwardEnv: string[]) {
+  if (!forwardEnv.length) return {};
+  
+  let envValues: Record<string, string> = {};
+  const missingValues = new Set<string>();
+  for (const key of forwardEnv) {
+    const value = process.env[key];
+    if (value) {
+      envValues[key] = value;
+    } else {
+      missingValues.add(key);
+    }
+  }
+  if (missingValues.size > 0 && isLocalOrRemoteEnv(env) === "remote") {
+    const secretsFromCluster = getMonorepoSecretObject(env, Array.from(missingValues));
+    for (const key of Object.keys(secretsFromCluster)) {
+      envValues[key] = secretsFromCluster[key];
+      missingValues.delete(key);
+    }
+  }
+  if (missingValues.size > 0) {
+    console.error(`Some forwardEnv variables are missing: ${Array.from(missingValues).join(", ")}`);
+    process.exit(1);
+  }
+  return envValues;
+}
+
+export async function buildDev(image: string) {
+  verifyCloudrunImage(image);
+  const env = "development";
+  const sha = randomBytes(12).toString("hex");
+
+  const buildDir = new CommandExecutor(`devops prep-build ${image}`, {
+    env,
+  }).exec().trim();
+
+  const tag = containerRegistryRepoPath(image, env, sha);
+  console.warn(`Building ${tag} from ${buildDir}`);
+
+  await new CommandExecutor(
+    `docker build --platform linux/amd64 -t ${tag} ${buildDir} --build-arg MONOREPO_ENV=${env}`,
+    { env }
+  ).spawn();
+
+  console.warn(`Pushing ${tag}`);
+  await new CommandExecutor(`docker push ${tag}`, { env }).spawn();
+
+  console.warn(`\n✅ Built and pushed ${tag}\n`);
+  console.warn('Run "devops cloudrun deploy" next. For example:')
+  console.warn(chalk.blue(`./devops cloudrun deploy ${image} ${sha} --env ${env} --allow-unauthenticated --region us-east1 --forward-env ENV1,ENV2`));
+  console.warn();
+  console.log(tag);
+}
+
+export async function deploy({
+  image,
+  env,
+  sha,
+  region,
+  forwardEnv = [],
+  allowUnauthenticated = false,
+  cpu = "0.25",
+  memory = "256Mi",
+  minInstances = 0,
+  maxInstances = 1,
+  timeout = "60s",
+  extraArgs = "",
+}: {
+  image: string;
+  env: string; 
+  sha: string; 
+  region: string;
+  forwardEnv?: string[];
+  allowUnauthenticated?: boolean;
+  cpu?: string;
+  memory?: string;
+  minInstances?: number;
+  maxInstances?: number;
+  timeout?: string;
+  extraArgs?: string;
+}) {
+  verifyCloudrunImage(image);
+  const repoPath = containerRegistryRepoPath(image, env, sha);
+  const envValues = getEnvValuesToForward(env, forwardEnv);
+  const envValuesCsv = Object.entries(envValues).map(([key, value]) => `${key}="${value}"`).join(",");
+  const serviceName = `${image}-${env}`;
+
+  const cmd = `
+    gcloud run deploy ${serviceName} 
+      --image ${repoPath} 
+      ${Object.keys(envValues).length > 0 ? `--set-env-vars ${envValuesCsv}` : ""} 
+      ${allowUnauthenticated ? "--allow-unauthenticated" : ""}
+      --region ${region}
+      --cpu ${cpu}
+      --memory ${memory}
+      --min-instances ${minInstances}
+      --max-instances ${maxInstances}
+      --timeout ${timeout}
+      ${extraArgs}
+  `.trim().replace(/\s+/g, " ");
+
+  await new CommandExecutor(cmd, { env }).spawn();
+}

package/src/libs/discovery/process-common.ts

@@ -34,7 +34,7 @@ export class PackageDataProcessor<T> {
         console.error(`Error parsing ${packageFilePath}: ${parsedRes.error}`);
         process.exit(1);
       }
-      const rootDir = path.dirname(packageFilePath);
+      const rootDir = path.relative(process.cwd(), path.dirname(packageFilePath));
       const name = this.nameExtractor(parsedRes.data)
       this.workspaceNames.add(name);
       this.loadedFiles[rootDir] = { name, data: parsedRes.data };

package/src/libs/k8s-constants.ts

@@ -24,13 +24,28 @@ export function allSupportedEnvs() {
   return [...remoteSupportedEnvs(), ...localSupportedEnvs()];
 }
 
-function validateEnv(monorepoEnv?: string) {
+export function isLocalOrRemoteEnv(monorepoEnv: string): "local" | "remote" {
+  if (remoteSupportedEnvs().includes(monorepoEnv)) return "remote";
+  if (localSupportedEnvs().includes(monorepoEnv)) return "local";
+  throw new Error(`Unsupported environment: ${monorepoEnv}`);
+}
+
+function validateEnv(monorepoEnv?: string, { allowLocal = false }: { allowLocal?: boolean } = {}) {
   if (!monorepoEnv) throw new Error("MONOREPO_ENV cannot be empty");
-  if (!remoteSupportedEnvs().includes(monorepoEnv)) {
-    console.error(
-      `MONOREPO_ENV must be one of: ${remoteSupportedEnvs().join(", ")}. Can be set using --env flag.`
-    );
-    process.exit(1);
+  if (allowLocal) {
+    if (!allSupportedEnvs().includes(monorepoEnv)) {
+      console.error(
+        `MONOREPO_ENV must be one of: ${allSupportedEnvs().join(", ")}. Can be set using --env flag.`
+      );
+      process.exit(1);
+    }
+  } else {
+    if (!remoteSupportedEnvs().includes(monorepoEnv)) {
+      console.error(
+        `MONOREPO_ENV must be one of: ${remoteSupportedEnvs().join(", ")}. Can be set using --env flag.`
+      );
+      process.exit(1);
+    }
   }
 }
 
@@ -52,7 +67,7 @@ export function imageConfigMap(image: string) {
 }
 
 export function containerRegistryImageName(image: string, monorepoEnv: string) {
-  validateEnv(monorepoEnv);
+  validateEnv(monorepoEnv, { allowLocal: true });
   return `${getConst("project-name")}-${monorepoEnv}-${image}`;
 }
 
@@ -65,11 +80,20 @@ export function containerRegistryRepoPath(
   monorepoEnv: string,
   gitSha: string
 ) {
-  return [
-    getConst("registry-base-url"),
-    getConst("registry-image-path-prefix", { ignoreIfInvalid: true }),
-    [containerRegistryImageName(image, monorepoEnv), gitSha].join(":"),
-  ].filter(Boolean).join("/");
+  const imageNameAndTag = [containerRegistryImageName(image, monorepoEnv), gitSha].join(":");
+  const imageData = getImageData(image);
+  if (imageData["cloudrun"]) {
+    return [
+      getConst("cloudrun-artifact-registry-repo-path"),
+      imageNameAndTag,
+    ].join("/");
+  } else {
+    return [
+      getConst("registry-base-url"),
+      getConst("registry-image-path-prefix", { ignoreIfInvalid: true }),
+      imageNameAndTag,
+    ].filter(Boolean).join("/");
+  }
 }
 
 export function domainNameForEnv(image: string, monorepoEnv: string) {

package/src/libs/k8s-generate.ts

@@ -6,7 +6,7 @@ import path from "path";
 import yaml from "yaml";
 import fs from 'fs';
 import { globSync } from "glob";
-import _, { template } from 'lodash';
+import _ from 'lodash';
 import Handlebars from "handlebars";
 import { getImageData } from "./config";
 import { getImageDescendentData } from "./discovery/images";
@@ -33,7 +33,7 @@ export function generateImageDeployments(
       return generateManifestForDeployment(projectData.rootPath, projectData.deployment!.template, renderFn);
     });
   const debug = generateDebugDeployment(monorepoEnv, image, gitSha);
-  const manifest = [debug, ...apps].join("\n---\n");
+  const manifest = [debug, ...apps].filter(Boolean).join("\n---\n");
   return ensureProperDomainsPresent(manifest, monorepoEnv, image);
 }
 
@@ -54,6 +54,7 @@ export function generateDebugDeployment(
   const context = generator.getDebug();
   const renderFn = (template: string) => Handlebars.compile(template)(context);
   const debugTemplate = getImageData(image)["debug-template"];
+  if (!debugTemplate) return;
   return generateManifestsFromTemplateName(debugTemplate, renderFn).map(x => yaml.stringify(x)).join("\n---\n");
 }
 

package/src/libs/k8s-secrets-manager.ts

@@ -16,7 +16,7 @@ function execUpdateSecret(
   new CommandExecutor(fullCommand, { quiet: true, redactedCommand }).exec();
 }
 
-function getSecret(monorepoEnv: string, keys: string[] = []) {
+export function getMonorepoSecretObject(monorepoEnv: string, keys: string[] = []) {
   // Dots in jsonpath can only be accessed with a \ prefix
   const escapedSecretFileName = SECRET_FILE_NAME.replaceAll(".", "\\.");
   // prettier-ignore
@@ -39,7 +39,7 @@ function updateSecret(monorepoEnv: string, vars: Record<string, string>) {
     );
     process.exit(1);
   }
-  const current = getSecret(monorepoEnv);
+  const current = getMonorepoSecretObject(monorepoEnv);
   const newVars = { ...current, ...vars };
   execUpdateSecret(monorepoEnv, newVars);
 }
@@ -49,15 +49,19 @@ function deleteSecretKeys(monorepoEnv: string, keys: string[] = []) {
     console.error("Keys to delete must be provided");
     process.exit(1);
   }
-  const secretValue = getSecret(monorepoEnv);
+  const secretValue = getMonorepoSecretObject(monorepoEnv);
   keys.forEach((key) => delete secretValue[key]);
   execUpdateSecret(monorepoEnv, secretValue);
 }
 
 //= Interface (L3)
 
-export function getMonorepoSecret(monorepoEnv: string, keys: string[] = []) {
-  const value = getSecret(monorepoEnv, keys);
+export function getMonorepoSecretStr(monorepoEnv: string, keys: string[] = []) {
+  const value = getMonorepoSecretObject(monorepoEnv, keys);
+  if (Object.keys(value).length === 1) {
+    return Object.values(value)[0];
+  }
+  
   return Object.entries(value)
     .map((pair) => pair.join("="))
     .join("\n");

package/src/target-templates/lang-variants-common/python/.devops/config/images.yaml

@@ -37,11 +37,13 @@
 #                       This is also the basis for the name of the image in the registry.
 # image-template        The build process copies .devops/docker-images/<image-template>.Dockerfile and .devops/docker-images/<image-template>/ to the image. 
 # language              The language of the image. Currently only "node" and "python" are supported.
-# debug-template        Each image comes with a debug pod that can be acccessed as a console. This is the name of the template to use. Only "debug-console" is currently supported.
+# debug-template        Each image comes with a debug pod that can be acccessed as a console. This is the name of the template to use. 
+#                       Only "debug-console" is currently supported. Can be left out if the image should not have a debug pod.
 # can-db-migrate        Whether this image can be used to run DB migrations. If set to true, the image could be used to run DB migrations if any project that uses this image depends 
 #                       on the db project and if the db project changed.
 # domains               The domains for the image. This is used to generate the ingress rules. 
 #                       This can be left out if the image does not have ingress rules, such as a worker image.
+# cloudrun              If this image should be pushed to cloudrun instead of the cluster registry, set to "true".
 # applications          List of applications that use this image. There is no need to specify dependencies - they will be derived as long as they are declared in package.json.
 #
 

package/src/target-templates/lang-variants-common/typescript/.devops/docker-images/cloudrun.Dockerfile

@@ -0,0 +1,31 @@
+FROM node:24-bookworm-slim AS builder
+
+RUN apt-get update && apt-get install -y jq curl
+
+WORKDIR /app
+
+ENV NODE_ENV=production
+
+ARG MONOREPO_ENV
+ENV MONOREPO_ENV=${MONOREPO_ENV}
+RUN echo "Building for environment: $MONOREPO_ENV"
+
+RUN npm install -g bun
+
+# This assumes devops prep-build was called by the host, which creates the config/ folder with necessary env variables
+# that are needed to be statitcally resolved by devops run-many build (e.g. NEXT_PUBLIC_*)
+COPY . .
+
+# Install dependencies using bun
+RUN --mount=type=cache,target=/root/.bun/install/cache bun install
+
+# This assumes the image has only one application that has a build script which outputs to dist/
+RUN ./devops run-many build
+
+
+FROM gcr.io/distroless/nodejs24-debian12 AS runner
+WORKDIR /app
+ENV NODE_ENV=production
+COPY --from=builder /app/dist ./dist
+EXPOSE 8080
+CMD ["dist/index.js"]

package/src/target-templates/lang-variants-common/typescript/.github/actions/deploy-image@v1/action.yaml

@@ -4,6 +4,10 @@ inputs:
   image_name:
     description: 'The image key in images.yaml'
     required: true
+outputs:
+  affected:
+    description: 'Whether the specified image is affected (computed before deploy)'
+    value: ${{ steps.check_affected.outputs.affected }}    
 runs:
   using: "composite"
   steps:

package/src/types/index.ts

@@ -11,6 +11,7 @@ export const constFileSchema = z.object({
   "image-versions-to-keep": z.number().optional(),
   "registry-base-url": z.string(),
   "registry-image-path-prefix": z.string().optional(),
+  "cloudrun-artifact-registry-repo-path": z.string().optional(),
   "extra-remote-environments": z.array(z.string()),
   "extra-local-environments": z.array(z.string()),
 })
@@ -26,7 +27,8 @@ const singleImageSchema = z.object({
   "image-template": z.string(),
   "language": z.enum(SUPPORTED_LANGUAGES),
   "domains": z.record(z.string()).optional(),
-  "debug-template": z.string(),
+  "debug-template": z.string().optional(),
+  "cloudrun": z.boolean().optional(),
   "can-db-migrate": z.boolean().optional(),
   applications: z.array(z.string()),
 });