@vaharoni/devops 1.1.7 → 1.1.9

package/src/cli/common.ts

@@ -4,10 +4,10 @@ import fs from "fs";
 import { globSync } from "glob";
 import { allSupportedEnvs } from "../libs/k8s-constants";
 
-type ParsedArgs = {
+type ParsedArgs<TBoolKeys extends readonly string[], TParamKeys extends readonly string[]> = {
   args: string[];
   argsStr: string;
-  options: { [key: string]: string | boolean };
+  options: Partial<Record<TBoolKeys[number], true>> & Partial<Record<TParamKeys[number], string>>;
   passthrough?: string[];
 };
 
@@ -27,8 +27,8 @@ export class CLICommandParser {
 
   constructor(cmdArray: string[]) {
     const parsedArgs = this._separateOptions(cmdArray.filter(Boolean), {
-      params: ["--env"],
-      booleans: ["--help", "--skip-env-check"],
+      params: ["--env"] as const,
+      booleans: ["--help", "--skip-env-check"] as const,
     });
     const [command, ...args] = parsedArgs.args;
     this.command = command;
@@ -67,18 +67,18 @@ export class CLICommandParser {
   //    # => { args: ['arg1'], options: { '--some-flag': true, '--in': 'workspace' } }
   //
   // Note that the global param --env is already extracted and can be accessed with cmd.env
-  parseOptions({
-    params = [],
-    booleans = [],
+  parseOptions<const TBoolKeys extends readonly string[], const TParamKeys extends readonly string[]>({
+    params,
+    booleans,
     passthroughArgs = false,
   }: {
     /** Param is used like so: --param value */
-    params?: string[];
+    params?: TParamKeys
     /** Boolean flag is used like so: --flag */
-    booleans?: string[];
+    booleans?: TBoolKeys;
     /** Pass through args are used like so: -- arg1 arg2 */
     passthroughArgs?: boolean;
-  } = {}): ParsedArgs {
+  } = {}): ParsedArgs<TBoolKeys, TParamKeys> {
     return this._separateOptions(this.args, {
       params,
       booleans,
@@ -97,46 +97,54 @@ export class CLICommandParser {
     return true;
   }
 
-  _separateOptions(
+  _separateOptions<const TBoolKeys extends readonly string[], const TParamKeys extends readonly string[]>(
     args: string[],
     {
-      params = [],
-      booleans = [],
+      params,
+      booleans,
       passthroughArgs = false,
     }: {
-      params?: string[];
-      booleans?: string[];
+      params?: TParamKeys;
+      booleans?: TBoolKeys;
       passthroughArgs?: boolean;
     } = {}
-  ): ParsedArgs {
-    const results: ParsedArgs = {
-      args: [],
-      argsStr: "",
-      options: {},
-      ...(passthroughArgs ? { passthrough: [] } : {}),
-    };
-    const paramsLookup = Object.fromEntries(params.map((x) => [x, true]));
-    const booleansLookup = Object.fromEntries(booleans.map((x) => [x, true]));
+  ): ParsedArgs<TBoolKeys, TParamKeys> {
+    const paramsLookup = new Set<TParamKeys[number]>(params ?? []);
+    const booleansLookup = new Set<TBoolKeys[number]>(booleans ?? []);
+    const isParam = (arg: string): arg is TParamKeys[number] => paramsLookup.has(arg);
+    const isBoolean = (arg: string): arg is TBoolKeys[number] => booleansLookup.has(arg);
+
     const passthroughArgsStart = passthroughArgs ? args.indexOf("--") : -1;
-    const numArgsToProcess =
-      passthroughArgsStart === -1 ? args.length : passthroughArgsStart;
+    // prettier-ignore
+    const numArgsToProcess = passthroughArgsStart === -1 ? args.length : passthroughArgsStart;
+
+    const getResPassthrough = () => {
+      if (!passthroughArgs || passthroughArgsStart < 0) return { passthrough: []};
+      return { passthrough: args.slice(passthroughArgsStart + 1) };
+    }
+
+    const resArgs: string[] = [];
+    const resParams: Partial<Record<TParamKeys[number], string>> = {};
+    const resOptions: Partial<Record<TBoolKeys[number], true>> = {};
+
     for (let i = 0; i < numArgsToProcess; ++i) {
       const curr = args[i];
-      if (paramsLookup[curr]) {
+      if (isParam(curr)) {
         const next = args[i + 1];
-        results.options[curr] = next;
+        resParams[curr] = next;
         ++i;
-      } else if (booleansLookup[curr]) {
-        results.options[curr] = true;
+      } else if (isBoolean(curr)) {
+        resOptions[curr] = true;
       } else {
-        results.args.push(curr);
+        resArgs.push(curr);
       }
     }
-    results.argsStr = results.args.join(" ");
-    if (passthroughArgs && passthroughArgsStart >= 0) {
-      results.passthrough = args.slice(passthroughArgsStart + 1);
-    }
-    return results;
+    return {
+      args: resArgs,
+      argsStr: resArgs.join(" "),
+      options: { ...resOptions, ...resParams },
+      ...getResPassthrough(),
+    };
   }
 }
 
@@ -220,10 +228,10 @@ export class CommandExecutor {
     const envToUse = this._getProcessEnv(env);
     return new Promise((resolve) => {
       try {
-        const [cmd, ...args] = fullCommand.split(" ").filter(Boolean);
-        const childProcess = spawn(cmd, args, {
+        const childProcess = spawn(fullCommand, {
           stdio: "inherit",
           env: envToUse,
+          shell: true,
         });
 
         childProcess.on("close", (code) => {

package/src/cli/db.ts

@@ -114,7 +114,7 @@ function run(cmdObj: CLICommandParser) {
   const parsed = cmdObj.parseOptions({ params: ["-p"] });
 
   const [command, namespace] = parsed.args;
-  const port = parsed.options["-p"] as string;
+  const port = parsed.options["-p"];
   // @ts-expect-error left as an exercise for the reader
   const handler = handlers[command];
   if (!handler) {

package/src/cli/dml.ts

@@ -107,7 +107,7 @@ function run(cmdObj: CLICommandParser) {
   });
   switch (parsed.args[0]) {
     case "create": {
-      const name = parsed.options["--name"] as string;
+      const name = parsed.options["--name"];
       if (!name) printUsageAndExit(usage);
       return createDml(name);
     }

package/src/cli/env.ts

@@ -1,7 +1,7 @@
 import { globSync } from "glob";
 import {
   deleteMonorepoSecret,
-  getMonorepoSecret,
+  getMonorepoSecretStr,
   setMonorepoSecret,
 } from "../libs/k8s-secrets-manager";
 import { CombinedEnvValidator } from "../libs/validate-env";
@@ -62,7 +62,7 @@ function run(cmdObj: CLICommandParser) {
     }
 
     case "get": {
-      console.log(getMonorepoSecret(cmdObj.env, rest));
+      console.log(getMonorepoSecretStr(cmdObj.env, rest));
       break;
     }
 

package/src/cli/exec.ts

@@ -32,7 +32,7 @@ function run(cmdObj: CLICommandParser) {
     params: ["--in"],
     booleans: ["--interactive"],
   });
-  const workspace = parsed.options["--in"] as string | undefined;
+  const workspace = parsed.options["--in"];
   let executor: CommandExecutor;
   if (workspace) {
     const rootPath = getWorkspace(workspace).rootPath;

package/src/cli/image.ts

@@ -2,9 +2,11 @@ import { deleteImageVersion, getImageVersion, getWorkspaceScale, resetWorkspaceS
 import { CLICommandParser, printUsageAndExit, StrongParams } from "../../src/cli/common";
 import { generateImageDeployments } from "../libs/k8s-generate";
 import { applyHandler } from "../libs/k8s-helpers";
+import { getImageType } from "../libs/config";
 
 const oneLiner = "Applies image-related manifests, retrieves or set the version deployed, and scales deployments of applications";
 const keyExamples = `
+    $ devops image get        type   main-node
     $ devops image deployment gen    main-node sha  --env staging
     $ devops image deployment create main-node sha  --env staging
     $ devops image deployment delete main-node      --env staging
@@ -21,6 +23,11 @@ const keyExamples = `
 const usage = `
 ${oneLiner}
 
+GET IMAGE TYPE
+    devops image get type <image-name>
+
+    Returns "k8s" or "cloudrun" depending on the image type.
+
 GENERATING DEPLOYMENT MANIFESTS
     devops image deployment gen|create|delete <image-name> <sha>
 
@@ -49,6 +56,11 @@ EXAMPLES
 `;
 
 const handlers = {
+  get: {
+    type: (opts: StrongParams) => {
+      console.log(getImageType(opts.required("image")));
+    },
+  },
   deployment: {
     gen: (opts: StrongParams) => {
       console.log(
@@ -176,7 +188,9 @@ function run(cmdObj: CLICommandParser) {
   }
 
   function getExtraParams() {
-    if (command === 'scale') {
+    if (command === 'get') {
+      return {};
+    } else if (command === 'scale') {
       return subcommand === 'set' ? { workspace: param1, replicas: param2 } : { workspace: param1 };
     } else {
       return { sha: param1 };

package/src/cli/job.ts

@@ -65,7 +65,7 @@ function run(cmdObj: CLICommandParser) {
     printUsageAndExit(usage);
   }
 
-  const timeout = parsedArgs.options["--timeout"] as string | undefined;
+  const timeout = parsedArgs.options["--timeout"];
   const params = new StrongParams(usage, {
     env: cmdObj.env,
     image,

package/src/cli/prep-build.ts

@@ -3,8 +3,10 @@ import os from "os";
 import path from "path";
 import { CLICommandParser, printUsageAndExit } from "./common";
 import { getImageData, getTemplateData } from "../libs/config";
-import { getMonorepoSecret } from "../libs/k8s-secrets-manager";
+import { getMonorepoSecretStr } from "../libs/k8s-secrets-manager";
 import { getImageDescendentData } from "../libs/discovery/images";
+import { isLocalOrRemoteEnv } from "../libs/k8s-constants";
+import chalk from "chalk";
 
 const oneLiner =
   "Copies all dependencies of an image to a temporary folder in preparation for a Docker build";
@@ -16,7 +18,10 @@ const usage = `
 ${oneLiner}
 
 USAGE
-    devops prep-build <image>
+    devops prep-build <image> --env <env>
+
+    If <env> is a remote environment (e.g. staging, production), the environment variables are 
+    fetched from the cluster and injected in case they are needed during the build process.
 
 EXAMPLES
     ${keyExamples}
@@ -61,8 +66,10 @@ async function run(cmdObj: CLICommandParser) {
     fs.copySync(dockerCommonPayloadPath, destFolder);
   }
 
-  console.warn(`COPYING Docker image payload`);
-  fs.copySync(dockerImagePayloadPath, destFolder);
+  if (fs.existsSync(dockerImagePayloadPath)) {
+    console.warn(`COPYING Docker image payload`);
+    fs.copySync(dockerImagePayloadPath, destFolder);
+  }
 
   console.warn(`COPYING .devops/config`);
   fs.mkdirSync(path.join(destFolder, ".devops"));
@@ -71,8 +78,29 @@ async function run(cmdObj: CLICommandParser) {
   // Create config directory. It should be deleted by the docker image so that it can be mounted as a volume when the pod is run
   console.warn(`CREATING config for the build process`);
   fs.mkdirSync(path.join(destFolder, "config"));
-  const envFileData = getMonorepoSecret(cmdObj.env);
-  fs.writeFileSync(path.join(destFolder, `config/.env.global`), envFileData);
+  const destGlobalEnvPath = path.join(destFolder, "config/.env.global");
+  if (isLocalOrRemoteEnv(cmdObj.env) === "remote") {
+    const envFileData = getMonorepoSecretStr(cmdObj.env);
+    fs.writeFileSync(destGlobalEnvPath, envFileData);
+  } else {
+    let anyCopied = false;
+    const localGlobalEnvPath = "config/.env.global";
+    const localEnvPath = `config/.env.${cmdObj.env}`;
+    const destEnvPath = path.join(destFolder, `config/.env.${cmdObj.env}`);
+    if (fs.existsSync(localGlobalEnvPath)) {
+      console.warn(`COPYING ${localGlobalEnvPath} to ${destGlobalEnvPath}`);
+      fs.copyFileSync(localGlobalEnvPath, destGlobalEnvPath);
+      anyCopied = true;
+    }
+    if (fs.existsSync(localEnvPath)) {
+      console.warn(`COPYING ${localEnvPath} to ${destEnvPath}`);
+      fs.copyFileSync(localEnvPath, destEnvPath);
+      anyCopied = true;
+    }
+    if (!anyCopied) {
+      console.warn(chalk.red(`\nWarning: local environment ${cmdObj.env} has no .env files. Environment variables will not be injected.\n`));
+    }
+  }
 
   // Copy all dependencies
   getImageDescendentData(image).forEach((project) => {

package/src/cli/redis.ts

@@ -64,7 +64,7 @@ function run(cmdObj: CLICommandParser) {
   const parsed = cmdObj.parseOptions({ params: ["-p"] });
 
   const [command, namespace] = parsed.args;
-  const port = parsed.options["-p"] as string;
+  const port = parsed.options["-p"];
   // @ts-expect-error left as an exercise for the reader
   const handler = handlers[command];
   if (!handler) {

package/src/cli/registry.ts

@@ -11,21 +11,27 @@ const oneLiner = "Manage container repositories";
 const keyExamples = `
     $ devops registry server-url
     $ devops registry reg-url
-    $ devops registry repo-url my-image sha
-    $ devops registry prune    my-image
+    $ devops registry repo-url   my-image sha
+    $ devops registry image-name my-image 
+    $ devops registry prune      my-image
 `.trim();
 
 const usage = `
 ${oneLiner}
 
 USAGE
-  Get base URLs
+  Get base URLs for the container registry of the cluster:
       devops registry server-url
       devops registry reg-url
 
+      Note: for cloudrun images these URLs are not relevant. 
+
   Gets the URL of an image in the container registry:
     devops registry repo-url <image> <sha> --env <env>
 
+  Gets the image name in the container registry:
+    devops registry image-name <image> --env <env>
+
   Prunes the repository of old images to enforce the "image-versions-to-keep" constant in config/constants.yaml:
     devops registry prune <image> --env <env>
 
@@ -47,13 +53,17 @@ const handlers = {
       )
     );
   },
+  "image-name": (opts: StrongParams) => {
+    console.log(containerRegistryImageName(opts.required("image"), opts.required("env")));
+  },
   prune: (opts: StrongParams) => {
     const regName = containerRegistryPath();
+    const image = opts.required("image");
     const repoName = containerRegistryImageName(
-      opts.required("image"),
+      image,
       opts.required("env")
     );
-    prune(regName, repoName);
+    prune(regName, repoName, image);
   },
 };
 

package/src/devops.ts

@@ -22,6 +22,7 @@ import namespace from "./cli/namespace";
 import image from "./cli/image";
 import template from "./cli/template";
 import job from "./cli/job";
+import cloudrun from "./cli/cloudrun";
 
 const [_node, _scriptPath, ...commandArgs] = process.argv;
 
@@ -45,11 +46,12 @@ const allImports = [
   job,
   //= Deployment
   prepBuild,
+  cloudrun,
   affected,
   constant,
   registry,
   internalCurl,
-  jwt
+  jwt,
 ];
 
 const commands: {

package/src/libs/cloudrun-helpers.ts

@@ -0,0 +1,118 @@
+import { randomBytes } from "crypto";
+import { CommandExecutor } from "../cli/common";
+import { containerRegistryRepoPath, isLocalOrRemoteEnv } from "./k8s-constants";
+import { getImageData } from "./config";
+import { getMonorepoSecretObject } from "./k8s-secrets-manager";
+import chalk from "chalk";
+
+function verifyCloudrunImage(image: string) {
+  const imageData = getImageData(image);
+  if (!imageData["cloudrun"]) {
+    console.error(`Image ${image} is not a cloudrun image. Add "cloudrun: true" in images.yaml`);
+    process.exit(1);
+  }
+}
+
+function getEnvValuesToForward(env: string, forwardEnv: string[]) {
+  if (!forwardEnv.length) return {};
+  
+  let envValues: Record<string, string> = {};
+  const missingValues = new Set<string>();
+  for (const key of forwardEnv) {
+    const value = process.env[key];
+    if (value) {
+      envValues[key] = value;
+    } else {
+      missingValues.add(key);
+    }
+  }
+  if (missingValues.size > 0 && isLocalOrRemoteEnv(env) === "remote") {
+    const secretsFromCluster = getMonorepoSecretObject(env, Array.from(missingValues));
+    for (const key of Object.keys(secretsFromCluster)) {
+      envValues[key] = secretsFromCluster[key];
+      missingValues.delete(key);
+    }
+  }
+  if (missingValues.size > 0) {
+    console.error(`Some forwardEnv variables are missing: ${Array.from(missingValues).join(", ")}`);
+    process.exit(1);
+  }
+  return envValues;
+}
+
+export async function buildDev(image: string) {
+  verifyCloudrunImage(image);
+  const env = "development";
+  const sha = randomBytes(12).toString("hex");
+
+  const buildDir = new CommandExecutor(`devops prep-build ${image}`, {
+    env,
+  }).exec().trim();
+
+  const tag = containerRegistryRepoPath(image, env, sha);
+  console.warn(`Building ${tag} from ${buildDir}`);
+
+  await new CommandExecutor(
+    `docker build --platform linux/amd64 -t ${tag} ${buildDir} --build-arg MONOREPO_ENV=${env}`,
+    { env }
+  ).spawn();
+
+  console.warn(`Pushing ${tag}`);
+  await new CommandExecutor(`docker push ${tag}`, { env }).spawn();
+
+  console.warn(`\n✅ Built and pushed ${tag}\n`);
+  console.warn('Run "devops cloudrun deploy" next. For example:')
+  console.warn(chalk.blue(`./devops cloudrun deploy ${image} ${sha} --env ${env} --allow-unauthenticated --region us-east1 --forward-env ENV1,ENV2`));
+  console.warn();
+  console.log(tag);
+}
+
+export async function deploy({
+  image,
+  env,
+  sha,
+  region,
+  forwardEnv = [],
+  allowUnauthenticated = false,
+  cpu = "0.25",
+  memory = "256Mi",
+  minInstances = 0,
+  maxInstances = 1,
+  timeout = "60s",
+  extraArgs = "",
+}: {
+  image: string;
+  env: string; 
+  sha: string; 
+  region: string;
+  forwardEnv?: string[];
+  allowUnauthenticated?: boolean;
+  cpu?: string;
+  memory?: string;
+  minInstances?: number;
+  maxInstances?: number;
+  timeout?: string;
+  extraArgs?: string;
+}) {
+  verifyCloudrunImage(image);
+  const repoPath = containerRegistryRepoPath(image, env, sha);
+  const envValues = getEnvValuesToForward(env, forwardEnv);
+  const envValuesCsv = Object.entries(envValues).map(([key, value]) => `${key}="${value}"`).join(",");
+  const serviceName = `${image}-${env}`;
+
+  const cmd = `
+    gcloud run deploy ${serviceName} 
+      --image ${repoPath} 
+      ${Object.keys(envValues).length > 0 ? `--set-env-vars ${envValuesCsv}` : ""} 
+      ${allowUnauthenticated ? "--allow-unauthenticated" : ""}
+      --region ${region}
+      --cpu ${cpu}
+      --memory ${memory}
+      --min-instances ${minInstances}
+      --max-instances ${maxInstances}
+      --timeout ${timeout}
+      ${extraArgs}
+  `.trim().replace(/\s+/g, " ");
+
+  await new CommandExecutor(cmd, { env }).spawn();
+}

package/src/libs/config.ts

@@ -11,6 +11,11 @@ const imagesFilePath = path.join(process.cwd(), ".devops/config/images.yaml");
 export const { getConst } = processConstFile();
 export const { getImageData, getImageNames, getTemplateData } = processImagesFile();
 
+export function getImageType(image: string) {
+  const imageData = getImageData(image);
+  return imageData["cloudrun"] ? "cloudrun" : "k8s";
+}
+
 // Process config/constants.yaml
 
 function processConstFile() {

package/src/libs/digital-ocean/container-reg.ts

@@ -1,6 +1,6 @@
 import { CommandExecutor } from "../../cli/common";
 import { z } from "zod";
-import { getConst } from "../config";
+import { getConst, getImageData } from "../config";
 
 const repoTagMetadataSchema = z.object({
   // What we rely on
@@ -60,7 +60,8 @@ export function prune(
   /** To keep the image-related constants simple, this accepts the full URL including the prefix registry.digitalocean.com */
   registryFullName: string,
   /** The name of the repository inside the registry */
-  repoName: string
+  repoName: string,
+  image: string
 ) {
   const infra = getConst("infra");
   if (infra !== "digitalocean") {
@@ -69,6 +70,13 @@ export function prune(
     );
     return;
   }
+  const imageData = getImageData(image);
+  if (imageData["cloudrun"]) {
+    console.warn(
+      "Pruning is skipped for cloudrun images"
+    );
+    return;
+  }
   const tags = getRepoTagMetadata(repoName);
   const versionsToKeep = Number(getConst("image-versions-to-keep"));
   if (!tags.length || tags.length <= versionsToKeep) return;

package/src/libs/k8s-constants.ts

@@ -24,13 +24,28 @@ export function allSupportedEnvs() {
   return [...remoteSupportedEnvs(), ...localSupportedEnvs()];
 }
 
-function validateEnv(monorepoEnv?: string) {
+export function isLocalOrRemoteEnv(monorepoEnv: string): "local" | "remote" {
+  if (remoteSupportedEnvs().includes(monorepoEnv)) return "remote";
+  if (localSupportedEnvs().includes(monorepoEnv)) return "local";
+  throw new Error(`Unsupported environment: ${monorepoEnv}`);
+}
+
+function validateEnv(monorepoEnv?: string, { allowLocal = false }: { allowLocal?: boolean } = {}) {
   if (!monorepoEnv) throw new Error("MONOREPO_ENV cannot be empty");
-  if (!remoteSupportedEnvs().includes(monorepoEnv)) {
-    console.error(
-      `MONOREPO_ENV must be one of: ${remoteSupportedEnvs().join(", ")}. Can be set using --env flag.`
-    );
-    process.exit(1);
+  if (allowLocal) {
+    if (!allSupportedEnvs().includes(monorepoEnv)) {
+      console.error(
+        `MONOREPO_ENV must be one of: ${allSupportedEnvs().join(", ")}. Can be set using --env flag.`
+      );
+      process.exit(1);
+    }
+  } else {
+    if (!remoteSupportedEnvs().includes(monorepoEnv)) {
+      console.error(
+        `MONOREPO_ENV must be one of: ${remoteSupportedEnvs().join(", ")}. Can be set using --env flag.`
+      );
+      process.exit(1);
+    }
   }
 }
 
@@ -52,7 +67,7 @@ export function imageConfigMap(image: string) {
 }
 
 export function containerRegistryImageName(image: string, monorepoEnv: string) {
-  validateEnv(monorepoEnv);
+  validateEnv(monorepoEnv, { allowLocal: true });
   return `${getConst("project-name")}-${monorepoEnv}-${image}`;
 }
 
@@ -65,11 +80,20 @@ export function containerRegistryRepoPath(
   monorepoEnv: string,
   gitSha: string
 ) {
-  return [
-    getConst("registry-base-url"),
-    getConst("registry-image-path-prefix", { ignoreIfInvalid: true }),
-    [containerRegistryImageName(image, monorepoEnv), gitSha].join(":"),
-  ].filter(Boolean).join("/");
+  const imageNameAndTag = [containerRegistryImageName(image, monorepoEnv), gitSha].join(":");
+  const imageData = getImageData(image);
+  if (imageData["cloudrun"]) {
+    return [
+      getConst("cloudrun-artifact-registry-repo-path"),
+      imageNameAndTag,
+    ].join("/");
+  } else {
+    return [
+      getConst("registry-base-url"),
+      getConst("registry-image-path-prefix", { ignoreIfInvalid: true }),
+      imageNameAndTag,
+    ].filter(Boolean).join("/");
+  }
 }
 
 export function domainNameForEnv(image: string, monorepoEnv: string) {