@vaharoni/devops 1.0.47

package/src/cli/prep-build.ts

@@ -0,0 +1,96 @@
+import fs from "fs-extra";
+import os from "os";
+import path from "path";
+import { CLICommandParser, printUsageAndExit } from "./common";
+import { getImageData, getTemplateData } from "../libs/config";
+import { getMonorepoSecret } from "../libs/k8s-secrets-manager";
+import { getImageDescendentData } from "../libs/discovery/images";
+
+const oneLiner =
+  "Copies all dependencies of an image to a temporary folder in preparation for a Docker build";
+const keyExamples = `
+    $ devops prep-build main-node
+`.trim();
+
+const usage = `
+${oneLiner}
+
+USAGE
+    devops prep-build <image>
+
+EXAMPLES
+    ${keyExamples}
+`;
+
+async function run(cmdObj: CLICommandParser) {
+  if (cmdObj.help || cmdObj.args.length === 0) printUsageAndExit(usage);
+  const [image] = cmdObj.args;
+  const imageData = getImageData(image);
+  const imageTemplate = imageData["image-template"];
+  const dockerFile = `${imageTemplate}.Dockerfile`;
+  const dockerFilePath = path.join(".devops/docker-images", dockerFile);
+  const dockerImagePayloadPath = path.join(".devops/docker-images", imageTemplate);
+  const dockerCommonPayloadPath = path.join(".devops/docker-images", "common");
+  const imageTemplateData = getTemplateData(imageTemplate);
+  const copyCommon = imageTemplateData["copy-common"] ?? false;
+  const imageExtraContent = imageTemplateData["extra-content"] ?? [];
+
+  if (!fs.existsSync(dockerFilePath)) {
+    console.error(`The dockerfile ${dockerFilePath} does not exist`);
+    process.exit(1);
+  }
+
+  imageExtraContent.forEach((file) => {
+    if (!fs.existsSync(file)) {
+      console.error(`The file ${file} is specified in the extra-content section of ${image} but does not exist`);
+      process.exit(1);
+    }
+  });
+
+  const destFolder = `${os.tmpdir()}/image-${image}-${Date.now()}`;
+  // Avoid clutter stdout, as the caller will want to capture the final output to cd into it
+  console.warn(`Creating build in ${destFolder}`);
+  fs.mkdirSync(destFolder);
+
+  // Copy Dockerfile
+  console.warn(`COPYING Dockerfile`);
+  fs.copySync(dockerFilePath, path.join(destFolder, "Dockerfile"));
+
+  if (copyCommon) {
+    console.warn(`COPYING Docker common`);
+    fs.copySync(dockerCommonPayloadPath, destFolder);
+  }
+
+  console.warn(`COPYING Docker image payload`);
+  fs.copySync(dockerImagePayloadPath, destFolder);
+
+  console.warn(`COPYING .devops/config`);
+  fs.mkdirSync(path.join(destFolder, ".devops"));
+  fs.copySync(".devops/config", path.join(destFolder, ".devops/config"));
+
+  // Create config directory. It should be deleted by the docker image so that it can be mounted as a volume when the pod is run
+  console.warn(`CREATING config for the build process`);
+  fs.mkdirSync(path.join(destFolder, "config"));
+  const envFileData = getMonorepoSecret(cmdObj.env);
+  fs.writeFileSync(path.join(destFolder, `config/.env.global`), envFileData);
+
+  // Copy all dependencies
+  getImageDescendentData(image).forEach((project) => {
+    console.warn(`COPYING ${project.rootPath}`);
+    if (project.rootPath === ".devops") return;
+    fs.copySync(project.rootPath, path.join(destFolder, project.rootPath));
+  });
+
+  // Copy image-extra content
+  console.warn(`COPYING files from image-extra-content`);
+  imageExtraContent.forEach((file) => {
+    fs.copySync(file, path.join(destFolder, file));
+    console.warn(`  ${file}`);
+  });
+
+  console.log(destFolder);
+}
+
+export default {
+  "prep-build": { oneLiner, keyExamples, run },
+};

package/src/cli/prisma.ts

@@ -0,0 +1,33 @@
+import { CLICommandParser, printUsageAndExit } from "./common";
+import url from "url";
+import path from "path";
+
+const __file__ = url.fileURLToPath(import.meta.url);
+const __src__ = path.join(path.dirname(__file__), "../..", "src");
+const execShPath = path.join(__src__, "cli/exec.sh");
+
+const oneLiner = "Runs prisma commands in the db project after injecting the environment variables";
+const keyExamples = `$ devops prisma migrate dev`;
+
+const usage = `
+${oneLiner}
+
+GENERAL USAGE
+    devops prisma <command>
+
+    <command> can be any command you normally set prisma for.
+
+EXAMPLES
+    ${keyExamples}
+`;
+
+async function run(cmdObj: CLICommandParser) {
+  cmdObj.executorFromEnv(
+    `${execShPath} db prisma ${cmdObj.args.join(" ")}`,
+    { checkEnvYaml: false }
+  ).spawn()
+}
+
+export default {
+  prisma: { oneLiner, keyExamples, run },
+};

package/src/cli/redis.ts

@@ -0,0 +1,83 @@
+import chalk from "chalk";
+import { establishRedisTunnel, getRedisList, getRedisPassword } from "../libs/k8s-redis";
+import { CLICommandParser, printUsageAndExit, StrongParams } from "./common";
+
+const oneLiner =
+  "Utilities to help accessing production and staging redis";
+const keyExamples = `
+    $ devops redis list
+    $ devops redis password  redis-staging
+    $ devops redis tunnel    redis-staging
+`.trim();
+
+const usage = `
+${oneLiner}
+
+COMMANDS
+    list                                  Lists the available redis installations
+    password <namespace>                  Shows the password for the Redis instance
+    tunnel   <namespace> [-p <port>]      Sets up a tunnel to the remote Redis instance so you can access the DB from your local machine on port 9379 by default
+
+NOTES
+    This command assumes the namespace and the helm release name are the same.
+    The --env flag should not be used with these commands, as the Redis namespaces follow different conventions than the monorepo env. 
+
+EXAMPLES
+    ${keyExamples}
+`;
+
+const handlers = {
+  list: () => {
+    const res = getRedisList();
+    console.log(res);
+  },
+  password: (opts: StrongParams) => {
+    const namespace = opts.required("namespace");
+    const res = getRedisPassword(namespace);
+    if (!res) {
+      console.error("Failed to get the secret");
+      process.exit(1);
+    } else {
+      console.log();
+      console.log(res.password);
+      console.log();
+    }
+  },
+  tunnel: (opts: StrongParams) => {
+    const namespace = opts.required("namespace");
+    const port = opts.optional("port") ?? '9379';
+    const res = getRedisPassword(namespace);
+    console.log(
+      chalk.blue('\nAfter the tunnel is established, connect to Redis by running:\n\t') + 
+      chalk.green.bold(`redis-cli -p ${port} --askpass`)
+    );
+    if (res) {
+      console.log(chalk.blue('\tPassword: ') + chalk.green.bold(res.password))
+      console.log();
+    } 
+    establishRedisTunnel(namespace, port);
+  },
+};
+
+function run(cmdObj: CLICommandParser) {
+  if (cmdObj.help || cmdObj.args.length < 1) printUsageAndExit(usage);
+  const parsed = cmdObj.parseOptions({ params: ["-p"] });
+
+  const [command, namespace] = parsed.args;
+  const port = parsed.options["-p"] as string;
+  // @ts-expect-error left as an exercise for the reader
+  const handler = handlers[command];
+  if (!handler) {
+    console.error(`Unknown command: ${command}`);
+    printUsageAndExit(usage);
+  }
+  const params = new StrongParams(usage, {
+    namespace,
+    port,
+  });
+  handler(params);
+}
+
+export default {
+  redis: { oneLiner, keyExamples, run },
+};

package/src/cli/registry.ts

@@ -0,0 +1,76 @@
+import { getConst, getImageData } from "../libs/config";
+import { prune } from "../libs/digital-ocean/container-reg";
+import {
+  containerRegistryPath,
+  containerRegistryRepoName,
+  containerRegistryRepoPath,
+} from "../libs/k8s-constants";
+import { CLICommandParser, StrongParams, printUsageAndExit } from "./common";
+
+const oneLiner = "Manage container repositories";
+const keyExamples = `
+    $ devops registry server-url
+    $ devops registry reg-url
+    $ devops registry repo-url my-image sha
+    $ devops registry prune    my-image
+`.trim();
+
+const usage = `
+${oneLiner}
+
+USAGE
+  Get base URLs
+      devops registry server-url
+      devops registry reg-url
+
+  Gets the URL of an image in the container registry:
+    devops registry repo-url <image> <sha> --env <env>
+
+  Prunes the repository of old images to enforce the "image-versions-to-keep" constant in config/constants.yaml:
+    devops registry prune <image> --env <env>
+
+    This is only relevant when the "infra" constant is set to "digitalocean".
+
+EXAMPLES
+    ${keyExamples}
+`;
+
+const handlers = {
+  "server-url": () => console.log(getConst("registry-base-url")),
+  "reg-url": () => console.log(containerRegistryPath()),
+  "repo-url": (opts: StrongParams) => {
+    console.log(
+      containerRegistryRepoPath(
+        opts.required("image"),
+        opts.required("env"),
+        opts.required("sha")
+      )
+    );
+  },
+  prune: (opts: StrongParams) => {
+    const regName = containerRegistryPath();
+    const repoName = containerRegistryRepoName(
+      opts.required("image"),
+      opts.required("env")
+    );
+    prune(regName, repoName);
+  },
+};
+
+function run(cmdObj: CLICommandParser) {
+  if (cmdObj.help || cmdObj.args.length < 1) printUsageAndExit(usage);
+
+  const [command, image, sha] = cmdObj.args;
+  // @ts-expect-error left as an exercise for the reader
+  const handler = handlers[command];
+  if (!handler) {
+    console.error(`Unknown command: ${command}`);
+    printUsageAndExit(usage);
+  }
+  const params = new StrongParams(usage, { image, env: cmdObj.env, sha });
+  handler(params);
+}
+
+export default {
+  registry: { oneLiner, keyExamples, run },
+};

package/src/cli/run-many.ts

@@ -0,0 +1,61 @@
+import { workspaceDirectoryForLanguage } from "../libs/discovery";
+import { CLICommandParser, printUsageAndExit } from "./common";
+import concurrently, {
+  type ConcurrentlyCommandInput,
+  type ConcurrentlyOptions,
+} from "concurrently";
+
+const oneLiner =
+  "Runs a script concurrently in all projects that define it in their package.json";
+const keyExamples = `
+    $ devops run-many build
+`.trim();
+
+const usage = `
+${oneLiner}
+
+USAGE
+    devops run-many <script-name> [--kill-others-on-fail]
+
+NOTE
+    Only works for node projects. Use 'devopspy' for python projects.
+
+EXAMPLES
+    ${keyExamples}
+`;
+
+async function run(cmdObj: CLICommandParser) {
+  if (cmdObj.help || cmdObj.args.length === 0) printUsageAndExit(usage);
+  const parsed = cmdObj.parseOptions({ booleans: ["--kill-others-on-fail"] });
+  const [script] = parsed.args;
+  const remaining = parsed.args.slice(1).join(" ");
+  const commands: ConcurrentlyCommandInput[] = [];
+
+  Object.values(workspaceDirectoryForLanguage('node')).forEach(async (packageData) => {
+    if (packageData.scripts?.[script]) {
+      commands.push({
+        name: packageData.name,
+        command: `devops --env ${cmdObj.env} run ${packageData.name}:${script} ${remaining}`,
+      });
+    }
+  });
+
+  if (!commands.length) {
+    console.error(`No workspaces define the script: ${script}`);
+    process.exit(0);
+  }
+
+  // prettier-ignore
+  const options: Partial<ConcurrentlyOptions> = parsed.options["--kill-others-on-fail"] ? { killOthers: "failure" } : {};
+
+  concurrently(commands, options)
+    .result.then(() => {})
+    .catch((error) => {
+      console.error("One of the commands failed");
+      process.exit(1);
+    });
+}
+
+export default {
+  "run-many": { oneLiner, keyExamples, run },
+};

package/src/cli/run.ts

@@ -0,0 +1,46 @@
+import { CLICommandParser, printUsageAndExit } from "./common";
+import url from "url";
+import path from "path";
+import { getWorkspace } from "../libs/discovery";
+
+const __file__ = url.fileURLToPath(import.meta.url);
+const __src__ = path.join(path.dirname(__file__), "../..", "src");
+const execShPath = path.join(__src__, "cli/exec.sh");
+
+const oneLiner =
+  "Runs a script defined in package.json after injecting env variables";
+const keyExamples = `$ devops run project:test`;
+
+const usage = `
+${oneLiner}
+
+GENERAL USAGE
+    devops run <project-name>:<script-name> [--] [options for script]
+
+NOTES
+    - Only works for node projects. Use 'devopspy' for python projects.
+    - Does not allow interactive mode. If you need interactivity, use devops exec instead.
+
+EXAMPLES
+    ${keyExamples}
+`;
+
+async function run(cmdObj: CLICommandParser) {
+  if (cmdObj.help || cmdObj.args.length === 0) printUsageAndExit(usage);
+  const [workspace, script] = cmdObj.args[0].split(":");
+  if (!workspace || !script) printUsageAndExit(usage);
+
+  const rootPath = getWorkspace(workspace).rootPath;
+  const remaining = cmdObj.args.slice(1).join(" ");
+
+  cmdObj
+    .executorFromEnv(
+      `${execShPath} ${rootPath} bun run ${script} ${remaining}`,
+      { checkEnvYaml: true }
+    )
+    .spawn();
+}
+
+export default {
+  run: { oneLiner, keyExamples, run },
+};

package/src/cli/template.ts

@@ -0,0 +1,169 @@
+import {
+} from "../libs/k8s-image-config";
+import { CLICommandParser, printUsageAndExit, StrongParams } from "../../src/cli/common";
+import { generateDbMigrateJob, generateDebugDeployment, generateWorkspaceDeployment, ImageContextGenerator } from "../libs/k8s-generate";
+import chalk from "chalk";
+import { getWorkspace } from "../libs/discovery";
+import { getWorkspaceImages } from "../libs/discovery/images";
+import { getImageData, getImageNames } from "../libs/config";
+
+const SUPPORTED_CONTEXT_TYPES = ['deployment', 'db-migrate', 'debug'];
+
+const oneLiner = "Utilities to help validate manifest templates rendering.";
+const keyExamples = `
+    $ devops template context deployment www
+    $ devops template context debug
+    $ devops template context db-migrate
+    $ devops template gen     deployment www
+    $ devops template gen     debug      main-node
+    $ devops template gen     db-migrate
+`.trim();
+
+const usage = `
+${oneLiner}
+
+IMPORTANT
+    This command generates examples only. It's intended to help design new templates by showing what context variables are available
+    and how they get rendered using handlebar replacement. They should not be used to apply changes to the cluster.
+
+SHOW CONTEXT OBJECT
+    devops template context deployment <workspace>
+    devops template context db-migrate-job|debug
+
+    Prints out a context object with dummy values for the specified template type.
+    For deployment, the workspace name is required.
+
+GENERATE TEMPLATES
+    devops template gen deployment <workspace>
+    devops template gen debug <image>
+    devops template gen db-migrate-job
+
+    For deployment, generates an example manifest of a workspace, including override files present under the 'manifests' folder.
+
+EXAMPLES
+    ${keyExamples}
+`;
+
+const handlers = {
+  context: {
+    'deployment': (opts: StrongParams) => {
+      const workspace = opts.required('workspaceOrImage');
+      const workspaceData = getWorkspace(workspace);
+      const packageDataWithDeployment = workspaceData.packageDataEntries.find((entry) => entry.deployment);
+      if (!packageDataWithDeployment) {
+        console.error(`No deployment found for workspace ${workspace}`);
+        process.exit(1);
+      }
+      const randomImage = getWorkspaceImages(workspace)[0];
+      console.warn(chalk.green("\nThis is a sample context object used to render a manifest template of type deployment:\n"))
+      console.log(
+        JSON.stringify(
+          new ImageContextGenerator(opts.required('env'), randomImage, 'dummy-sha').getDeployment(packageDataWithDeployment),
+          null,
+          2
+        )
+      )
+    },
+    'db-migrate': (opts: StrongParams) => {
+      const randomImage = getWorkspaceImages('db').filter(image => getImageData(image)['can-db-migrate'])[0];
+      if (!randomImage) {
+        console.error("No image found with can-db-migrate=true in the db workspace.");
+        process.exit(1);
+      }
+      console.warn(chalk.green("\nThis is a sample context object used to render a manifest template of type db-migrate:\n"))
+      console.log(
+        JSON.stringify(
+          new ImageContextGenerator(opts.required('env'), randomImage, 'dummy-sha').getDbMigrate(),
+          null,
+          2
+        )
+      )
+    },
+    'debug': (opts: StrongParams) => {
+      const randomImage = getImageNames()[0];
+      console.warn(chalk.green("\nThis is a sample context object used to render a manifest template of type debug:\n"))
+      console.log(
+        JSON.stringify(
+          new ImageContextGenerator(opts.required('env'), randomImage, 'dummy-sha').getDebug(),
+          null,
+          2
+        )
+      )
+    },
+  },
+  gen: {
+    'deployment': (opts: StrongParams) => {
+      const workspace = opts.required('workspaceOrImage');
+      const workspaceData = getWorkspace(workspace);
+      const packageDataWithDeployment = workspaceData.packageDataEntries.find((entry) => entry.deployment);
+      if (!packageDataWithDeployment) {
+        console.error(`No deployment found for workspace ${workspace}`);
+        process.exit(1);
+      }
+      const randomImage = getWorkspaceImages(workspace)[0];
+      console.warn(chalk.green(`\nThis is a sample of generated manifests for the ${workspace} workspace:\n`))
+      console.log(
+        generateWorkspaceDeployment(
+          packageDataWithDeployment,
+          opts.required('env'),
+          randomImage,
+          'dummy-sha'
+        )
+      )
+    },
+    'db-migrate': (opts: StrongParams) => {
+      const randomImage = getWorkspaceImages('db').filter(image => getImageData(image)['can-db-migrate'])[0];
+      if (!randomImage) {
+        console.error("No image found with can-db-migrate=true in the db workspace.");
+        process.exit(1);
+      }
+      console.warn(chalk.green("\nThis is a sample of generated manifests for the db-migrate job:\n"))
+      console.log(
+        generateDbMigrateJob(
+          opts.required('env'),
+          randomImage,
+          'dummy-sha'
+        )
+      )
+    },
+    'debug': (opts: StrongParams) => {
+      const image = opts.required('workspaceOrImage');
+      console.warn(chalk.green(`\nThis is a sample of generated manifests for the debug image ${image}:\n`))
+      console.log(
+        generateDebugDeployment(
+          opts.required('env'),
+          image,
+          'dummy-sha'
+        )
+      )
+    },
+  },
+};
+
+function run(cmdObj: CLICommandParser) {
+  if (cmdObj.help || cmdObj.args.length < 1) printUsageAndExit(usage);
+
+  const [command, contextType, param] = cmdObj.args;
+  const commandHandler = handlers[command as keyof typeof handlers];
+  if (!commandHandler) {
+    console.error(`Unknown command: ${command}`);
+    printUsageAndExit(usage);
+  }
+
+  if (!SUPPORTED_CONTEXT_TYPES.includes(contextType)) {
+    console.error(`Unknown context type: ${contextType}. Supported types: ${SUPPORTED_CONTEXT_TYPES.join(', ')}`);
+    process.exit(1);
+  }
+  const handler = commandHandler[contextType as keyof typeof commandHandler];
+
+  const params = new StrongParams(usage, {
+    env: cmdObj.env === 'development' ? 'staging' : cmdObj.env,
+    contextType,
+    workspaceOrImage: param
+  });
+  handler(params);
+}
+
+export default {
+  template: { oneLiner, keyExamples, run },
+};

package/src/cli/test.ts

@@ -0,0 +1,30 @@
+import { CLICommandParser, CommandExecutor, printUsageAndExit } from "./common";
+
+const oneLiner = "Runs tests in all projects or one specific project";
+const keyExamples = `
+$ devops test
+$ devops test project
+`;
+
+const usage = `
+${oneLiner}
+
+USAGE
+  ${keyExamples}
+`;
+
+function run(cmdObj: CLICommandParser) {
+  const options = cmdObj.parseOptions({ params: ["--in"] });
+  if (cmdObj.help || options.args.length > 1) printUsageAndExit(usage);
+  const workspace = options.args[0];
+  const env = cmdObj.envForced ? cmdObj.env : "test";
+  if (workspace) {
+    new CommandExecutor(`devops run ${workspace}:test --env ${env}`).spawn();
+  } else {
+    new CommandExecutor(`devops run-many test --env ${env}`).spawn();
+  }
+}
+
+export default {
+  test: { oneLiner, keyExamples, run },
+};