@vaharoni/devops 1.0.47

package/src/cli/common.ts

@@ -0,0 +1,323 @@
+import chalk from "chalk";
+import { execSync, spawn } from "child_process";
+import fs from "fs";
+import { globSync } from "glob";
+import { allSupportedEnvs } from "../libs/k8s-constants";
+
+type ParsedArgs = {
+  args: string[];
+  argsStr: string;
+  options: { [key: string]: string | boolean };
+  passthrough?: string[];
+};
+
+interface ExecSyncError extends Error {
+  status?: number; // Exit code of the subprocess
+  stdout?: Buffer; // Standard output
+  stderr?: Buffer; // Standard error
+}
+
+export class CLICommandParser {
+  command: string;
+  args: string[];
+  env: string;
+  envForced: boolean;
+  help: boolean;
+  skipEnvCheck: boolean;
+
+  constructor(cmdArray: string[]) {
+    const parsedArgs = this._separateOptions(cmdArray.filter(Boolean), {
+      params: ["--env"],
+      booleans: ["--help", "--skip-env-check"],
+    });
+    const [command, ...args] = parsedArgs.args;
+    this.command = command;
+    this.args = args;
+    this.envForced = Boolean(parsedArgs.options["--env"]);
+    this.env =
+      (parsedArgs.options["--env"] as string) ??
+      process.env["MONOREPO_ENV"] ??
+      "development";
+    this.help = Boolean(parsedArgs.options["--help"]);
+    this.skipEnvCheck = Boolean(parsedArgs.options["--skip-env-check"]);
+    // We need to hardcode this to avoid chicken-and-egg problem (validate env depends on constants.yaml existence)
+    if (this.command !== 'init') {
+      this._validateEnv(this.env);
+    }
+  }
+
+  // Copies the env
+  executorFromEnv(
+    commandStr: string,
+    options: Omit<CommandExecutorOptions, "env"> = {}
+  ): CommandExecutor {
+    const checkEnvYamlOverride = this.skipEnvCheck
+      ? { checkEnvYaml: false }
+      : {};
+    return new CommandExecutor(commandStr, {
+      env: this.env,
+      ...options,
+      ...checkEnvYamlOverride,
+    });
+  }
+
+  // Example:
+  //    const cmd = new CLICommandParser('devops run --some-flag --in workspace arg1'.split(' '));
+  //    cmd.parseOptions({ boolean: ['--some-flag'], params: ['--in'] })
+  //    # => { args: ['arg1'], options: { '--some-flag': true, '--in': 'workspace' } }
+  //
+  // Note that the global param --env is already extracted and can be accessed with cmd.env
+  parseOptions({
+    params = [],
+    booleans = [],
+    passthroughArgs = false,
+  }: {
+    /** Param is used like so: --param value */
+    params?: string[];
+    /** Boolean flag is used like so: --flag */
+    booleans?: string[];
+    /** Pass through args are used like so: -- arg1 arg2 */
+    passthroughArgs?: boolean;
+  } = {}): ParsedArgs {
+    return this._separateOptions(this.args, {
+      params,
+      booleans,
+      passthroughArgs,
+    });
+  }
+
+  _validateEnv(env: string) {
+    if (!allSupportedEnvs().includes(env)) {
+      console.error(
+        // prettier-ignore
+        `Environment must be one of: ${allSupportedEnvs().join(", ")}. Received: ${env}`
+      );
+      process.exit(1);
+    }
+    return true;
+  }
+
+  _separateOptions(
+    args: string[],
+    {
+      params = [],
+      booleans = [],
+      passthroughArgs = false,
+    }: {
+      params?: string[];
+      booleans?: string[];
+      passthroughArgs?: boolean;
+    } = {}
+  ): ParsedArgs {
+    const results: ParsedArgs = {
+      args: [],
+      argsStr: "",
+      options: {},
+      ...(passthroughArgs ? { passthrough: [] } : {}),
+    };
+    const paramsLookup = Object.fromEntries(params.map((x) => [x, true]));
+    const booleansLookup = Object.fromEntries(booleans.map((x) => [x, true]));
+    const passthroughArgsStart = passthroughArgs ? args.indexOf("--") : -1;
+    const numArgsToProcess =
+      passthroughArgsStart === -1 ? args.length : passthroughArgsStart;
+    for (let i = 0; i < numArgsToProcess; ++i) {
+      const curr = args[i];
+      if (paramsLookup[curr]) {
+        const next = args[i + 1];
+        results.options[curr] = next;
+        ++i;
+      } else if (booleansLookup[curr]) {
+        results.options[curr] = true;
+      } else {
+        results.args.push(curr);
+      }
+    }
+    results.argsStr = results.args.join(" ");
+    if (passthroughArgs && passthroughArgsStart >= 0) {
+      results.passthrough = args.slice(passthroughArgsStart + 1);
+    }
+    return results;
+  }
+}
+
+type CommandExecutorOptions = {
+  env?: string;
+  quiet?: boolean;
+  checkEnvYaml?: boolean;
+  redactedCommand?: string;
+};
+export class CommandExecutor {
+  commandStr: string;
+  env?: string;
+  quiet: boolean;
+  redactedCommand?: string;
+  checkEnvYaml: boolean;
+
+  constructor(
+    commandStr: string,
+    {
+      env,
+      quiet = false,
+      redactedCommand,
+      checkEnvYaml = false,
+    }: CommandExecutorOptions = {}
+  ) {
+    this.env = env;
+    this.quiet = quiet ?? false;
+    this.commandStr = commandStr;
+    this.checkEnvYaml = checkEnvYaml;
+    this.redactedCommand = redactedCommand;
+  }
+
+  /** Non-interactive use only. stdout is returned. */
+  exec(options?: {
+    onlyStatusCode?: false;
+    asObject?: false;
+    env?: object;
+  }): string;
+  exec(options: { onlyStatusCode?: false; asObject: true; env?: object }): {
+    statusCode: number;
+    stdout: string;
+    stderr: string;
+  };
+  exec(options: {
+    onlyStatusCode: true;
+    asObject?: boolean;
+    env?: object;
+  }): number;
+  exec({
+    onlyStatusCode = false,
+    asObject = false,
+    env = {},
+  }: { onlyStatusCode?: boolean; asObject?: boolean; env?: object } = {}) {
+    this._checkEnvYamlFiles();
+    const fullCommand = this._prepareFullCommand();
+    const envToUse = this._getProcessEnv(env);
+    try {
+      const output = execSync(fullCommand, { env: envToUse });
+      if (onlyStatusCode) return 0;
+      if (!this.quiet) console.log(output.toString());
+      if (asObject) return { statusCode: 0, stdout: output.toString() };
+      return output.toString();
+    } catch (error) {
+      const typedError = error as ExecSyncError;
+      if (onlyStatusCode) return typedError.status;
+      const stdout = typedError.stdout?.toString().trim();
+      const stderr = typedError.stderr?.toString().trim();
+      if (!this.quiet) {
+        console.warn(stdout);
+        console.error(stderr);
+      }
+      if (asObject) return { statusCode: typedError.status, stdout, stderr };
+      process.exit(typedError.status);
+    }
+  }
+
+  /** Should be used for CLI commands intended to be used locally. Provides interactivity. Unlike exec(), stdout is not returned. */
+  spawn({ env = {} } = {}) {
+    this._checkEnvYamlFiles();
+    const fullCommand = this._prepareFullCommand();
+    const envToUse = this._getProcessEnv(env);
+    return new Promise((resolve) => {
+      try {
+        const [cmd, ...args] = fullCommand.split(" ").filter(Boolean);
+        const childProcess = spawn(cmd, args, {
+          stdio: "inherit",
+          env: envToUse,
+        });
+
+        childProcess.on("close", (code) => {
+          if (code !== 0) {
+            console.error(chalk.red(`Process exited with code ${code}`));
+            process.exit(code);
+          }
+          resolve(code);
+        });
+
+        childProcess.on("error", (error) => {
+          console.error(chalk.red(`Error: ${error.message}`));
+          process.exit(1);
+        });
+      } catch (error) {
+        if (error instanceof Error) {
+          console.error(chalk.red(`${error.message}`));
+        }
+        process.exit(1);
+      }
+    });
+  }
+
+  _prepareFullCommand() {
+    const envPrefix = this.env ? this._envInjectorPrefix() : "";
+    const fullCommand = [envPrefix, this.commandStr].join(" ").trim();
+
+    const envPrefixLog = this.env ? `MONOREPO_ENV=${this.env}` : "";
+    const fullCommandLog = [envPrefixLog, fullCommand].join(" ").trim();
+    if (this.redactedCommand) {
+      console.warn(
+        chalk.yellow(
+          fullCommandLog.replace(this.commandStr, this.redactedCommand)
+        )
+      );
+    } else {
+      console.warn(chalk.yellow(fullCommandLog));
+    }
+
+    return fullCommand;
+  }
+
+  _getProcessEnv(envOverride = {}) {
+    return { ...process.env, MONOREPO_ENV: this.env, ...envOverride };
+  }
+
+  _envInjectorPrefix() {
+    const envFiles = dotEnvFilesForEnv(this.env);
+    if (envFiles.length === 0) {
+      return "";
+    } else {
+      return `bunx dotenvx -q run -f ${envFiles.join(" ")} -- `;
+    }
+  }
+
+  _checkEnvYamlFiles() {
+    if (!this.checkEnvYaml) return;
+    const envYamlFiles = globSync("**/env.yaml");
+    const checkEnvCmd = new CommandExecutor(
+      `devops env _validate ${envYamlFiles.join(" ")}`,
+      { env: this.env, quiet: true, checkEnvYaml: false }
+    );
+    checkEnvCmd.exec();
+  }
+}
+
+export function dotEnvFilesForEnv(env?: string) {
+  const globalEnvFilePath = "config/.env.global";
+  const specificEnvFilePath = `config/.env.${env}`;
+
+  const envFiles: string[] = [];
+  if (env && fs.existsSync(specificEnvFilePath))
+    envFiles.push(specificEnvFilePath);
+  if (fs.existsSync(globalEnvFilePath)) envFiles.push(globalEnvFilePath);
+  return envFiles;
+}
+
+export function printUsageAndExit(text: string): never {
+  console.warn(text);
+  process.exit(1);
+}
+
+export class StrongParams {
+  constructor(private usage: string, private args: Record<string, string | undefined>) {}
+
+  required(key: string) {
+    if (!this.args[key]) {
+      console.error(`Missing required argument: ${key}`);
+      printUsageAndExit(this.usage);
+    }
+    return this.args[key];
+  }
+
+  optional(key: string) {
+    return this.args[key];
+  }
+}

package/src/cli/console.ts

@@ -0,0 +1,46 @@
+import { getConst } from "../libs/config";
+import {
+  envToNamespace,
+  imageDebugName,
+} from "../libs/k8s-constants";
+import { kubectlCommand } from "../libs/k8s-helpers";
+import { CLICommandParser, printUsageAndExit } from "./common";
+
+const oneLiner = "Get a shell into the debug pod of an image";
+const keyExamples = `
+$ devops console main-node
+`.trim();
+
+const usage = `
+${oneLiner}
+
+Each image has a debug pod. This command gets a shell into the debug pod of the specified image.
+
+EXAMPLES
+  ${keyExamples}
+`;
+
+function run(cmdObj: CLICommandParser) {
+  if (cmdObj.help || cmdObj.args.length === 0) printUsageAndExit(usage);
+  const image = cmdObj.args[0];
+  const debugName = imageDebugName(image);
+  const namespace = envToNamespace(cmdObj.env);
+  const podName = cmdObj
+    .executorFromEnv(
+      kubectlCommand(`get pod -n ${namespace} -l app=${debugName} -o name`, {
+        namespace,
+      }),
+      { quiet: true }
+    )
+    .exec()
+    .trim();
+  cmdObj
+    .executorFromEnv(
+      kubectlCommand(`exec -it ${podName} -- /bin/bash`, { namespace })
+    )
+    .spawn();
+}
+
+export default {
+  console: { oneLiner, keyExamples, run },
+};

package/src/cli/constant.ts

@@ -0,0 +1,25 @@
+import { getConst } from "../libs/config";
+import { CLICommandParser, printUsageAndExit } from "./common";
+
+const oneLiner = "Prints to stdout a constant from constant.yaml";
+const keyExamples = `$ devops constant infra`;
+
+const usage = `
+${oneLiner}
+
+GENERAL USAGE
+    devops constant <constant-name>
+
+EXAMPLES
+    ${keyExamples}
+`;
+
+async function run(cmdObj: CLICommandParser) {
+  if (cmdObj.help || cmdObj.args.length === 0) printUsageAndExit(usage);
+  const [constant] = cmdObj.args;
+  console.log(getConst(constant as any));
+}
+
+export default {
+  constant: { oneLiner, keyExamples, run },
+};

package/src/cli/db.ts

@@ -0,0 +1,133 @@
+import {
+  connectToPatroni,
+  connectToPsql,
+  establishTunnel,
+  getDbAdminPassword,
+  getDbBackups,
+  getDbList,
+  getDbPasswords,
+} from "../libs/k8s-db";
+import { CLICommandParser, printUsageAndExit, StrongParams } from "./common";
+
+const oneLiner =
+  "Utilities to help day to day operations of production and staging databases";
+const keyExamples = `
+    $ devops db list
+    $ devops db backups
+    $ devops db password  ui
+    $ devops db password  db-staging
+    $ devops db tunnel    db-staging
+    $ devops db patroni   db-staging
+    $ devops db psql      db-staging
+`.trim();
+
+const usage = `
+${oneLiner}
+
+NOTES
+    The admin UI provided by Stackgres is great. It allows you to do most of the operations you need, such as 
+    restarting the cluster, upgrading postgres versions, and restoring from backups with point in time recovery (PITR).
+
+    This utility complements the admin UI with a few helpful shortcuts.
+
+    Note that the --env flag should not be used with these commands, as the DB namespaces follow different
+    conventions than the monorepo env. 
+
+    This utility assumes that the cluster name and the namespace are always the same.
+
+COMMANDS
+    list                                  Lists the available clusters
+    backups                               Lists all available backups
+    password ui                           Shows the password to the admin UI
+    password <namespace>                  Shows the superuser, replication, and authenticator password of the remote database
+    patroni  <namespace>                  Obtain a shell to the primary pod's patroni container, where you can run 'patronictl'
+    psql     <namespace>                  Runs 'psql' in the primary pod's postgres-utils container
+    tunnel   <namespace> [-p <port>]      Sets up a tunnel to the remote database so you can access the DB from your local machine. 
+                                          By default, the port is taken from the namespace to make it easier to create connection profiles locally: 
+                                          db-staging: 7432, db-production: 8432, otherwise: 9432
+
+EXAMPLES
+    ${keyExamples}
+`;
+
+const DEFAULT_PORTS = {
+  "db-staging": "7432",
+  "db-production": "8432",
+};
+
+const handlers = {
+  list: () => {
+    const res = getDbList();
+    console.log(res);
+  },
+  backups: () => {
+    const res = getDbBackups();
+    console.log(res);
+  },
+  password: (opts: StrongParams) => {
+    const namespace = opts.required("namespace");
+    if (namespace === "ui") {
+      const res = getDbAdminPassword();
+      if (!res) {
+        console.error("Failed to get the secret");
+        process.exit(1);
+      } else {
+        console.log(`User: ${res.user}`);
+        console.log(`Password: ${res.password}`);
+      }
+      return;
+    }
+
+    const res = getDbPasswords(namespace);
+    if (!res) {
+      console.error("Failed to get the secret");
+      process.exit(1);
+    } else {
+      console.log("\nSuperuser");
+      console.log(`  ${res.superUser}`);
+      console.log(`  ${res.superPassword}`);
+      console.log("\nAuthenticator");
+      console.log(`  ${res.authenticatorUser}`);
+      console.log(`  ${res.authenticatorPassword}`);
+      console.log("\nReplication");
+      console.log(`  ${res.replicationUser}`);
+      console.log(`  ${res.replicationPassword}`);
+      console.log();
+    }
+  },
+  tunnel: (opts: StrongParams) => {
+    // prettier-ignore
+    const defaultPort = DEFAULT_PORTS[opts.required("namespace") as keyof typeof DEFAULT_PORTS] || "9432";
+    const port = opts.optional("port") || defaultPort;
+    establishTunnel(opts.required("namespace"), port);
+  },
+  patroni: (opts: StrongParams) => {
+    connectToPatroni(opts.required("namespace"));
+  },
+  psql: (opts: StrongParams) => {
+    connectToPsql(opts.required("namespace"));
+  },
+};
+
+function run(cmdObj: CLICommandParser) {
+  if (cmdObj.help || cmdObj.args.length < 1) printUsageAndExit(usage);
+  const parsed = cmdObj.parseOptions({ params: ["-p"] });
+
+  const [command, namespace] = parsed.args;
+  const port = parsed.options["-p"] as string;
+  // @ts-expect-error left as an exercise for the reader
+  const handler = handlers[command];
+  if (!handler) {
+    console.error(`Unknown command: ${command}`);
+    printUsageAndExit(usage);
+  }
+  const params = new StrongParams(usage, {
+    namespace,
+    port,
+  });
+  handler(params);
+}
+
+export default {
+  db: { oneLiner, keyExamples, run },
+};

package/src/cli/dml.ts

@@ -0,0 +1,126 @@
+import fs from "fs";
+import { CLICommandParser, printUsageAndExit } from "./common";
+
+const oneLiner = "Utilities to manage and run DML scripts in the db project";
+const keyExamples = `
+    $ devops dml create --name my-dml-name
+    $ devops dml run 20250113153318_my_dml_name
+`;
+
+const usage = `
+${oneLiner}
+
+CREATE DML SCRIPTS
+    devops dml create --name <dml-semantic-name>
+
+    This command creates a new folder under /dml using the current timestamp and a 
+    snake-case version of the name. Inside the folder, a file called migrate.ts is created. 
+    You should write your DML script in this file.
+    
+    You can add additional artifacts to the folder, such as a README.md file, sql files, json 
+    files, csv files, etc. You can also add optional scripts, such as rollback.ts.
+
+RUN DML SCRIPTS
+    devops dml run <dml-folder-name> [script-file-name] [-- arg1 arg2 ...]
+
+    The dml-folder-name must be the full name, including the timestamp. This follows prisma 
+    conventions. 
+    If the optional script-file-name is omitted, 'migrate' is used by default. The name should 
+    not include the '.ts' suffix. 
+    Optionally, args can be passed to the script as command line arguments after double
+    dash (--).
+    The runner first changes the working directory to 'dml/', then executes the script using 
+    'bunx tsx'.
+    
+    Note: DML scripts are typically run inside the debug container of the image.
+
+EXAMPLES
+    ${keyExamples.trim()}
+    $ devops dml run 20250113153318_my_dml_name rollback
+    $ devops dml run 20250113153318_my_dml_name -- staging
+`;
+
+const dmlFileTemplate = `
+/**
+ * Header code that retrieves the context of the DML script.
+ * Feel free to modify this code to suit your needs.
+ * 
+ * fullDmlFilePath  - path to the current file
+ * fullDmlDirPath   - path to the current directory
+ * dmlFile          - name of the current DML script file
+ * dmlDir           - name of the directory containing the DML scripts
+ * args             - command line arguments passed to the script
+ * 
+ * Notes: 
+ * - the script runs with the cwd set to the dml/ directory
+ * - remove unused variables from this template, otherwise the linter will complain 
+*/
+
+import { prisma } from 'db';
+import { basename, dirname, sep } from 'path';
+import { fileURLToPath } from 'url';
+
+const fullDmlFilePath = fileURLToPath(import.meta.url);
+const fullDmlDirPath = dirname(fullDmlFilePath);
+const dmlFile = basename(fullDmlFilePath);
+const dmlDir = fullDmlDirPath.split(sep).pop();
+
+const args = process.argv.slice(2);
+`.trim();
+
+function createDml(name: string) {
+  const timestamp = new Date()
+    .toISOString()
+    .replace(/[^0-9]/g, "")
+    .slice(0, 14);
+  const nameSnake = name.toLowerCase().replace(/[^a-z0-9]/g, "_");
+  const folderName = `${timestamp}_${nameSnake}`;
+  fs.mkdirSync(`dml/${folderName}`);
+  fs.writeFileSync(`dml/${folderName}/migrate.ts`, dmlFileTemplate);
+  console.log(`\nCreated DML folder: dml/${folderName}\n`);
+}
+
+function runDml(
+  cmdObj: CLICommandParser,
+  folderName: string,
+  scriptFileName?: string,
+  args?: string[]
+) {
+  scriptFileName ??= "migrate";
+  args ??= [];
+  const script = scriptFileName.endsWith(".ts")
+    ? scriptFileName
+    : `${scriptFileName}.ts`;
+  cmdObj
+    .executorFromEnv(
+      // prettier-ignore
+      `devops exec --in dml bun ${folderName}/${script} ${args.join(" ")}`
+    )
+    .exec();
+}
+
+function run(cmdObj: CLICommandParser) {
+  if (cmdObj.help || cmdObj.args.length < 1) printUsageAndExit(usage);
+  const parsed = cmdObj.parseOptions({
+    passthroughArgs: true,
+    params: ["--name"],
+  });
+  switch (parsed.args[0]) {
+    case "create": {
+      const name = parsed.options["--name"] as string;
+      if (!name) printUsageAndExit(usage);
+      return createDml(name);
+    }
+    case "run": {
+      const [_, folderName, scriptFileName] = parsed.args;
+      if (!folderName) printUsageAndExit(usage);
+      return runDml(cmdObj, folderName, scriptFileName, parsed.passthrough);
+    }
+    default:
+      printUsageAndExit(usage);
+  }
+}
+
+export default {
+  dml: { oneLiner, keyExamples, run },
+};