@vaharoni/devops 1.0.47

package/src/cli/env.ts

@@ -0,0 +1,87 @@
+import { globSync } from "glob";
+import {
+  deleteMonorepoSecret,
+  getMonorepoSecret,
+  setMonorepoSecret,
+} from "../libs/k8s-secrets-manager";
+import { CombinedEnvValidator } from "../libs/validate-env";
+import {
+  CLICommandParser,
+  dotEnvFilesForEnv,
+  printUsageAndExit,
+} from "./common";
+
+const oneLiner = "Commands to manipulate env variables";
+const keyExamples = `
+$ devops env get --env staging
+$ devops env get KEY1 KEY2 --env staging
+$ devops env set KEY1=123 KEY2=345 --env staging
+$ devops env delete KEY1 KEY2 --env staging
+$ devops env validate
+`;
+
+const usage = `
+${oneLiner}
+
+COMMANDS
+  get         Fetches secrets for the chosen environment and printes them to console
+  set         Sets specific secrets for the chosen environment
+  delete      Deletes specific secrets for the chosen environment
+  validate    Validate locally, verifying the existence and type of environment variables against all env.yaml files
+
+EXAMPLES
+  ${keyExamples}
+`;
+
+function run(cmdObj: CLICommandParser) {
+  if (cmdObj.help || cmdObj.args.length === 0) printUsageAndExit(usage);
+  const [command, ...rest] = cmdObj.args;
+  switch (command) {
+    case "validate": {
+      const envYamlFiles = globSync("**/env.yaml");
+
+      // We have to have a _validate so that we go through a CommandExecutor which injects env variables into the process
+      cmdObj
+        .executorFromEnv(
+          `devops env _validate ${envYamlFiles.join(" ")}`,
+          { quiet: false }
+        )
+        .exec();
+      break;
+    }
+
+    case "_validate": {
+      const options = cmdObj.parseOptions({ booleans: ["--skip-dotenv"] });
+      const [_subcmd, ...envYamlFiles] = options.args;
+      const envFiles = options.options["--skip-dotenv"]
+        ? []
+        : dotEnvFilesForEnv(cmdObj.env);
+      const validator = new CombinedEnvValidator(envYamlFiles, envFiles);
+      validator.validate();
+      break;
+    }
+
+    case "get": {
+      console.log(getMonorepoSecret(cmdObj.env, rest));
+      break;
+    }
+
+    case "set": {
+      setMonorepoSecret(cmdObj.env, rest);
+      break;
+    }
+
+    case "delete": {
+      deleteMonorepoSecret(cmdObj.env, rest);
+      break;
+    }
+
+    default: {
+      printUsageAndExit(usage);
+    }
+  }
+}
+
+export default {
+  env: { oneLiner, keyExamples, run },
+};

package/src/cli/exec.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+# The following:
+#   run.sh some-dir some-command arg1 args
+# Changes the current directory to some-dir and then runs the following command before returning to the original directory:
+#   some-command arg1 arg2
+#
+# We cannot use `cwd` in exec or spawn since dotenvx won't inject the env variables properly
+
+# Change to the provided directory
+pushd "$1" &> /dev/null || exit
+
+# Execute the rest of the arguments
+"${@:2}"
+
+EXIT_CODE=$?
+
+# Return to the previous directory
+popd &> /dev/null 
+
+exit $EXIT_CODE

package/src/cli/exec.ts

@@ -0,0 +1,57 @@
+import { CLICommandParser, CommandExecutor, printUsageAndExit } from "./common";
+import url from "url";
+import path from "path";
+import { getWorkspace } from "../libs/discovery";
+
+const __file__ = url.fileURLToPath(import.meta.url);
+const __src__ = path.join(path.dirname(__file__), "../..", "src");
+const execShPath = path.join(__src__, "cli/exec.sh");
+
+const oneLiner =
+  "Executes a command after injecting env variables, either globally or in a workspace";
+const keyExamples = `
+    $ devops exec tmp/test.sh
+    $ devops exec bun test.ts --in myworkspace --env staging
+`.trim();
+
+const usage = `
+${oneLiner}
+
+USAGE
+    devops exec <command>
+    devops exec --in <workspace> <command>
+    devops exec --in <workspace> <command> --interactive
+
+EXAMPLES
+    ${keyExamples}
+`;
+
+function run(cmdObj: CLICommandParser) {
+  if (cmdObj.help || cmdObj.args.length === 0) printUsageAndExit(usage);
+  const parsed = cmdObj.parseOptions({
+    params: ["--in"],
+    booleans: ["--interactive"],
+  });
+  const workspace = parsed.options["--in"] as string | undefined;
+  let executor: CommandExecutor;
+  if (workspace) {
+    const rootPath = getWorkspace(workspace).rootPath;
+    executor = cmdObj.executorFromEnv(
+      `${execShPath} ${rootPath} ${parsed.argsStr}`,
+      { checkEnvYaml: true }
+    );
+  } else {
+    executor = cmdObj.executorFromEnv(parsed.argsStr, { checkEnvYaml: true });
+  }
+
+  const interactive = Boolean(parsed.options["--interactive"]);
+  if (interactive) {
+    executor.spawn();
+  } else {
+    executor.exec();
+  }
+}
+
+export default {
+  exec: { oneLiner, keyExamples, run },
+};

package/src/cli/image.ts

@@ -0,0 +1,197 @@
+import { deleteImageVersion, getImageVersion, getWorkspaceScale, resetWorkspaceScale, setImageVersion, setWorkspaceScale } from "../libs/k8s-image-config";
+import { CLICommandParser, printUsageAndExit, StrongParams } from "../../src/cli/common";
+import { generateImageDeployments } from "../libs/k8s-generate";
+import { applyHandler } from "../libs/k8s-helpers";
+
+const oneLiner = "Applies image-related manifests, retrieves or set the version deployed, and scales deployments of applications";
+const keyExamples = `
+    $ devops image deployment gen    main-node sha  --env staging
+    $ devops image deployment create main-node sha  --env staging
+    $ devops image deployment delete main-node      --env staging
+    $ devops image version    get    main-node      --env staging
+    $ devops image version    set    main-node sha  --env staging
+    $ devops image version    unset  main-node      --env staging
+    $ devops image scale      get    main-node       --env staging
+    $ devops image scale      get    main-node www   --env staging
+    $ devops image scale      set    main-node www 3 --env staging
+    $ devops image scale      unset  main-node       --env staging
+    $ devops image scale      unset  main-node www   --env staging
+`.trim();
+
+const usage = `
+${oneLiner}
+
+GENERATING DEPLOYMENT MANIFESTS
+    devops image deployment gen|create|delete <image-name> <sha>
+
+    gen     - generates the manifest file with all applications that are deployed with the image
+    create  - generates the manifest file and then runs kubectl apply
+    delete  - generates the manifest file and then runs kubectl delete
+
+MANAGINE APPLICATION SCALE
+    devops image scale get|unset <image-name> [<workspace-name>]
+    devops image scale set <image-name> <workspace-name> <replica-count>
+
+    set    - sets the scale count in the image's config map so that it persists across deployments. 
+    get    - retrieves the current scale count. If workspace is not provided, all workspaces are returned.
+    unset  - resets the scale count by removing the key from the image's config map. If workspace is not provided, all workspaces are reset.
+
+    Both 'set' and 'unset' return the previous scale count prior to the operation.
+
+MANAGING IMAGE VERSIONS
+    devops image version get|unset <image> 
+    devops image version set <image> <sha>
+
+    Meant to be called during deployment in order to maintain a record of the most recent deployed version.
+
+EXAMPLES
+    ${keyExamples}
+`;
+
+const handlers = {
+  deployment: {
+    gen: (opts: StrongParams) => {
+      console.log(
+        generateImageDeployments(
+          opts.required("env"), 
+          opts.required("image"),
+          opts.required("sha")
+        )
+      )
+    },
+    create: (opts: StrongParams) => {
+      applyHandler(
+        'apply-deployment-', 
+        'apply', 
+        generateImageDeployments(
+          opts.required("env"), 
+          opts.required("image"),
+          opts.required("sha")
+        )
+      )
+    },
+    delete: (opts: StrongParams) => {
+      applyHandler(
+        'delete-deployment-', 
+        'delete', 
+        generateImageDeployments(
+          opts.required("env"), 
+          opts.required("image"),
+          "dummy-sha"
+        )
+      )
+    },
+  },
+  scale: {
+    set: (opts: StrongParams) => {
+      const workspace = opts.required("workspace");
+      const image = opts.required("image");
+      const replicas = Number(opts.required("replicas"));
+      const res = setWorkspaceScale(
+        opts.required("env"),
+        image,
+        workspace,
+        replicas
+      );
+      if (res) {
+        console.warn(
+          `Scale for ${workspace} in ${image} set to ${replicas}. Previous value:`
+        );
+        console.log(res);
+      }
+    },
+    get: (opts: StrongParams) => {
+      const workspace = opts.optional("workspace");
+      // Satiate the type checker
+      if (!workspace) {
+        console.log(
+          getWorkspaceScale(
+            opts.required("env"),
+            opts.required("image"),
+          )
+        )
+      } else {
+        console.log(
+          getWorkspaceScale(
+            opts.required("env"),
+            opts.required("image"),
+            workspace
+          )
+        )
+      }
+    },
+    unset: (opts: StrongParams) => {
+      const image = opts.required("image");
+      const workspace = opts.optional("workspace");
+  
+      const prev = resetWorkspaceScale(opts.required("env"), image, workspace);
+      if (workspace) {
+        console.warn(`Scale for ${workspace} in ${image} unset. Previous scale:`);
+        console.log(prev);
+      } else {
+        console.warn(
+          `Scale for all workspaces in ${image} unset. Previous scale:`
+        );
+        console.log(prev);
+      }
+    },
+  },
+  version: {
+    get: (opts: StrongParams) => {
+      const version = getImageVersion(
+        opts.required("env"),
+        opts.required("image")
+      )
+      console.log(version ?? "")
+    },
+    set: (opts: StrongParams) => {
+      setImageVersion(
+        opts.required("env"),
+        opts.required("image"),
+        opts.required("sha")
+      )
+    },
+    unset: (opts: StrongParams) => {
+      deleteImageVersion(
+        opts.required("env"),
+        opts.required("image")
+      )
+    },
+  },
+};
+
+function run(cmdObj: CLICommandParser) {
+  if (cmdObj.help || cmdObj.args.length < 1) printUsageAndExit(usage);
+
+  const [command, subcommand, image, param1, param2] = cmdObj.args;
+  const commandHandler = handlers[command as keyof typeof handlers];
+  if (!commandHandler) {
+    console.error(`Unknown command: ${command}`);
+    printUsageAndExit(usage);
+  }
+  const handler = commandHandler[subcommand as keyof typeof commandHandler] as (opts: StrongParams) => void;
+  if (!handler) {
+    console.error(`Unknown subcommand: ${subcommand}`);
+    printUsageAndExit(usage);
+  }
+
+  function getExtraParams() {
+    if (command === 'scale') {
+      return subcommand === 'set' ? { workspace: param1, replicas: param2 } : { workspace: param1 };
+    } else {
+      return { sha: param1 };
+    }
+  }
+
+  const params = new StrongParams(usage, {
+    env: cmdObj.env,
+    subcommand,
+    image,
+    ...getExtraParams()
+  });
+  handler(params);
+}
+
+export default {
+  image: { oneLiner, keyExamples, run },
+};

package/src/cli/init.ts

@@ -0,0 +1,75 @@
+import url from "url";
+import path from "path";
+import fs from "fs-extra";
+
+const __file__ = url.fileURLToPath(import.meta.url);
+const __root__ = path.join(path.dirname(__file__), "../..");
+const templatesDir = path.join(__root__, "src/target-templates");
+const targetDir = process.cwd(); // User's current working directory
+
+import { CLICommandParser, printUsageAndExit } from "./common";
+
+const oneLiner =
+  "Initializes the devops utility by copying template files to the current folder";
+const keyExamples = `$ devops init`;
+
+const usage = `
+${oneLiner}
+
+NOTE
+    No files are overwritten.
+
+EXAMPLES
+    ${keyExamples}
+`;
+
+async function run(cmdObj: CLICommandParser) {
+  if (cmdObj.help) printUsageAndExit(usage);
+  copyTemplates();
+}
+
+export default {
+  init: { oneLiner, keyExamples, run },
+};
+
+async function copyTemplates() {
+  console.log(`Initializing devops files from ${templatesDir} to ${targetDir}`);
+
+  try {
+    // Copy files without overriding existing ones
+    await fs.copy(templatesDir, targetDir, {
+      overwrite: false,
+      errorOnExist: false,
+      dereference: false,
+    });
+
+    console.log(successMessage);
+  } catch (error) {
+    console.error("❌ Failed to initialize devops files:", error);
+    process.exit(1);
+  }
+}
+
+const successMessage = `
+✅ Devops files initialized successfully!
+
+To finish the setup: 
+
+1. add the following entry to the main package.json:
+  "workspaces": [
+    "libs/**",
+    "applications/**",
+    "db/**",
+    "dml/**"
+  ],
+
+2. add the following to your .gitignore:
+.envrc
+**/.DS_Store
+**/.env*
+config/kubeconfig
+tmp/**
+!tmp/**/.gitkeep
+venv/
+**/__pycache__
+`;

package/src/cli/internal-curl.ts

@@ -0,0 +1,48 @@
+import { InternalToken } from "../app-support/crypto";
+import { CLICommandParser, printUsageAndExit } from "./common";
+
+const oneLiner =
+  "Runs a curl command bearing an internal jwt-like token to allow verifying internal communication within the namespace";
+const keyExamples = `$ devops internal-curl jobs http://service-name:port/path`;
+
+const usage = `
+${oneLiner}
+
+GENERAL USAGE
+    devops internal-curl <subject> [curl-options] <url>
+
+NOTE
+    --env should not be used with this command. It is expected to be run inside pods in the namespace.
+    Relies on the MONOREPO_BASE_SECRET environment variable for signing the token.
+    
+    'subject' is the subject of the token. Receiving endpoints should verify it matches the expected value using the 
+    'InternalToken' class exposed by @vaharoni/devops.
+
+EXAMPLES
+    ${keyExamples}
+`;
+
+async function run(cmdObj: CLICommandParser) {
+  if (cmdObj.help || cmdObj.args.length === 0 || cmdObj.envForced) printUsageAndExit(usage);
+  const [subject, ...rest] = cmdObj.args;
+
+  // A trick to allow running this command locally while setting the MONOREPO_BASE_SECRET in .env.global.
+  // Basically we have to inject the env variables into the shell, which is why we execute 'devops jwt' first
+  let token;
+  if (process.env.MONOREPO_BASE_SECRET) {
+    token = new InternalToken(subject).generate();
+  } else {
+    const res = cmdObj.executorFromEnv(`devops jwt ${subject}`).exec({ asObject: true });
+    if (res.statusCode !== 0) {
+      console.error("Failed to generate token. Aborting.");
+      process.exit(1);
+    }
+    token = res.stdout.trim();
+  }
+
+  cmdObj.executorFromEnv(`curl -H "Authorization: Bearer ${token}" ${rest.join(" ")}`).exec();
+}
+
+export default {
+  'internal-curl': { oneLiner, keyExamples, run },
+};

package/src/cli/job.ts

@@ -0,0 +1,80 @@
+import {
+} from "../libs/k8s-image-config";
+import { CLICommandParser, printUsageAndExit, StrongParams } from "../../src/cli/common";
+import { generateDbMigrateJob } from "../libs/k8s-generate";
+import { applyHandler } from "../libs/k8s-helpers";
+import { k8sJobWaiter, printJobStatuses } from "../libs/k8s-job-waiter";
+import { dbMigrateJobName } from "../libs/k8s-constants";
+
+const oneLiner = "Creates a k8s job and waits for it to run";
+const keyExamples = `
+    $ devops job db-migrate gen    main-node <sha> --env staging
+    $ devops job db-migrate create main-node <sha> --env staging --timeout 120
+`.trim();
+
+const usage = `
+${oneLiner}
+
+GENERAL USAGE
+    devops job db-migrate gen    <image> <sha>
+    devops job db-migrate create <image> <sha> --timeout <timeout>
+
+EXAMPLES
+    ${keyExamples}
+`;
+
+const handlers = {
+  gen: (opts: StrongParams) => {
+    console.log(
+      generateDbMigrateJob(opts.required("env"), opts.required("image"), opts.required("sha"))
+    )
+  },
+  create: async (opts: StrongParams) => {
+    const env = opts.required("env");
+    const image = opts.required("image");
+    const sha = opts.required("sha");
+    const timeout = opts.optional("timeout") ?? '240';
+    const manifest = generateDbMigrateJob(env, image, sha);
+    const jobName = dbMigrateJobName(sha);
+
+    applyHandler(`apply-${jobName}`, 'apply', manifest);
+    const statuses = await k8sJobWaiter(env, Number(timeout ?? '240'), [jobName]);
+    if (statuses && Object.keys(statuses).length > 0) {
+      printJobStatuses(statuses);
+    }
+  },
+};
+
+function run(cmdObj: CLICommandParser) {
+  if (cmdObj.help || cmdObj.args.length < 1) printUsageAndExit(usage);
+
+  const parsedArgs = cmdObj.parseOptions({
+    params: ["--timeout"]
+  });
+
+  const [jobName, command, image, sha] = parsedArgs.args;
+
+  if (jobName !== 'db-migrate') {
+    console.error(`Unknown job: ${jobName}. Only db-migrate is supported at this time.`);
+    process.exit(1)
+  }
+
+  const handler = handlers[command as keyof typeof handlers];
+  if (!handler) {
+    console.error(`Unknown command: ${command}`);
+    printUsageAndExit(usage);
+  }
+
+  const timeout = parsedArgs.options["--timeout"] as string | undefined;
+  const params = new StrongParams(usage, {
+    env: cmdObj.env,
+    image,
+    sha,
+    timeout
+  });
+  handler(params);
+}
+
+export default {
+  job: { oneLiner, keyExamples, run },
+};

package/src/cli/jwt.ts

@@ -0,0 +1,32 @@
+import { InternalToken } from "../app-support/crypto";
+import { CLICommandParser, printUsageAndExit } from "./common";
+
+const oneLiner =
+  "Returns a JWT-like token to allow verifying internal communication within the namespace";
+const keyExamples = `$ devops jwt jobs`;
+
+const usage = `
+${oneLiner}
+
+GENERAL USAGE
+    devops jwt <subject>
+
+NOTE
+    The token is valid for 60 seconds and bears the specified subject.
+    --env should not be used with this command. It is expected to be run inside pods in the namespace.
+    Relies on the MONOREPO_BASE_SECRET environment variable for signing the token.
+
+EXAMPLES
+    ${keyExamples}
+`;
+
+async function run(cmdObj: CLICommandParser) {
+  if (cmdObj.help || cmdObj.args.length === 0 || cmdObj.envForced) printUsageAndExit(usage);
+  const subject = cmdObj.args[0];
+
+  console.log(new InternalToken(subject).generate());
+}
+
+export default {
+  jwt: { oneLiner, keyExamples, run },
+};

package/src/cli/namespace.ts

@@ -0,0 +1,78 @@
+import {
+} from "../libs/k8s-image-config";
+import { CLICommandParser, printUsageAndExit, StrongParams } from "../../src/cli/common";
+import {
+  copySecretHarborToNamespace,
+  patchServiceAccountImagePullSecret,
+} from "../../src/libs/hetzner/reg-secret";
+import { checkEnvSetup, createEmptyEnvSecret, createNamespace, deleteNamespace, patchBaseSecret } from "../libs/k8s-namespace";
+
+const oneLiner = "Creates the basic prerequisites for a monorepo";
+const keyExamples = `
+    $ devops namespace create --env staging
+    $ devops namespace delete --env staging
+    $ devops namespace check  --env staging
+`.trim();
+
+const usage = `
+${oneLiner}
+
+GENERAL USAGE
+    devops namespcae create|delete|check --env <env>
+
+    'create' does the following:
+      1. Creates the namepace
+      2. Creates a secret to hold environment variables (used by devops env) and the base cryptographic secret 
+      3. On Hetzner, copies the Harbor secret to the namespace and patches the default service account to use it
+
+    'delete' removes the namespace in kubernetes, which deletes all entities within it.
+
+    'check' returns exit code 0 if the namespace exists in kubernetes, 1 otherwise.
+
+EXAMPLES
+    ${keyExamples}
+`;
+
+const handlers = {
+  create (opts: StrongParams) {
+    const env = opts.required("env");
+    createNamespace(env);
+    createEmptyEnvSecret(env);
+    patchBaseSecret(env);
+    copySecretHarborToNamespace(env);
+    patchServiceAccountImagePullSecret(env);
+  },
+  delete (opts: StrongParams) {
+    deleteNamespace(opts.required("env"));
+  },
+  check (opts: StrongParams) {
+    const exists = checkEnvSetup(opts.required("env"));
+    if (exists) return;
+    console.error(`
+      The environment does not exist in the cluster.
+      In order to create resources for it in the cluster, it must be first set up. This is done to protect from unintentional resource creation.
+      To set up the environment, run the following from your dev machine:
+      $ devops namespace create --env <env>
+    `);
+    process.exit(1);
+  },
+};
+
+function run(cmdObj: CLICommandParser) {
+  if (cmdObj.help || cmdObj.args.length < 1) printUsageAndExit(usage);
+
+  const [command] = cmdObj.args;
+  const handler = handlers[command as keyof typeof handlers];
+  if (!handler) {
+    console.error(`Unknown command: ${command}`);
+    printUsageAndExit(usage);
+  }
+  const params = new StrongParams(usage, {
+    env: cmdObj.env,
+  });
+  handler(params);
+}
+
+export default {
+  namespace: { oneLiner, keyExamples, run },
+};