@catladder/cli 0.0.0-test7 → 0.0.0-waitfor-deploy-e64a33ba

package/src/apps/{shell → cli}/commands/general/index.ts

@@ -1,12 +1,7 @@
 import memoizee from "memoizee";
 import Vorpal from "vorpal";
-import k8sApi from "../../../../k8sApi";
-import {
-  connectToCluster,
-  getAllClusterNames,
-  getCurrentConnectedClusterName,
-  getCurrentContext,
-} from "../../../../utils/cluster";
+import { getk8sApi } from "../../../../k8sApi";
+import { getCurrentContext } from "../../../../utils/cluster";
 import { logError } from "../../../../utils/log";
 import { syncBitwarden } from "../../../../utils/passwordstore";
 import {
@@ -14,16 +9,12 @@ import {
   stopPortForward,
 } from "../../../../utils/portForward";
 import { getShell } from "../../../../utils/shell";
-import { getGoogleAuthUserNumber } from "../../utils/getGoogleAuthUserNumber";
-import {
-  openGoogleCloudKubernetesDashboard,
-  openGoogleCloudLogs,
-} from "../shared";
 import { namespaceAutoCompletion } from "./namespaceAutoCompletion";
 import portForward from "./portForward";
 
 const getAllNamespaces = memoizee(
   async () => {
+    const k8sApi = getk8sApi();
     const res = await k8sApi.listNamespace();
     return res.body.items;
   },
@@ -34,14 +25,7 @@ export const getAllNamespacesNames = async () => {
   const namespaces = await getAllNamespaces();
   return namespaces.map((n) => n.metadata.name);
 };
-export default (vorpal: Vorpal) => {
-  vorpal
-    .command("connect-cluster <clustername>")
-    .autocomplete(getAllClusterNames())
-    .action(async function ({ clustername }) {
-      this.log(`connecting to ${clustername}`);
-      await connectToCluster(clustername);
-    });
+export default async (vorpal: Vorpal) => {
   vorpal.command("current-context").action(async function () {
     this.log(await getCurrentContext());
   });
@@ -57,6 +41,7 @@ export default (vorpal: Vorpal) => {
     .command("list-secrets <namespace>", "show secrets")
     .autocomplete(namespaceAutoCompletion)
     .action(async function ({ namespace }) {
+      const k8sApi = getk8sApi();
       const res = await k8sApi.listNamespacedSecret(namespace);
 
       this.log(res.body.items.map((n) => n.metadata.name).join("\n"));
@@ -71,6 +56,7 @@ export default (vorpal: Vorpal) => {
     .command("list-pods <namespace>", "list all pods of namespace")
     .autocomplete(namespaceAutoCompletion)
     .action(async function ({ namespace }) {
+      const k8sApi = getk8sApi();
       const res = await k8sApi.listNamespacedPod(namespace);
       this.log(res.body.items.map((n) => n.metadata.name).join("\n"));
     });
@@ -82,36 +68,11 @@ export default (vorpal: Vorpal) => {
       stopPortForward(name.trim());
     });
 
-  vorpal
-    .command("open-dashboard <namespace>", "open kubernetes dashboard")
-    .autocomplete(namespaceAutoCompletion)
-    .action(async function ({ namespace }) {
-      const clustername = await getCurrentConnectedClusterName();
-      const authGoogleNumber = await getGoogleAuthUserNumber.call(this, vorpal);
-
-      await openGoogleCloudKubernetesDashboard(
-        authGoogleNumber,
-        clustername,
-        namespace
-      );
-    });
-  vorpal
-    .command(
-      "open-logs <namespace>",
-      "open google cloud logs (stackdriver stuff)"
-    )
-    .autocomplete(namespaceAutoCompletion)
-    .action(async function ({ namespace }) {
-      const clustername = await getCurrentConnectedClusterName();
-      const authGoogleNumber = await getGoogleAuthUserNumber.call(this, vorpal);
-
-      await openGoogleCloudLogs(authGoogleNumber, clustername, namespace);
-    });
-
   vorpal
     .command("get-shell <namespace>", "get a shell to a pod in the environment")
     .autocomplete(namespaceAutoCompletion)
     .action(async function ({ namespace }) {
+      const k8sApi = getk8sApi();
       const res = await k8sApi.listNamespacedPod(namespace);
       if (res.body.items.length === 0) {
         logError(this, "sorry, no pods found");

package/src/apps/{shell → cli}/commands/general/portForward.ts

@@ -3,14 +3,15 @@ import { logError } from "../../../../utils/log";
 
 import { startPortForward } from "../../../../utils/portForward";
 
-import k8sApi from "../../../../k8sApi";
+import { getk8sApi } from "../../../../k8sApi";
 import { namespaceAutoCompletion } from "./namespaceAutoCompletion";
 
-export default (vorpal: Vorpal) =>
+export default async (vorpal: Vorpal) =>
   vorpal
     .command("port-forward <namespace>", "start port-forwarding")
     .autocomplete(namespaceAutoCompletion)
     .action(async function ({ namespace }) {
+      const k8sApi = getk8sApi();
       const res = await k8sApi.listNamespacedPod(namespace);
       if (res.body.items.length === 0) {
         logError(this, "sorry, no pods found");

package/src/apps/{shell → cli}/commands/mongodb/index.ts

@@ -3,7 +3,7 @@ import projectMongoDestroyMember from "./projectMongoDestroyMember";
 import projectMongoGetShell from "./projectMongoGetShell";
 import projectMongoPortForward from "./projectMongoPortForward";
 
-export default (vorpal: Vorpal) => {
+export default async (vorpal: Vorpal) => {
   projectMongoGetShell(vorpal);
   projectMongoPortForward(vorpal);
   projectMongoDestroyMember(vorpal);

package/src/apps/{shell → cli}/commands/mongodb/projectMongoDestroyMember.ts

@@ -3,9 +3,9 @@ import Vorpal from "vorpal";
 import { logError } from "../../../../utils/log";
 import {
   getProjectNamespace,
-  getProjectPvcs
+  getProjectPvcs,
 } from "../../../../utils/projects";
-import { envAutocompletion } from "../project/utils/autocompletions";
+import { envAndComponents } from "../project/utils/autocompletions";
 import ensureCluster from "../project/utils/ensureCluster";
 import { getMongoDbPodsWithReplInfo } from "./utils";
 
@@ -31,18 +31,18 @@ const removeFinalizerAndDelete = async (
 ) => {
   return Promise.all([
     removeFinalizer(namespace, type, name),
-    await deleteResource(namespace, type, name)
+    await deleteResource(namespace, type, name),
   ]);
 };
-export default (vorpal: Vorpal) =>
+export default async (vorpal: Vorpal) =>
   vorpal
     .command(
-      "project-mongo-destroy-member <env>",
+      "project-mongo-destroy-member <envComponent>",
       "DESTROY a member of a replicaset in order to reinitialize it"
     )
-    .autocomplete(envAutocompletion)
-    .action(async function({ env }) {
-      await ensureCluster.call(this);
+    .autocomplete(await envAndComponents())
+    .action(async function ({ envComponent }) {
+      await ensureCluster.call(this, envComponent);
       this.log(
         "this command tries to delete a (secondary) member of the replicaset, it's persistent volume claim (pvc) and the volume"
       );
@@ -67,18 +67,18 @@ export default (vorpal: Vorpal) =>
         default: true,
         message: "DO YOU UNDERSTAND?",
         name: "understood",
-        type: "confirm"
+        type: "confirm",
       });
 
       if (!understood) {
         throw new Error("abort");
       }
 
-      const namespace = await getProjectNamespace(env);
-      const mongodbPods = await getMongoDbPodsWithReplInfo(env);
+      const namespace = await getProjectNamespace(envComponent);
+      const mongodbPods = await getMongoDbPodsWithReplInfo(envComponent);
 
-      const pvcs = await getProjectPvcs(env);
-      const secondaries = mongodbPods.filter(pod => !pod.isMaster);
+      const pvcs = await getProjectPvcs(envComponent);
+      const secondaries = mongodbPods.filter((pod) => !pod.isMaster);
 
       if (secondaries.length === 0) {
         logError(this, "sorry, no pods found");
@@ -89,16 +89,16 @@ export default (vorpal: Vorpal) =>
         await this.prompt({
           type: "list",
           name: "podName",
-          choices: secondaries.map(p => ({
+          choices: secondaries.map((p) => ({
             name: `[ secondary ] ${p.podName}`,
-            value: p.podName
+            value: p.podName,
           })),
-          message: "Which pod? 🤔"
+          message: "Which pod? 🤔",
         })
       ).podName;
 
       const thePvc = pvcs.find(
-        pvc => pvc.metadata.name === `datadir-${podName}`
+        (pvc) => pvc.metadata.name === `datadir-${podName}`
       );
       if (!thePvc) {
         logError(this, `sorry, no pvc found for ${podName}`);
@@ -109,7 +109,7 @@ export default (vorpal: Vorpal) =>
         message:
           "THIS WILL DESTROY THE POD, ITS VOLUME AND ALL ITS DATA 🙀🙀🙀!!! continue? 🤔",
         name: "shouldContinue",
-        type: "confirm"
+        type: "confirm",
       });
 
       if (!shouldContinue) {

package/src/apps/{shell → cli}/commands/mongodb/projectMongoGetShell.ts

@@ -1,24 +1,26 @@
 import Vorpal from "vorpal";
 import { logError } from "../../../../utils/log";
 import { getProjectNamespace } from "../../../../utils/projects";
-import { envAutocompletion } from "../project/utils/autocompletions";
+import { envAndComponents } from "../project/utils/autocompletions";
 import ensureCluster from "../project/utils/ensureCluster";
 import {
   getMongodbShell,
-  getProjectMongodbAllPodsSortedWithLabel
+  getProjectMongodbAllPodsSortedWithLabel,
 } from "./utils";
 
-export default (vorpal: Vorpal) =>
+export default async (vorpal: Vorpal) =>
   vorpal
     .command(
-      "project-mongo-get-shell <env>",
+      "project-mongo-get-shell <envComponent>",
       "get a shell to a mongodb in the environment"
     )
-    .autocomplete(envAutocompletion)
-    .action(async function({ env }) {
-      await ensureCluster.call(this);
-      const namespace = await getProjectNamespace(env);
-      const podNames = await getProjectMongodbAllPodsSortedWithLabel(env);
+    .autocomplete(await envAndComponents())
+    .action(async function ({ envComponent }) {
+      await ensureCluster.call(this, envComponent);
+      const namespace = await getProjectNamespace(envComponent);
+      const podNames = await getProjectMongodbAllPodsSortedWithLabel(
+        envComponent
+      );
       if (podNames.length === 0) {
         logError(this, "sorry, no pods found");
         return;
@@ -32,7 +34,7 @@ export default (vorpal: Vorpal) =>
             type: "list",
             name: "podName",
             choices: podNames,
-            message: "Which pod? 🤔"
+            message: "Which pod? 🤔",
           })
         ).podName;
       }

package/src/apps/{shell → cli}/commands/mongodb/projectMongoPortForward.ts

@@ -2,18 +2,23 @@ import Vorpal from "vorpal";
 import { logError } from "../../../../utils/log";
 import { startPortForward } from "../../../../utils/portForward";
 import { getProjectNamespace } from "../../../../utils/projects";
-import { envAutocompletion } from "../project/utils/autocompletions";
+import { envAndComponents } from "../project/utils/autocompletions";
 import ensureCluster from "../project/utils/ensureCluster";
 import { getProjectMongodbAllPodsSortedWithLabel } from "./utils";
 
-export default (vorpal: Vorpal) =>
+export default async (vorpal: Vorpal) =>
   vorpal
-    .command("project-mongo-port-forward <env>", "port foward to a mongodb")
-    .autocomplete(envAutocompletion)
-    .action(async function ({ env }) {
-      await ensureCluster.call(this);
-      const namespace = await getProjectNamespace(env);
-      const podNames = await getProjectMongodbAllPodsSortedWithLabel(env);
+    .command(
+      "project-mongo-port-forward <envComponent>",
+      "port foward to a mongodb"
+    )
+    .autocomplete(await envAndComponents())
+    .action(async function ({ envComponent }) {
+      await ensureCluster.call(this, envComponent);
+      const namespace = await getProjectNamespace(envComponent);
+      const podNames = await getProjectMongodbAllPodsSortedWithLabel(
+        envComponent
+      );
       if (podNames.length === 0) {
         logError(this, "sorry, no pods found");
         return;

package/src/apps/{shell → cli}/commands/mongodb/utils/index.ts

@@ -1,5 +1,4 @@
 import { exec, spawn } from "child-process-promise";
-import { Env } from "../../../../../types/types";
 import {
   getProjectNamespace,
   getProjectPodNames,
@@ -8,8 +7,8 @@ import {
 const filterMongoDbs = (podNames: string[]) =>
   podNames.filter((name) => name.includes("mongodb-replicaset"));
 
-export const getProjectMongodbAllPods = async (env: Env) =>
-  filterMongoDbs(await getProjectPodNames(env));
+export const getProjectMongodbAllPods = async (envComponent: string) =>
+  filterMongoDbs(await getProjectPodNames(envComponent));
 
 export const getMongodbShell = async (namespace: string, podName: string) => {
   const command = `kubectl exec -it ${podName} --namespace ${namespace} mongo`;
@@ -55,12 +54,12 @@ export const podIsMaster = async (namespace: string, podName: string) => {
 
 const spaces = (n: number) => " ".repeat(n);
 
-export const getMongoDbPodsWithReplInfo = async (env: Env) => {
-  const namespace = await getProjectNamespace(env);
+export const getMongoDbPodsWithReplInfo = async (envComponent: string) => {
+  const namespace = await getProjectNamespace(envComponent);
   return (
     await Promise.all(
       (
-        await getProjectMongodbAllPods(env)
+        await getProjectMongodbAllPods(envComponent)
       ).map(async (podName) => ({
         podName,
         componentName: podName.replace(/-mongodb-replicaset-[0-9]+/, ""),
@@ -70,8 +69,10 @@ export const getMongoDbPodsWithReplInfo = async (env: Env) => {
   ).sort((podA, podB) => (podA.isMaster ? (podB.isMaster ? 0 : -1) : 1));
 };
 
-export const getProjectMongodbAllPodsSortedWithLabel = async (env: Env) => {
-  const pods = await getMongoDbPodsWithReplInfo(env);
+export const getProjectMongodbAllPodsSortedWithLabel = async (
+  envComponent: string
+) => {
+  const pods = await getMongoDbPodsWithReplInfo(envComponent);
   const maxComponentNameLength = Math.max(
     ...pods.map((c) => c.componentName.length)
   );

package/src/apps/cli/commands/project/commandCloudSqlProxy.ts

@@ -0,0 +1,88 @@
+import { isOfDeployType } from "@catladder/pipeline";
+import { spawn } from "child-process-promise";
+import { writeFile } from "fs-extra";
+import { withFile } from "tmp-promise";
+import Vorpal from "vorpal";
+import {
+  getEnvVars,
+  getGitlabVar,
+  getPipelineContextByChoice,
+  getProjectConfig,
+  parseChoice,
+} from "../../../../config/getProjectConfig";
+import { envAndComponents } from "./utils/autocompletions";
+
+export default async (vorpal: Vorpal) =>
+  vorpal
+    .command("project-cloud-sql-proxy <envComponent>", "proxy to cloud sql db")
+    .autocomplete(await envAndComponents())
+    .action(async function ({ envComponent }) {
+      const { env, componentName } = parseChoice(envComponent);
+
+      const config = await getProjectConfig();
+      // skynet-164509:europe-west6:pvl-cyclomania-review=tcp:5432
+
+      const { localPort } = await this.prompt({
+        type: "number",
+        name: "localPort",
+        default: "54320",
+        message: "Local port: ",
+      });
+
+      const envVars = await getEnvVars(this, env, componentName);
+      const POSTGRESQL_PASSWORD = envVars?.POSTGRESQL_PASSWORD;
+
+      const context = await getPipelineContextByChoice(env, componentName);
+      if (!isOfDeployType(context.componentConfig.deploy, "kubernetes")) {
+        throw new Error("currently only supported for kubernetes deployment");
+      }
+      this.log("");
+      this.log(`postgres-PW: ${POSTGRESQL_PASSWORD}`);
+      this.log("");
+      this.log(
+        `POSTGRESQL_URL=postgresql://postgres:${POSTGRESQL_PASSWORD}@localhost:${localPort}/${context.environment.envVars.KUBE_APP_NAME}?schema=public`
+      );
+      this.log("");
+
+      const values = context.componentConfig.deploy.values;
+
+      const projectId =
+        values?.cloudsql?.projectId ||
+        context.componentConfig.deploy.cluster?.projectId;
+
+      const defaultInstanceId = `${config.customerName}-${config.appName}-${env}`;
+      const instanceId = values?.cloudsql?.instanceId || defaultInstanceId;
+
+      const defaultRegion = "europe-west6"; // currently hardcoded
+      const region = values?.cloudsql?.region || defaultRegion;
+
+      const instanceName = `${projectId}:${region}:${instanceId}=tcp:${localPort}`;
+
+      const cloudsqlProxyCredentials = await getGitlabVar(
+        this,
+        env,
+        componentName,
+        "cloudsqlProxyCredentials"
+      );
+
+      if (!cloudsqlProxyCredentials) {
+        // we store cloudsqlProxyCredentials on gitlab, but its currently get pushed via bitwarden due to legacy reasons
+        // this will be fixed with when https://git.panter.ch/catladder/catladder/-/merge_requests/32/ is merged
+        this.log(
+          "cloudsqlProxyCredentials env var missing in gitlab. Please contact gilde-ci-cd about that."
+        );
+        throw new Error("cloudsqlProxyCredentials missing in secrets");
+      }
+      await withFile(async ({ path: tmpFilePath }) => {
+        await writeFile(tmpFilePath, cloudsqlProxyCredentials);
+
+        await spawn(
+          "cloud_sql_proxy",
+          ["-instances", instanceName, "-credential_file", tmpFilePath],
+          {
+            stdio: "inherit",
+            shell: true,
+          }
+        );
+      });
+    });

package/src/apps/cli/commands/project/commandConfigSecrets.ts

@@ -0,0 +1,220 @@
+/* eslint-disable no-constant-condition */
+import { stripIndents } from "common-tags";
+import { difference } from "lodash";
+import Vorpal, { CommandInstance } from "vorpal";
+import { GOOGLE_CLOUD_SQL_PASS_PATH } from "../../../../config/constants";
+import {
+  getAllComponentsWithAllEnvsHierarchical,
+  getEnvironment,
+  getEnvVars,
+  getPipelineContextByChoice,
+  getProjectComponents,
+  parseChoice,
+} from "../../../../config/getProjectConfig";
+import { editAsFile } from "../../../../utils/editAsFile";
+import { upsertAllVariables } from "../../../../utils/gitlab";
+import { hasBitwarden, readPass } from "../../../../utils/passwordstore";
+import { delay } from "../../../../utils/promise";
+import { allEnvsAndAllComponents } from "./utils/autocompletions";
+
+type Vars = {
+  [env: string]: {
+    [componentName: string]: Record<string, string>;
+  };
+};
+/* for convenience, parse json objects. that makes it easier to edit secrets that are object */
+const resolveJson = (v: Vars) =>
+  Object.fromEntries(
+    Object.entries(v).map(([componentName, envs]) => {
+      return [
+        componentName,
+        Object.fromEntries(
+          Object.entries(envs).map(([env, secrets]) => [
+            env,
+            Object.fromEntries(
+              Object.entries(secrets).map(([key, value]) => {
+                try {
+                  return [key, JSON.parse(value)];
+                } catch (e) {
+                  return [key, value];
+                }
+              })
+            ),
+          ])
+        ),
+      ];
+    })
+  );
+const getEnvVarsToEdit = async (
+  instance: CommandInstance,
+  env: string,
+  componentName: string
+) => {
+  const { secretEnvVarKeys } = await getEnvironment(env, componentName);
+
+  const allEnvVars = await getEnvVars(instance, env, componentName);
+  return Object.fromEntries(
+    secretEnvVarKeys.map((key) => [key, allEnvVars[key]])
+  );
+};
+const doItFor = async (
+  instance: CommandInstance,
+  envAndComponents: {
+    [componentName: string]: string[];
+  }
+) => {
+  let valuesToEdit: Vars = Object.fromEntries(
+    await Promise.all(
+      Object.entries(envAndComponents).map(async ([componentName, envs]) => [
+        componentName,
+        Object.fromEntries(
+          await Promise.all(
+            envs.map(async (env) => [
+              env,
+              await getEnvVarsToEdit(instance, env, componentName),
+            ])
+          )
+        ),
+      ])
+    )
+  );
+  let hasErrors = true;
+  while (hasErrors) {
+    valuesToEdit = await editAsFile(
+      resolveJson(valuesToEdit),
+      stripIndents`
+        Please fill in all secrets for: 
+
+        ${Object.entries(envAndComponents)
+          .map(
+            ([componentName, envs]) => `- ${componentName}: ${envs.join(", ")}`
+          )
+          .join("\n")}
+
+        `
+    );
+    // check for errors
+    hasErrors = false;
+    for (const [componentName, envs] of Object.entries(envAndComponents)) {
+      for (const env of envs) {
+        const usedKeys = valuesToEdit[componentName][env]
+          ? Object.keys(valuesToEdit[componentName][env])
+          : [];
+        // check whether newValues have the exact number of keys
+        const { secretEnvVarKeys } = await getEnvironment(env, componentName);
+        const extranous = difference(usedKeys, secretEnvVarKeys);
+        const missing = difference(secretEnvVarKeys, usedKeys);
+
+        if (extranous.length > 0 || missing.length > 0) {
+          instance.log("");
+          instance.log(
+            `😿 Oh no! There is something wrong with "${componentName}"`
+          );
+          instance.log("");
+          if (extranous.length > 0) {
+            instance.log("these secrets are not declared in the config");
+            extranous.forEach((key) => instance.log(key));
+            instance.log("");
+          }
+          if (missing.length > 0) {
+            instance.log("these secrets have not been provided:");
+            missing.forEach((key) => instance.log(key));
+            instance.log("");
+          }
+
+          await delay(1000);
+          const { shouldContinue } = await instance.prompt({
+            default: true,
+            message: "Try again? 🤔",
+            name: "shouldContinue",
+            type: "confirm",
+          });
+
+          if (!shouldContinue) {
+            throw new Error("abort");
+          }
+          hasErrors = true;
+        }
+      }
+    }
+  }
+  instance.log("upserting all variables, please wait...");
+  instance.log("");
+  for (const [componentName, envs] of Object.entries(envAndComponents)) {
+    for (const env of envs) {
+      await upsertAllVariables(
+        instance,
+        valuesToEdit[componentName][env],
+        env,
+        componentName
+      );
+
+      if (hasBitwarden()) {
+        // add cloud sql secret if needed.
+        // TODO: this is legacy, in the future we want to have one service account per app
+        try {
+          const context = await getPipelineContextByChoice(env, componentName);
+          if (
+            context.componentConfig.deploy &&
+            context.componentConfig.deploy.type === "kubernetes" &&
+            context.componentConfig.deploy.values?.cloudsql?.enabled
+          ) {
+            await upsertAllVariables(
+              instance,
+              {
+                cloudsqlProxyCredentials: await readPass(
+                  GOOGLE_CLOUD_SQL_PASS_PATH
+                ),
+              },
+              env,
+              componentName
+            );
+          }
+        } catch (e) {
+          // ignore atm
+        }
+      }
+      instance.log("✅ " + env + ":" + componentName);
+    }
+  }
+  instance.log("done! 😻");
+  instance.log("");
+};
+
+export const projectConfigSecrets = async (
+  vorpal: CommandInstance,
+  envComponent?: string
+) => {
+  if (!envComponent) {
+    const allEnvAndcomponents = await getAllComponentsWithAllEnvsHierarchical();
+    await doItFor(vorpal, allEnvAndcomponents);
+  } else {
+    const { env, componentName } = parseChoice(envComponent);
+
+    // componentName can be null. in this case, iterate over all  components
+    if (!componentName) {
+      const components = await getProjectComponents();
+      await doItFor(
+        vorpal,
+        Object.fromEntries(components.map((c) => [c, [env]]))
+      );
+    }
+    if (componentName) {
+      await doItFor(vorpal, {
+        [componentName]: [env],
+      });
+    }
+  }
+};
+
+export default async (vorpal: Vorpal) => {
+  vorpal
+    .command(
+      "project-config-secrets [envComponent]",
+      "setup/update secrets stored in pass"
+    )
+    .autocomplete(await allEnvsAndAllComponents())
+    .action(async function ({ envComponent }) {
+      return await projectConfigSecrets(this, envComponent);
+    });
+};