@catladder/cli 0.0.0-vars-07a720d0 → 0.0.0-wait-for-completion-88d93ae7

package/src/apps/cli/commands/general/namespaceAutoCompletion.ts

@@ -3,5 +3,5 @@ export const namespaceAutoCompletion = {
   async data() {
     const namespaces = await getAllNamespacesNames();
     return namespaces;
-  }
+  },
 };

package/src/apps/cli/commands/general/portForward.ts

@@ -1,14 +1,14 @@
-import Vorpal from "vorpal";
+import type Vorpal from "vorpal";
 import { logError } from "../../../../utils/log";
 
-import { startPortForward } from "../../../../utils/portForward";
+import { startKubePortForward } from "../../../../kubernetes/portForward";
 
 import { getk8sApi } from "../../../../k8sApi";
 import { namespaceAutoCompletion } from "./namespaceAutoCompletion";
 
 export default async (vorpal: Vorpal) =>
   vorpal
-    .command("port-forward <namespace>", "start port-forwarding")
+    .command("kube-port-forward <namespace>", "start port-forwarding")
     .autocomplete(namespaceAutoCompletion)
     .action(async function ({ namespace }) {
       const k8sApi = getk8sApi();
@@ -44,5 +44,5 @@ export default async (vorpal: Vorpal) =>
         message: "Remote port: ",
       });
 
-      return startPortForward(podName, localPort, remotePort, namespace);
+      return startKubePortForward(podName, localPort, remotePort, namespace);
     });

package/src/apps/cli/commands/mongodb/index.ts

@@ -1,4 +1,4 @@
-import Vorpal from "vorpal";
+import type Vorpal from "vorpal";
 import projectMongoDestroyMember from "./projectMongoDestroyMember";
 import projectMongoGetShell from "./projectMongoGetShell";
 import projectMongoPortForward from "./projectMongoPortForward";

package/src/apps/cli/commands/mongodb/projectMongoDestroyMember.ts

@@ -1,10 +1,8 @@
 import { exec } from "child-process-promise";
-import Vorpal from "vorpal";
+import type Vorpal from "vorpal";
+import { getProjectPvcs } from "../../../../kubernetes";
 import { logError } from "../../../../utils/log";
-import {
-  getProjectNamespace,
-  getProjectPvcs,
-} from "../../../../utils/projects";
+import { getProjectNamespace } from "../../../../utils/projects";
 import { envAndComponents } from "../project/utils/autocompletions";
 import ensureCluster from "../project/utils/ensureCluster";
 import { getMongoDbPodsWithReplInfo } from "./utils";
@@ -12,22 +10,22 @@ import { getMongoDbPodsWithReplInfo } from "./utils";
 const removeFinalizer = async (
   namespace: string,
   type: "pod" | "pv" | "pvc",
-  name: string
+  name: string,
 ) =>
   exec(
-    `kubectl patch --namespace ${namespace} ${type} ${name} -p '{"metadata":{"finalizers":null}}'`
+    `kubectl patch --namespace ${namespace} ${type} ${name} -p '{"metadata":{"finalizers":null}}'`,
   );
 
 const deleteResource = async (
   namespace: string,
   type: "pod" | "pv" | "pvc",
-  name: string
+  name: string,
 ) => exec(`kubectl delete --namespace ${namespace} ${type} ${name}`);
 
 const removeFinalizerAndDelete = async (
   namespace: string,
   type: "pod" | "pv" | "pvc",
-  name: string
+  name: string,
 ) => {
   return Promise.all([
     removeFinalizer(namespace, type, name),
@@ -38,29 +36,29 @@ export default async (vorpal: Vorpal) =>
   vorpal
     .command(
       "project-mongo-destroy-member <envComponent>",
-      "DESTROY a member of a replicaset in order to reinitialize it"
+      "DESTROY a member of a replicaset in order to reinitialize it",
     )
     .autocomplete(await envAndComponents())
     .action(async function ({ envComponent }) {
       await ensureCluster.call(this, envComponent);
       this.log(
-        "this command tries to delete a (secondary) member of the replicaset, it's persistent volume claim (pvc) and the volume"
+        "this command tries to delete a (secondary) member of the replicaset, it's persistent volume claim (pvc) and the volume",
       );
       this.log("");
       this.log(
-        "this is useful, if you update the stateful set with new volume configuration (different size or storage class)"
+        "this is useful, if you update the stateful set with new volume configuration (different size or storage class)",
       );
       this.log("");
       this.log(
-        "Kubernetes will usually recreate the missing member with the updated config and mongodb will start synchronizing the new member."
+        "Kubernetes will usually recreate the missing member with the updated config and mongodb will start synchronizing the new member.",
       );
       this.log("");
       this.log(
-        "This works without downtime, but should be done when load on the db is low. Also it has not been tested with large dbs (> 10gi)"
+        "This works without downtime, but should be done when load on the db is low. Also it has not been tested with large dbs (> 10gi)",
       );
       this.log("");
       this.log(
-        "Deleting the volume and claim often stuck, just wait or cancel and restart."
+        "Deleting the volume and claim often stuck, just wait or cancel and restart.",
       );
       this.log("");
       const { understood } = await this.prompt({
@@ -98,7 +96,7 @@ export default async (vorpal: Vorpal) =>
       ).podName;
 
       const thePvc = pvcs.find(
-        (pvc) => pvc.metadata.name === `datadir-${podName}`
+        (pvc) => pvc.metadata.name === `datadir-${podName}`,
       );
       if (!thePvc) {
         logError(this, `sorry, no pvc found for ${podName}`);

package/src/apps/cli/commands/mongodb/projectMongoGetShell.ts

@@ -1,4 +1,4 @@
-import Vorpal from "vorpal";
+import type Vorpal from "vorpal";
 import { logError } from "../../../../utils/log";
 import { getProjectNamespace } from "../../../../utils/projects";
 import { envAndComponents } from "../project/utils/autocompletions";
@@ -12,15 +12,14 @@ export default async (vorpal: Vorpal) =>
   vorpal
     .command(
       "project-mongo-get-shell <envComponent>",
-      "get a shell to a mongodb in the environment"
+      "get a shell to a mongodb in the environment",
     )
     .autocomplete(await envAndComponents())
     .action(async function ({ envComponent }) {
       await ensureCluster.call(this, envComponent);
       const namespace = await getProjectNamespace(envComponent);
-      const podNames = await getProjectMongodbAllPodsSortedWithLabel(
-        envComponent
-      );
+      const podNames =
+        await getProjectMongodbAllPodsSortedWithLabel(envComponent);
       if (podNames.length === 0) {
         logError(this, "sorry, no pods found");
         return;

package/src/apps/cli/commands/mongodb/projectMongoPortForward.ts

@@ -1,24 +1,28 @@
-import Vorpal from "vorpal";
+import type Vorpal from "vorpal";
+import {
+  getEnvVarsResolved,
+  parseChoice,
+} from "../../../../config/getProjectConfig";
 import { logError } from "../../../../utils/log";
-import { startPortForward } from "../../../../utils/portForward";
+import { startKubePortForward } from "../../../../kubernetes/portForward";
 import { getProjectNamespace } from "../../../../utils/projects";
 import { envAndComponents } from "../project/utils/autocompletions";
 import ensureCluster from "../project/utils/ensureCluster";
 import { getProjectMongodbAllPodsSortedWithLabel } from "./utils";
+import clipboard from "clipboardy";
 
 export default async (vorpal: Vorpal) =>
   vorpal
     .command(
       "project-mongo-port-forward <envComponent>",
-      "port foward to a mongodb"
+      "port foward to a mongodb",
     )
     .autocomplete(await envAndComponents())
     .action(async function ({ envComponent }) {
       await ensureCluster.call(this, envComponent);
       const namespace = await getProjectNamespace(envComponent);
-      const podNames = await getProjectMongodbAllPodsSortedWithLabel(
-        envComponent
-      );
+      const podNames =
+        await getProjectMongodbAllPodsSortedWithLabel(envComponent);
       if (podNames.length === 0) {
         logError(this, "sorry, no pods found");
         return;
@@ -43,5 +47,18 @@ export default async (vorpal: Vorpal) =>
         default: "30000",
         message: "Local port: ",
       });
-      return startPortForward(podName, localPort, 27017, namespace);
+      const { env, componentName } = parseChoice(envComponent);
+      const envVars = await getEnvVarsResolved(this, env, componentName);
+      const MONGODB_ROOT_PASSWORD = envVars?.MONGODB_ROOT_PASSWORD;
+      const connectionUrl = `mongodb://root:${MONGODB_ROOT_PASSWORD}@localhost:${localPort}`;
+      clipboard.writeSync(connectionUrl);
+      this.log("");
+      this.log("username: root");
+      this.log(`password: ${MONGODB_ROOT_PASSWORD}`);
+      this.log(`connection string: ${connectionUrl}`);
+      this.log("");
+      this.log("👆 connection string has been copied to your clipboard!");
+      this.log("");
+
+      return startKubePortForward(podName, localPort, 27017, namespace);
     });

package/src/apps/cli/commands/mongodb/utils/index.ts

@@ -1,11 +1,14 @@
 import { exec, spawn } from "child-process-promise";
-import {
-  getProjectNamespace,
-  getProjectPodNames,
-} from "../../../../../utils/projects";
+import { getProjectPodNames } from "../../../../../kubernetes";
+import { getProjectNamespace } from "../../../../../utils/projects";
 
 const filterMongoDbs = (podNames: string[]) =>
-  podNames.filter((name) => name.includes("mongodb-replicaset"));
+  podNames.filter(
+    (name) =>
+      name.includes("mongodb") &&
+      !name.includes("mongodb-backup") &&
+      !name.includes("arbiter"),
+  );
 
 export const getProjectMongodbAllPods = async (envComponent: string) =>
   filterMongoDbs(await getProjectPodNames(envComponent));
@@ -29,7 +32,7 @@ export const getMongodbShell = async (namespace: string, podName: string) => {
 export const executeMongodbCommand = async (
   namespace: string,
   podName: string,
-  mongoCommand: string
+  mongoCommand: string,
 ) => {
   const fullCommand = `kubectl exec -it ${podName} --namespace ${namespace} -- mongo --quiet --eval "JSON.stringify(${mongoCommand})"`;
   const { stdout } = await exec(fullCommand, {
@@ -43,13 +46,18 @@ export const executeMongodbCommand = async (
 };
 
 export const podIsMaster = async (namespace: string, podName: string) => {
-  const result = await executeMongodbCommand(
-    namespace,
-    podName,
-    "db.isMaster()"
-  );
+  try {
+    const result = await executeMongodbCommand(
+      namespace,
+      podName,
+      "db.isMaster()",
+    );
 
-  return result.ismaster;
+    return result.ismaster;
+  } catch (e) {
+    // maybe shutting down ?
+    return null;
+  }
 };
 
 const spaces = (n: number) => " ".repeat(n);
@@ -58,28 +66,26 @@ export const getMongoDbPodsWithReplInfo = async (envComponent: string) => {
   const namespace = await getProjectNamespace(envComponent);
   return (
     await Promise.all(
-      (
-        await getProjectMongodbAllPods(envComponent)
-      ).map(async (podName) => ({
+      (await getProjectMongodbAllPods(envComponent)).map(async (podName) => ({
         podName,
         componentName: podName.replace(/-mongodb-replicaset-[0-9]+/, ""),
         isMaster: await podIsMaster(namespace, podName),
-      }))
+      })),
     )
   ).sort((podA, podB) => (podA.isMaster ? (podB.isMaster ? 0 : -1) : 1));
 };
 
 export const getProjectMongodbAllPodsSortedWithLabel = async (
-  envComponent: string
+  envComponent: string,
 ) => {
   const pods = await getMongoDbPodsWithReplInfo(envComponent);
   const maxComponentNameLength = Math.max(
-    ...pods.map((c) => c.componentName.length)
+    ...pods.map((c) => c.componentName.length),
   );
   return pods.map(({ podName, isMaster, componentName }) => ({
     value: podName,
     name: `[ ${componentName}${spaces(
-      maxComponentNameLength - componentName.length
+      maxComponentNameLength - componentName.length,
     )} ${isMaster ? "  PRIMARY  " : " secondary "}] ${podName}`,
   }));
 };

package/src/apps/cli/commands/project/cloudSql/commandProjectRestoreDb.ts

@@ -0,0 +1,181 @@
+import type Vorpal from "vorpal";
+import {
+  getEnvVarsResolved,
+  getPipelineContextByChoice,
+} from "../../../../../config/getProjectConfig";
+import type { CloudSqlBackgroundProxy } from "../../../../../gcloud/cloudSql/startProxy";
+import { startCloudSqlProxyInBackground } from "../../../../../gcloud/cloudSql/startProxy";
+import { envAndComponents } from "../utils/autocompletions";
+
+import { ConnectionStringParser } from "connection-string-parser";
+import { spawnCopyDb } from "../../../../../gcloud/cloudSql/copyDb";
+
+export default async (vorpal: Vorpal) =>
+  vorpal
+    .command(
+      "project-cloud-sql-restore-db",
+      "restores a project db from one source to another target",
+    )
+    .action(async function restoreDb() {
+      const envs = await envAndComponents();
+
+      const { sourceEnvAndComponent } = await this.prompt({
+        type: "list",
+        name: "sourceEnvAndComponent",
+        choices: envs,
+
+        message: "Source instance (connection string or 'local')? 🤔 ",
+      });
+
+      const [sourceEnv, sourceComponent] = sourceEnvAndComponent.split(":");
+
+      const sourceContext = await getPipelineContextByChoice(
+        sourceEnv,
+        sourceComponent,
+      );
+
+      const sourceEnvVars = await getEnvVarsResolved(
+        this,
+        sourceContext.env,
+        sourceContext.name,
+      );
+
+      let sourceProxy: CloudSqlBackgroundProxy;
+      let sourceDbName: string;
+
+      let sourceUsername: string;
+      let sourcePassword: string;
+
+      let targetUsername: string;
+      let targetPassword: string;
+      let targetProxy: CloudSqlBackgroundProxy;
+
+      let sourcePort: number;
+      let targetPort: number;
+      let targetDbName: string;
+
+      const closeAll = () => {
+        sourceProxy?.stop();
+        targetProxy?.stop();
+      };
+
+      if (sourceEnv === "local") {
+        const parser = new ConnectionStringParser({
+          scheme: "postgres",
+          hosts: [],
+        });
+
+        const parsersResult = parser.parse(
+          sourceEnvVars.DATABASE_URL.toString(),
+        );
+        sourcePort = parsersResult.hosts?.[0]?.port;
+        sourceUsername = parsersResult.username;
+        sourcePassword = parsersResult.password;
+        sourceDbName = parsersResult.endpoint;
+      } else {
+        sourcePort = 54399;
+        sourceProxy = await startCloudSqlProxyInBackground({
+          instanceName:
+            sourceEnvVars.CLOUD_SQL_INSTANCE_CONNECTION_NAME.toString(),
+          localPort: sourcePort,
+        });
+
+        sourceUsername = sourceEnvVars.DB_USER.toString();
+        sourcePassword = sourceEnvVars.DB_PASSWORD.toString();
+        sourceDbName = sourceEnvVars.DB_NAME?.toString();
+      }
+
+      const { targetEnvAndComponent } = await this.prompt({
+        type: "list",
+        name: "targetEnvAndComponent",
+        choices: envs,
+
+        message: "target env? 🤔 ",
+      });
+
+      const [targetEnv, targetComponent] = targetEnvAndComponent.split(":");
+
+      const targetContext = await getPipelineContextByChoice(
+        targetEnv,
+        targetComponent,
+      );
+
+      const targetEnvVars = await getEnvVarsResolved(
+        this,
+        targetContext.env,
+        targetContext.name,
+      );
+
+      if (targetEnv === "local") {
+        const parser = new ConnectionStringParser({
+          scheme: "postgres",
+          hosts: [],
+        });
+
+        const parsersResult = parser.parse(
+          targetEnvVars.DATABASE_URL.toString(),
+        );
+
+        targetPort = parsersResult.hosts?.[0]?.port;
+        targetUsername = parsersResult.username;
+        targetPassword = parsersResult.password;
+        targetDbName = parsersResult.endpoint;
+      } else {
+        targetPort = 54499;
+        targetProxy = await startCloudSqlProxyInBackground({
+          instanceName:
+            targetEnvVars.CLOUD_SQL_INSTANCE_CONNECTION_NAME.toString(),
+          localPort: targetPort,
+        });
+
+        targetUsername = targetEnvVars.DB_USER.toString();
+        targetPassword = targetEnvVars.DB_PASSWORD.toString();
+        targetDbName = targetEnvVars.DB_NAME.toString();
+      }
+
+      const { shouldContinue } = await this.prompt({
+        type: "confirm",
+        name: "shouldContinue",
+        message: `This will drop ${targetEnv}/${targetDbName} and replace it with ${sourceEnv}/${sourceDbName}. Continue? 🤔 `,
+      });
+
+      if (!shouldContinue) {
+        this.log("abort");
+        closeAll();
+        return;
+      }
+
+      if (targetContext.environment.envType === "prod") {
+        this.log(
+          `\n🚨 You are overriding a production environment. Please type in ${targetEnvVars.CLOUD_SQL_INSTANCE_CONNECTION_NAME} to continue\n\n`,
+        );
+        const { confirmInstance } = await this.prompt({
+          type: "input",
+          name: "confirmInstance",
+          message: "confirm: ",
+        });
+
+        if (
+          confirmInstance !== targetEnvVars.CLOUD_SQL_INSTANCE_CONNECTION_NAME
+        ) {
+          this.log("abort");
+          closeAll();
+          return;
+        }
+      }
+
+      try {
+        await spawnCopyDb({
+          targetPassword,
+          targetPort,
+          targetUsername,
+          sourceUsername,
+          sourcePassword,
+          sourcePort,
+          sourceDbName,
+          targetDbName,
+        });
+      } finally {
+        closeAll();
+      }
+    });

package/src/apps/cli/commands/project/commandCloudSqlProxy.ts

@@ -1,28 +1,53 @@
-import { isOfDeployType } from "@catladder/pipeline";
-import { spawn } from "child-process-promise";
-import { writeFile } from "fs-extra";
-import { withFile } from "tmp-promise";
-import Vorpal from "vorpal";
-import { GOOGLE_CLOUD_SQL_PASS_PATH } from "../../../../config/constants";
+import type { ComponentContext } from "@catladder/pipeline";
 import {
-  getEnvVars,
+  createKubernetesCloudsqlBaseValues,
+  isOfDeployType,
+} from "@catladder/pipeline";
+import type Vorpal from "vorpal";
+import type { CommandInstance } from "vorpal";
+import {
+  getEnvVarsResolved,
   getPipelineContextByChoice,
-  getProjectConfig,
   parseChoice,
 } from "../../../../config/getProjectConfig";
-import { readPass } from "../../../../utils/passwordstore";
 import { envAndComponents } from "./utils/autocompletions";
+import { startCloudSqlProxyInCurrentShell } from "../../../../gcloud/cloudSql/startProxy";
 
+type ProxyInfo = {
+  instanceName: string;
+  DB_NAME: string;
+  DB_PASSWORD: string;
+  DB_USER: string;
+};
 export default async (vorpal: Vorpal) =>
   vorpal
     .command("project-cloud-sql-proxy <envComponent>", "proxy to cloud sql db")
     .autocomplete(await envAndComponents())
     .action(async function ({ envComponent }) {
       const { env, componentName } = parseChoice(envComponent);
+      if (!componentName) {
+        this.log("need componentName");
+        return;
+      }
 
-      const config = await getProjectConfig();
-      // skynet-164509:europe-west6:pvl-cyclomania-review=tcp:5432
+      const context = await getPipelineContextByChoice(env, componentName);
+      let proxyInfo: ProxyInfo;
 
+      if (env === "review") {
+        vorpal.log(
+          "⚠️ connection string does not include mr information on review environments",
+        );
+      }
+      if (isOfDeployType(context.deploy?.config, "kubernetes")) {
+        proxyInfo = await getProxyInfoForKubernetes(this, context);
+      } else if (isOfDeployType(context.deploy?.config, "google-cloudrun")) {
+        proxyInfo = await getProxyInfoForCloudRun(this, context);
+      } else {
+        throw new Error("unsupported environment");
+      }
+
+      // skynet-164509:europe-west6:pvl-cyclomania-review=tcp:5432
+      const { DB_PASSWORD, DB_NAME, DB_USER, instanceName } = proxyInfo;
       const { localPort } = await this.prompt({
         type: "number",
         name: "localPort",
@@ -30,46 +55,76 @@ export default async (vorpal: Vorpal) =>
         message: "Local port: ",
       });
 
-      const POSTGRESQL_PASSWORD = (await getEnvVars(this, env, componentName))
-        ?.POSTGRESQL_PASSWORD;
-
-      const context = await getPipelineContextByChoice(env, componentName);
-      if (!isOfDeployType(context.componentConfig.deploy, "kubernetes")) {
-        throw new Error("currently only supported for kubernetes deployment");
-      }
       this.log("");
-      this.log(`postgres-PW: ${POSTGRESQL_PASSWORD}`);
+      this.log(`postgres-PW: ${DB_PASSWORD}`);
+      this.log("");
+      this.log("connection strings:");
       this.log("");
       this.log(
-        `POSTGRESQL_URL=postgresql://postgres:${POSTGRESQL_PASSWORD}@localhost:${localPort}/${context.environment.envVars.KUBE_APP_NAME}?schema=public`
+        `DATABASE_URL=postgresql://${DB_USER}:${DB_PASSWORD}@localhost:${localPort}/${DB_NAME}?schema=public`,
+      );
+      this.log("");
+      this.log(
+        `DATABASE_JDBC_URL=jdbc:postgresql://localhost:${localPort}/${DB_NAME}?schema=public&user=${DB_USER}&password=${DB_PASSWORD}`,
       );
       this.log("");
 
-      const values = context.componentConfig.deploy.values;
+      await startCloudSqlProxyInCurrentShell({
+        instanceName,
+        localPort,
+      });
+    });
 
-      const projectId =
-        values?.cloudsql?.projectId ||
-        context.componentConfig.deploy.cluster?.projectId;
+const getProxyInfoForKubernetes = async (
+  vorpal: CommandInstance,
+  context: ComponentContext,
+): Promise<ProxyInfo> => {
+  if (!isOfDeployType(context.deploy?.config, "kubernetes")) {
+    throw new Error("unsupported");
+  }
 
-      const defaultInstanceId = `${config.customerName}-${config.appName}-${env}`;
-      const instanceId = values?.cloudsql?.instanceId || defaultInstanceId;
+  const envVars = await getEnvVarsResolved(vorpal, context.env, context.name);
+  // bit hacky, would be nicer if we would also declare this through env vars
+  const cloudSqlValues = createKubernetesCloudsqlBaseValues(context);
 
-      const defaultRegion = "europe-west6"; // currently hardcoded
-      const region = values?.cloudsql?.region || defaultRegion;
+  const DB_PASSWORD = (
+    envVars?.DB_PASSWORD || envVars?.POSTGRESQL_PASSWORD
+  )?.toString();
 
-      const instanceName = `${projectId}:${region}:${instanceId}=tcp:${localPort}`;
+  const DB_NAME = cloudSqlValues.cloudsql.fullDbName.toString();
 
-      const cloudsqlCredentials = await readPass(GOOGLE_CLOUD_SQL_PASS_PATH);
-      await withFile(async ({ path: tmpFilePath }) => {
-        await writeFile(tmpFilePath, cloudsqlCredentials);
+  const instanceName = cloudSqlValues.cloudsql.instanceConnectionName;
 
-        await spawn(
-          "cloud_sql_proxy",
-          ["-instances", instanceName, "-credential_file", tmpFilePath],
-          {
-            stdio: "inherit",
-            shell: true,
-          }
-        );
-      });
-    });
+  return {
+    instanceName,
+    DB_PASSWORD,
+    DB_NAME,
+    DB_USER: "postgres",
+  };
+};
+
+const getProxyInfoForCloudRun = async (
+  vorpal: CommandInstance,
+  context: ComponentContext,
+): Promise<ProxyInfo> => {
+  if (
+    !isOfDeployType(context.deploy?.config, "google-cloudrun") ||
+    !context.deploy?.config.cloudSql
+  ) {
+    throw new Error("unsupported");
+  }
+
+  const envVars = await getEnvVarsResolved(vorpal, context.env, context.name);
+
+  const DB_PASSWORD = envVars?.DB_PASSWORD?.toString();
+  const DB_USER = envVars?.DB_USER?.toString();
+
+  const DB_NAME = context.environment.envVars.DB_NAME.toString();
+
+  return {
+    instanceName: context.deploy?.config.cloudSql.instanceConnectionName,
+    DB_PASSWORD,
+    DB_NAME,
+    DB_USER,
+  };
+};