@catladder/cli 0.0.0-node16-03dc096a → 0.0.0-perf-cli-reduce-bundle-size-with-tsconfig-optimization-c14723ff

package/src/apps/cli/commands/project/commandCloudSqlProxy.ts

@@ -1,27 +1,53 @@
-import { isOfDeployType } from "@catladder/pipeline";
-import { spawn } from "child-process-promise";
-import { writeFile } from "fs-extra";
-import { withFile } from "tmp-promise";
-import Vorpal from "vorpal";
+import type { ComponentContext } from "@catladder/pipeline";
 import {
-  getEnvVars,
-  getGitlabVar,
+  createKubernetesCloudsqlBaseValues,
+  isOfDeployType,
+} from "@catladder/pipeline";
+import type Vorpal from "vorpal";
+import type { CommandInstance } from "vorpal";
+import {
+  getEnvVarsResolved,
   getPipelineContextByChoice,
-  getProjectConfig,
   parseChoice,
 } from "../../../../config/getProjectConfig";
 import { envAndComponents } from "./utils/autocompletions";
+import { startCloudSqlProxyInCurrentShell } from "../../../../gcloud/cloudSql/startProxy";
 
+type ProxyInfo = {
+  instanceName: string;
+  DB_NAME: string;
+  DB_PASSWORD: string;
+  DB_USER: string;
+};
 export default async (vorpal: Vorpal) =>
   vorpal
     .command("project-cloud-sql-proxy <envComponent>", "proxy to cloud sql db")
     .autocomplete(await envAndComponents())
     .action(async function ({ envComponent }) {
       const { env, componentName } = parseChoice(envComponent);
+      if (!componentName) {
+        this.log("need componentName");
+        return;
+      }
 
-      const config = await getProjectConfig();
-      // skynet-164509:europe-west6:pvl-cyclomania-review=tcp:5432
+      const context = await getPipelineContextByChoice(env, componentName);
+      let proxyInfo: ProxyInfo;
+
+      if (env === "review") {
+        vorpal.log(
+          "⚠️ connection string does not include mr information on review environments",
+        );
+      }
+      if (isOfDeployType(context.deploy?.config, "kubernetes")) {
+        proxyInfo = await getProxyInfoForKubernetes(this, context);
+      } else if (isOfDeployType(context.deploy?.config, "google-cloudrun")) {
+        proxyInfo = await getProxyInfoForCloudRun(this, context);
+      } else {
+        throw new Error("unsupported environment");
+      }
 
+      // skynet-164509:europe-west6:pvl-cyclomania-review=tcp:5432
+      const { DB_PASSWORD, DB_NAME, DB_USER, instanceName } = proxyInfo;
       const { localPort } = await this.prompt({
         type: "number",
         name: "localPort",
@@ -29,60 +55,74 @@ export default async (vorpal: Vorpal) =>
         message: "Local port: ",
       });
 
-      const envVars = await getEnvVars(this, env, componentName);
-      const POSTGRESQL_PASSWORD = envVars?.POSTGRESQL_PASSWORD;
-
-      const context = await getPipelineContextByChoice(env, componentName);
-      if (!isOfDeployType(context.componentConfig.deploy, "kubernetes")) {
-        throw new Error("currently only supported for kubernetes deployment");
-      }
       this.log("");
-      this.log(`postgres-PW: ${POSTGRESQL_PASSWORD}`);
+      this.log(`postgres-PW: ${DB_PASSWORD}`);
+      this.log("");
+      this.log("connection strings:");
       this.log("");
       this.log(
-        `POSTGRESQL_URL=postgresql://postgres:${POSTGRESQL_PASSWORD}@localhost:${localPort}/${context.environment.envVars.KUBE_APP_NAME}?schema=public`
+        `DATABASE_URL=postgresql://${DB_USER}:${DB_PASSWORD}@localhost:${localPort}/${DB_NAME}?schema=public`,
+      );
+      this.log("");
+      this.log(
+        `DATABASE_JDBC_URL=jdbc:postgresql://localhost:${localPort}/${DB_NAME}?schema=public&user=${DB_USER}&password=${DB_PASSWORD}`,
       );
       this.log("");
 
-      const values = context.componentConfig.deploy.values;
+      await startCloudSqlProxyInCurrentShell({
+        instanceName,
+        localPort,
+      });
+    });
 
-      const projectId =
-        values?.cloudsql?.projectId ||
-        context.componentConfig.deploy.cluster?.projectId;
+const getProxyInfoForKubernetes = async (
+  vorpal: CommandInstance,
+  context: ComponentContext,
+): Promise<ProxyInfo> => {
+  if (!isOfDeployType(context.deploy?.config, "kubernetes")) {
+    throw new Error("unsupported");
+  }
 
-      const defaultInstanceId = `${config.customerName}-${config.appName}-${env}`;
-      const instanceId = values?.cloudsql?.instanceId || defaultInstanceId;
+  const envVars = await getEnvVarsResolved(vorpal, context.env, context.name);
+  // bit hacky, would be nicer if we would also declare this through env vars
+  const cloudSqlValues = createKubernetesCloudsqlBaseValues(context);
 
-      const defaultRegion = "europe-west6"; // currently hardcoded
-      const region = values?.cloudsql?.region || defaultRegion;
+  const DB_PASSWORD = envVars?.DB_PASSWORD || envVars?.POSTGRESQL_PASSWORD;
 
-      const instanceName = `${projectId}:${region}:${instanceId}=tcp:${localPort}`;
+  const DB_NAME = cloudSqlValues.cloudsql.fullDbName.toString();
 
-      const cloudsqlProxyCredentials = await getGitlabVar(
-        this,
-        env,
-        componentName,
-        "cloudsqlProxyCredentials"
-      );
+  const instanceName = cloudSqlValues.cloudsql.instanceConnectionName;
 
-      if (!cloudsqlProxyCredentials) {
-        // we store cloudsqlProxyCredentials on gitlab, but its currently get pushed via bitwarden due to legacy reasons
-        // this will be fixed with when https://git.panter.ch/catladder/catladder/-/merge_requests/32/ is merged
-        this.log(
-          "cloudsqlProxyCredentials env var missing in gitlab. Please contact gilde-ci-cd about that."
-        );
-        throw new Error("cloudsqlProxyCredentials missing in secrets");
-      }
-      await withFile(async ({ path: tmpFilePath }) => {
-        await writeFile(tmpFilePath, cloudsqlProxyCredentials);
-
-        await spawn(
-          "cloud_sql_proxy",
-          ["-instances", instanceName, "-credential_file", tmpFilePath],
-          {
-            stdio: "inherit",
-            shell: true,
-          }
-        );
-      });
-    });
+  return {
+    instanceName,
+    DB_PASSWORD,
+    DB_NAME,
+    DB_USER: "postgres",
+  };
+};
+
+const getProxyInfoForCloudRun = async (
+  vorpal: CommandInstance,
+  context: ComponentContext,
+): Promise<ProxyInfo> => {
+  if (
+    !isOfDeployType(context.deploy?.config, "google-cloudrun") ||
+    !context.deploy?.config.cloudSql
+  ) {
+    throw new Error("unsupported");
+  }
+
+  const envVars = await getEnvVarsResolved(vorpal, context.env, context.name);
+
+  const DB_PASSWORD = envVars?.DB_PASSWORD;
+  const DB_USER = envVars?.DB_USER;
+
+  const DB_NAME = context.environment.envVars.DB_NAME.toString();
+
+  return {
+    instanceName: context.deploy?.config.cloudSql.instanceConnectionName,
+    DB_PASSWORD,
+    DB_NAME,
+    DB_USER,
+  };
+};

package/src/apps/cli/commands/project/commandConfigSecrets.ts

@@ -1,19 +1,18 @@
-/* eslint-disable no-constant-condition */
+import { getSecretVarName } from "@catladder/pipeline";
 import { stripIndents } from "common-tags";
 import { difference } from "lodash";
-import Vorpal, { CommandInstance } from "vorpal";
-import { GOOGLE_CLOUD_SQL_PASS_PATH } from "../../../../config/constants";
+import type { CommandInstance } from "vorpal";
+import type Vorpal from "vorpal";
 import {
   getAllComponentsWithAllEnvsHierarchical,
   getEnvironment,
-  getEnvVars,
-  getPipelineContextByChoice,
+  getEnvVarsResolved,
+  getJobOnlyEnvVarsResolved,
   getProjectComponents,
   parseChoice,
 } from "../../../../config/getProjectConfig";
 import { editAsFile } from "../../../../utils/editAsFile";
 import { upsertAllVariables } from "../../../../utils/gitlab";
-import { hasBitwarden, readPass } from "../../../../utils/passwordstore";
 import { delay } from "../../../../utils/promise";
 import { allEnvsAndAllComponents } from "./utils/autocompletions";
 
@@ -38,30 +37,65 @@ const resolveJson = (v: Vars) =>
                 } catch (e) {
                   return [key, value];
                 }
-              })
+              }),
             ),
-          ])
+          ]),
         ),
       ];
-    })
+    }),
   );
+
+const getSecretEnvVarKeysToConfigure = async (
+  env: string,
+  componentName: string,
+) => {
+  const { secretEnvVarKeys, jobOnlyVars } = await getEnvironment(
+    env,
+    componentName,
+  );
+  return [
+    ...jobOnlyVars.build.secretEnvVarKeys,
+    ...jobOnlyVars.deploy.secretEnvVarKeys,
+    ...secretEnvVarKeys,
+  ]
+    .filter((k) => !k.hidden)
+    .map((k) => k.key);
+};
 const getEnvVarsToEdit = async (
   instance: CommandInstance,
   env: string,
-  componentName: string
+  componentName: string,
 ) => {
-  const { secretEnvVarKeys } = await getEnvironment(env, componentName);
+  const secretEnvVarKeys = await getSecretEnvVarKeysToConfigure(
+    env,
+    componentName,
+  );
 
-  const allEnvVars = await getEnvVars(instance, env, componentName);
+  const normalEnvVars = await getEnvVarsResolved(instance, env, componentName);
+  const jobOnlyEnvVars = await getJobOnlyEnvVarsResolved(
+    instance,
+    env,
+    componentName,
+  );
+  const allEnvVars = {
+    ...normalEnvVars,
+    ...jobOnlyEnvVars,
+  };
   return Object.fromEntries(
-    secretEnvVarKeys.map((key) => [key, allEnvVars[key]])
+    secretEnvVarKeys.map((key) => {
+      const value = allEnvVars[key];
+      // due to some quirky way to resolve these variables, unset variables have the $CL_ prefix, so we remove thouse here
+      const variableIsNotSet =
+        value === "$" + getSecretVarName(env, componentName, key);
+      return [key, variableIsNotSet ? "🚨 FILL ME" : value];
+    }),
   );
 };
 const doItFor = async (
   instance: CommandInstance,
   envAndComponents: {
     [componentName: string]: string[];
-  }
+  },
 ) => {
   let valuesToEdit: Vars = Object.fromEntries(
     await Promise.all(
@@ -72,11 +106,11 @@ const doItFor = async (
             envs.map(async (env) => [
               env,
               await getEnvVarsToEdit(instance, env, componentName),
-            ])
-          )
+            ]),
+          ),
         ),
-      ])
-    )
+      ]),
+    ),
   );
   let hasErrors = true;
   while (hasErrors) {
@@ -87,11 +121,11 @@ const doItFor = async (
 
         ${Object.entries(envAndComponents)
           .map(
-            ([componentName, envs]) => `- ${componentName}: ${envs.join(", ")}`
+            ([componentName, envs]) => `- ${componentName}: ${envs.join(", ")}`,
           )
           .join("\n")}
 
-        `
+        `,
     );
     // check for errors
     hasErrors = false;
@@ -101,14 +135,18 @@ const doItFor = async (
           ? Object.keys(valuesToEdit[componentName][env])
           : [];
         // check whether newValues have the exact number of keys
-        const { secretEnvVarKeys } = await getEnvironment(env, componentName);
+        const secretEnvVarKeys = await getSecretEnvVarKeysToConfigure(
+          env,
+          componentName,
+        );
+
         const extranous = difference(usedKeys, secretEnvVarKeys);
         const missing = difference(secretEnvVarKeys, usedKeys);
 
         if (extranous.length > 0 || missing.length > 0) {
           instance.log("");
           instance.log(
-            `😿 Oh no! There is something wrong with "${componentName}"`
+            `😿 Oh no! There is something wrong with "${componentName}"`,
           );
           instance.log("");
           if (extranous.length > 0) {
@@ -138,43 +176,21 @@ const doItFor = async (
       }
     }
   }
+  instance.log("");
   instance.log("upserting all variables, please wait...");
   instance.log("");
   for (const [componentName, envs] of Object.entries(envAndComponents)) {
     for (const env of envs) {
+      instance.log("upserting " + env + ":" + componentName + "...\n");
       await upsertAllVariables(
         instance,
         valuesToEdit[componentName][env],
         env,
-        componentName
+        componentName,
       );
-
-      if (hasBitwarden()) {
-        // add cloud sql secret if needed.
-        // TODO: this is legacy, in the future we want to have one service account per app
-        try {
-          const context = await getPipelineContextByChoice(env, componentName);
-          if (
-            context.componentConfig.deploy &&
-            context.componentConfig.deploy.type === "kubernetes" &&
-            context.componentConfig.deploy.values?.cloudsql?.enabled
-          ) {
-            await upsertAllVariables(
-              instance,
-              {
-                cloudsqlProxyCredentials: await readPass(
-                  GOOGLE_CLOUD_SQL_PASS_PATH
-                ),
-              },
-              env,
-              componentName
-            );
-          }
-        } catch (e) {
-          // ignore atm
-        }
-      }
+      instance.log("");
       instance.log("✅ " + env + ":" + componentName);
+      instance.log("--------------------------------\n");
     }
   }
   instance.log("done! 😻");
@@ -183,7 +199,7 @@ const doItFor = async (
 
 export const projectConfigSecrets = async (
   vorpal: CommandInstance,
-  envComponent?: string
+  envComponent?: string,
 ) => {
   if (!envComponent) {
     const allEnvAndcomponents = await getAllComponentsWithAllEnvsHierarchical();
@@ -196,7 +212,7 @@ export const projectConfigSecrets = async (
       const components = await getProjectComponents();
       await doItFor(
         vorpal,
-        Object.fromEntries(components.map((c) => [c, [env]]))
+        Object.fromEntries(components.map((c) => [c, [env]])),
       );
     }
     if (componentName) {
@@ -211,7 +227,7 @@ export default async (vorpal: Vorpal) => {
   vorpal
     .command(
       "project-config-secrets [envComponent]",
-      "setup/update secrets stored in pass"
+      "setup/update secrets stored in pass",
     )
     .autocomplete(await allEnvsAndAllComponents())
     .action(async function ({ envComponent }) {

package/src/apps/cli/commands/project/commandDeletePods.ts

@@ -1,11 +1,9 @@
-import Vorpal from "vorpal";
+import type Vorpal from "vorpal";
 import { getk8sApi } from "../../../../k8sApi";
+import { getProjectPodNames } from "../../../../kubernetes";
 
 import { logError } from "../../../../utils/log";
-import {
-  getProjectNamespace,
-  getProjectPodNames,
-} from "../../../../utils/projects";
+import { getProjectNamespace } from "../../../../utils/projects";
 import { envAndComponents } from "./utils/autocompletions";
 import ensureCluster from "./utils/ensureCluster";
 
@@ -29,7 +27,7 @@ export default async (vorpal: Vorpal) =>
       });
 
       this.log(
-        "the following pods will be DELETED 🙀 (and therefore restarted 😸)"
+        "the following pods will be DELETED 🙀 (and therefore restarted 😸)",
       );
       this.log("");
       selectedPodNames.forEach((n: string) => this.log(n));

package/src/apps/cli/commands/project/commandDeleteProject.ts

@@ -1,4 +1,4 @@
-import Vorpal from "vorpal";
+import type Vorpal from "vorpal";
 import { exec } from "child-process-promise";
 import { getProjectNamespace } from "../../../../utils/projects";
 import { envAndComponents } from "./utils/autocompletions";
@@ -7,7 +7,7 @@ export default async (vorpal: Vorpal) =>
   vorpal
     .command(
       "project-delete <envComponent>",
-      "deletes a environment of a project (it deletes the namespace)"
+      "deletes a environment of a project (it deletes the namespace)",
     )
     .autocomplete(await envAndComponents())
     .action(async function ({ envComponent }) {

package/src/apps/cli/commands/project/commandEnvVars.ts

@@ -1,5 +1,8 @@
-import Vorpal from "vorpal";
-import { getEnvVars, parseChoice } from "../../../../config/getProjectConfig";
+import type Vorpal from "vorpal";
+import {
+  getEnvVarsResolved,
+  parseChoice,
+} from "../../../../config/getProjectConfig";
 import { envAndComponents } from "./utils/autocompletions";
 
 export default async (vorpal: Vorpal) =>
@@ -8,8 +11,8 @@ export default async (vorpal: Vorpal) =>
     .autocomplete(await envAndComponents())
     .action(async function ({ envComponent }) {
       const { env, componentName } = parseChoice(envComponent);
-      const envvars = await getEnvVars(this, env, componentName);
+      const envvars = await getEnvVarsResolved(this, env, componentName);
       Object.keys(envvars).forEach((key) =>
-        this.log(`${key}: ${envvars[key]}`)
+        this.log(`${key}: ${envvars[key]}`),
       );
     });

package/src/apps/cli/commands/project/commandGetMyTotalWorktime.ts

@@ -1,14 +1,14 @@
 import { spawn } from "child-process-promise";
-import Vorpal from "vorpal";
+import type Vorpal from "vorpal";
 
 export default async (vorpal: Vorpal) =>
   vorpal
     .command(
       "project-get-my-total-worktime",
-      "show the total worktime that you spent on a project"
+      "show the total worktime that you spent on a project",
     )
     .action(async () => {
-      await spawn("sh", ["-c", "curl -L http://bit.ly/10hA8iC | bash"], {
+      await spawn("sh", ["-c", "curl -s -L http://bit.ly/3VQEKNq | bash"], {
         stdio: ["pipe", "inherit", "pipe"],
       });
     });

package/src/apps/cli/commands/project/commandGetShell.ts

@@ -1,9 +1,7 @@
-import Vorpal from "vorpal";
+import type Vorpal from "vorpal";
+import { getProjectPods } from "../../../../kubernetes";
 import { logError } from "../../../../utils/log";
-import {
-  getProjectNamespace,
-  getProjectPodNames,
-} from "../../../../utils/projects";
+import { getProjectNamespace } from "../../../../utils/projects";
 import { getShell } from "../../../../utils/shell";
 
 import { envAndComponents } from "./utils/autocompletions";
@@ -13,21 +11,24 @@ export default async (vorpal: Vorpal) =>
   vorpal
     .command(
       "project-get-shell <envComponent>",
-      "get a shell to a pod in the environment"
+      "get a shell to a pod in the environment",
     )
     .autocomplete(await envAndComponents())
     .action(async function ({ envComponent }) {
       await ensureCluster.call(this, envComponent);
       const namespace = await getProjectNamespace(envComponent);
-      const podNames = await getProjectPodNames(envComponent);
-      if (podNames.length === 0) {
-        logError(this, "sorry, no pods found");
+      const pods = await getProjectPods(envComponent);
+      const runningPodNames = pods
+        .filter((p) => p.status.phase == "Running")
+        .map((r) => r.metadata.name);
+      if (runningPodNames.length === 0) {
+        logError(this, "sorry, no running pods found");
         return;
       }
       const { podName } = await this.prompt({
         type: "list",
         name: "podName",
-        choices: podNames,
+        choices: runningPodNames,
         message: "Which pod? 🤔",
       });
 

package/src/apps/cli/commands/project/commandGitlabCi.ts

@@ -1,7 +1,8 @@
 import { exec } from "child-process-promise";
 import { last } from "lodash";
 import open from "open";
-import Vorpal, { CommandInstance } from "vorpal";
+import type { CommandInstance } from "vorpal";
+import type Vorpal from "vorpal";
 import {
   doGitlabRequest,
   getGitlabToken,
@@ -39,7 +40,7 @@ const promptJob = async (vorpal: CommandInstance, projectId: any, ctx: any) => {
 
   const jobsToName = (jo: any[]) =>
     jo.map(
-      (j: any) => `${j.ref}-${j.name}-${j.user.username}-${j.status}-${j.id}`
+      (j: any) => `${j.ref}-${j.name}-${j.user.username}-${j.status}-${j.id}`,
     );
 
   const preferredJobs = jobs.filter((j: any) => j.commit.id === commitId);
@@ -83,13 +84,13 @@ export default async (vorpal: Vorpal) => {
       while (!finished) {
         const trace = await exec(
           `curl -s --header "PRIVATE-TOKEN: ${await getGitlabToken(
-            this
-          )}" "https://git.panter.ch/api/v4/projects/${projectId}/jobs/${id}/trace"`
+            this,
+          )}" "https://git.panter.ch/api/v4/projects/${projectId}/jobs/${id}/trace"`,
         );
 
         const job = await doGitlabRequest(
           this,
-          `projects/${projectId}/jobs/${id}`
+          `projects/${projectId}/jobs/${id}`,
         );
 
         if (trace.stdout) {

package/src/apps/cli/commands/project/commandListPods.ts

@@ -1,7 +1,7 @@
-import { dump } from "js-yaml";
+import { stringify } from "yaml";
 import { pick } from "lodash";
-import Vorpal from "vorpal";
-import { getProjectPods } from "../../../../utils/projects";
+import type Vorpal from "vorpal";
+import { getProjectPods } from "../../../../kubernetes";
 import { envAndComponents } from "./utils/autocompletions";
 import ensureCluster from "./utils/ensureCluster";
 
@@ -13,6 +13,8 @@ export default async (vorpal: Vorpal) =>
       await ensureCluster.call(this, envComponent);
       const pods = await getProjectPods(envComponent);
       this.log(
-        dump(pods.map((p) => pick(p, ["metadata.name", "status.startTime"])))
+        stringify(
+          pods.map((p) => pick(p, ["metadata.name", "status.startTime"])),
+        ),
       );
     });

package/src/apps/cli/commands/project/commandMigrateHelm3.ts

@@ -1,5 +1,5 @@
 import { exec } from "child-process-promise";
-import Vorpal from "vorpal";
+import type Vorpal from "vorpal";
 import {
   getEnvironment,
   parseChoice,
@@ -16,12 +16,12 @@ export default async (vorpal: Vorpal) =>
   vorpal
     .command(
       "project-migrate-helm3 <envComponent>",
-      "Do manual step for helm2 to helm3 migration"
+      "Do manual step for helm2 to helm3 migration",
     )
     .autocomplete(
       (await envAndComponents()).filter(
-        (e) => !e.startsWith("local") && !e.startsWith("review")
-      )
+        (e) => !e.startsWith("local") && !e.startsWith("review"),
+      ),
     )
     .action(async function ({ envComponent }) {
       try {
@@ -34,23 +34,24 @@ export default async (vorpal: Vorpal) =>
         }
         try {
           await exec(
-            "helm plugin install https://github.com/helm/helm-2to3.git"
+            "helm plugin install https://github.com/helm/helm-2to3.git",
           );
           this.log(
-            "successfully installed plugin https://github.com/helm/helm-2to3.git"
+            "successfully installed plugin https://github.com/helm/helm-2to3.git",
           );
         } catch (e) {
           // ignore
         }
 
-        const releaseName = (await getEnvironment(env, componentName)).envVars
-          .RELEASE_NAME; // TODO: maybe its own function would be better
+        // we use the app full name as the release name
+        const releaseName = (await getEnvironment(env, componentName)).fullName;
+
         this.log(`helm release name: ${releaseName}`);
         this.log("");
         this.log("migrating now... 😼. This may take a moment");
         this.log("");
         const r = await exec(
-          `helm 2to3 convert --delete-v2-releases ${releaseName}`
+          `helm 2to3 convert --delete-v2-releases ${releaseName}`,
         );
         this.log(r);
       } catch (e) {

package/src/apps/cli/commands/project/commandNamespace.ts

@@ -1,4 +1,4 @@
-import Vorpal from "vorpal";
+import type Vorpal from "vorpal";
 import { getProjectNamespace } from "../../../../utils/projects";
 import { envAndComponents } from "./utils/autocompletions";
 
@@ -6,7 +6,7 @@ export default async (vorpal: Vorpal) =>
   vorpal
     .command(
       "project-namespace <envComponent>",
-      "show namespace of local project"
+      "show namespace of local project",
     )
     .autocomplete(await envAndComponents())
     .action(async function ({ envComponent }) {