@catladder/cli 0.0.0-vars-07a720d0 → 0.0.0-wait-for-completion-88d93ae7

package/src/apps/cli/commands/project/commandConfigSecrets.ts

@@ -1,19 +1,18 @@
-/* eslint-disable no-constant-condition */
+import { getSecretVarName } from "@catladder/pipeline";
 import { stripIndents } from "common-tags";
 import { difference } from "lodash";
-import Vorpal, { CommandInstance } from "vorpal";
-import { GOOGLE_CLOUD_SQL_PASS_PATH } from "../../../../config/constants";
+import type { CommandInstance } from "vorpal";
+import type Vorpal from "vorpal";
 import {
   getAllComponentsWithAllEnvsHierarchical,
   getEnvironment,
-  getEnvVars,
-  getPipelineContextByChoice,
+  getEnvVarsResolved,
+  getJobOnlyEnvVarsResolved,
   getProjectComponents,
   parseChoice,
 } from "../../../../config/getProjectConfig";
 import { editAsFile } from "../../../../utils/editAsFile";
 import { upsertAllVariables } from "../../../../utils/gitlab";
-import { hasBitwarden, readPass } from "../../../../utils/passwordstore";
 import { delay } from "../../../../utils/promise";
 import { allEnvsAndAllComponents } from "./utils/autocompletions";
 
@@ -38,30 +37,65 @@ const resolveJson = (v: Vars) =>
                 } catch (e) {
                   return [key, value];
                 }
-              })
+              }),
             ),
-          ])
+          ]),
         ),
       ];
-    })
+    }),
   );
+
+const getSecretEnvVarKeysToConfigure = async (
+  env: string,
+  componentName: string,
+) => {
+  const { secretEnvVarKeys, jobOnlyVars } = await getEnvironment(
+    env,
+    componentName,
+  );
+  return [
+    ...jobOnlyVars.build.secretEnvVarKeys,
+    ...jobOnlyVars.deploy.secretEnvVarKeys,
+    ...secretEnvVarKeys,
+  ]
+    .filter((k) => !k.hidden)
+    .map((k) => k.key);
+};
 const getEnvVarsToEdit = async (
   instance: CommandInstance,
   env: string,
-  componentName: string
+  componentName: string,
 ) => {
-  const { secretEnvVarKeys } = await getEnvironment(env, componentName);
+  const secretEnvVarKeys = await getSecretEnvVarKeysToConfigure(
+    env,
+    componentName,
+  );
 
-  const allEnvVars = await getEnvVars(instance, env, componentName);
+  const normalEnvVars = await getEnvVarsResolved(instance, env, componentName);
+  const jobOnlyEnvVars = await getJobOnlyEnvVarsResolved(
+    instance,
+    env,
+    componentName,
+  );
+  const allEnvVars = {
+    ...normalEnvVars,
+    ...jobOnlyEnvVars,
+  };
   return Object.fromEntries(
-    secretEnvVarKeys.map((key) => [key, allEnvVars[key]])
+    secretEnvVarKeys.map((key) => {
+      const value = allEnvVars[key];
+      // due to some quirky way to resolve these variables, unset variables have the $CL_ prefix, so we remove thouse here
+      const variableIsNotSet =
+        value === "$" + getSecretVarName(env, componentName, key);
+      return [key, variableIsNotSet ? "🚨 FILL ME" : value];
+    }),
   );
 };
 const doItFor = async (
   instance: CommandInstance,
   envAndComponents: {
     [componentName: string]: string[];
-  }
+  },
 ) => {
   let valuesToEdit: Vars = Object.fromEntries(
     await Promise.all(
@@ -72,11 +106,11 @@ const doItFor = async (
             envs.map(async (env) => [
               env,
               await getEnvVarsToEdit(instance, env, componentName),
-            ])
-          )
+            ]),
+          ),
         ),
-      ])
-    )
+      ]),
+    ),
   );
   let hasErrors = true;
   while (hasErrors) {
@@ -87,11 +121,11 @@ const doItFor = async (
 
         ${Object.entries(envAndComponents)
           .map(
-            ([componentName, envs]) => `- ${componentName}: ${envs.join(", ")}`
+            ([componentName, envs]) => `- ${componentName}: ${envs.join(", ")}`,
           )
           .join("\n")}
 
-        `
+        `,
     );
     // check for errors
     hasErrors = false;
@@ -101,14 +135,18 @@ const doItFor = async (
           ? Object.keys(valuesToEdit[componentName][env])
           : [];
         // check whether newValues have the exact number of keys
-        const { secretEnvVarKeys } = await getEnvironment(env, componentName);
+        const secretEnvVarKeys = await getSecretEnvVarKeysToConfigure(
+          env,
+          componentName,
+        );
+
         const extranous = difference(usedKeys, secretEnvVarKeys);
         const missing = difference(secretEnvVarKeys, usedKeys);
 
         if (extranous.length > 0 || missing.length > 0) {
           instance.log("");
           instance.log(
-            `😿 Oh no! There is something wrong with "${componentName}"`
+            `😿 Oh no! There is something wrong with "${componentName}"`,
           );
           instance.log("");
           if (extranous.length > 0) {
@@ -138,39 +176,21 @@ const doItFor = async (
       }
     }
   }
+  instance.log("");
   instance.log("upserting all variables, please wait...");
   instance.log("");
   for (const [componentName, envs] of Object.entries(envAndComponents)) {
     for (const env of envs) {
+      instance.log("upserting " + env + ":" + componentName + "...\n");
       await upsertAllVariables(
         instance,
         valuesToEdit[componentName][env],
         env,
-        componentName
+        componentName,
       );
-
-      if (hasBitwarden()) {
-        // add cloud sql secret if needed.
-        // TODO: this is legacy, in the future we want to have one service account per app
-
-        const context = await getPipelineContextByChoice(env, componentName);
-        if (
-          context.componentConfig.deploy &&
-          context.componentConfig.deploy.values?.cloudsql?.enabled
-        ) {
-          await upsertAllVariables(
-            instance,
-            {
-              cloudsqlProxyCredentials: await readPass(
-                GOOGLE_CLOUD_SQL_PASS_PATH
-              ),
-            },
-            env,
-            componentName
-          );
-        }
-      }
+      instance.log("");
       instance.log("✅ " + env + ":" + componentName);
+      instance.log("--------------------------------\n");
     }
   }
   instance.log("done! 😻");
@@ -179,7 +199,7 @@ const doItFor = async (
 
 export const projectConfigSecrets = async (
   vorpal: CommandInstance,
-  envComponent?: string
+  envComponent?: string,
 ) => {
   if (!envComponent) {
     const allEnvAndcomponents = await getAllComponentsWithAllEnvsHierarchical();
@@ -192,7 +212,7 @@ export const projectConfigSecrets = async (
       const components = await getProjectComponents();
       await doItFor(
         vorpal,
-        Object.fromEntries(components.map((c) => [c, [env]]))
+        Object.fromEntries(components.map((c) => [c, [env]])),
       );
     }
     if (componentName) {
@@ -207,7 +227,7 @@ export default async (vorpal: Vorpal) => {
   vorpal
     .command(
       "project-config-secrets [envComponent]",
-      "setup/update secrets stored in pass"
+      "setup/update secrets stored in pass",
     )
     .autocomplete(await allEnvsAndAllComponents())
     .action(async function ({ envComponent }) {

package/src/apps/cli/commands/project/commandDeletePods.ts

@@ -1,11 +1,9 @@
-import Vorpal from "vorpal";
+import type Vorpal from "vorpal";
 import { getk8sApi } from "../../../../k8sApi";
+import { getProjectPodNames } from "../../../../kubernetes";
 
 import { logError } from "../../../../utils/log";
-import {
-  getProjectNamespace,
-  getProjectPodNames,
-} from "../../../../utils/projects";
+import { getProjectNamespace } from "../../../../utils/projects";
 import { envAndComponents } from "./utils/autocompletions";
 import ensureCluster from "./utils/ensureCluster";
 
@@ -29,7 +27,7 @@ export default async (vorpal: Vorpal) =>
       });
 
       this.log(
-        "the following pods will be DELETED 🙀 (and therefore restarted 😸)"
+        "the following pods will be DELETED 🙀 (and therefore restarted 😸)",
       );
       this.log("");
       selectedPodNames.forEach((n: string) => this.log(n));

package/src/apps/cli/commands/project/commandDeleteProject.ts

@@ -1,4 +1,4 @@
-import Vorpal from "vorpal";
+import type Vorpal from "vorpal";
 import { exec } from "child-process-promise";
 import { getProjectNamespace } from "../../../../utils/projects";
 import { envAndComponents } from "./utils/autocompletions";
@@ -7,7 +7,7 @@ export default async (vorpal: Vorpal) =>
   vorpal
     .command(
       "project-delete <envComponent>",
-      "deletes a environment of a project (it deletes the namespace)"
+      "deletes a environment of a project (it deletes the namespace)",
     )
     .autocomplete(await envAndComponents())
     .action(async function ({ envComponent }) {

package/src/apps/cli/commands/project/commandEnvVars.ts

@@ -1,5 +1,8 @@
-import Vorpal from "vorpal";
-import { getEnvVars, parseChoice } from "../../../../config/getProjectConfig";
+import type Vorpal from "vorpal";
+import {
+  getEnvVarsResolved,
+  parseChoice,
+} from "../../../../config/getProjectConfig";
 import { envAndComponents } from "./utils/autocompletions";
 
 export default async (vorpal: Vorpal) =>
@@ -8,8 +11,8 @@ export default async (vorpal: Vorpal) =>
     .autocomplete(await envAndComponents())
     .action(async function ({ envComponent }) {
       const { env, componentName } = parseChoice(envComponent);
-      const envvars = await getEnvVars(this, env, componentName);
+      const envvars = await getEnvVarsResolved(this, env, componentName);
       Object.keys(envvars).forEach((key) =>
-        this.log(`${key}: ${envvars[key]}`)
+        this.log(`${key}: ${envvars[key]}`),
       );
     });

package/src/apps/cli/commands/project/commandGetMyTotalWorktime.ts

@@ -1,14 +1,14 @@
 import { spawn } from "child-process-promise";
-import Vorpal from "vorpal";
+import type Vorpal from "vorpal";
 
 export default async (vorpal: Vorpal) =>
   vorpal
     .command(
       "project-get-my-total-worktime",
-      "show the total worktime that you spent on a project"
+      "show the total worktime that you spent on a project",
     )
     .action(async () => {
-      await spawn("sh", ["-c", "curl -L http://bit.ly/10hA8iC | bash"], {
+      await spawn("sh", ["-c", "curl -s -L http://bit.ly/3VQEKNq | bash"], {
         stdio: ["pipe", "inherit", "pipe"],
       });
     });

package/src/apps/cli/commands/project/commandGetShell.ts

@@ -1,9 +1,7 @@
-import Vorpal from "vorpal";
+import type Vorpal from "vorpal";
+import { getProjectPods } from "../../../../kubernetes";
 import { logError } from "../../../../utils/log";
-import {
-  getProjectNamespace,
-  getProjectPodNames,
-} from "../../../../utils/projects";
+import { getProjectNamespace } from "../../../../utils/projects";
 import { getShell } from "../../../../utils/shell";
 
 import { envAndComponents } from "./utils/autocompletions";
@@ -13,21 +11,24 @@ export default async (vorpal: Vorpal) =>
   vorpal
     .command(
       "project-get-shell <envComponent>",
-      "get a shell to a pod in the environment"
+      "get a shell to a pod in the environment",
     )
     .autocomplete(await envAndComponents())
     .action(async function ({ envComponent }) {
       await ensureCluster.call(this, envComponent);
       const namespace = await getProjectNamespace(envComponent);
-      const podNames = await getProjectPodNames(envComponent);
-      if (podNames.length === 0) {
-        logError(this, "sorry, no pods found");
+      const pods = await getProjectPods(envComponent);
+      const runningPodNames = pods
+        .filter((p) => p.status.phase == "Running")
+        .map((r) => r.metadata.name);
+      if (runningPodNames.length === 0) {
+        logError(this, "sorry, no running pods found");
         return;
       }
       const { podName } = await this.prompt({
         type: "list",
         name: "podName",
-        choices: podNames,
+        choices: runningPodNames,
         message: "Which pod? 🤔",
       });
 

package/src/apps/cli/commands/project/commandGitlabCi.ts

@@ -1,7 +1,8 @@
 import { exec } from "child-process-promise";
 import { last } from "lodash";
 import open from "open";
-import Vorpal, { CommandInstance } from "vorpal";
+import type { CommandInstance } from "vorpal";
+import type Vorpal from "vorpal";
 import {
   doGitlabRequest,
   getGitlabToken,
@@ -39,7 +40,7 @@ const promptJob = async (vorpal: CommandInstance, projectId: any, ctx: any) => {
 
   const jobsToName = (jo: any[]) =>
     jo.map(
-      (j: any) => `${j.ref}-${j.name}-${j.user.username}-${j.status}-${j.id}`
+      (j: any) => `${j.ref}-${j.name}-${j.user.username}-${j.status}-${j.id}`,
     );
 
   const preferredJobs = jobs.filter((j: any) => j.commit.id === commitId);
@@ -83,13 +84,13 @@ export default async (vorpal: Vorpal) => {
       while (!finished) {
         const trace = await exec(
           `curl -s --header "PRIVATE-TOKEN: ${await getGitlabToken(
-            this
-          )}" "https://git.panter.ch/api/v4/projects/${projectId}/jobs/${id}/trace"`
+            this,
+          )}" "https://git.panter.ch/api/v4/projects/${projectId}/jobs/${id}/trace"`,
         );
 
         const job = await doGitlabRequest(
           this,
-          `projects/${projectId}/jobs/${id}`
+          `projects/${projectId}/jobs/${id}`,
         );
 
         if (trace.stdout) {

package/src/apps/cli/commands/project/commandListPods.ts

@@ -1,7 +1,7 @@
-import { dump } from "js-yaml";
+import { stringify } from "yaml";
 import { pick } from "lodash";
-import Vorpal from "vorpal";
-import { getProjectPods } from "../../../../utils/projects";
+import type Vorpal from "vorpal";
+import { getProjectPods } from "../../../../kubernetes";
 import { envAndComponents } from "./utils/autocompletions";
 import ensureCluster from "./utils/ensureCluster";
 
@@ -13,6 +13,8 @@ export default async (vorpal: Vorpal) =>
       await ensureCluster.call(this, envComponent);
       const pods = await getProjectPods(envComponent);
       this.log(
-        dump(pods.map((p) => pick(p, ["metadata.name", "status.startTime"])))
+        stringify(
+          pods.map((p) => pick(p, ["metadata.name", "status.startTime"])),
+        ),
       );
     });

package/src/apps/cli/commands/project/commandNamespace.ts

@@ -1,4 +1,4 @@
-import Vorpal from "vorpal";
+import type Vorpal from "vorpal";
 import { getProjectNamespace } from "../../../../utils/projects";
 import { envAndComponents } from "./utils/autocompletions";
 
@@ -6,7 +6,7 @@ export default async (vorpal: Vorpal) =>
   vorpal
     .command(
       "project-namespace <envComponent>",
-      "show namespace of local project"
+      "show namespace of local project",
     )
     .autocomplete(await envAndComponents())
     .action(async function ({ envComponent }) {

package/src/apps/cli/commands/project/commandOpenEnv.ts

@@ -1,5 +1,5 @@
 import open from "open";
-import Vorpal from "vorpal";
+import type Vorpal from "vorpal";
 import {
   getEnvironment,
   parseChoice,
@@ -16,7 +16,7 @@ export default async (vorpal: Vorpal) =>
       const { env, componentName } = parseChoice(envComponent);
       await ensureCluster.call(this, envComponent);
       const environment = await getEnvironment(env, componentName);
-      const url = environment.url;
+      const url = environment.envVars.ROOT_URL;
 
-      open(url);
+      open(url.toString());
     });

package/src/apps/cli/commands/project/commandOpenGit.ts

@@ -1,12 +1,12 @@
-import Vorpal from "vorpal";
-import { $ } from "zx";
+import type Vorpal from "vorpal";
+import { exec } from "child-process-promise";
 
 export default async (vorpal: Vorpal) =>
   vorpal
     .command(
       "project-open-git",
-      "open the repo on gitlab / github in your browser"
+      "open the repo on gitlab / github in your browser",
     )
     .action(async () => {
-      await $`npx git-open`;
+      await exec("npx git-open");
     });

package/src/apps/cli/commands/project/commandOpenGrafana.ts

@@ -1,6 +1,6 @@
 import open from "open";
-import { startPortForward } from "../../../../utils/portForward";
-import Vorpal from "vorpal";
+import { startKubePortForward } from "../../../../kubernetes/portForward";
+import type Vorpal from "vorpal";
 import {
   GRAFANA_PROXY_LOCAL_PORT,
   GRAFANA_PROXY_TARGET_PORT,
@@ -14,18 +14,18 @@ export default async (vorpal: Vorpal) =>
   vorpal
     .command(
       "project-open-grafana <envComponent>",
-      "open Grafana dashboard for your namespace"
+      "open Grafana dashboard for your namespace",
     )
     .autocomplete(await envAndComponents())
     .action(async function ({ envComponent }) {
       await ensureCluster.call(this, envComponent);
       const namespace = await getProjectNamespace(envComponent);
       const url = `http://localhost:${GRAFANA_PROXY_LOCAL_PORT}/grafana/d/at-cost-analysis-namespace2/namespace-utilization-metrics?var-namespace=${namespace}`;
-      await startPortForward(
+      await startKubePortForward(
         "deployment/kubecost-cost-analyzer",
         GRAFANA_PROXY_LOCAL_PORT,
         GRAFANA_PROXY_TARGET_PORT,
-        "kubecost"
+        "kubecost",
       );
       open(url);
     });

package/src/apps/cli/commands/project/commandOpenGrafanaPod.ts

@@ -1,15 +1,13 @@
 import open from "open";
-import Vorpal from "vorpal";
+import type Vorpal from "vorpal";
 import {
   GRAFANA_PROXY_LOCAL_PORT,
   GRAFANA_PROXY_TARGET_PORT,
 } from "../../../../config/constants";
+import { getProjectPodNames } from "../../../../kubernetes";
 import { logError } from "../../../../utils/log";
-import { startPortForward } from "../../../../utils/portForward";
-import {
-  getProjectNamespace,
-  getProjectPodNames,
-} from "../../../../utils/projects";
+import { startKubePortForward } from "../../../../kubernetes/portForward";
+import { getProjectNamespace } from "../../../../utils/projects";
 import { envAndComponents } from "./utils/autocompletions";
 import ensureCluster from "./utils/ensureCluster";
 
@@ -17,7 +15,7 @@ export default async (vorpal: Vorpal) =>
   vorpal
     .command(
       "project-open-grafana-pod <envComponent>",
-      "open Grafana dashboard for a specific pod"
+      "open Grafana dashboard for a specific pod",
     )
     .autocomplete(await envAndComponents())
     .action(async function ({ envComponent }) {
@@ -35,11 +33,11 @@ export default async (vorpal: Vorpal) =>
       });
       const namespace = await getProjectNamespace(envComponent);
       const url = `http://localhost:${GRAFANA_PROXY_LOCAL_PORT}/grafana/d/at-cost-analysis-pod/pod-cost-and-utilization-metrics?var-namespace=${namespace}&var-pod=${podName}`;
-      await startPortForward(
+      await startKubePortForward(
         "deployment/kubecost-cost-analyzer",
         GRAFANA_PROXY_LOCAL_PORT,
         GRAFANA_PROXY_TARGET_PORT,
-        "kubecost"
+        "kubecost",
       );
       open(url);
     });

package/src/apps/cli/commands/project/commandOpenLogs.ts

@@ -1,52 +1,21 @@
-import Vorpal from "vorpal";
+import type Vorpal from "vorpal";
 import {
   getPipelineContextByChoice,
   parseChoice,
 } from "../../../../config/getProjectConfig";
-import { getCurrentConnectedClusterName } from "../../../../utils/cluster";
-import { getProjectNamespace } from "../../../../utils/projects";
-import { getGoogleAuthUserNumber } from "../../utils/getGoogleAuthUserNumber";
-import { openGoogleCloudLogs } from "../shared";
+import { openGoogleCloudLogs } from "../../../../kubernetes/openKubernetesDashboards";
 import { envAndComponents } from "./utils/autocompletions";
-import ensureCluster from "./utils/ensureCluster";
 
 export default async (vorpal: Vorpal) =>
   vorpal
     .command(
       "project-open-logs <envComponent>",
-      "open google cloud logs (stackdriver logs)"
+      "open google cloud logs (stackdriver logs)",
     )
     .autocomplete(await envAndComponents())
     .action(async function ({ envComponent }) {
       const { env, componentName } = parseChoice(envComponent);
-      const { componentConfig, environment } = await getPipelineContextByChoice(
-        env,
-        componentName
-      );
+      const context = await getPipelineContextByChoice(env, componentName);
 
-      if (
-        !componentConfig.deploy ||
-        componentConfig.deploy.type !== "kubernetes"
-      ) {
-        throw new Error(
-          "only kubernetes deployments are supported at the moment"
-        );
-      }
-
-      if (
-        !componentConfig.deploy.cluster ||
-        componentConfig.deploy.cluster.type !== "gcloud"
-      ) {
-        throw new Error("no gcloud custer configured");
-      }
-      // currently only supports kubernetes
-      const namespace = environment.envVars.KUBE_NAMESPACE;
-
-      const authGoogleNumber = await getGoogleAuthUserNumber.call(this, vorpal);
-
-      await openGoogleCloudLogs(
-        componentConfig.deploy.cluster,
-        namespace,
-        authGoogleNumber
-      );
+      await openGoogleCloudLogs(this, context);
     });

package/src/apps/cli/commands/project/commandPauseProject.ts

@@ -1,4 +1,4 @@
-import Vorpal from "vorpal";
+import type Vorpal from "vorpal";
 import { exec } from "child-process-promise";
 import { getProjectNamespace } from "../../../../utils/projects";
 import { envAndComponents } from "./utils/autocompletions";
@@ -7,7 +7,7 @@ export default async (vorpal: Vorpal) =>
   vorpal
     .command(
       "project-pause <envComponent>",
-      "halts all running pods (scales to 0)"
+      "halts all running pods (scales to 0)",
     )
     .autocomplete(await envAndComponents())
     .action(async function ({ envComponent }) {