@catladder/cli 0.0.0-test7 → 0.0.0-update-meteor-53480fed

package/src/apps/cli/commands/project/commandCloudSqlProxy.ts

@@ -0,0 +1,139 @@
+import type { Context } from "@catladder/pipeline";
+import {
+  createKubernetesCloudsqlBaseValues,
+  isOfDeployType,
+} from "@catladder/pipeline";
+import type Vorpal from "vorpal";
+import type { CommandInstance } from "vorpal";
+import {
+  getEnvVarsResolved,
+  getPipelineContextByChoice,
+  parseChoice,
+} from "../../../../config/getProjectConfig";
+import { envAndComponents } from "./utils/autocompletions";
+import { startCloudSqlProxyInCurrentShell } from "../../../../gcloud/cloudSql/startProxy";
+
+type ProxyInfo = {
+  instanceName: string;
+  DB_NAME: string;
+  DB_PASSWORD: string;
+  DB_USER: string;
+};
+export default async (vorpal: Vorpal) =>
+  vorpal
+    .command("project-cloud-sql-proxy <envComponent>", "proxy to cloud sql db")
+    .autocomplete(await envAndComponents())
+    .action(async function ({ envComponent }) {
+      const { env, componentName } = parseChoice(envComponent);
+      if (!componentName) {
+        this.log("need componentName");
+        return;
+      }
+
+      const context = await getPipelineContextByChoice(env, componentName);
+      let proxyInfo: ProxyInfo;
+
+      if (env === "review") {
+        vorpal.log(
+          "⚠️ connection string does not include mr information on review environments"
+        );
+      }
+      if (isOfDeployType(context.componentConfig.deploy, "kubernetes")) {
+        proxyInfo = await getProxyInfoForKubernetes(this, context);
+      } else if (
+        isOfDeployType(context.componentConfig.deploy, "google-cloudrun")
+      ) {
+        proxyInfo = await getProxyInfoForCloudRun(this, context);
+      } else {
+        throw new Error("unsupported environment");
+      }
+
+      // skynet-164509:europe-west6:pvl-cyclomania-review=tcp:5432
+      const { DB_PASSWORD, DB_NAME, DB_USER, instanceName } = proxyInfo;
+      const { localPort } = await this.prompt({
+        type: "number",
+        name: "localPort",
+        default: "54320",
+        message: "Local port: ",
+      });
+
+      this.log("");
+      this.log(`postgres-PW: ${DB_PASSWORD}`);
+      this.log("");
+      this.log("connection strings:");
+      this.log("");
+      this.log(
+        `DATABASE_URL=postgresql://${DB_USER}:${DB_PASSWORD}@localhost:${localPort}/${DB_NAME}?schema=public`
+      );
+      this.log("");
+      this.log(
+        `DATABASE_JDBC_URL=jdbc:postgresql://localhost:${localPort}/${DB_NAME}?schema=public&user=${DB_USER}&password=${DB_PASSWORD}`
+      );
+      this.log("");
+
+      await startCloudSqlProxyInCurrentShell({
+        instanceName,
+        localPort,
+      });
+    });
+
+const getProxyInfoForKubernetes = async (
+  vorpal: CommandInstance,
+  context: Context
+): Promise<ProxyInfo> => {
+  if (!isOfDeployType(context.componentConfig.deploy, "kubernetes")) {
+    throw new Error("unsupported");
+  }
+
+  const envVars = await getEnvVarsResolved(
+    vorpal,
+    context.environment.shortName,
+    context.componentName
+  );
+  // bit hacky, would be nicer if we would also declare this through env vars
+  const cloudSqlValues = createKubernetesCloudsqlBaseValues(context);
+
+  const DB_PASSWORD = envVars?.DB_PASSWORD || envVars?.POSTGRESQL_PASSWORD;
+
+  const DB_NAME = cloudSqlValues.cloudsql.fullDbName;
+
+  const instanceName = cloudSqlValues.cloudsql.instanceConnectionName;
+
+  return {
+    instanceName,
+    DB_PASSWORD,
+    DB_NAME,
+    DB_USER: "postgres",
+  };
+};
+
+const getProxyInfoForCloudRun = async (
+  vorpal: CommandInstance,
+  context: Context
+): Promise<ProxyInfo> => {
+  if (
+    !isOfDeployType(context.componentConfig.deploy, "google-cloudrun") ||
+    !context.componentConfig.deploy.cloudSql
+  ) {
+    throw new Error("unsupported");
+  }
+
+  const envVars = await getEnvVarsResolved(
+    vorpal,
+    context.environment.shortName,
+    context.componentName
+  );
+
+  const DB_PASSWORD = envVars?.DB_PASSWORD;
+  const DB_USER = envVars?.DB_USER;
+
+  const DB_NAME = context.environment.envVars.DB_NAME;
+
+  return {
+    instanceName:
+      context.componentConfig.deploy.cloudSql.instanceConnectionName,
+    DB_PASSWORD,
+    DB_NAME,
+    DB_USER,
+  };
+};

package/src/apps/cli/commands/project/commandConfigSecrets.ts

@@ -0,0 +1,234 @@
+/* eslint-disable no-constant-condition */
+import { getSecretVarName } from "@catladder/pipeline";
+import { stripIndents } from "common-tags";
+import { difference } from "lodash";
+import type { CommandInstance } from "vorpal";
+import type Vorpal from "vorpal";
+import {
+  getAllComponentsWithAllEnvsHierarchical,
+  getEnvironment,
+  getEnvVarsResolved,
+  getJobOnlyEnvVarsResolved,
+  getProjectComponents,
+  parseChoice,
+} from "../../../../config/getProjectConfig";
+import { editAsFile } from "../../../../utils/editAsFile";
+import { upsertAllVariables } from "../../../../utils/gitlab";
+import { delay } from "../../../../utils/promise";
+import { allEnvsAndAllComponents } from "./utils/autocompletions";
+
+type Vars = {
+  [env: string]: {
+    [componentName: string]: Record<string, string>;
+  };
+};
+/* for convenience, parse json objects. that makes it easier to edit secrets that are object */
+const resolveJson = (v: Vars) =>
+  Object.fromEntries(
+    Object.entries(v).map(([componentName, envs]) => {
+      return [
+        componentName,
+        Object.fromEntries(
+          Object.entries(envs).map(([env, secrets]) => [
+            env,
+            Object.fromEntries(
+              Object.entries(secrets).map(([key, value]) => {
+                try {
+                  return [key, JSON.parse(value)];
+                } catch (e) {
+                  return [key, value];
+                }
+              })
+            ),
+          ])
+        ),
+      ];
+    })
+  );
+
+const getSecretEnvVarKeysToConfigure = async (
+  env: string,
+  componentName: string
+) => {
+  const { secretEnvVarKeys, jobOnlyVars } = await getEnvironment(
+    env,
+    componentName
+  );
+  return [
+    ...jobOnlyVars.build.secretEnvVarKeys,
+    ...jobOnlyVars.deploy.secretEnvVarKeys,
+    ...secretEnvVarKeys,
+  ]
+    .filter((k) => !k.hidden)
+    .map((k) => k.key);
+};
+const getEnvVarsToEdit = async (
+  instance: CommandInstance,
+  env: string,
+  componentName: string
+) => {
+  const secretEnvVarKeys = await getSecretEnvVarKeysToConfigure(
+    env,
+    componentName
+  );
+
+  const normalEnvVars = await getEnvVarsResolved(instance, env, componentName);
+  const jobOnlyEnvVars = await getJobOnlyEnvVarsResolved(
+    instance,
+    env,
+    componentName
+  );
+  const allEnvVars = {
+    ...normalEnvVars,
+    ...jobOnlyEnvVars,
+  };
+  return Object.fromEntries(
+    secretEnvVarKeys.map((key) => {
+      const value = allEnvVars[key];
+      // due to some quirky way to resolve these variables, unset variables have the $CL_ prefix, so we remove thouse here
+      const variableIsNotSet =
+        value === "$" + getSecretVarName(env, componentName, key);
+      return [key, variableIsNotSet ? "🚨 FILL ME" : value];
+    })
+  );
+};
+const doItFor = async (
+  instance: CommandInstance,
+  envAndComponents: {
+    [componentName: string]: string[];
+  }
+) => {
+  let valuesToEdit: Vars = Object.fromEntries(
+    await Promise.all(
+      Object.entries(envAndComponents).map(async ([componentName, envs]) => [
+        componentName,
+        Object.fromEntries(
+          await Promise.all(
+            envs.map(async (env) => [
+              env,
+              await getEnvVarsToEdit(instance, env, componentName),
+            ])
+          )
+        ),
+      ])
+    )
+  );
+  let hasErrors = true;
+  while (hasErrors) {
+    valuesToEdit = await editAsFile(
+      resolveJson(valuesToEdit),
+      stripIndents`
+        Please fill in all secrets for: 
+
+        ${Object.entries(envAndComponents)
+          .map(
+            ([componentName, envs]) => `- ${componentName}: ${envs.join(", ")}`
+          )
+          .join("\n")}
+
+        `
+    );
+    // check for errors
+    hasErrors = false;
+    for (const [componentName, envs] of Object.entries(envAndComponents)) {
+      for (const env of envs) {
+        const usedKeys = valuesToEdit[componentName][env]
+          ? Object.keys(valuesToEdit[componentName][env])
+          : [];
+        // check whether newValues have the exact number of keys
+        const secretEnvVarKeys = await getSecretEnvVarKeysToConfigure(
+          env,
+          componentName
+        );
+
+        const extranous = difference(usedKeys, secretEnvVarKeys);
+        const missing = difference(secretEnvVarKeys, usedKeys);
+
+        if (extranous.length > 0 || missing.length > 0) {
+          instance.log("");
+          instance.log(
+            `😿 Oh no! There is something wrong with "${componentName}"`
+          );
+          instance.log("");
+          if (extranous.length > 0) {
+            instance.log("these secrets are not declared in the config");
+            extranous.forEach((key) => instance.log(key));
+            instance.log("");
+          }
+          if (missing.length > 0) {
+            instance.log("these secrets have not been provided:");
+            missing.forEach((key) => instance.log(key));
+            instance.log("");
+          }
+
+          await delay(1000);
+          const { shouldContinue } = await instance.prompt({
+            default: true,
+            message: "Try again? 🤔",
+            name: "shouldContinue",
+            type: "confirm",
+          });
+
+          if (!shouldContinue) {
+            throw new Error("abort");
+          }
+          hasErrors = true;
+        }
+      }
+    }
+  }
+  instance.log("upserting all variables, please wait...");
+  instance.log("");
+  for (const [componentName, envs] of Object.entries(envAndComponents)) {
+    for (const env of envs) {
+      await upsertAllVariables(
+        instance,
+        valuesToEdit[componentName][env],
+        env,
+        componentName
+      );
+      instance.log("");
+      instance.log("✅ " + env + ":" + componentName);
+    }
+  }
+  instance.log("done! 😻");
+  instance.log("");
+};
+
+export const projectConfigSecrets = async (
+  vorpal: CommandInstance,
+  envComponent?: string
+) => {
+  if (!envComponent) {
+    const allEnvAndcomponents = await getAllComponentsWithAllEnvsHierarchical();
+    await doItFor(vorpal, allEnvAndcomponents);
+  } else {
+    const { env, componentName } = parseChoice(envComponent);
+
+    // componentName can be null. in this case, iterate over all  components
+    if (!componentName) {
+      const components = await getProjectComponents();
+      await doItFor(
+        vorpal,
+        Object.fromEntries(components.map((c) => [c, [env]]))
+      );
+    }
+    if (componentName) {
+      await doItFor(vorpal, {
+        [componentName]: [env],
+      });
+    }
+  }
+};
+
+export default async (vorpal: Vorpal) => {
+  vorpal
+    .command(
+      "project-config-secrets [envComponent]",
+      "setup/update secrets stored in pass"
+    )
+    .autocomplete(await allEnvsAndAllComponents())
+    .action(async function ({ envComponent }) {
+      return await projectConfigSecrets(this, envComponent);
+    });
+};

package/src/apps/{shell → cli}/commands/project/commandDeletePods.ts

@@ -1,23 +1,20 @@
-import { V1DeleteOptions } from "@kubernetes/client-node";
-import Vorpal from "vorpal";
-import k8sApi from "../../../../k8sApi";
+import type Vorpal from "vorpal";
+import { getk8sApi } from "../../../../k8sApi";
+import { getProjectPodNames } from "../../../../kubernetes";
 
 import { logError } from "../../../../utils/log";
-import {
-  getProjectNamespace,
-  getProjectPodNames,
-} from "../../../../utils/projects";
-import { envAutocompletion } from "./utils/autocompletions";
+import { getProjectNamespace } from "../../../../utils/projects";
+import { envAndComponents } from "./utils/autocompletions";
 import ensureCluster from "./utils/ensureCluster";
 
-export default (vorpal: Vorpal) =>
+export default async (vorpal: Vorpal) =>
   vorpal
-    .command("project-delete-pods <env>", "delete / restart pods")
-    .autocomplete(envAutocompletion)
-    .action(async function ({ env }) {
-      await ensureCluster.call(this);
-      const namespace = await getProjectNamespace(env);
-      const podNames = await getProjectPodNames(env);
+    .command("project-delete-pods <envComponent>", "delete / restart pods")
+    .autocomplete(await envAndComponents())
+    .action(async function ({ envComponent }) {
+      await ensureCluster.call(this, envComponent);
+      const namespace = await getProjectNamespace(envComponent);
+      const podNames = await getProjectPodNames(envComponent);
       if (podNames.length === 0) {
         logError(this, "sorry, no pods found");
         return;
@@ -41,6 +38,7 @@ export default (vorpal: Vorpal) =>
         message: "Continue ? 🤔  ",
       });
       this.log("");
+      const k8sApi = getk8sApi();
       if (shouldContinue) {
         for (const podName of selectedPodNames) {
           await k8sApi.deleteNamespacedPod(podName, namespace, "true");

package/src/apps/{shell → cli}/commands/project/commandDeleteProject.ts

@@ -1,18 +1,17 @@
-import Vorpal from "vorpal";
-import { exec, spawn } from "child-process-promise";
-import { Env } from "../../../../types/types";
+import type Vorpal from "vorpal";
+import { exec } from "child-process-promise";
 import { getProjectNamespace } from "../../../../utils/projects";
-import { envAutocompletion } from "./utils/autocompletions";
+import { envAndComponents } from "./utils/autocompletions";
 
-export default (vorpal: Vorpal) =>
+export default async (vorpal: Vorpal) =>
   vorpal
     .command(
-      "project-delete <env>",
+      "project-delete <envComponent>",
       "deletes a environment of a project (it deletes the namespace)"
     )
-    .autocomplete(envAutocompletion)
-    .action(async function ({ env }) {
-      const namespace = await getProjectNamespace(env as Env);
+    .autocomplete(await envAndComponents())
+    .action(async function ({ envComponent }) {
+      const namespace = await getProjectNamespace(envComponent);
       const { shouldContinue } = await this.prompt({
         type: "confirm",
         name: "shouldContinue",

package/src/apps/cli/commands/project/commandEnvVars.ts

@@ -0,0 +1,18 @@
+import type Vorpal from "vorpal";
+import {
+  getEnvVarsResolved,
+  parseChoice,
+} from "../../../../config/getProjectConfig";
+import { envAndComponents } from "./utils/autocompletions";
+
+export default async (vorpal: Vorpal) =>
+  vorpal
+    .command("project-env-vars <envComponent>", "list env vars")
+    .autocomplete(await envAndComponents())
+    .action(async function ({ envComponent }) {
+      const { env, componentName } = parseChoice(envComponent);
+      const envvars = await getEnvVarsResolved(this, env, componentName);
+      Object.keys(envvars).forEach((key) =>
+        this.log(`${key}: ${envvars[key]}`)
+      );
+    });

package/src/apps/{shell → cli}/commands/project/commandGetMyTotalWorktime.ts

@@ -1,14 +1,14 @@
 import { spawn } from "child-process-promise";
-import Vorpal from "vorpal";
+import type Vorpal from "vorpal";
 
-export default (vorpal: Vorpal) =>
+export default async (vorpal: Vorpal) =>
   vorpal
     .command(
       "project-get-my-total-worktime",
       "show the total worktime that you spent on a project"
     )
     .action(async () => {
-      await spawn("sh", ["-c", "curl -L http://bit.ly/10hA8iC | bash"], {
-        stdio: ["pipe", "inherit", "pipe"]
+      await spawn("sh", ["-c", "curl -s -L http://bit.ly/3VQEKNq | bash"], {
+        stdio: ["pipe", "inherit", "pipe"],
       });
     });

package/src/apps/cli/commands/project/commandGetShell.ts

@@ -0,0 +1,36 @@
+import type Vorpal from "vorpal";
+import { getProjectPods } from "../../../../kubernetes";
+import { logError } from "../../../../utils/log";
+import { getProjectNamespace } from "../../../../utils/projects";
+import { getShell } from "../../../../utils/shell";
+
+import { envAndComponents } from "./utils/autocompletions";
+import ensureCluster from "./utils/ensureCluster";
+
+export default async (vorpal: Vorpal) =>
+  vorpal
+    .command(
+      "project-get-shell <envComponent>",
+      "get a shell to a pod in the environment"
+    )
+    .autocomplete(await envAndComponents())
+    .action(async function ({ envComponent }) {
+      await ensureCluster.call(this, envComponent);
+      const namespace = await getProjectNamespace(envComponent);
+      const pods = await getProjectPods(envComponent);
+      const runningPodNames = pods
+        .filter((p) => p.status.phase == "Running")
+        .map((r) => r.metadata.name);
+      if (runningPodNames.length === 0) {
+        logError(this, "sorry, no running pods found");
+        return;
+      }
+      const { podName } = await this.prompt({
+        type: "list",
+        name: "podName",
+        choices: runningPodNames,
+        message: "Which pod? 🤔",
+      });
+
+      return getShell(namespace, podName);
+    });

package/src/apps/{shell → cli}/commands/project/commandGitlabCi.ts

@@ -1,7 +1,8 @@
 import { exec } from "child-process-promise";
 import { last } from "lodash";
 import open from "open";
-import Vorpal, { CommandInstance } from "vorpal";
+import type { CommandInstance } from "vorpal";
+import type Vorpal from "vorpal";
 import {
   doGitlabRequest,
   getGitlabToken,
@@ -64,7 +65,7 @@ const promptJob = async (vorpal: CommandInstance, projectId: any, ctx: any) => {
   return jobs.find((j: any) => j.id === jobId);
 };
 
-export default (vorpal: Vorpal) => {
+export default async (vorpal: Vorpal) => {
   vorpal.command("project-ci-job-open", "Open a Job").action(async function () {
     const { id: projectId } = await getProjectInfo(this);
 

package/src/apps/{shell/commands/project/commandInitProject.ts → cli/commands/project/commandInitProject.old.ts}

@@ -1,8 +1,8 @@
 import { readFile, writeFile } from "fs-extra";
-import { safeDump } from "js-yaml";
+import { dump } from "js-yaml";
 import { mapKeys, snakeCase, toUpper } from "lodash";
 import path from "path";
-import Vorpal from "vorpal";
+import type Vorpal from "vorpal";
 import { hasGitlabCiFile } from "../../../../utils/projects";
 
 const transformVar = (key: string) => toUpper(snakeCase(key));
@@ -29,7 +29,7 @@ const DEFAULT_VARIABLES = {
   clusterName: "production",
 };
 // TODO: we should find a way how to fetch the variables of the inclues directly from gitlab
-const DEFAULT_NODE_APPLICATION = {};
+const DEFAULT_NODE_asdfAPPLICATION = {};
 
 const DEFAULT_FILES_TO_CREATE: FileToCreate[] = [
   {
@@ -38,6 +38,9 @@ const DEFAULT_FILES_TO_CREATE: FileToCreate[] = [
       readFile(path.resolve(__dirname, "../includes/envrc"), "utf8"),
   },
 ];
+/*
+
+TODO: reimplement as PRESETS ? 
 const PLATFORMS: { [platformName: string]: PlatformDefinition } = {
   nextJS: {
     variables: {},
@@ -154,7 +157,7 @@ const PLATFORMS: { [platformName: string]: PlatformDefinition } = {
       {
         filename: "values-review.yml",
         content: async () =>
-          safeDump({
+          dump({
             jobs: {
               "db-prepare-seed": {
                 hook: "post-install",
@@ -180,10 +183,15 @@ const PLATFORMS: { [platformName: string]: PlatformDefinition } = {
     values: {},
   },
 };
-export default (vorpal: Vorpal) =>
+
+*/
+
+const PLATFORMS: any = {};
+export default async (vorpal: Vorpal) =>
   vorpal
     .command("project-init", "Inits a new project")
     .action(async function () {
+      throw new Error("needs reimplementation");
       const hasGitlabFile = await hasGitlabCiFile();
       if (hasGitlabFile) {
         this.log("there is already a gitlab-ci.yml file. Skipping");
@@ -251,7 +259,7 @@ export default (vorpal: Vorpal) =>
           variables: transformedVariables,
         };
 
-        const gitlabCiContent = safeDump(gitlabCiContentObj);
+        const gitlabCiContent = dump(gitlabCiContentObj);
         await writeFile(".gitlab-ci.yml", gitlabCiContent);
 
         allFilesToCreate.forEach(async ({ filename, content }) => {
@@ -260,7 +268,7 @@ export default (vorpal: Vorpal) =>
           await writeFile(filename, theContent);
         });
 
-        await writeFile("values.yml", safeDump(values));
+        await writeFile("values.yml", dump(values));
 
         this.log("");
         this.log("gitlab-ci created! 💪😻");

package/src/apps/cli/commands/project/commandListPods.ts

@@ -0,0 +1,18 @@
+import { dump } from "js-yaml";
+import { pick } from "lodash";
+import type Vorpal from "vorpal";
+import { getProjectPods } from "../../../../kubernetes";
+import { envAndComponents } from "./utils/autocompletions";
+import ensureCluster from "./utils/ensureCluster";
+
+export default async (vorpal: Vorpal) =>
+  vorpal
+    .command("project-list-pods <envComponent>", "list pods of local project")
+    .autocomplete(await envAndComponents())
+    .action(async function ({ envComponent }) {
+      await ensureCluster.call(this, envComponent);
+      const pods = await getProjectPods(envComponent);
+      this.log(
+        dump(pods.map((p) => pick(p, ["metadata.name", "status.startTime"])))
+      );
+    });

package/src/apps/{shell → cli}/commands/project/commandMigrateHelm3.ts

@@ -1,24 +1,31 @@
 import { exec } from "child-process-promise";
-import Vorpal from "vorpal";
+import type Vorpal from "vorpal";
+import {
+  getEnvironment,
+  parseChoice,
+} from "../../../../config/getProjectConfig";
 import { logError } from "../../../../utils/log";
-import { getProjectHelmReleaseName } from "../../../../utils/projects";
-import { envAutocompletion } from "./utils/autocompletions";
+
+import { envAndComponents } from "./utils/autocompletions";
 
 /**
  * Error: rendered manifests contain a resource that already exists.
  * Unable to continue with install: ConfigMap "web-app-env" in namespace "pvl-bike2school-dev" exists and cannot be imported into the current release: invalid ownership metadata; label validation error: missing key "app.kubernetes.io/managed-by": must be set to "Helm"; annotation validation error: missing key "meta.helm.sh/release-name": must be set to "pvl-bike2school-dev-web"; annotation validation error: missing key "meta.helm.sh/release-namespace": must be set to "pvl-bike2school-dev"
  */
-export default (vorpal: Vorpal) =>
+export default async (vorpal: Vorpal) =>
   vorpal
     .command(
-      "project-migrate-helm3 <env>",
+      "project-migrate-helm3 <envComponent>",
       "Do manual step for helm2 to helm3 migration"
     )
     .autocomplete(
-      envAutocompletion.filter((e) => e !== "dev-local" && e !== "review")
+      (await envAndComponents()).filter(
+        (e) => !e.startsWith("local") && !e.startsWith("review")
+      )
     )
-    .action(async function ({ env }) {
+    .action(async function ({ envComponent }) {
       try {
+        const { env, componentName } = parseChoice(envComponent);
         const result = await exec("helm version --short");
         const version = result.stdout as string;
         this.log(`your helm version: ${version}`);
@@ -35,7 +42,10 @@ export default (vorpal: Vorpal) =>
         } catch (e) {
           // ignore
         }
-        const releaseName = await getProjectHelmReleaseName(env);
+
+        // we use the app full name as the release name
+        const releaseName = (await getEnvironment(env, componentName)).fullName;
+
         this.log(`helm release name: ${releaseName}`);
         this.log("");
         this.log("migrating now... 😼. This may take a moment");