@catladder/cli 1.5.8 → 1.9.0

package/package.json

@@ -16,7 +16,7 @@
     "catladder": "./bin/catladder"
   },
   "dependencies": {
-    "@catladder/pipeline": "1.5.8",
+    "@catladder/pipeline": "1.9.0",
     "@kubernetes/client-node": "^0.16.2",
     "child-process-promise": "^2.2.1",
     "command-exists-promise": "^2.0.2",
@@ -55,5 +55,5 @@
     "eslint": "^8.7.0",
     "typescript": "^4.5.4"
   },
-  "version": "1.5.8"
+  "version": "1.9.0"
 }

package/src/apps/cli/commands/project/commandCloudSqlProxy.ts

@@ -37,8 +37,13 @@ export default async (vorpal: Vorpal) =>
       if (!isOfDeployType(context.componentConfig.deploy, "kubernetes")) {
         throw new Error("currently only supported for kubernetes deployment");
       }
+      this.log("");
       this.log(`postgres-PW: ${POSTGRESQL_PASSWORD}`);
       this.log("");
+      this.log(
+        `POSTGRESQL_URL=postgresql://postgres:${POSTGRESQL_PASSWORD}@localhost:${localPort}/${context.environment.envVars.KUBE_APP_NAME}?schema=public`
+      );
+      this.log("");
 
       const values = context.componentConfig.deploy.values;
 

package/src/apps/cli/commands/project/commandConfigSecrets.ts

@@ -4,6 +4,7 @@ import { difference } from "lodash";
 import Vorpal, { CommandInstance } from "vorpal";
 import { GOOGLE_CLOUD_SQL_PASS_PATH } from "../../../../config/constants";
 import {
+  getAllComponentsWithAllEnvsHierarchical,
   getEnvironment,
   getEnvVars,
   getPipelineContextByChoice,
@@ -16,20 +17,30 @@ import { hasBitwarden, readPass } from "../../../../utils/passwordstore";
 import { delay } from "../../../../utils/promise";
 import { allEnvsAndAllComponents } from "./utils/autocompletions";
 
+type Vars = {
+  [env: string]: {
+    [componentName: string]: Record<string, string>;
+  };
+};
 /* for convenience, parse json objects. that makes it easier to edit secrets that are object */
-const resolveJson = (v: Record<string, Record<string, string>>) =>
+const resolveJson = (v: Vars) =>
   Object.fromEntries(
-    Object.entries(v).map(([c, secrets]) => {
+    Object.entries(v).map(([componentName, envs]) => {
       return [
-        c,
+        componentName,
         Object.fromEntries(
-          Object.entries(secrets).map(([key, value]) => {
-            try {
-              return [key, JSON.parse(value)];
-            } catch (e) {
-              return [key, value];
-            }
-          })
+          Object.entries(envs).map(([env, secrets]) => [
+            env,
+            Object.fromEntries(
+              Object.entries(secrets).map(([key, value]) => {
+                try {
+                  return [key, JSON.parse(value)];
+                } catch (e) {
+                  return [key, value];
+                }
+              })
+            ),
+          ])
         ),
       ];
     })
@@ -48,14 +59,22 @@ const getEnvVarsToEdit = async (
 };
 const doItFor = async (
   instance: CommandInstance,
-  env: string,
-  components: string[]
+  envAndComponents: {
+    [componentName: string]: string[];
+  }
 ) => {
-  let valuesToEdit: Record<string, Record<string, string>> = Object.fromEntries(
+  let valuesToEdit: Vars = Object.fromEntries(
     await Promise.all(
-      components.map(async (componentName) => [
+      Object.entries(envAndComponents).map(async ([componentName, envs]) => [
         componentName,
-        await getEnvVarsToEdit(instance, env, componentName),
+        Object.fromEntries(
+          await Promise.all(
+            envs.map(async (env) => [
+              env,
+              await getEnvVarsToEdit(instance, env, componentName),
+            ])
+          )
+        ),
       ])
     )
   );
@@ -64,81 +83,90 @@ const doItFor = async (
     valuesToEdit = await editAsFile(
       resolveJson(valuesToEdit),
       stripIndents`
-        Please fill in all secrets for: ${components.join(", ")}
+        Please fill in all secrets for: 
+
+        ${Object.entries(envAndComponents)
+          .map(
+            ([componentName, envs]) => `- ${componentName}: ${envs.join(", ")}`
+          )
+          .join("\n")}
 
         `
     );
     // check for errors
     hasErrors = false;
-    for (const componentName of components) {
-      const usedKeys = valuesToEdit[componentName]
-        ? Object.keys(valuesToEdit[componentName])
-        : [];
-      // check whether newValues have the exact number of keys
-      const { secretEnvVarKeys } = await getEnvironment(env, componentName);
-      const extranous = difference(usedKeys, secretEnvVarKeys);
-      const missing = difference(secretEnvVarKeys, usedKeys);
+    for (const [componentName, envs] of Object.entries(envAndComponents)) {
+      for (const env of envs) {
+        const usedKeys = valuesToEdit[componentName][env]
+          ? Object.keys(valuesToEdit[componentName][env])
+          : [];
+        // check whether newValues have the exact number of keys
+        const { secretEnvVarKeys } = await getEnvironment(env, componentName);
+        const extranous = difference(usedKeys, secretEnvVarKeys);
+        const missing = difference(secretEnvVarKeys, usedKeys);
 
-      if (extranous.length > 0 || missing.length > 0) {
-        instance.log("");
-        instance.log(
-          `😿 Oh no! There is something wrong with "${componentName}"`
-        );
-        instance.log("");
-        if (extranous.length > 0) {
-          instance.log("these secrets are not declared in the config");
-          extranous.forEach((key) => instance.log(key));
+        if (extranous.length > 0 || missing.length > 0) {
           instance.log("");
-        }
-        if (missing.length > 0) {
-          instance.log("these secrets have not been provided:");
-          missing.forEach((key) => instance.log(key));
+          instance.log(
+            `😿 Oh no! There is something wrong with "${componentName}"`
+          );
           instance.log("");
-        }
+          if (extranous.length > 0) {
+            instance.log("these secrets are not declared in the config");
+            extranous.forEach((key) => instance.log(key));
+            instance.log("");
+          }
+          if (missing.length > 0) {
+            instance.log("these secrets have not been provided:");
+            missing.forEach((key) => instance.log(key));
+            instance.log("");
+          }
 
-        await delay(1000);
-        const { shouldContinue } = await instance.prompt({
-          default: true,
-          message: "Try again? 🤔",
-          name: "shouldContinue",
-          type: "confirm",
-        });
+          await delay(1000);
+          const { shouldContinue } = await instance.prompt({
+            default: true,
+            message: "Try again? 🤔",
+            name: "shouldContinue",
+            type: "confirm",
+          });
 
-        if (!shouldContinue) {
-          throw new Error("abort");
+          if (!shouldContinue) {
+            throw new Error("abort");
+          }
+          hasErrors = true;
         }
-        hasErrors = true;
       }
     }
   }
+  for (const [componentName, envs] of Object.entries(envAndComponents)) {
+    for (const env of envs) {
+      await upsertAllVariables(
+        instance,
+        valuesToEdit[componentName][env],
+        env,
+        componentName
+      );
 
-  for (const componentName of components) {
-    await upsertAllVariables(
-      instance,
-      valuesToEdit[componentName],
-      env,
-      componentName
-    );
-
-    if (hasBitwarden()) {
-      // add cloud sql secret if needed.
-      // TODO: this is legacy, in the future we want to have one service account per app
+      if (hasBitwarden()) {
+        // add cloud sql secret if needed.
+        // TODO: this is legacy, in the future we want to have one service account per app
 
-      const context = await getPipelineContextByChoice(env, componentName);
-      if (
-        context.componentConfig.deploy &&
-        context.componentConfig.deploy.values?.cloudsql?.enabled
-      ) {
-        await upsertAllVariables(
-          instance,
-          {
-            cloudsqlProxyCredentials: await readPass(
-              GOOGLE_CLOUD_SQL_PASS_PATH
-            ),
-          },
-          env,
-          componentName
-        );
+        const context = await getPipelineContextByChoice(env, componentName);
+        if (
+          context.componentConfig.deploy &&
+          context.componentConfig.deploy.values?.cloudsql?.enabled
+        ) {
+          await upsertAllVariables(
+            instance,
+            {
+              cloudsqlProxyCredentials: await readPass(
+                GOOGLE_CLOUD_SQL_PASS_PATH
+              ),
+            },
+            env,
+            componentName
+          );
+        }
       }
     }
   }
@@ -147,62 +175,31 @@ const doItFor = async (
 export default async (vorpal: Vorpal) => {
   vorpal
     .command(
-      "project-config-secrets <envComponent>",
+      "project-config-secrets [envComponent]",
       "setup/update secrets stored in pass"
     )
     .autocomplete(await allEnvsAndAllComponents())
     .action(async function ({ envComponent }) {
-      const { env, componentName } = parseChoice(envComponent);
-
-      // componentName can be null. in this case, iterate over all  components
-      if (!componentName) {
-        const components = await getProjectComponents();
-        await doItFor(this, env, components);
-      }
-      if (componentName) {
-        await doItFor(this, env, [componentName]);
-      }
+      if (!envComponent) {
+        const allEnvAndcomponents =
+          await getAllComponentsWithAllEnvsHierarchical();
+        await doItFor(this, allEnvAndcomponents);
+      } else {
+        const { env, componentName } = parseChoice(envComponent);
 
-      /*
-
-      
-        // adding gcloud sql proxy secret
-        const cloudsqlCredentials = await readPass(GOOGLE_CLOUD_SQL_PASS_PATH);
-        await createKubernetesSecret.call(
-          this,
-          namespace,
-          "cloudsql-instance-credentials",
-          {
-            "credentials.json": cloudsqlCredentials,
-          }
-        );
-        this.log("");
-        this.log(
-          "⚠️  You need to delete/restart pods in order to make them pick up the new config"
-        );
-        this.log(`you can use project-delete-pods ${env} to do that`);
-        this.log("");
-        this.log("");
-        await delay(1000);
+        // componentName can be null. in this case, iterate over all  components
+        if (!componentName) {
+          const components = await getProjectComponents();
+          await doItFor(
+            this,
+            Object.fromEntries(components.map((c) => [c, [env]]))
+          );
+        }
+        if (componentName) {
+          await doItFor(this, {
+            [componentName]: [env],
+          });
+        }
       }
-      this.log("");
-      this.log("😻 success!!!!!");
-      this.log("");
-      */
     });
-}; /*
-async function createNewEnvInPass(env: any, secretEnvVarsMapping: ISecrets) {
-  // const passPath = await getPassPath(env);
-  this.log(
-    "Your selected env is not yet in pass. Do you want to copy it from another env? "
-  );
-  const noAnswer = "No, I will create a new one from scratch.";
-  const { sourceEnv } = await this.prompt({
-    type: "list",
-    name: "sourceEnv",
-    choices: [...(await envAndComponents()).filter((e) => e !== env), noAnswer],
-    message: "Do you want to copy an env?",
-  });
-  // TODO: reimplenent
-}
-*/
+};

package/src/apps/cli/commands/project/commandInitGitlab.ts

@@ -1,14 +1,18 @@
 import {
   getFullKubernetesClusterName,
   isOfDeployType,
-  getKubernetesNamespace,
 } from "@catladder/pipeline";
 import Vorpal from "vorpal";
 import { $ } from "zx";
 import { getAllPipelineContexts } from "../../../../config/getProjectConfig";
 import { connectToCluster } from "../../../../utils/cluster";
-import { upsertAllVariables } from "../../../../utils/gitlab";
+import {
+  doGitlabRequest,
+  getProjectInfo,
+  upsertAllVariables,
+} from "../../../../utils/gitlab";
 import ensureNamespace from "./utils/ensureNamespace";
+import open from "open";
 
 export default async (vorpal: Vorpal) =>
   vorpal
@@ -109,69 +113,9 @@ EOF
         }
       }
 
-      // there is a constraint, see https://git.panter.ch/catladder/catladder/-/issues/2#note_345677
-      // any two components have to use the same custer per env, so e.g. dev:api and dev:www cannot use two different namespaces
-      // in practise, that is often not an issue, but it might happen because the config allows it
-      /*
-      Object.entries(configuredClusters).forEach(([fullname, config]) =>
-        this.log(` - ${config.cluster.name || "unknown"} (${fullname})`)
-      );
-      this.log("");
-
-      const missingClusters = Object.fromEntries(
-        Object.entries(configuredClusters).filter(
-          ([c]) => !existingClusters.some((exist) => exist === c)
-        )
-      );
-
-      this.log("");
-      this.log("These clusters are not configured yet on gitlab:");
-      this.log("");
-
-      Object.entries(missingClusters).forEach(([fullname, config]) =>
-        this.log(` - ${config.cluster.name || "unknown"} (${fullname})`)
+      const { id: projectId, web_url: projectWebUrl } = await getProjectInfo(
+        this
       );
-      this.log("");
-
-      for (const [fullname, config] of Object.entries(missingClusters)) {
-        this.log(`${config.name} (${fullname})`);
-        this.log("");
-        const { shouldContinue } = await this.prompt({
-          type: "confirm",
-          name: "shouldContinue",
-          message: "Should I add the this cluster ? 🤔  ",
-        });
-        this.log("");
-        if (shouldContinue) {
-          await connectToCluster(fullname);
-          const { stdout: api_url } =
-            await $`kubectl cluster-info | grep -E 'Kubernetes master|Kubernetes control plane' | awk '/http/ {print $NF}'`;
-          const { stdout: ca_cert } =
-            await $`kubectl get secret default-token-69xv4 -o jsonpath="{['data']['ca\.crt']}" | base64 --decode`;
-          const { stdout: token } =
-            await $`kubectl get secret default-token-69xv4 -o jsonpath="{['data']['token']}" | base64 --decode`;
-          const postResult = await doGitlabRequest(
-            this,
-            `projects/${projectId}/clusters/user`,
-            {
-              name: fullname,
-              managed: false,
-              environment_scope: "*",
-              platform_kubernetes_attributes: {
-                api_url,
-                ca_cert,
-                token,
-                namespace: await getProjectNamespace("prod"),
-              },
-            }
-          );
-          const { message } = postResult;
-          if (message) {
-            this.log(`Message from gitlab: ${message}`);
-          }
-        }
-      }
-
       const variables = await doGitlabRequest(
         this,
         `projects/${projectId}/variables`
@@ -244,5 +188,4 @@ EOF
       ].forEach((tip) => this.log(` - ${tip}`));
       this.log("\n");
       this.log("\n");
-      */
     });

package/src/apps/cli/commands/project/utils/autocompletions.ts

@@ -1,6 +1,6 @@
 import { getAllEnvs, getAllEnvsInAllComponents } from "@catladder/pipeline";
 import {
-  getAllComponentsWithAllEnvs,
+  getAllComponentsWithAllEnvsFlat,
   getProjectConfig,
 } from "../../../../../config/getProjectConfig";
 
@@ -13,7 +13,7 @@ export const allEnvs = async () => {
 };
 
 export const envAndComponents = async () => {
-  const allEnvAndcomponents = await getAllComponentsWithAllEnvs();
+  const allEnvAndcomponents = await getAllComponentsWithAllEnvsFlat();
 
   return allEnvAndcomponents.reduce<string[]>(
     (acc, { env, componentName }) => [...acc, env + ":" + componentName],

package/src/config/getProjectConfig.ts

@@ -58,21 +58,37 @@ export const getPipelineContextByChoice = async (
   const config = await getProjectConfig();
   return createContext(config, componentName, env);
 };
-export const getAllComponentsWithAllEnvs = async () => {
+export const getAllComponentsWithAllEnvsFlat = async (): Promise<
+  Array<{ env: string; componentName: string }>
+> => {
   const config = await getProjectConfig();
   if (!config) {
     return [];
   }
-  return Promise.all(
-    Object.keys(config.components).flatMap((componentName) =>
-      getAllEnvs(config, componentName).map((env) => ({ env, componentName }))
-    )
+  return Object.keys(config.components).flatMap((componentName) =>
+    getAllEnvs(config, componentName).map((env) => ({ env, componentName }))
+  );
+};
+
+export const getAllComponentsWithAllEnvsHierarchical = async (): Promise<{
+  [componentName: string]: string[];
+}> => {
+  const config = await getProjectConfig();
+  if (!config) {
+    return {};
+  }
+
+  return Object.fromEntries(
+    Object.keys(config.components).map((componentName) => [
+      componentName,
+      getAllEnvs(config, componentName),
+    ])
   );
 };
 
 export const getAllPipelineContexts = async () => {
   return Promise.all(
-    (await getAllComponentsWithAllEnvs())
+    (await getAllComponentsWithAllEnvsFlat())
       .filter((c) => c.env !== "local")
       .map(({ env, componentName }) =>
         getPipelineContextByChoice(env, componentName)