@catladder/cli 0.0.0-test4 → 0.0.0-updatemongo6-52a2af91

package/src/utils/gitlab.ts

@@ -1,52 +1,68 @@
 import { exec } from "child-process-promise";
+import { getSecretVarName } from "@catladder/pipeline";
+import { has, isObject, times } from "lodash";
+import memoizee from "memoizee";
 import fetch from "node-fetch";
 import open from "open";
 import { CommandInstance } from "vorpal";
 import { getPreference, hasPreference, setPreference } from "./preferences";
 
 const TOKEN_KEY = "gitlab-personal-access-token";
+
+export const hasGitlabToken = async () => await hasPreference(TOKEN_KEY);
+export const setupGitlabToken = async (vorpal: CommandInstance) => {
+  vorpal.log("");
+  vorpal.log("☝ in order to access the api, we need a personal access token");
+  vorpal.log("Its best to create one specifically for catladder");
+  vorpal.log("");
+  vorpal.log("☝ we open up the settings page for you!");
+  vorpal.log("");
+  const { shouldContinue } = await vorpal.prompt({
+    default: true,
+    message: "Ok",
+    name: "shouldContinue",
+    type: "prompt",
+  });
+
+  open("https://git.panter.ch/-/profile/personal_access_tokens");
+
+  vorpal.log("Please type in gitlab's personal access token");
+
+  const { personalToken } = await vorpal.prompt({
+    type: "string",
+    name: "personalToken",
+    default: "",
+    message: "Your personal access token ",
+  });
+  if (personalToken) {
+    await setPreference(TOKEN_KEY, personalToken);
+  }
+};
 export const getGitlabToken = async (vorpal: CommandInstance) => {
-  if (!(await hasPreference(TOKEN_KEY))) {
-    vorpal.log("");
-    vorpal.log(
-      "☝ in order to access the api, we need a personal access token"
-    );
-    vorpal.log("Its best to create one specifically for catladder");
-    vorpal.log("");
-    vorpal.log("☝ we open up the settings page for you!");
-    vorpal.log("");
-    const { shouldContinue } = await vorpal.prompt({
-      default: true,
-      message: "Ok",
-      name: "shouldContinue",
-      type: "prompt",
-    });
-
-    open("https://git.panter.ch/-/profile/personal_access_tokens");
-
-    vorpal.log("Please type in gitlab's personal access token");
-
-    const { personalToken } = await vorpal.prompt({
-      type: "string",
-      name: "personalToken",
-      default: "",
-      message: "Your personal access token ",
-    });
-    if (personalToken) {
-      await setPreference(TOKEN_KEY, personalToken);
+  if (!(await hasGitlabToken())) {
+    if (!vorpal) {
+      console.error(
+        "⚠️ gitlab token missing, please run catladder to set it up"
+      );
+      process.exit(1);
     }
+    await setupGitlabToken(vorpal);
   }
   return getPreference(TOKEN_KEY);
 };
 
-export const doGitlabRequest = async (
+export const doGitlabRequest = async <T = any>(
   vorpal: CommandInstance,
   path: string,
-  data?: any
-) => {
+  data: any = undefined,
+  update?: boolean
+): Promise<T> => {
   const rootToken = await getGitlabToken(vorpal);
+
+  const method = data ? (update ? "PUT" : "POST") : "GET";
+
   const result = await fetch(`https://git.panter.ch/api/v4/${path}`, {
-    method: data ? "POST" : "GET",
+    method,
     headers: {
       "Content-Type": "application/json",
       "Private-Token": rootToken,
@@ -57,15 +73,20 @@ export const doGitlabRequest = async (
   if (result.status >= 200 && result.status < 400) {
     return result.json();
   }
+  if (result.status === 404) {
+    throw new Error("not found");
+  }
 
   throw new Error(
-    `Could not send request to gitlab api: ${result.status} "${
+    `Could not send request to gitlab api ${path}: ${result.status} "${
       result.statusText
     }".\nResponse: ${JSON.stringify(await result.json(), null, 2)}`
   );
 };
 
-export const getProjectInfo = async (vorpal: CommandInstance): Promise<any> => {
+export const getProjectInfo = async (
+  vorpal: CommandInstance
+): Promise<{ id: string; web_url: string }> => {
   const gitRemoteOriginUrl = (
     await exec("git config --get remote.origin.url")
   ).stdout.trim();
@@ -78,3 +99,160 @@ export const getProjectInfo = async (vorpal: CommandInstance): Promise<any> => {
   );
   return project;
 };
+
+type GitlabVariable = {
+  variable_type: string;
+  key: string;
+  value: string;
+  protected: boolean;
+  masked: boolean;
+  environment_scope: string;
+};
+export const getAllVariables = memoizee(
+  async (vorpal: CommandInstance): Promise<Array<GitlabVariable>> => {
+    const { id } = await getProjectInfo(vorpal);
+    let all: Array<GitlabVariable> = [];
+    let result: Array<GitlabVariable>;
+    let page = 1;
+    do {
+      result = await doGitlabRequest(
+        vorpal,
+        // 100 is max page size
+        `projects/${id}/variables?per_page=100&page=${page}`
+      );
+      page++;
+      all = [...all, ...result];
+    } while (result?.length > 0);
+    return all;
+  },
+  { promise: true }
+);
+
+export const getVariableValueByRawName = async (
+  vorpal: CommandInstance,
+  rawName: string
+) => {
+  const allVariables = await getAllVariables(vorpal);
+  return allVariables.find((v) => v.key === rawName)?.value;
+};
+
+const maskableRegex = new RegExp("^[a-zA-Z0-9_+=/@:.~-]{8,}$"); // SEE https://gitlab.com/gitlab-org/gitlab-foss/-/blob/master/spec/frontend/ci_variable_list/components/ci_variable_modal_spec.js#L20
+const isMaskable = (value: string): boolean => maskableRegex.test(value);
+
+const createVariable = async (
+  vorpal: CommandInstance,
+  projectId: string,
+  key: string,
+  value: string,
+  environment_scope = "*"
+) => {
+  return await doGitlabRequest(vorpal, `projects/${projectId}/variables`, {
+    key,
+    value,
+    masked: isMaskable(value),
+    environment_scope,
+  });
+};
+
+const updateVariable = async (
+  vorpal: CommandInstance,
+  projectId: string,
+  key: string,
+  value: string
+) => {
+  return await doGitlabRequest(
+    vorpal,
+    `projects/${projectId}/variables/${key}`,
+    {
+      value,
+      masked: isMaskable(value),
+    },
+    true
+  );
+};
+
+const getAllCatladderEnvVarsInGitlab = async (vorpal: CommandInstance) => {
+  const allVariables = await getAllVariables(vorpal).then((v) =>
+    v.reduce<{
+      [key: string]: {
+        value?: string;
+        backups: number[];
+      };
+    }>((acc, variable) => {
+      const { key } = variable;
+
+      if (key.startsWith("CL_")) {
+        const matchBackup = key.match(/(CL_.*)_backup_([0-9]+)/);
+
+        if (matchBackup) {
+          const key = matchBackup[1];
+          const timestamp = Number(matchBackup[2]);
+          const backups = [...(acc[key]?.backups ?? []), timestamp];
+          return {
+            ...acc,
+            [key]: {
+              ...(acc[key] ?? {}), // add value
+              backups,
+            },
+          };
+        }
+
+        return {
+          ...acc,
+          [key]: {
+            backups: [],
+            ...(acc[key] ?? {}), // may add backups
+            value: variable.value,
+          },
+        };
+      }
+
+      return acc;
+    }, {})
+  );
+  return allVariables;
+};
+
+export const upsertAllVariables = async (
+  vorpal: CommandInstance,
+  variables: Record<string, any>,
+  env: string,
+  componentName: string,
+  backup = true
+): Promise<void> => {
+  const { id } = await getProjectInfo(vorpal);
+
+  // we list all existing variables. We also gather backup versions, but its currently unused
+  const existingVariables = await getAllCatladderEnvVarsInGitlab(vorpal);
+  for (const [key, value] of Object.entries(variables ?? {})) {
+    const fullKey = getSecretVarName(env, componentName, key);
+    const valueSanitized = isObject(value) ? JSON.stringify(value) : `${value}`;
+
+    const exists = has(existingVariables, fullKey);
+    const oldValue = existingVariables[fullKey]?.value;
+    const changed = oldValue !== valueSanitized;
+    if (changed) {
+      if (exists) {
+        vorpal.log(`changed: ${key}`);
+
+        await updateVariable(vorpal, id, fullKey, valueSanitized);
+        // write backup
+        if (backup) {
+          await createVariable(
+            vorpal,
+            id,
+            fullKey + "_backup_" + new Date().getTime(),
+            oldValue,
+            "_backup"
+          );
+        }
+      } else {
+        vorpal.log(`new    : ${key}`);
+        await createVariable(vorpal, id, fullKey, valueSanitized);
+      }
+    } else {
+      vorpal.log(`skip   : ${key}`);
+    }
+  }
+  getAllVariables.clear();
+};

package/src/utils/passwordstore/index.ts

@@ -1,12 +1,6 @@
 import { exec, spawn } from "child-process-promise";
 import commandExists from "command-exists-promise";
 import dayjs from "dayjs";
-
-import { readFile, writeFile } from "fs-extra";
-import yaml from "js-yaml";
-import { withFile } from "tmp-promise";
-import formatEnvVars from "../formatEnvVars";
-import getEditor from "../getEditor";
 import { getPreference, hasPreference, setPreference } from "../preferences";
 
 const DEBUG = false;
@@ -78,9 +72,7 @@ const execBitwardenCommand = async (command: string): Promise<any> => {
       await loginBitwarden();
       return execBitwardenCommand(command);
     } else {
-      console.error(e);
-
-      console.log("wooops", e.message);
+      throw e;
     }
   }
 };
@@ -125,14 +117,6 @@ export const readPass = async (path: string) => {
   return result.notes || result.login?.password;
 };
 
-const update = async (type: string, itemId: string, value: any) => {
-  const result = await execBitwardenCommand(
-    `edit ${type} ${itemId} ${encode(value)}`
-  );
-
-  return result;
-};
-
 const MAX_SYNC_AGE_IN_MINUTES = 30;
 export const syncBitwarden = async (force = true) => {
   const lastSync = (await hasPreference("bwLastSync"))
@@ -169,24 +153,10 @@ const share = async (itemId: string) =>
     ])}`
   );
 
-export const editPass = async (path: string) => {
+export const trashItem = async (path: string) => {
   const item = await getItem(path);
 
-  await withFile(async ({ path: tmpFilePath }) => {
-    await writeFile(tmpFilePath, item.notes);
-    await (await getEditor()).open(tmpFilePath);
-    const newContent = (await readFile(tmpFilePath)).toString("utf-8");
+  const result = await execBitwardenCommand(`delete item ${item.id}`);
 
-    await update("item", item.id, {
-      ...item,
-      notes: newContent,
-    });
-  }, { postfix: ".yml" });
-};
-
-export const readPassEnvVars = async (path: string) => {
-  // make sure that you have pulled pass beforehand
-  const yamlstring = await readPass(path);
-  // if a value is an object, we convert it to strings
-  return formatEnvVars(yaml.load(yamlstring));
+  return result;
 };

package/src/utils/preferences/index.ts

@@ -1,7 +1,7 @@
 import fs from "fs-extra";
 import { homedir } from "os";
 
-import yaml from "js-yaml";
+import { load, dump } from "js-yaml";
 const directory = `${homedir()}/.catladder`;
 const file = `${directory}/preferences.yml`;
 
@@ -9,7 +9,10 @@ const getPreferences = async () => {
   if (!(await fs.pathExists(file))) {
     await fs.createFile(file);
   }
-  return yaml.load(await fs.readFile(file, { encoding: "utf-8" })) ?? {};
+  return (load(await fs.readFile(file, { encoding: "utf-8" })) ?? {}) as Record<
+    string,
+    string
+  >;
 };
 
 export const hasPreference = async (key: string) => {
@@ -29,5 +32,5 @@ export const setPreference = async (key: string, value: string | number) => {
     [key]: value,
   };
 
-  await fs.writeFile(file, yaml.safeDump(newPreferences));
+  await fs.writeFile(file, dump(newPreferences));
 };

package/src/utils/projects/index.ts

@@ -1,19 +1,11 @@
-import { exec } from "child-process-promise";
+import { $ } from "zx";
 import { join } from "path";
-import k8sApi from "../../k8sApi";
-import { Env, ISecrets, IValueFile } from "../../types/types";
-import { readFileOrError, readYaml } from "../files";
-import formatEnvVars from "../formatEnvVars";
-import { hasBitwarden, readPassEnvVars, syncBitwarden } from "../passwordstore";
-import { filter } from "../promise";
+import { getProjectConfig, parseChoice } from "../../config/getProjectConfig";
+import { getk8sApi } from "../../k8sApi";
+import { readFileOrError } from "../files";
 
-import { merge } from "lodash";
-
-// eslint-disable-next-line @typescript-eslint/no-var-requires
-const YAWN = require("yawn-yaml/cjs");
-
-export const getGitRoot = async () => {
-  return (await exec("git rev-parse --show-toplevel")).stdout?.trim();
+export const getGitRoot = async (): Promise<string> => {
+  return (await $`git rev-parse --show-toplevel`).stdout.trim();
 };
 
 export const getRootGitlabCiFile = async () => {
@@ -31,141 +23,33 @@ export const hasGitlabCiFile = async () => {
   }
   return true;
 };
-export const getLocalProjectVariables = async () => {
-  const [error, file] = await readRootGitlabCiFile();
-  if (error) {
-    throw new Error("there is no '.gitlab-ci.yml' in the current project");
-  }
-  // yawn allows to modify yaml while keeping comments!
-  const yawn = new YAWN(file);
-  const { variables } = yawn.json;
-  if (!variables) {
-    throw new Error("your '.gitlab-ci.yml' does not define `variables`");
-  }
-  const defaults = {
-    CLUSTER_NAME: "production",
-  };
-  return {
-    ...defaults,
-    ...variables,
-  };
-};
 
-export const getProjectNamespace = async (env: Env) => {
-  const { CUSTOMER_NAME, APP_NAME } = await getLocalProjectVariables();
-  return `${CUSTOMER_NAME}-${APP_NAME}-${env}`;
+export const getProjectNamespace = async (envComponent: string) => {
+  const { env } = parseChoice(envComponent);
+  const config = await getProjectConfig();
+  return `${config.customerName}-${config.appName}-${env}`;
 };
 
-export const getProjectHelmReleaseName = async (env: Env) => {
-  // can't properly get the release name for review as it contains an additional branch slug
-  if (env === "review" || env === "dev-local") {
-    throw new Error(`can't get helm release name for ${env}`);
-  }
-  const {
-    CUSTOMER_NAME,
-    APP_NAME,
-    COMPONENT_NAME = "web",
-  } = await getLocalProjectVariables();
-  return `${CUSTOMER_NAME}-${APP_NAME}-${env}-${COMPONENT_NAME}`;
-};
-export const getProjectPods = async (env: Env) => {
-  const namespace = await getProjectNamespace(env);
+export const getProjectPods = async (envComponent: string) => {
+  const namespace = await getProjectNamespace(envComponent);
+  const k8sApi = getk8sApi();
   const res = await k8sApi.listNamespacedPod(namespace);
 
-  return res.body.items;
+  const { componentName } = parseChoice(envComponent);
+  return res.body.items.filter((item) =>
+    componentName ? item.metadata?.name?.includes(componentName + "-") : true
+  );
 };
 
-export const getProjectPvcs = async (env: Env) => {
-  const namespace = await getProjectNamespace(env);
+export const getProjectPvcs = async (envComponent: string) => {
+  const namespace = await getProjectNamespace(envComponent);
+  const k8sApi = getk8sApi();
   const res = await k8sApi.listNamespacedPersistentVolumeClaim(namespace);
 
   return res.body.items;
 };
 
-export const getProjectPodNames = async (env: Env) => {
-  const pods = await getProjectPods(env);
+export const getProjectPodNames = async (envComponent: string) => {
+  const pods = await getProjectPods(envComponent);
   return pods.map((n) => n.metadata.name);
 };
-
-export const getPassPath = async (env: Env) => {
-  const { CUSTOMER_NAME, APP_NAME } = await getLocalProjectVariables();
-  return `${CUSTOMER_NAME}/${APP_NAME}/${env}/secrets.yml`;
-};
-
-export const getProjectValuesFiles = async (env: Env, subApp?: string) => {
-  const gitRoot = await getGitRoot();
-  const possibleFiles = ["values.yml", `values-${env}.yml`].map((file) =>
-    subApp ? join(gitRoot, subApp, file) : join(gitRoot, file)
-  );
-  return filter(possibleFiles, async (file) => {
-    const [error] = await readFileOrError(file);
-    return !error;
-  });
-};
-
-export const getAllValues = async (env: Env, subApp?: string) => {
-  const valuesFilePaths = await getProjectValuesFiles(env, subApp);
-  return Promise.all(
-    valuesFilePaths.map(async (file) => (await readYaml(file)) as IValueFile)
-  );
-};
-
-export const getProjectValues = async (env: Env, subApp?: string) => {
-  const values = await getAllValues(env, subApp);
-  return merge({}, ...values) as IValueFile;
-};
-
-export const getAllPublicEnvVars = async (env: Env, subApp?: string) => {
-  return formatEnvVars(
-    (await getAllValues(env, subApp)).reduce<ISecrets>((acc, value) => {
-      if (value.env && value.env.public) {
-        return {
-          ...acc,
-          ...value.env.public,
-        };
-      }
-      return acc;
-    }, {})
-  );
-};
-export const getAllSecretsEnvVarsMapping = async (
-  env: Env,
-  subApp?: string
-) => {
-  return (await getAllValues(env, subApp)).reduce<ISecrets>((acc, value) => {
-    if (value.env && value.env.secret) {
-      return {
-        ...acc,
-        ...value.env.secret,
-      };
-    }
-    return acc;
-  }, {});
-};
-
-export const hasSecrets = async (env: Env, subApp?: string) => {
-  return Object.keys(await getAllSecretsEnvVarsMapping(env, subApp)).length > 0;
-};
-
-export const getAllEnvVars = async (
-  env: Env,
-  subApp?: string
-): Promise<Record<string, string>> => {
-  return {
-    ...(await getAllPublicEnvVars(env, subApp)),
-    ...(await getAllSecretEnvVars(env, subApp)),
-  };
-};
-
-export const getAllSecretEnvVars = async (env: Env, subApp?: string) => {
-  if (!(await hasSecrets(env, subApp))) {
-    return {};
-  }
-  if (await hasBitwarden()) {
-    await syncBitwarden(false /* do not force sync */);
-    const passPath = await getPassPath(env);
-    return readPassEnvVars(passPath);
-  } else {
-    return {};
-  }
-};

package/tsconfig.json

@@ -1,6 +1,6 @@
 {
-  "extends": "@tsconfig/node14/tsconfig.json",
   "compilerOptions": {
+    "lib": ["es2020", "DOM"],
     "module": "CommonJS",
     "declaration": true,
     "noImplicitAny": true,
@@ -9,12 +9,8 @@
     "sourceMap": true,
     "outDir": "dist",
     "strict": false,
-    "baseUrl": ".",
     "incremental": true,
-    "resolveJsonModule": true,
-    "paths": {
-      "*": ["node_modules/*", "src/types/*"]
-    }
+    "resolveJsonModule": true
   },
-  "include": ["src/**/*"]
+  "include": ["./src/**/*"]
 }

package/bin/catenv.sh

@@ -1 +0,0 @@
-eval $(catladder --catenv )

package/src/apps/shell/commands/mongodb/projectMongoPortForward.ts

@@ -1,42 +0,0 @@
-import Vorpal from "vorpal";
-import { logError } from "../../../../utils/log";
-import { startPortForward } from "../../../../utils/portForward";
-import { getProjectNamespace } from "../../../../utils/projects";
-import { envAutocompletion } from "../project/utils/autocompletions";
-import ensureCluster from "../project/utils/ensureCluster";
-import { getProjectMongodbAllPodsSortedWithLabel } from "./utils";
-
-export default (vorpal: Vorpal) =>
-  vorpal
-    .command("project-mongo-port-forward <env>", "port foward to a mongodb")
-    .autocomplete(envAutocompletion)
-    .action(async function ({ env }) {
-      await ensureCluster.call(this);
-      const namespace = await getProjectNamespace(env);
-      const podNames = await getProjectMongodbAllPodsSortedWithLabel(env);
-      if (podNames.length === 0) {
-        logError(this, "sorry, no pods found");
-        return;
-      }
-      let podName;
-      if (podNames.length === 1) {
-        podName = podNames[0].value;
-      } else {
-        podName = (
-          await this.prompt({
-            type: "list",
-            name: "podName",
-            choices: podNames,
-            message: "Which pod? 🤔",
-          })
-        ).podName;
-      }
-
-      const { localPort } = await this.prompt({
-        type: "number",
-        name: "localPort",
-        default: "30000",
-        message: "Local port: ",
-      });
-      return startPortForward(podName, localPort, 27017, namespace);
-    });

package/src/apps/shell/commands/project/commandCloudSqlProxy.ts

@@ -1,65 +0,0 @@
-import { spawn } from "child-process-promise";
-import { writeFile } from "fs-extra";
-import { withFile } from "tmp-promise";
-import Vorpal from "vorpal";
-import {
-  GOOGLE_CLOUD_SQL_PASS_PATH,
-  GOOGLE_PROJECT,
-} from "../../../../config/constants";
-import { readPass } from "../../../../utils/passwordstore";
-import {
-  getAllEnvVars,
-  getLocalProjectVariables,
-  getProjectValues,
-} from "../../../../utils/projects";
-import { envAutocompletion } from "./utils/autocompletions";
-import { promptForSubAppIfAny } from "./utils/monorepo";
-
-export default (vorpal: Vorpal) =>
-  vorpal
-    .command("project-cloud-sql-proxy <env>", "proxy to cloud sql db")
-    .autocomplete(envAutocompletion)
-    .action(async function ({ env }) {
-      const { CUSTOMER_NAME, APP_NAME } = await getLocalProjectVariables();
-      // skynet-164509:europe-west6:pvl-cyclomania-review=tcp:5432
-
-      const { localPort } = await this.prompt({
-        type: "number",
-        name: "localPort",
-        default: "54320",
-        message: "Local port: ",
-      });
-
-      const subapp = await promptForSubAppIfAny(this);
-      const POSTGRESQL_PASSWORD = (await getAllEnvVars(env, subapp))
-        ?.POSTGRESQL_PASSWORD;
-
-      const values = await getProjectValues(env, subapp);
-      this.log("");
-      this.log(`postgres-PW: ${POSTGRESQL_PASSWORD}`);
-      this.log("");
-
-      const projectId = values?.cloudsql?.projectId || GOOGLE_PROJECT;
-
-      const defaultInstanceId = `${CUSTOMER_NAME}-${APP_NAME}-${env}`;
-      const instanceId = values?.cloudsql?.instanceId || defaultInstanceId;
-
-      const defaultRegion = "europe-west6"; // currently hardcoded
-      const region = values?.cloudsql?.region || defaultRegion;
-
-      const instanceName = `${projectId}:${region}:${instanceId}=tcp:${localPort}`;
-
-      const cloudsqlCredentials = await readPass(GOOGLE_CLOUD_SQL_PASS_PATH);
-      await withFile(async ({ path: tmpFilePath }) => {
-        await writeFile(tmpFilePath, cloudsqlCredentials);
-
-        await spawn(
-          "cloud_sql_proxy",
-          ["-instances", instanceName, "-credential_file", tmpFilePath],
-          {
-            stdio: "inherit",
-            shell: true,
-          }
-        );
-      });
-    });