@catladder/cli 0.0.0-test7 → 0.0.0-update-meteor-53480fed

package/src/utils/gitlab.ts

@@ -1,80 +1,301 @@
+import { getSecretVarName } from "@catladder/pipeline";
 import { exec } from "child-process-promise";
+import { has, isObject } from "lodash";
+import memoizee from "memoizee";
 import fetch from "node-fetch";
 import open from "open";
-import { CommandInstance } from "vorpal";
+import type { CommandInstance } from "vorpal";
 import { getPreference, hasPreference, setPreference } from "./preferences";
 
 const TOKEN_KEY = "gitlab-personal-access-token";
-export const getGitlabToken = async (vorpal: CommandInstance) => {
-  if (!(await hasPreference(TOKEN_KEY))) {
-    vorpal.log("");
-    vorpal.log(
-      "☝ in order to access the api, we need a personal access token"
-    );
-    vorpal.log("Its best to create one specifically for catladder");
-    vorpal.log("");
-    vorpal.log("☝ we open up the settings page for you!");
-    vorpal.log("");
-    const { shouldContinue } = await vorpal.prompt({
-      default: true,
-      message: "Ok",
-      name: "shouldContinue",
-      type: "prompt",
-    });
-
-    open("https://git.panter.ch/-/profile/personal_access_tokens");
-
-    vorpal.log("Please type in gitlab's personal access token");
-
-    const { personalToken } = await vorpal.prompt({
-      type: "string",
-      name: "personalToken",
-      default: "",
-      message: "Your personal access token ",
-    });
-    if (personalToken) {
-      await setPreference(TOKEN_KEY, personalToken);
+
+export const hasGitlabToken = async () => await hasPreference(TOKEN_KEY);
+export const setupGitlabToken = async (vorpal: CommandInstance) => {
+  vorpal.log("");
+  vorpal.log("☝ in order to access the api, we need a personal access token");
+  vorpal.log("Its best to create one specifically for catladder");
+  vorpal.log("");
+  vorpal.log("☝ we open up the settings page for you!");
+  vorpal.log("");
+  const { shouldContinue } = await vorpal.prompt({
+    default: true,
+    message: "Ok",
+    name: "shouldContinue",
+    type: "prompt",
+  });
+
+  open("https://git.panter.ch/-/profile/personal_access_tokens");
+
+  vorpal.log("Please type in gitlab's personal access token");
+
+  const { personalToken } = await vorpal.prompt({
+    type: "string",
+    name: "personalToken",
+    default: "",
+    message: "Your personal access token ",
+  });
+  if (personalToken) {
+    await setPreference(TOKEN_KEY, personalToken);
+  }
+};
+export const getGitlabToken = async (vorpal: CommandInstance | null) => {
+  if (!(await hasGitlabToken())) {
+    if (!vorpal) {
+      console.error(
+        "⚠️ gitlab token missing, please run catladder to set it up"
+      );
+      process.exit(1);
     }
+    await setupGitlabToken(vorpal);
   }
   return getPreference(TOKEN_KEY);
 };
 
-export const doGitlabRequest = async (
-  vorpal: CommandInstance,
+type Method = "GET" | "PUT" | "POST" | "DELETE";
+export const doGitlabRequest = async <T = any>(
+  vorpal: CommandInstance | null,
   path: string,
-  data?: any
-) => {
+  data: any = undefined,
+  method: Method = "GET"
+): Promise<T> => {
   const rootToken = await getGitlabToken(vorpal);
+
+  //const method = data ? (update ? "PUT" : "POST") : "GET";
+
   const result = await fetch(`https://git.panter.ch/api/v4/${path}`, {
-    method: data ? "POST" : "GET",
+    method,
     headers: {
       "Content-Type": "application/json",
       "Private-Token": rootToken,
     },
-    body: JSON.stringify(data),
+    body: data ? JSON.stringify(data) : undefined,
   });
 
   if (result.status >= 200 && result.status < 400) {
-    return result.json();
+    if (result.headers.get("content-type") === "application/json") {
+      return result.json();
+    }
+    return null;
+  }
+  if (result.status === 404) {
+    throw new Error("not found");
   }
 
   throw new Error(
-    `Could not send request to gitlab api: ${result.status} "${
+    `Could not send request to gitlab api ${path}: ${result.status} "${
       result.statusText
     }".\nResponse: ${JSON.stringify(await result.json(), null, 2)}`
   );
 };
 
-export const getProjectInfo = async (vorpal: CommandInstance): Promise<any> => {
+export const getProjectInfo = async (
+  vorpal: CommandInstance | null
+): Promise<{ id: string; web_url: string }> => {
   const gitRemoteOriginUrl = (
     await exec("git config --get remote.origin.url")
   ).stdout.trim();
-  const projectPath = /(https:\/\/|git@)git\.panter\.ch[:/](.*)\.git/g.exec(
-    gitRemoteOriginUrl
-  );
+  const projectPath =
+    /(https:\/\/|git@)git\.panter\.ch[:/]([^.]*)(\.git)?/g.exec(
+      gitRemoteOriginUrl
+    );
   const project = await doGitlabRequest(
     vorpal,
     `projects/${encodeURIComponent(projectPath[2])}`
   );
   return project;
 };
+
+type GitlabVariable = {
+  variable_type: string;
+  key: string;
+  value: string;
+  protected: boolean;
+  masked: boolean;
+  environment_scope: string;
+};
+export const getAllVariables = memoizee(
+  async (vorpal: CommandInstance | null): Promise<Array<GitlabVariable>> => {
+    const { id } = await getProjectInfo(vorpal);
+    let all: Array<GitlabVariable> = [];
+    let result: Array<GitlabVariable>;
+    let page = 1;
+    do {
+      result = await doGitlabRequest(
+        vorpal,
+        // 100 is max page size
+        `projects/${id}/variables?per_page=100&page=${page}`
+      );
+      page++;
+      all = [...all, ...result];
+    } while (result?.length > 0);
+    return all;
+  },
+  { promise: true }
+);
+
+export const getVariableValueByRawName = async (
+  vorpal: CommandInstance,
+  rawName: string
+) => {
+  const allVariables = await getAllVariables(vorpal);
+  return allVariables.find((v) => v.key === rawName)?.value;
+};
+
+const maskableRegex = new RegExp("^[a-zA-Z0-9_+=/@:.~-]{8,}$"); // SEE https://gitlab.com/gitlab-org/gitlab-foss/-/blob/master/spec/frontend/ci_variable_list/components/ci_variable_modal_spec.js#L20
+const isMaskable = (value: string): boolean => maskableRegex.test(value);
+
+const createVariable = async (
+  vorpal: CommandInstance,
+  projectId: string,
+  key: string,
+  value: string,
+  masked = true,
+  environment_scope = "*"
+) => {
+  return await doGitlabRequest(
+    vorpal,
+    `projects/${projectId}/variables`,
+    {
+      key,
+      value,
+      masked: masked && isMaskable(value),
+      environment_scope,
+    },
+    "POST"
+  );
+};
+
+const updateVariable = async (
+  vorpal: CommandInstance,
+  projectId: string,
+  key: string,
+  value: string,
+  masked = true
+) => {
+  return await doGitlabRequest(
+    vorpal,
+    `projects/${projectId}/variables/${key}`,
+    {
+      value,
+      masked: masked && isMaskable(value),
+    },
+    "PUT"
+  );
+};
+
+const deleteVariable = async (
+  vorpal: CommandInstance,
+  projectId: string,
+  key: string
+) => {
+  return await doGitlabRequest(
+    vorpal,
+    `projects/${projectId}/variables/${key}`,
+    undefined,
+    "DELETE"
+  );
+};
+
+const getAllCatladderEnvVarsInGitlab = async (vorpal: CommandInstance) => {
+  const allVariables = await getAllVariables(vorpal).then((v) =>
+    v.reduce<{
+      [key: string]: {
+        value?: string;
+        backups: number[];
+      };
+    }>((acc, variable) => {
+      const { key } = variable;
+
+      if (key.startsWith("CL_")) {
+        const matchBackup = key.match(/(CL_.*)_backup_([0-9]+)/);
+
+        if (matchBackup) {
+          const key = matchBackup[1];
+          const timestamp = Number(matchBackup[2]);
+          const backups = [...(acc[key]?.backups ?? []), timestamp];
+          return {
+            ...acc,
+            [key]: {
+              ...(acc[key] ?? {}), // add value
+              backups,
+            },
+          };
+        }
+
+        return {
+          ...acc,
+          [key]: {
+            backups: [],
+            ...(acc[key] ?? {}), // may add backups
+            value: variable.value,
+          },
+        };
+      }
+
+      return acc;
+    }, {})
+  );
+  return allVariables;
+};
+
+const getBackupKey = (fullKey: string, timestamp: number) =>
+  `${fullKey}_backup_${timestamp}`;
+export const clearBackups = async (vorpal: CommandInstance, keep: number) => {
+  const existingVariables = await getAllCatladderEnvVarsInGitlab(vorpal);
+  const { id } = await getProjectInfo(vorpal);
+  for (const [key, { backups }] of Object.entries(existingVariables)) {
+    const backupsSorted = backups.sort((a, b) => b - a);
+    //const toKeep = backupsSorted.slice(0, keep);
+    const toDelete = backupsSorted.slice(keep);
+
+    for (const timestamp of toDelete) {
+      await deleteVariable(vorpal, id, getBackupKey(key, timestamp));
+    }
+  }
+};
+
+export const upsertAllVariables = async (
+  vorpal: CommandInstance,
+  variables: Record<string, any>,
+  env: string,
+  componentName: string,
+  backup = true,
+  masked = true // FIXME: would be better to have this per variable
+): Promise<void> => {
+  const { id } = await getProjectInfo(vorpal);
+
+  // we list all existing variables. We also gather backup versions, but its currently unused
+  const existingVariables = await getAllCatladderEnvVarsInGitlab(vorpal);
+  for (const [key, value] of Object.entries(variables ?? {})) {
+    const fullKey = getSecretVarName(env, componentName, key);
+    const valueSanitized = isObject(value) ? JSON.stringify(value) : `${value}`;
+
+    const exists = has(existingVariables, fullKey);
+    const oldValue = existingVariables[fullKey]?.value;
+    const changed = oldValue !== valueSanitized;
+    if (changed) {
+      if (exists) {
+        vorpal.log(`changed: ${key}`);
+
+        await updateVariable(vorpal, id, fullKey, valueSanitized, masked);
+        // write backup
+        if (backup) {
+          await createVariable(
+            vorpal,
+            id,
+            getBackupKey(fullKey, new Date().getTime()),
+            oldValue,
+            masked,
+            "_backup"
+          );
+        }
+      } else {
+        vorpal.log(`new    : ${key}`);
+        await createVariable(vorpal, id, fullKey, valueSanitized, masked);
+      }
+    } else {
+      vorpal.log(`skip   : ${key}`);
+    }
+  }
+  getAllVariables.clear();
+};

package/src/utils/passwordstore/index.ts

@@ -1,12 +1,6 @@
 import { exec, spawn } from "child-process-promise";
 import commandExists from "command-exists-promise";
 import dayjs from "dayjs";
-
-import { readFile, writeFile } from "fs-extra";
-import yaml from "js-yaml";
-import { withFile } from "tmp-promise";
-import formatEnvVars from "../formatEnvVars";
-import getEditor from "../getEditor";
 import { getPreference, hasPreference, setPreference } from "../preferences";
 
 const DEBUG = false;
@@ -21,7 +15,7 @@ const unlockBitwarden = async () => {
   let session = null;
   promise.childProcess.stdout.on(
     "data",
-    (d: any) => (session = d.toString("utf-8"))
+    (d) => (session = d.toString("utf-8"))
   );
   await promise;
   await setPreference("bwsession", session);
@@ -36,7 +30,7 @@ const loginBitwarden = async () => {
   let session = null;
   promise.childProcess.stdout.on(
     "data",
-    (d: any) => (session = d.toString("utf-8"))
+    (d) => (session = d.toString("utf-8"))
   );
   await promise;
   await setPreference("bwsession", session);
@@ -78,9 +72,7 @@ const execBitwardenCommand = async (command: string): Promise<any> => {
       await loginBitwarden();
       return execBitwardenCommand(command);
     } else {
-      console.error(e);
-
-      console.log("wooops", e.message);
+      throw e;
     }
   }
 };
@@ -125,14 +117,6 @@ export const readPass = async (path: string) => {
   return result.notes || result.login?.password;
 };
 
-const update = async (type: string, itemId: string, value: any) => {
-  const result = await execBitwardenCommand(
-    `edit ${type} ${itemId} ${encode(value)}`
-  );
-
-  return result;
-};
-
 const MAX_SYNC_AGE_IN_MINUTES = 30;
 export const syncBitwarden = async (force = true) => {
   const lastSync = (await hasPreference("bwLastSync"))
@@ -169,24 +153,10 @@ const share = async (itemId: string) =>
     ])}`
   );
 
-export const editPass = async (path: string) => {
+export const trashItem = async (path: string) => {
   const item = await getItem(path);
 
-  await withFile(async ({ path: tmpFilePath }) => {
-    await writeFile(tmpFilePath, item.notes);
-    await (await getEditor()).open(tmpFilePath);
-    const newContent = (await readFile(tmpFilePath)).toString("utf-8");
+  const result = await execBitwardenCommand(`delete item ${item.id}`);
 
-    await update("item", item.id, {
-      ...item,
-      notes: newContent,
-    });
-  }, { postfix: ".yml" });
-};
-
-export const readPassEnvVars = async (path: string) => {
-  // make sure that you have pulled pass beforehand
-  const yamlstring = await readPass(path);
-  // if a value is an object, we convert it to strings
-  return formatEnvVars(yaml.load(yamlstring));
+  return result;
 };

package/src/utils/{portForward.ts → portForwards.ts}

@@ -1,6 +1,14 @@
 import { spawn } from "child-process-promise";
+import type { ReadStream } from "fs-extra";
+import open from "open";
 
-const portForwards = new Map();
+export type PortForward = Promise<unknown> & {
+  childProcess: {
+    stdout: ReadStream;
+    kill: () => any;
+  };
+};
+const portForwards = new Map<string, PortForward>();
 
 export const stopPortForward = async (name: string) => {
   const old = portForwards.get(name);
@@ -19,32 +27,29 @@ export const stopPortForward = async (name: string) => {
 export const getAllRunningPortForwards = () => {
   return Array.from(portForwards.keys());
 };
-export const startPortForward = async (
-  podname: string,
-  localPort: number,
-  remotePort: number,
-  namespace: string
+
+const addPortForward = (name: string, portForward: PortForward) => {
+  portForwards.set(name, portForward);
+};
+
+export const startPortForwardCommand = async (
+  name: string,
+  command: string
 ) => {
-  const name = `${namespace}/${podname}/${localPort}:${remotePort}`;
   // stop if already there
-  if (portForwards.has(name)) {
-    stopPortForward(name);
-  }
-  const promise = spawn(
-    "kubectl",
-    ["port-forward", podname, `${localPort}:${remotePort}`, "-n", namespace],
-    {
-      env: {
-        ...process.env,
-        DEBUG: "",
-      },
-    }
-  );
-  portForwards.set(name, promise);
+  await stopPortForward(name);
+  const [cmd, ...args] = command.split(" ");
+
+  const promise = spawn(cmd, args, {
+    env: {
+      ...process.env,
+      DEBUG: "",
+    },
+  }) as PortForward;
+  addPortForward(name, promise);
   // wait a moment so that is surley started, unfortunatly we don't know that
   await new Promise((r) => setTimeout(r, 1000));
 };
-
 export const stopAllPortForwards = async () => {
   getAllRunningPortForwards().forEach((name) => {
     stopPortForward(name);

package/src/utils/preferences/index.ts

@@ -1,7 +1,7 @@
 import fs from "fs-extra";
 import { homedir } from "os";
 
-import yaml from "js-yaml";
+import { load, dump } from "js-yaml";
 const directory = `${homedir()}/.catladder`;
 const file = `${directory}/preferences.yml`;
 
@@ -9,7 +9,10 @@ const getPreferences = async () => {
   if (!(await fs.pathExists(file))) {
     await fs.createFile(file);
   }
-  return yaml.load(await fs.readFile(file, { encoding: "utf-8" })) ?? {};
+  return (load(await fs.readFile(file, { encoding: "utf-8" })) ?? {}) as Record<
+    string,
+    string
+  >;
 };
 
 export const hasPreference = async (key: string) => {
@@ -29,5 +32,5 @@ export const setPreference = async (key: string, value: string | number) => {
     [key]: value,
   };
 
-  await fs.writeFile(file, yaml.safeDump(newPreferences));
+  await fs.writeFile(file, dump(newPreferences));
 };

package/src/utils/projects/index.ts

@@ -1,19 +1,11 @@
-import { exec } from "child-process-promise";
 import { join } from "path";
-import k8sApi from "../../k8sApi";
-import { Env, ISecrets, IValueFile } from "../../types/types";
-import { readFileOrError, readYaml } from "../files";
-import formatEnvVars from "../formatEnvVars";
-import { hasBitwarden, readPassEnvVars, syncBitwarden } from "../passwordstore";
-import { filter } from "../promise";
-
-import { merge } from "lodash";
+import { getProjectConfig, parseChoice } from "../../config/getProjectConfig";
 
-// eslint-disable-next-line @typescript-eslint/no-var-requires
-const YAWN = require("yawn-yaml/cjs");
+import { readFileOrError } from "../files";
+import { exec } from "child-process-promise";
 
-export const getGitRoot = async () => {
-  return (await exec("git rev-parse --show-toplevel")).stdout?.trim();
+export const getGitRoot = async (): Promise<string> => {
+  return (await exec(`git rev-parse --show-toplevel`)).stdout.trim();
 };
 
 export const getRootGitlabCiFile = async () => {
@@ -31,141 +23,9 @@ export const hasGitlabCiFile = async () => {
   }
   return true;
 };
-export const getLocalProjectVariables = async () => {
-  const [error, file] = await readRootGitlabCiFile();
-  if (error) {
-    throw new Error("there is no '.gitlab-ci.yml' in the current project");
-  }
-  // yawn allows to modify yaml while keeping comments!
-  const yawn = new YAWN(file);
-  const { variables } = yawn.json;
-  if (!variables) {
-    throw new Error("your '.gitlab-ci.yml' does not define `variables`");
-  }
-  const defaults = {
-    CLUSTER_NAME: "production",
-  };
-  return {
-    ...defaults,
-    ...variables,
-  };
-};
-
-export const getProjectNamespace = async (env: Env) => {
-  const { CUSTOMER_NAME, APP_NAME } = await getLocalProjectVariables();
-  return `${CUSTOMER_NAME}-${APP_NAME}-${env}`;
-};
-
-export const getProjectHelmReleaseName = async (env: Env) => {
-  // can't properly get the release name for review as it contains an additional branch slug
-  if (env === "review" || env === "dev-local") {
-    throw new Error(`can't get helm release name for ${env}`);
-  }
-  const {
-    CUSTOMER_NAME,
-    APP_NAME,
-    COMPONENT_NAME = "web",
-  } = await getLocalProjectVariables();
-  return `${CUSTOMER_NAME}-${APP_NAME}-${env}-${COMPONENT_NAME}`;
-};
-export const getProjectPods = async (env: Env) => {
-  const namespace = await getProjectNamespace(env);
-  const res = await k8sApi.listNamespacedPod(namespace);
-
-  return res.body.items;
-};
-
-export const getProjectPvcs = async (env: Env) => {
-  const namespace = await getProjectNamespace(env);
-  const res = await k8sApi.listNamespacedPersistentVolumeClaim(namespace);
-
-  return res.body.items;
-};
-
-export const getProjectPodNames = async (env: Env) => {
-  const pods = await getProjectPods(env);
-  return pods.map((n) => n.metadata.name);
-};
-
-export const getPassPath = async (env: Env) => {
-  const { CUSTOMER_NAME, APP_NAME } = await getLocalProjectVariables();
-  return `${CUSTOMER_NAME}/${APP_NAME}/${env}/secrets.yml`;
-};
-
-export const getProjectValuesFiles = async (env: Env, subApp?: string) => {
-  const gitRoot = await getGitRoot();
-  const possibleFiles = ["values.yml", `values-${env}.yml`].map((file) =>
-    subApp ? join(gitRoot, subApp, file) : join(gitRoot, file)
-  );
-  return filter(possibleFiles, async (file) => {
-    const [error] = await readFileOrError(file);
-    return !error;
-  });
-};
-
-export const getAllValues = async (env: Env, subApp?: string) => {
-  const valuesFilePaths = await getProjectValuesFiles(env, subApp);
-  return Promise.all(
-    valuesFilePaths.map(async (file) => (await readYaml(file)) as IValueFile)
-  );
-};
-
-export const getProjectValues = async (env: Env, subApp?: string) => {
-  const values = await getAllValues(env, subApp);
-  return merge({}, ...values) as IValueFile;
-};
 
-export const getAllPublicEnvVars = async (env: Env, subApp?: string) => {
-  return formatEnvVars(
-    (await getAllValues(env, subApp)).reduce<ISecrets>((acc, value) => {
-      if (value.env && value.env.public) {
-        return {
-          ...acc,
-          ...value.env.public,
-        };
-      }
-      return acc;
-    }, {})
-  );
-};
-export const getAllSecretsEnvVarsMapping = async (
-  env: Env,
-  subApp?: string
-) => {
-  return (await getAllValues(env, subApp)).reduce<ISecrets>((acc, value) => {
-    if (value.env && value.env.secret) {
-      return {
-        ...acc,
-        ...value.env.secret,
-      };
-    }
-    return acc;
-  }, {});
-};
-
-export const hasSecrets = async (env: Env, subApp?: string) => {
-  return Object.keys(await getAllSecretsEnvVarsMapping(env, subApp)).length > 0;
-};
-
-export const getAllEnvVars = async (
-  env: Env,
-  subApp?: string
-): Promise<Record<string, string>> => {
-  return {
-    ...(await getAllPublicEnvVars(env, subApp)),
-    ...(await getAllSecretEnvVars(env, subApp)),
-  };
-};
-
-export const getAllSecretEnvVars = async (env: Env, subApp?: string) => {
-  if (!(await hasSecrets(env, subApp))) {
-    return {};
-  }
-  if (await hasBitwarden()) {
-    await syncBitwarden(false /* do not force sync */);
-    const passPath = await getPassPath(env);
-    return readPassEnvVars(passPath);
-  } else {
-    return {};
-  }
+export const getProjectNamespace = async (envComponent: string) => {
+  const { env } = parseChoice(envComponent);
+  const config = await getProjectConfig();
+  return `${config.customerName}-${config.appName}-${env}`;
 };