@catladder/cli 0.0.0-test7 → 0.0.0-waitfor-deploy-e64a33ba

package/src/utils/cluster.ts

@@ -1,21 +1,12 @@
-import { exec } from "child-process-promise";
-import { findKey } from "lodash";
-import clusters from "../config/clusters";
+import { $ } from "zx";
+
 export const getCurrentContext = async () =>
-  (await exec("kubectl config current-context")).stdout.trim();
+  (await $`kubectl config current-context`).stdout.trim();
 
 export const getCurrentConnectedClusterName = async () => {
-  const currentContext = await getCurrentContext();
-  return findKey(clusters, { fullName: currentContext });
-};
-
-export const getClusterByName = (name: string) => {
-  return clusters[name];
+  return await getCurrentContext();
 };
 
-export const getAllClusterNames = () => Object.keys(clusters);
-
-export const connectToCluster = async (clusterName: string) => {
-  const { connect } = getClusterByName(clusterName);
-  await connect();
+export const connectToCluster = async (fullname: string) => {
+  await $`kubectl config use-context ${fullname}`;
 };

package/src/utils/editAsFile.ts

@@ -0,0 +1,31 @@
+import { withFile } from "tmp-promise";
+import { dump, load } from "js-yaml";
+
+import { readFile, writeFile } from "fs-extra";
+import getEditor from "./getEditor";
+
+export const editAsFile = async <T>(
+  inObject: T,
+  preamble?: string
+): Promise<T> => {
+  const fullPreamble = preamble
+    ? `#
+# ${preamble.split("\n").join("\n# ")}
+#
+
+`
+    : "\n";
+  const asString = fullPreamble + dump(inObject, { noRefs: true });
+  let newContent: T;
+
+  await withFile(
+    async ({ path: tmpFilePath }) => {
+      await writeFile(tmpFilePath, asString);
+      await (await getEditor()).open(tmpFilePath);
+      newContent = load((await readFile(tmpFilePath)).toString("utf-8")) as T;
+    },
+    { postfix: ".yml" }
+  );
+
+  return newContent;
+};

package/src/utils/files.ts

@@ -13,6 +13,10 @@ export const readFileOrError = async (filePath: string) => {
   }
 };
 
-export const readYaml = async (filename: string) => {
-  return yaml.load(await readFile(filename));
+export const readYaml = async <T = Record<string, any>>(filename: string) => {
+  try {
+    return yaml.load(await readFile(filename)) as T;
+  } catch (e) {
+    return null;
+  }
 };

package/src/utils/gitlab.ts

@@ -1,52 +1,68 @@
 import { exec } from "child-process-promise";
+import { getSecretVarName } from "@catladder/pipeline";
+import { isObject } from "lodash";
+import memoizee from "memoizee";
 import fetch from "node-fetch";
 import open from "open";
 import { CommandInstance } from "vorpal";
 import { getPreference, hasPreference, setPreference } from "./preferences";
 
 const TOKEN_KEY = "gitlab-personal-access-token";
+
+export const hasGitlabToken = async () => await hasPreference(TOKEN_KEY);
+export const setupGitlabToken = async (vorpal: CommandInstance) => {
+  vorpal.log("");
+  vorpal.log("☝ in order to access the api, we need a personal access token");
+  vorpal.log("Its best to create one specifically for catladder");
+  vorpal.log("");
+  vorpal.log("☝ we open up the settings page for you!");
+  vorpal.log("");
+  const { shouldContinue } = await vorpal.prompt({
+    default: true,
+    message: "Ok",
+    name: "shouldContinue",
+    type: "prompt",
+  });
+
+  open("https://git.panter.ch/-/profile/personal_access_tokens");
+
+  vorpal.log("Please type in gitlab's personal access token");
+
+  const { personalToken } = await vorpal.prompt({
+    type: "string",
+    name: "personalToken",
+    default: "",
+    message: "Your personal access token ",
+  });
+  if (personalToken) {
+    await setPreference(TOKEN_KEY, personalToken);
+  }
+};
 export const getGitlabToken = async (vorpal: CommandInstance) => {
-  if (!(await hasPreference(TOKEN_KEY))) {
-    vorpal.log("");
-    vorpal.log(
-      "☝ in order to access the api, we need a personal access token"
-    );
-    vorpal.log("Its best to create one specifically for catladder");
-    vorpal.log("");
-    vorpal.log("☝ we open up the settings page for you!");
-    vorpal.log("");
-    const { shouldContinue } = await vorpal.prompt({
-      default: true,
-      message: "Ok",
-      name: "shouldContinue",
-      type: "prompt",
-    });
-
-    open("https://git.panter.ch/-/profile/personal_access_tokens");
-
-    vorpal.log("Please type in gitlab's personal access token");
-
-    const { personalToken } = await vorpal.prompt({
-      type: "string",
-      name: "personalToken",
-      default: "",
-      message: "Your personal access token ",
-    });
-    if (personalToken) {
-      await setPreference(TOKEN_KEY, personalToken);
+  if (!(await hasGitlabToken())) {
+    if (!vorpal) {
+      console.error(
+        "⚠️ gitlab token missing, please run catladder to set it up"
+      );
+      process.exit(1);
     }
+    await setupGitlabToken(vorpal);
   }
   return getPreference(TOKEN_KEY);
 };
 
-export const doGitlabRequest = async (
+export const doGitlabRequest = async <T = any>(
   vorpal: CommandInstance,
   path: string,
-  data?: any
-) => {
+  data: any = undefined,
+  update?: boolean
+): Promise<T> => {
   const rootToken = await getGitlabToken(vorpal);
+
+  const method = data ? (update ? "PUT" : "POST") : "GET";
+
   const result = await fetch(`https://git.panter.ch/api/v4/${path}`, {
-    method: data ? "POST" : "GET",
+    method,
     headers: {
       "Content-Type": "application/json",
       "Private-Token": rootToken,
@@ -57,15 +73,20 @@ export const doGitlabRequest = async (
   if (result.status >= 200 && result.status < 400) {
     return result.json();
   }
+  if (result.status === 404) {
+    throw new Error("not found");
+  }
 
   throw new Error(
-    `Could not send request to gitlab api: ${result.status} "${
+    `Could not send request to gitlab api ${path}: ${result.status} "${
       result.statusText
     }".\nResponse: ${JSON.stringify(await result.json(), null, 2)}`
   );
 };
 
-export const getProjectInfo = async (vorpal: CommandInstance): Promise<any> => {
+export const getProjectInfo = async (
+  vorpal: CommandInstance
+): Promise<{ id: string; web_url: string }> => {
   const gitRemoteOriginUrl = (
     await exec("git config --get remote.origin.url")
   ).stdout.trim();
@@ -78,3 +99,92 @@ export const getProjectInfo = async (vorpal: CommandInstance): Promise<any> => {
   );
   return project;
 };
+
+type GitlabVariable = {
+  variable_type: string;
+  key: string;
+  value: string;
+  protected: boolean;
+  masked: boolean;
+  environment_scope: string;
+};
+export const getAllVariables = memoizee(
+  async (vorpal: CommandInstance): Promise<Array<GitlabVariable>> => {
+    const { id } = await getProjectInfo(vorpal);
+    let all: Array<GitlabVariable> = [];
+    let result: Array<GitlabVariable>;
+    let page = 1;
+    do {
+      result = await doGitlabRequest(
+        vorpal,
+        // 100 is max page size
+        `projects/${id}/variables?per_page=100&page=${page}`
+      );
+      page++;
+      all = [...all, ...result];
+    } while (result?.length > 0);
+    return all;
+  },
+  { promise: true }
+);
+
+export const getVariableValueByRawName = async (
+  vorpal: CommandInstance,
+  rawName: string
+) => {
+  const allVariables = await getAllVariables(vorpal);
+  return allVariables.find((v) => v.key === rawName)?.value;
+};
+
+const maskableRegex = new RegExp("^[a-zA-Z0-9_+=/@:.~-]{8,}$"); // SEE https://gitlab.com/gitlab-org/gitlab-foss/-/blob/master/spec/frontend/ci_variable_list/components/ci_variable_modal_spec.js#L20
+const isMaskable = (value: string): boolean => maskableRegex.test(value);
+
+const createVariable = async (
+  vorpal: CommandInstance,
+  projectId: string,
+  key: string,
+  value: string
+) => {
+  return await doGitlabRequest(vorpal, `projects/${projectId}/variables`, {
+    key,
+    value,
+    masked: isMaskable(value),
+  });
+};
+
+const updateVariable = async (
+  vorpal: CommandInstance,
+  projectId: string,
+  key: string,
+  value: string
+) => {
+  return await doGitlabRequest(
+    vorpal,
+    `projects/${projectId}/variables/${key}`,
+    {
+      value,
+      masked: isMaskable(value),
+    },
+    true
+  );
+};
+export const upsertAllVariables = async (
+  vorpal: CommandInstance,
+  variables: Record<string, any>,
+  env: string,
+  componentName: string
+): Promise<void> => {
+  const { id } = await getProjectInfo(vorpal);
+
+  for (const [key, value] of Object.entries(variables ?? {})) {
+    const fullKey = getSecretVarName(env, componentName, key);
+    const valueSanitized = isObject(value) ? JSON.stringify(value) : `${value}`;
+    try {
+      await updateVariable(vorpal, id, fullKey, valueSanitized);
+    } catch (e) {
+      await createVariable(vorpal, id, fullKey, valueSanitized);
+    } finally {
+      getAllVariables.clear();
+    }
+  }
+};

package/src/utils/passwordstore/index.ts

@@ -1,11 +1,8 @@
 import { exec, spawn } from "child-process-promise";
 import commandExists from "command-exists-promise";
 import dayjs from "dayjs";
-
 import { readFile, writeFile } from "fs-extra";
-import yaml from "js-yaml";
 import { withFile } from "tmp-promise";
-import formatEnvVars from "../formatEnvVars";
 import getEditor from "../getEditor";
 import { getPreference, hasPreference, setPreference } from "../preferences";
 
@@ -78,9 +75,7 @@ const execBitwardenCommand = async (command: string): Promise<any> => {
       await loginBitwarden();
       return execBitwardenCommand(command);
     } else {
-      console.error(e);
-
-      console.log("wooops", e.message);
+      throw e;
     }
   }
 };
@@ -169,24 +164,27 @@ const share = async (itemId: string) =>
     ])}`
   );
 
-export const editPass = async (path: string) => {
+export const trashItem = async (path: string) => {
   const item = await getItem(path);
 
-  await withFile(async ({ path: tmpFilePath }) => {
-    await writeFile(tmpFilePath, item.notes);
-    await (await getEditor()).open(tmpFilePath);
-    const newContent = (await readFile(tmpFilePath)).toString("utf-8");
+  const result = await execBitwardenCommand(`delete item ${item.id}`);
 
-    await update("item", item.id, {
-      ...item,
-      notes: newContent,
-    });
-  }, { postfix: ".yml" });
+  return result;
 };
+export const editPass = async (path: string) => {
+  const item = await getItem(path);
 
-export const readPassEnvVars = async (path: string) => {
-  // make sure that you have pulled pass beforehand
-  const yamlstring = await readPass(path);
-  // if a value is an object, we convert it to strings
-  return formatEnvVars(yaml.load(yamlstring));
+  await withFile(
+    async ({ path: tmpFilePath }) => {
+      await writeFile(tmpFilePath, item.notes);
+      await (await getEditor()).open(tmpFilePath);
+      const newContent = (await readFile(tmpFilePath)).toString("utf-8");
+
+      await update("item", item.id, {
+        ...item,
+        notes: newContent,
+      });
+    },
+    { postfix: ".yml" }
+  );
 };

package/src/utils/preferences/index.ts

@@ -1,7 +1,7 @@
 import fs from "fs-extra";
 import { homedir } from "os";
 
-import yaml from "js-yaml";
+import { load, dump } from "js-yaml";
 const directory = `${homedir()}/.catladder`;
 const file = `${directory}/preferences.yml`;
 
@@ -9,7 +9,10 @@ const getPreferences = async () => {
   if (!(await fs.pathExists(file))) {
     await fs.createFile(file);
   }
-  return yaml.load(await fs.readFile(file, { encoding: "utf-8" })) ?? {};
+  return (load(await fs.readFile(file, { encoding: "utf-8" })) ?? {}) as Record<
+    string,
+    string
+  >;
 };
 
 export const hasPreference = async (key: string) => {
@@ -29,5 +32,5 @@ export const setPreference = async (key: string, value: string | number) => {
     [key]: value,
   };
 
-  await fs.writeFile(file, yaml.safeDump(newPreferences));
+  await fs.writeFile(file, dump(newPreferences));
 };

package/src/utils/projects/index.ts

@@ -1,19 +1,11 @@
-import { exec } from "child-process-promise";
+import { $ } from "zx";
 import { join } from "path";
-import k8sApi from "../../k8sApi";
-import { Env, ISecrets, IValueFile } from "../../types/types";
-import { readFileOrError, readYaml } from "../files";
-import formatEnvVars from "../formatEnvVars";
-import { hasBitwarden, readPassEnvVars, syncBitwarden } from "../passwordstore";
-import { filter } from "../promise";
+import { getProjectConfig, parseChoice } from "../../config/getProjectConfig";
+import { getk8sApi } from "../../k8sApi";
+import { readFileOrError } from "../files";
 
-import { merge } from "lodash";
-
-// eslint-disable-next-line @typescript-eslint/no-var-requires
-const YAWN = require("yawn-yaml/cjs");
-
-export const getGitRoot = async () => {
-  return (await exec("git rev-parse --show-toplevel")).stdout?.trim();
+export const getGitRoot = async (): Promise<string> => {
+  return (await $`git rev-parse --show-toplevel`).stdout.trim();
 };
 
 export const getRootGitlabCiFile = async () => {
@@ -31,141 +23,30 @@ export const hasGitlabCiFile = async () => {
   }
   return true;
 };
-export const getLocalProjectVariables = async () => {
-  const [error, file] = await readRootGitlabCiFile();
-  if (error) {
-    throw new Error("there is no '.gitlab-ci.yml' in the current project");
-  }
-  // yawn allows to modify yaml while keeping comments!
-  const yawn = new YAWN(file);
-  const { variables } = yawn.json;
-  if (!variables) {
-    throw new Error("your '.gitlab-ci.yml' does not define `variables`");
-  }
-  const defaults = {
-    CLUSTER_NAME: "production",
-  };
-  return {
-    ...defaults,
-    ...variables,
-  };
-};
 
-export const getProjectNamespace = async (env: Env) => {
-  const { CUSTOMER_NAME, APP_NAME } = await getLocalProjectVariables();
-  return `${CUSTOMER_NAME}-${APP_NAME}-${env}`;
+export const getProjectNamespace = async (envComponent: string) => {
+  const { env } = parseChoice(envComponent);
+  const config = await getProjectConfig();
+  return `${config.customerName}-${config.appName}-${env}`;
 };
 
-export const getProjectHelmReleaseName = async (env: Env) => {
-  // can't properly get the release name for review as it contains an additional branch slug
-  if (env === "review" || env === "dev-local") {
-    throw new Error(`can't get helm release name for ${env}`);
-  }
-  const {
-    CUSTOMER_NAME,
-    APP_NAME,
-    COMPONENT_NAME = "web",
-  } = await getLocalProjectVariables();
-  return `${CUSTOMER_NAME}-${APP_NAME}-${env}-${COMPONENT_NAME}`;
-};
-export const getProjectPods = async (env: Env) => {
-  const namespace = await getProjectNamespace(env);
+export const getProjectPods = async (envComponent: string) => {
+  const namespace = await getProjectNamespace(envComponent);
+  const k8sApi = getk8sApi();
   const res = await k8sApi.listNamespacedPod(namespace);
 
   return res.body.items;
 };
 
-export const getProjectPvcs = async (env: Env) => {
-  const namespace = await getProjectNamespace(env);
+export const getProjectPvcs = async (envComponent: string) => {
+  const namespace = await getProjectNamespace(envComponent);
+  const k8sApi = getk8sApi();
   const res = await k8sApi.listNamespacedPersistentVolumeClaim(namespace);
 
   return res.body.items;
 };
 
-export const getProjectPodNames = async (env: Env) => {
-  const pods = await getProjectPods(env);
+export const getProjectPodNames = async (envComponent: string) => {
+  const pods = await getProjectPods(envComponent);
   return pods.map((n) => n.metadata.name);
 };
-
-export const getPassPath = async (env: Env) => {
-  const { CUSTOMER_NAME, APP_NAME } = await getLocalProjectVariables();
-  return `${CUSTOMER_NAME}/${APP_NAME}/${env}/secrets.yml`;
-};
-
-export const getProjectValuesFiles = async (env: Env, subApp?: string) => {
-  const gitRoot = await getGitRoot();
-  const possibleFiles = ["values.yml", `values-${env}.yml`].map((file) =>
-    subApp ? join(gitRoot, subApp, file) : join(gitRoot, file)
-  );
-  return filter(possibleFiles, async (file) => {
-    const [error] = await readFileOrError(file);
-    return !error;
-  });
-};
-
-export const getAllValues = async (env: Env, subApp?: string) => {
-  const valuesFilePaths = await getProjectValuesFiles(env, subApp);
-  return Promise.all(
-    valuesFilePaths.map(async (file) => (await readYaml(file)) as IValueFile)
-  );
-};
-
-export const getProjectValues = async (env: Env, subApp?: string) => {
-  const values = await getAllValues(env, subApp);
-  return merge({}, ...values) as IValueFile;
-};
-
-export const getAllPublicEnvVars = async (env: Env, subApp?: string) => {
-  return formatEnvVars(
-    (await getAllValues(env, subApp)).reduce<ISecrets>((acc, value) => {
-      if (value.env && value.env.public) {
-        return {
-          ...acc,
-          ...value.env.public,
-        };
-      }
-      return acc;
-    }, {})
-  );
-};
-export const getAllSecretsEnvVarsMapping = async (
-  env: Env,
-  subApp?: string
-) => {
-  return (await getAllValues(env, subApp)).reduce<ISecrets>((acc, value) => {
-    if (value.env && value.env.secret) {
-      return {
-        ...acc,
-        ...value.env.secret,
-      };
-    }
-    return acc;
-  }, {});
-};
-
-export const hasSecrets = async (env: Env, subApp?: string) => {
-  return Object.keys(await getAllSecretsEnvVarsMapping(env, subApp)).length > 0;
-};
-
-export const getAllEnvVars = async (
-  env: Env,
-  subApp?: string
-): Promise<Record<string, string>> => {
-  return {
-    ...(await getAllPublicEnvVars(env, subApp)),
-    ...(await getAllSecretEnvVars(env, subApp)),
-  };
-};
-
-export const getAllSecretEnvVars = async (env: Env, subApp?: string) => {
-  if (!(await hasSecrets(env, subApp))) {
-    return {};
-  }
-  if (await hasBitwarden()) {
-    await syncBitwarden(false /* do not force sync */);
-    const passPath = await getPassPath(env);
-    return readPassEnvVars(passPath);
-  } else {
-    return {};
-  }
-};

package/tsconfig.json

@@ -1,6 +1,6 @@
 {
-  "extends": "@tsconfig/node14/tsconfig.json",
   "compilerOptions": {
+    "lib": ["es2020", "DOM"],
     "module": "CommonJS",
     "declaration": true,
     "noImplicitAny": true,
@@ -9,12 +9,8 @@
     "sourceMap": true,
     "outDir": "dist",
     "strict": false,
-    "baseUrl": ".",
     "incremental": true,
-    "resolveJsonModule": true,
-    "paths": {
-      "*": ["node_modules/*", "src/types/*"]
-    }
+    "resolveJsonModule": true
   },
-  "include": ["src/**/*"]
+  "include": ["./src/**/*"]
 }

package/bin/catenv.sh

@@ -1 +0,0 @@
-eval $(catladder --catenv )

package/src/apps/shell/commands/project/commandCloudSqlProxy.ts

@@ -1,65 +0,0 @@
-import { spawn } from "child-process-promise";
-import { writeFile } from "fs-extra";
-import { withFile } from "tmp-promise";
-import Vorpal from "vorpal";
-import {
-  GOOGLE_CLOUD_SQL_PASS_PATH,
-  GOOGLE_PROJECT,
-} from "../../../../config/constants";
-import { readPass } from "../../../../utils/passwordstore";
-import {
-  getAllEnvVars,
-  getLocalProjectVariables,
-  getProjectValues,
-} from "../../../../utils/projects";
-import { envAutocompletion } from "./utils/autocompletions";
-import { promptForSubAppIfAny } from "./utils/monorepo";
-
-export default (vorpal: Vorpal) =>
-  vorpal
-    .command("project-cloud-sql-proxy <env>", "proxy to cloud sql db")
-    .autocomplete(envAutocompletion)
-    .action(async function ({ env }) {
-      const { CUSTOMER_NAME, APP_NAME } = await getLocalProjectVariables();
-      // skynet-164509:europe-west6:pvl-cyclomania-review=tcp:5432
-
-      const { localPort } = await this.prompt({
-        type: "number",
-        name: "localPort",
-        default: "54320",
-        message: "Local port: ",
-      });
-
-      const subapp = await promptForSubAppIfAny(this);
-      const POSTGRESQL_PASSWORD = (await getAllEnvVars(env, subapp))
-        ?.POSTGRESQL_PASSWORD;
-
-      const values = await getProjectValues(env, subapp);
-      this.log("");
-      this.log(`postgres-PW: ${POSTGRESQL_PASSWORD}`);
-      this.log("");
-
-      const projectId = values?.cloudsql?.projectId || GOOGLE_PROJECT;
-
-      const defaultInstanceId = `${CUSTOMER_NAME}-${APP_NAME}-${env}`;
-      const instanceId = values?.cloudsql?.instanceId || defaultInstanceId;
-
-      const defaultRegion = "europe-west6"; // currently hardcoded
-      const region = values?.cloudsql?.region || defaultRegion;
-
-      const instanceName = `${projectId}:${region}:${instanceId}=tcp:${localPort}`;
-
-      const cloudsqlCredentials = await readPass(GOOGLE_CLOUD_SQL_PASS_PATH);
-      await withFile(async ({ path: tmpFilePath }) => {
-        await writeFile(tmpFilePath, cloudsqlCredentials);
-
-        await spawn(
-          "cloud_sql_proxy",
-          ["-instances", instanceName, "-credential_file", tmpFilePath],
-          {
-            stdio: "inherit",
-            shell: true,
-          }
-        );
-      });
-    });