@catladder/cli 0.0.1-bla → 1.0.0

package/src/config/getProjectConfig.ts

@@ -0,0 +1,118 @@
+import {
+  readConfigSync,
+  getAllEnvs,
+  getEnvironment as _getEnvironment,
+  createContext,
+} from "@catladder/pipeline";
+
+import { CommandInstance } from "vorpal";
+import { getAllVariables } from "../utils/gitlab";
+import memoizee from "memoizee";
+import { getGitRoot } from "../utils/projects";
+import { readYaml } from "../utils/files";
+// currently cant change
+
+export const getProjectConfig = memoizee(
+  async () => {
+    try {
+      const gitRoot = await getGitRoot();
+      return readConfigSync(gitRoot);
+    } catch (e) {
+      // ignore
+      return null;
+    }
+  },
+  { promise: true }
+);
+
+export const reloadConfig = () => getProjectConfig.clear();
+
+export const getGitlabCiFilePath = async () => {
+  const gitRoot = await getGitRoot();
+  return gitRoot + "/.gitlab-ci.yml";
+};
+export const getGitlabCi = async <T = Record<string, any>>() => {
+  try {
+    return readYaml(await getGitlabCiFilePath()) as Promise<T>;
+  } catch (e) {
+    // ignore
+    return null;
+  }
+};
+
+export const getProjectComponents = async () => {
+  const config = await getProjectConfig();
+  if (!config) return [];
+  return Object.keys(config.components);
+};
+
+export const parseChoice = (envComponent: string) => {
+  const [env, componentName] = envComponent.split(":");
+  return { env, componentName };
+};
+
+export const getPipelineContextByChoice = async (
+  env: string,
+  componentName: string
+) => {
+  const config = await getProjectConfig();
+  return createContext(config, componentName, env);
+};
+export const getAllComponentsWithAllEnvs = async () => {
+  const config = await getProjectConfig();
+  if (!config) {
+    return [];
+  }
+  return Promise.all(
+    Object.keys(config.components).flatMap((componentName) =>
+      getAllEnvs(config, componentName).map((env) => ({ env, componentName }))
+    )
+  );
+};
+
+export const getAllPipelineContexts = async () => {
+  return Promise.all(
+    (await getAllComponentsWithAllEnvs()).map(({ env, componentName }) =>
+      getPipelineContextByChoice(env, componentName)
+    )
+  );
+};
+
+export const getEnvironment = async (env: string, componentName: string) => {
+  const config = await getProjectConfig();
+  return _getEnvironment(config, componentName, env);
+};
+
+const resolveSecrets = async (
+  vorpal: CommandInstance,
+  allEnvVars: Record<string, string>
+) => {
+  const allVariablesInGitlab = await getAllVariables(vorpal);
+
+  return Object.fromEntries(
+    Object.entries(allEnvVars).map(([key, value]) => {
+      const isSecret = value?.startsWith?.("$CL_");
+      if (isSecret) {
+        // secrets have CL_XXXX structure
+        const found = allVariablesInGitlab.find((v) => "$" + v.key === value);
+
+        if (found) {
+          return [key, found.value];
+        } else {
+          return [key, ""];
+        }
+      }
+      return [key, value];
+    })
+  );
+};
+
+export const getEnvVars = async (
+  vorpal: CommandInstance,
+  env: string,
+  componentName: string
+) => {
+  const envionment = await getEnvironment(env, componentName);
+
+  return resolveSecrets(vorpal, envionment.envVars);
+};

package/src/types/types.ts

@@ -1,5 +1,3 @@
-export type Env = "dev-local" | "prod" | "dev" | "stage" | "review";
-
 export type KubernetesSecretName = string;
 export interface ISecrets {
   [key: string]: KubernetesSecretName;

package/src/utils/cluster.ts

@@ -9,8 +9,21 @@ export const getCurrentConnectedClusterName = async () => {
   return findKey(clusters, { fullName: currentContext });
 };
 
-export const getClusterByName = (name: string) => {
-  return clusters[name];
+export const getClusterByFullName = (fullName: string) => {
+  const found = Object.entries(getAllClusters()).find(
+    ([, config]) => config.fullName === fullName
+  );
+  if (found) {
+    return { name: found[0], cluster: found[1] };
+  } else {
+    return null;
+  }
+};
+
+export const getClusterByName = (name: string) => getAllClusters()[name];
+
+export const getAllClusters = () => {
+  return clusters;
 };
 
 export const getAllClusterNames = () => Object.keys(clusters);

package/src/utils/editAsFile.ts

@@ -0,0 +1,31 @@
+import { withFile } from "tmp-promise";
+import { dump, load } from "js-yaml";
+
+import { readFile, writeFile } from "fs-extra";
+import getEditor from "./getEditor";
+
+export const editAsFile = async <T>(
+  inObject: T,
+  preamble?: string
+): Promise<T> => {
+  const fullPreamble = preamble
+    ? `#
+# ${preamble.split("\n").join("\n# ")}
+#
+
+`
+    : "\n";
+  const asString = fullPreamble + dump(inObject, { noRefs: true });
+  let newContent: T;
+
+  await withFile(
+    async ({ path: tmpFilePath }) => {
+      await writeFile(tmpFilePath, asString);
+      await (await getEditor()).open(tmpFilePath);
+      newContent = load((await readFile(tmpFilePath)).toString("utf-8")) as T;
+    },
+    { postfix: ".yml" }
+  );
+
+  return newContent;
+};

package/src/utils/files.ts

@@ -13,6 +13,10 @@ export const readFileOrError = async (filePath: string) => {
   }
 };
 
-export const readYaml = async (filename: string) => {
-  return yaml.load(await readFile(filename));
+export const readYaml = async <T = Record<string, any>>(filename: string) => {
+  try {
+    return yaml.load(await readFile(filename)) as T;
+  } catch (e) {
+    return null;
+  }
 };

package/src/utils/gitlab.ts

@@ -1,52 +1,68 @@
 import { exec } from "child-process-promise";
+import { getSecretVarName } from "@catladder/pipeline";
+import { isObject } from "lodash";
+import memoizee from "memoizee";
 import fetch from "node-fetch";
 import open from "open";
 import { CommandInstance } from "vorpal";
 import { getPreference, hasPreference, setPreference } from "./preferences";
 
 const TOKEN_KEY = "gitlab-personal-access-token";
-export const getGitlabToken = async (vorpal: CommandInstance) => {
-  if (!(await hasPreference(TOKEN_KEY))) {
-    vorpal.log("");
-    vorpal.log(
-      "☝ in order to access the api, we need a personal access token"
-    );
-    vorpal.log("Its best to create one specifically for catladder");
-    vorpal.log("");
-    vorpal.log("☝ we open up the settings page for you!");
-    vorpal.log("");
-    const { shouldContinue } = await vorpal.prompt({
-      default: true,
-      message: "Ok",
-      name: "shouldContinue",
-      type: "prompt",
-    });
 
-    open("https://git.panter.ch/-/profile/personal_access_tokens");
+export const hasGitlabToken = async () => await hasPreference(TOKEN_KEY);
+export const setupGitlabToken = async (vorpal: CommandInstance) => {
+  vorpal.log("");
+  vorpal.log("☝ in order to access the api, we need a personal access token");
+  vorpal.log("Its best to create one specifically for catladder");
+  vorpal.log("");
+  vorpal.log("☝ we open up the settings page for you!");
+  vorpal.log("");
+  const { shouldContinue } = await vorpal.prompt({
+    default: true,
+    message: "Ok",
+    name: "shouldContinue",
+    type: "prompt",
+  });
+
+  open("https://git.panter.ch/-/profile/personal_access_tokens");
 
-    vorpal.log("Please type in gitlab's personal access token");
+  vorpal.log("Please type in gitlab's personal access token");
 
-    const { personalToken } = await vorpal.prompt({
-      type: "string",
-      name: "personalToken",
-      default: "",
-      message: "Your personal access token ",
-    });
-    if (personalToken) {
-      await setPreference(TOKEN_KEY, personalToken);
+  const { personalToken } = await vorpal.prompt({
+    type: "string",
+    name: "personalToken",
+    default: "",
+    message: "Your personal access token ",
+  });
+  if (personalToken) {
+    await setPreference(TOKEN_KEY, personalToken);
+  }
+};
+export const getGitlabToken = async (vorpal: CommandInstance) => {
+  if (!(await hasGitlabToken())) {
+    if (!vorpal) {
+      console.error(
+        "⚠️ gitlab token missing, please run catladder to set it up"
+      );
+      process.exit(1);
     }
+    await setupGitlabToken(vorpal);
   }
   return getPreference(TOKEN_KEY);
 };
 
-export const doGitlabRequest = async (
+export const doGitlabRequest = async <T = any>(
   vorpal: CommandInstance,
   path: string,
-  data?: any
-) => {
+  data: any = undefined,
+  update?: boolean
+): Promise<T> => {
   const rootToken = await getGitlabToken(vorpal);
+
+  const method = data ? (update ? "PUT" : "POST") : "GET";
+
   const result = await fetch(`https://git.panter.ch/api/v4/${path}`, {
-    method: data ? "POST" : "GET",
+    method,
     headers: {
       "Content-Type": "application/json",
       "Private-Token": rootToken,
@@ -59,13 +75,15 @@ export const doGitlabRequest = async (
   }
 
   throw new Error(
-    `Could not send request to gitlab api: ${result.status} "${
+    `Could not send request to gitlab api ${path}: ${result.status} "${
       result.statusText
     }".\nResponse: ${JSON.stringify(await result.json(), null, 2)}`
   );
 };
 
-export const getProjectInfo = async (vorpal: CommandInstance): Promise<any> => {
+export const getProjectInfo = async (
+  vorpal: CommandInstance
+): Promise<{ id: string; web_url: string }> => {
   const gitRemoteOriginUrl = (
     await exec("git config --get remote.origin.url")
   ).stdout.trim();
@@ -78,3 +96,79 @@ export const getProjectInfo = async (vorpal: CommandInstance): Promise<any> => {
   );
   return project;
 };
+
+type GitlabVariable = {
+  variable_type: string;
+  key: string;
+  value: string;
+  protected: boolean;
+  masked: boolean;
+  environment_scope: string;
+};
+export const getAllVariables = memoizee(
+  async (vorpal: CommandInstance): Promise<Array<GitlabVariable>> => {
+    const { id } = await getProjectInfo(vorpal);
+    let all: Array<GitlabVariable> = [];
+    let result: Array<GitlabVariable>;
+    let page = 1;
+    do {
+      result = await doGitlabRequest(
+        vorpal,
+        // 100 is max page size
+        `projects/${id}/variables?per_page=100&page=${page}`
+      );
+      page++;
+      all = [...all, ...result];
+    } while (result?.length > 0);
+    return all;
+  },
+  { promise: true }
+);
+
+const createVariable = async (
+  vorpal: CommandInstance,
+  projectId: string,
+  key: string,
+  value: string
+) => {
+  return await doGitlabRequest(vorpal, `projects/${projectId}/variables`, {
+    key,
+    value,
+  });
+};
+
+const updateVariable = async (
+  vorpal: CommandInstance,
+  projectId: string,
+  key: string,
+  value: string
+) => {
+  return await doGitlabRequest(
+    vorpal,
+    `projects/${projectId}/variables/${key}`,
+    {
+      value,
+    },
+    true
+  );
+};
+export const upsertAllVariables = async (
+  vorpal: CommandInstance,
+  variables: Record<string, any>,
+  env: string,
+  componentName: string
+): Promise<void> => {
+  const { id } = await getProjectInfo(vorpal);
+
+  for (const [key, value] of Object.entries(variables ?? {})) {
+    const fullKey = getSecretVarName(env, componentName, key);
+    const valueSanitized = isObject(value) ? JSON.stringify(value) : `${value}`;
+    try {
+      await updateVariable(vorpal, id, fullKey, valueSanitized);
+    } catch (e) {
+      await createVariable(vorpal, id, fullKey, valueSanitized);
+    } finally {
+      getAllVariables.clear();
+    }
+  }
+};

package/src/utils/passwordstore/index.ts

@@ -1,11 +1,8 @@
 import { exec, spawn } from "child-process-promise";
 import commandExists from "command-exists-promise";
 import dayjs from "dayjs";
-
 import { readFile, writeFile } from "fs-extra";
-import yaml from "js-yaml";
 import { withFile } from "tmp-promise";
-import formatEnvVars from "../formatEnvVars";
 import getEditor from "../getEditor";
 import { getPreference, hasPreference, setPreference } from "../preferences";
 
@@ -78,9 +75,7 @@ const execBitwardenCommand = async (command: string): Promise<any> => {
       await loginBitwarden();
       return execBitwardenCommand(command);
     } else {
-      console.error(e);
-
-      console.log("wooops", e.message);
+      throw e;
     }
   }
 };
@@ -169,24 +164,27 @@ const share = async (itemId: string) =>
     ])}`
   );
 
-export const editPass = async (path: string) => {
+export const trashItem = async (path: string) => {
   const item = await getItem(path);
 
-  await withFile(async ({ path: tmpFilePath }) => {
-    await writeFile(tmpFilePath, item.notes);
-    await (await getEditor()).open(tmpFilePath);
-    const newContent = (await readFile(tmpFilePath)).toString("utf-8");
+  const result = await execBitwardenCommand(`delete item ${item.id}`);
 
-    await update("item", item.id, {
-      ...item,
-      notes: newContent,
-    });
-  }, { postfix: ".yml" });
+  return result;
 };
+export const editPass = async (path: string) => {
+  const item = await getItem(path);
 
-export const readPassEnvVars = async (path: string) => {
-  // make sure that you have pulled pass beforehand
-  const yamlstring = await readPass(path);
-  // if a value is an object, we convert it to strings
-  return formatEnvVars(yaml.load(yamlstring));
+  await withFile(
+    async ({ path: tmpFilePath }) => {
+      await writeFile(tmpFilePath, item.notes);
+      await (await getEditor()).open(tmpFilePath);
+      const newContent = (await readFile(tmpFilePath)).toString("utf-8");
+
+      await update("item", item.id, {
+        ...item,
+        notes: newContent,
+      });
+    },
+    { postfix: ".yml" }
+  );
 };

package/src/utils/preferences/index.ts

@@ -1,7 +1,7 @@
 import fs from "fs-extra";
 import { homedir } from "os";
 
-import yaml from "js-yaml";
+import { load, dump } from "js-yaml";
 const directory = `${homedir()}/.catladder`;
 const file = `${directory}/preferences.yml`;
 
@@ -9,7 +9,10 @@ const getPreferences = async () => {
   if (!(await fs.pathExists(file))) {
     await fs.createFile(file);
   }
-  return yaml.load(await fs.readFile(file, { encoding: "utf-8" })) ?? {};
+  return (load(await fs.readFile(file, { encoding: "utf-8" })) ?? {}) as Record<
+    string,
+    string
+  >;
 };
 
 export const hasPreference = async (key: string) => {
@@ -29,5 +32,5 @@ export const setPreference = async (key: string, value: string | number) => {
     [key]: value,
   };
 
-  await fs.writeFile(file, yaml.safeDump(newPreferences));
+  await fs.writeFile(file, dump(newPreferences));
 };

package/src/utils/projects/index.ts

@@ -1,18 +1,11 @@
 import { exec } from "child-process-promise";
+
 import { join } from "path";
+import { getProjectConfig, parseChoice } from "../../config/getProjectConfig";
 import k8sApi from "../../k8sApi";
-import { Env, ISecrets, IValueFile } from "../../types/types";
-import { readFileOrError, readYaml } from "../files";
-import formatEnvVars from "../formatEnvVars";
-import { hasBitwarden, readPassEnvVars, syncBitwarden } from "../passwordstore";
-import { filter } from "../promise";
-
-import { merge } from "lodash";
+import { readFileOrError } from "../files";
 
-// eslint-disable-next-line @typescript-eslint/no-var-requires
-const YAWN = require("yawn-yaml/cjs");
-
-export const getGitRoot = async () => {
+export const getGitRoot = async (): Promise<string> => {
   return (await exec("git rev-parse --show-toplevel")).stdout?.trim();
 };
 
@@ -31,141 +24,28 @@ export const hasGitlabCiFile = async () => {
   }
   return true;
 };
-export const getLocalProjectVariables = async () => {
-  const [error, file] = await readRootGitlabCiFile();
-  if (error) {
-    throw new Error("there is no '.gitlab-ci.yml' in the current project");
-  }
-  // yawn allows to modify yaml while keeping comments!
-  const yawn = new YAWN(file);
-  const { variables } = yawn.json;
-  if (!variables) {
-    throw new Error("your '.gitlab-ci.yml' does not define `variables`");
-  }
-  const defaults = {
-    CLUSTER_NAME: "production",
-  };
-  return {
-    ...defaults,
-    ...variables,
-  };
-};
 
-export const getProjectNamespace = async (env: Env) => {
-  const { CUSTOMER_NAME, APP_NAME } = await getLocalProjectVariables();
-  return `${CUSTOMER_NAME}-${APP_NAME}-${env}`;
+export const getProjectNamespace = async (envComponent: string) => {
+  const { env } = parseChoice(envComponent);
+  const config = await getProjectConfig();
+  return `${config.customerName}-${config.appName}-${env}`;
 };
 
-export const getProjectHelmReleaseName = async (env: Env) => {
-  // can't properly get the release name for review as it contains an additional branch slug
-  if (env === "review" || env === "dev-local") {
-    throw new Error(`can't get helm release name for ${env}`);
-  }
-  const {
-    CUSTOMER_NAME,
-    APP_NAME,
-    COMPONENT_NAME = "web",
-  } = await getLocalProjectVariables();
-  return `${CUSTOMER_NAME}-${APP_NAME}-${env}-${COMPONENT_NAME}`;
-};
-export const getProjectPods = async (env: Env) => {
-  const namespace = await getProjectNamespace(env);
+export const getProjectPods = async (envComponent: string) => {
+  const namespace = await getProjectNamespace(envComponent);
   const res = await k8sApi.listNamespacedPod(namespace);
 
   return res.body.items;
 };
 
-export const getProjectPvcs = async (env: Env) => {
-  const namespace = await getProjectNamespace(env);
+export const getProjectPvcs = async (envComponent: string) => {
+  const namespace = await getProjectNamespace(envComponent);
   const res = await k8sApi.listNamespacedPersistentVolumeClaim(namespace);
 
   return res.body.items;
 };
 
-export const getProjectPodNames = async (env: Env) => {
-  const pods = await getProjectPods(env);
+export const getProjectPodNames = async (envComponent: string) => {
+  const pods = await getProjectPods(envComponent);
   return pods.map((n) => n.metadata.name);
 };
-
-export const getPassPath = async (env: Env) => {
-  const { CUSTOMER_NAME, APP_NAME } = await getLocalProjectVariables();
-  return `${CUSTOMER_NAME}/${APP_NAME}/${env}/secrets.yml`;
-};
-
-export const getProjectValuesFiles = async (env: Env, subApp?: string) => {
-  const gitRoot = await getGitRoot();
-  const possibleFiles = ["values.yml", `values-${env}.yml`].map((file) =>
-    subApp ? join(gitRoot, subApp, file) : join(gitRoot, file)
-  );
-  return filter(possibleFiles, async (file) => {
-    const [error] = await readFileOrError(file);
-    return !error;
-  });
-};
-
-export const getAllValues = async (env: Env, subApp?: string) => {
-  const valuesFilePaths = await getProjectValuesFiles(env, subApp);
-  return Promise.all(
-    valuesFilePaths.map(async (file) => (await readYaml(file)) as IValueFile)
-  );
-};
-
-export const getProjectValues = async (env: Env, subApp?: string) => {
-  const values = await getAllValues(env, subApp);
-  return merge({}, ...values) as IValueFile;
-};
-
-export const getAllPublicEnvVars = async (env: Env, subApp?: string) => {
-  return formatEnvVars(
-    (await getAllValues(env, subApp)).reduce<ISecrets>((acc, value) => {
-      if (value.env && value.env.public) {
-        return {
-          ...acc,
-          ...value.env.public,
-        };
-      }
-      return acc;
-    }, {})
-  );
-};
-export const getAllSecretsEnvVarsMapping = async (
-  env: Env,
-  subApp?: string
-) => {
-  return (await getAllValues(env, subApp)).reduce<ISecrets>((acc, value) => {
-    if (value.env && value.env.secret) {
-      return {
-        ...acc,
-        ...value.env.secret,
-      };
-    }
-    return acc;
-  }, {});
-};
-
-export const hasSecrets = async (env: Env, subApp?: string) => {
-  return Object.keys(await getAllSecretsEnvVarsMapping(env, subApp)).length > 0;
-};
-
-export const getAllEnvVars = async (
-  env: Env,
-  subApp?: string
-): Promise<Record<string, string>> => {
-  return {
-    ...(await getAllPublicEnvVars(env, subApp)),
-    ...(await getAllSecretEnvVars(env, subApp)),
-  };
-};
-
-export const getAllSecretEnvVars = async (env: Env, subApp?: string) => {
-  if (!(await hasSecrets(env, subApp))) {
-    return {};
-  }
-  if (await hasBitwarden()) {
-    await syncBitwarden(false /* do not force sync */);
-    const passPath = await getPassPath(env);
-    return readPassEnvVars(passPath);
-  } else {
-    return {};
-  }
-};

package/tsconfig.json

@@ -1,6 +1,6 @@
 {
-  "extends": "@tsconfig/node14/tsconfig.json",
   "compilerOptions": {
+    "lib": ["es2020", "DOM"],
     "module": "CommonJS",
     "declaration": true,
     "noImplicitAny": true,
@@ -9,12 +9,8 @@
     "sourceMap": true,
     "outDir": "dist",
     "strict": false,
-    "baseUrl": ".",
     "incremental": true,
-    "resolveJsonModule": true,
-    "paths": {
-      "*": ["node_modules/*", "src/types/*"]
-    }
+    "resolveJsonModule": true
   },
-  "include": ["src/**/*"]
+  "include": ["./src/**/*"]
 }

package/bin/catenv.sh

@@ -1 +0,0 @@
-eval $(catladder --catenv )