@catladder/cli 0.0.0-vars-07a720d0 → 0.0.0-vitest-8a770211

package/src/utils/gitlab.ts

@@ -1,11 +1,11 @@
-import { exec } from "child-process-promise";
 import { getSecretVarName } from "@catladder/pipeline";
-import { isObject } from "lodash";
+import { has, isObject } from "lodash";
 import memoizee from "memoizee";
 import fetch from "node-fetch";
 import open from "open";
-import { CommandInstance } from "vorpal";
+import type { CommandInstance } from "vorpal";
 import { getPreference, hasPreference, setPreference } from "./preferences";
+import { getGitRemoteHostAndPath } from "../git/gitProjectInformation";
 
 const TOKEN_KEY = "gitlab-personal-access-token";
 
@@ -14,17 +14,21 @@ export const setupGitlabToken = async (vorpal: CommandInstance) => {
   vorpal.log("");
   vorpal.log("☝ in order to access the api, we need a personal access token");
   vorpal.log("Its best to create one specifically for catladder");
+  vorpal.log("Scopes needed: api");
   vorpal.log("");
   vorpal.log("☝ we open up the settings page for you!");
   vorpal.log("");
-  const { shouldContinue } = await vorpal.prompt({
-    default: true,
-    message: "Ok",
-    name: "shouldContinue",
-    type: "prompt",
-  });
+  const [{ shouldContinue }, { gitRemoteHost }] = await Promise.all([
+    vorpal.prompt({
+      default: true,
+      message: "Ok",
+      name: "shouldContinue",
+      type: "prompt",
+    }),
+    getGitRemoteHostAndPath(),
+  ]);
 
-  open("https://git.panter.ch/-/profile/personal_access_tokens");
+  open(`https://${gitRemoteHost}/-/user_settings/personal_access_tokens`);
 
   vorpal.log("Please type in gitlab's personal access token");
 
@@ -38,11 +42,11 @@ export const setupGitlabToken = async (vorpal: CommandInstance) => {
     await setPreference(TOKEN_KEY, personalToken);
   }
 };
-export const getGitlabToken = async (vorpal: CommandInstance) => {
+export const getGitlabToken = async (vorpal: CommandInstance | null) => {
   if (!(await hasGitlabToken())) {
     if (!vorpal) {
       console.error(
-        "⚠️ gitlab token missing, please run catladder to set it up"
+        "⚠️ gitlab token missing, please run catladder to set it up",
       );
       process.exit(1);
     }
@@ -51,27 +55,34 @@ export const getGitlabToken = async (vorpal: CommandInstance) => {
   return getPreference(TOKEN_KEY);
 };
 
+type Method = "GET" | "PUT" | "POST" | "DELETE";
 export const doGitlabRequest = async <T = any>(
-  vorpal: CommandInstance,
+  vorpal: CommandInstance | null,
   path: string,
   data: any = undefined,
-  update?: boolean
+  method: Method = "GET",
 ): Promise<T> => {
-  const rootToken = await getGitlabToken(vorpal);
+  const [rootToken, { gitRemoteHost }] = await Promise.all([
+    getGitlabToken(vorpal),
+    getGitRemoteHostAndPath(),
+  ]);
 
-  const method = data ? (update ? "PUT" : "POST") : "GET";
+  //const method = data ? (update ? "PUT" : "POST") : "GET";
 
-  const result = await fetch(`https://git.panter.ch/api/v4/${path}`, {
+  const result = await fetch(`https://${gitRemoteHost}/api/v4/${path}`, {
     method,
     headers: {
       "Content-Type": "application/json",
       "Private-Token": rootToken,
     },
-    body: JSON.stringify(data),
+    body: data ? JSON.stringify(data) : undefined,
   });
 
   if (result.status >= 200 && result.status < 400) {
-    return result.json();
+    if (result.headers.get("content-type") === "application/json") {
+      return result.json();
+    }
+    return null;
   }
   if (result.status === 404) {
     throw new Error("not found");
@@ -80,22 +91,17 @@ export const doGitlabRequest = async <T = any>(
   throw new Error(
     `Could not send request to gitlab api ${path}: ${result.status} "${
       result.statusText
-    }".\nResponse: ${JSON.stringify(await result.json(), null, 2)}`
+    }".\nResponse: ${JSON.stringify(await result.json(), null, 2)}`,
   );
 };
 
 export const getProjectInfo = async (
-  vorpal: CommandInstance
+  vorpal: CommandInstance | null,
 ): Promise<{ id: string; web_url: string }> => {
-  const gitRemoteOriginUrl = (
-    await exec("git config --get remote.origin.url")
-  ).stdout.trim();
-  const projectPath = /(https:\/\/|git@)git\.panter\.ch[:/](.*)\.git/g.exec(
-    gitRemoteOriginUrl
-  );
+  const { gitRemotePath } = await getGitRemoteHostAndPath();
   const project = await doGitlabRequest(
     vorpal,
-    `projects/${encodeURIComponent(projectPath[2])}`
+    `projects/${encodeURIComponent(gitRemotePath)}`,
   );
   return project;
 };
@@ -109,25 +115,49 @@ type GitlabVariable = {
   environment_scope: string;
 };
 export const getAllVariables = memoizee(
-  async (vorpal: CommandInstance): Promise<Array<GitlabVariable>> => {
+  async (
+    vorpal: CommandInstance | null,
+    n = 5, // how many requests to do in parallel, 5 seems a good value for many projects, ideally we would do one request in parallel per component
+  ): Promise<Array<GitlabVariable>> => {
     const { id } = await getProjectInfo(vorpal);
+
     let all: Array<GitlabVariable> = [];
-    let result: Array<GitlabVariable>;
+    let result: Array<Array<GitlabVariable>> = [];
     let page = 1;
+
     do {
-      result = await doGitlabRequest(
-        vorpal,
-        // 100 is max page size
-        `projects/${id}/variables?per_page=100&page=${page}`
-      );
-      page++;
-      all = [...all, ...result];
-    } while (result?.length > 0);
+      // Create an array of promises for N pages
+      const promises = Array.from({ length: n }, (_, i) => {
+        return doGitlabRequest(
+          vorpal,
+          `projects/${id}/variables?per_page=100&page=${page + i}`,
+        );
+      });
+
+      // Wait for all promises to resolve
+      result = await Promise.all(promises);
+
+      // Increment the page by N
+      page += n;
+
+      // Flatten the result array and add it to 'all'
+      all = [...all, ...result.flat()];
+      // Continue only if the last page had results
+    } while (result.length > 0 && result[result.length - 1].length > 0);
+
     return all;
   },
-  { promise: true }
+  { promise: true },
 );
 
+export const getVariableValueByRawName = async (
+  vorpal: CommandInstance,
+  rawName: string,
+) => {
+  const allVariables = await getAllVariables(vorpal);
+  return allVariables.find((v) => v.key === rawName)?.value;
+};
+
 const maskableRegex = new RegExp("^[a-zA-Z0-9_+=/@:.~-]{8,}$"); // SEE https://gitlab.com/gitlab-org/gitlab-foss/-/blob/master/spec/frontend/ci_variable_list/components/ci_variable_modal_spec.js#L20
 const isMaskable = (value: string): boolean => maskableRegex.test(value);
 
@@ -135,48 +165,154 @@ const createVariable = async (
   vorpal: CommandInstance,
   projectId: string,
   key: string,
-  value: string
+  value: string,
+  masked = true,
+  environment_scope = "*",
 ) => {
-  return await doGitlabRequest(vorpal, `projects/${projectId}/variables`, {
-    key,
-    value,
-    masked: isMaskable(value),
-  });
+  return await doGitlabRequest(
+    vorpal,
+    `projects/${projectId}/variables`,
+    {
+      key,
+      value,
+      masked: masked && isMaskable(value),
+      environment_scope,
+    },
+    "POST",
+  );
 };
 
 const updateVariable = async (
   vorpal: CommandInstance,
   projectId: string,
   key: string,
-  value: string
+  value: string,
+  masked = true,
 ) => {
   return await doGitlabRequest(
     vorpal,
     `projects/${projectId}/variables/${key}`,
     {
       value,
-      masked: isMaskable(value),
+      masked: masked && isMaskable(value),
     },
-    true
+    "PUT",
+  );
+};
+
+const deleteVariable = async (
+  vorpal: CommandInstance,
+  projectId: string,
+  key: string,
+) => {
+  return await doGitlabRequest(
+    vorpal,
+    `projects/${projectId}/variables/${key}`,
+    undefined,
+    "DELETE",
+  );
+};
+
+const getAllCatladderEnvVarsInGitlab = async (vorpal: CommandInstance) => {
+  const allVariables = await getAllVariables(vorpal).then((v) =>
+    v.reduce<{
+      [key: string]: {
+        value?: string;
+        backups: number[];
+      };
+    }>((acc, variable) => {
+      const { key } = variable;
+
+      if (key.startsWith("CL_")) {
+        const matchBackup = key.match(/(CL_.*)_backup_([0-9]+)/);
+
+        if (matchBackup) {
+          const key = matchBackup[1];
+          const timestamp = Number(matchBackup[2]);
+          const backups = [...(acc[key]?.backups ?? []), timestamp];
+          return {
+            ...acc,
+            [key]: {
+              ...(acc[key] ?? {}), // add value
+              backups,
+            },
+          };
+        }
+
+        return {
+          ...acc,
+          [key]: {
+            backups: [],
+            ...(acc[key] ?? {}), // may add backups
+            value: variable.value,
+          },
+        };
+      }
+
+      return acc;
+    }, {}),
   );
+  return allVariables;
+};
+
+const getBackupKey = (fullKey: string, timestamp: number) =>
+  `${fullKey}_backup_${timestamp}`;
+export const clearBackups = async (vorpal: CommandInstance, keep: number) => {
+  const existingVariables = await getAllCatladderEnvVarsInGitlab(vorpal);
+  const { id } = await getProjectInfo(vorpal);
+  for (const [key, { backups }] of Object.entries(existingVariables)) {
+    const backupsSorted = backups.sort((a, b) => b - a);
+    //const toKeep = backupsSorted.slice(0, keep);
+    const toDelete = backupsSorted.slice(keep);
+
+    for (const timestamp of toDelete) {
+      await deleteVariable(vorpal, id, getBackupKey(key, timestamp));
+    }
+  }
 };
+
 export const upsertAllVariables = async (
   vorpal: CommandInstance,
   variables: Record<string, any>,
   env: string,
-  componentName: string
+  componentName: string,
+  backup = true,
+  masked = true, // FIXME: would be better to have this per variable
 ): Promise<void> => {
   const { id } = await getProjectInfo(vorpal);
 
+  // we list all existing variables. We also gather backup versions, but its currently unused
+  const existingVariables = await getAllCatladderEnvVarsInGitlab(vorpal);
   for (const [key, value] of Object.entries(variables ?? {})) {
     const fullKey = getSecretVarName(env, componentName, key);
     const valueSanitized = isObject(value) ? JSON.stringify(value) : `${value}`;
-    try {
-      await updateVariable(vorpal, id, fullKey, valueSanitized);
-    } catch (e) {
-      await createVariable(vorpal, id, fullKey, valueSanitized);
-    } finally {
-      getAllVariables.clear();
+
+    const exists = has(existingVariables, fullKey);
+    const oldValue = existingVariables[fullKey]?.value;
+    const changed = oldValue !== valueSanitized;
+    if (changed) {
+      if (exists) {
+        vorpal.log(`changed: ${key}`);
+
+        await updateVariable(vorpal, id, fullKey, valueSanitized, masked);
+        // write backup
+        if (backup) {
+          await createVariable(
+            vorpal,
+            id,
+            getBackupKey(fullKey, new Date().getTime()),
+            oldValue,
+            masked,
+            "_backup",
+          );
+        }
+      } else {
+        vorpal.log(`new    : ${key}`);
+        await createVariable(vorpal, id, fullKey, valueSanitized, masked);
+      }
+    } else {
+      vorpal.log(`skip   : ${key}`);
     }
   }
+  getAllVariables.clear();
 };

package/src/utils/log.ts

@@ -1,13 +1,113 @@
+import type { CommandInstance } from "vorpal";
+
+type AnyMessagesFn = (message?: any, ...optionalParams: any[]) => void;
+type StringMessagesFn = (message?: string, ...optionalParams: string[]) => void;
+type AnyMessageFn = (message?: any) => void;
+type StringMessageFn = (message?: string) => void;
+type StringMessageSpreadFn = (...message: string[]) => void;
+type AnyMessageSpreadFn = (...message: any[]) => void;
+type SomeKindOfLogFn =
+  | AnyMessagesFn
+  | StringMessagesFn
+  | AnyMessageFn
+  | StringMessageFn
+  | StringMessageSpreadFn
+  | AnyMessageSpreadFn;
+type HasSomeKindOfLogMethod = { log: SomeKindOfLogFn };
+
+/**
+ * Something that has a .log method taking a string argument.
+ *
+ * null and undefined will fall back to console.log.
+ */
+export type LoggerCmdInstance =
+  | CommandInstance
+  | Console
+  | HasSomeKindOfLogMethod
+  | null
+  | undefined;
+
+type LinesArgItem =
+  | string
+  | number
+  | boolean
+  | Date
+  | RegExp
+  | null
+  | undefined;
+type LinesArg = LinesArgItem | LinesArgItem[];
+
+/**
+ * Logs multiple lines using the log method of the provided {@link LoggerCmdInstance}.
+ *
+ * Line items that are `null` or `undefined` are logged as empty lines.
+ *
+ * @param cmd - The logger instance to use. If `null` or `undefined`, `console` will be used.
+ *
+ * @example
+ * ```ts
+ * logLines(cmd, "line 1", "line 2", null, "line 3");
+ * logLines(cmd, ...linesArray); // arrays can be spread into the function
+ * logLines(cmd, linesArray); // but it will flatten the plain array for convenience
+ * ```
+ */
+export function logLines(
+  cmd: LoggerCmdInstance = console,
+  ...lines: LinesArg[]
+) {
+  for (const line of lines.flat()) {
+    (cmd ?? console).log(`${line ?? ""}`);
+  }
+}
+
+/**
+ * Logs an error message using the log method of the provided {@link LoggerCmdInstance}.
+ *
+ * Line items that are `null` or `undefined` are logged as empty lines.
+ *
+ * @example
+ * ```ts
+ * logError(cmd, "Message");
+ * logError(cmd, "Message", "Additional message");
+ * logError(cmd, "Message", "Additional message", "Another additional message line");
+ * logError(cmd, "Message", ...additionalMessages); // arrays can be spread into the function
+ * logError(cmd, "Message", additionalMessages); // but it will flatten the plain array for convenience
+ * ```
+ */
 export const logError = (
-  cmd: any,
+  cmd: LoggerCmdInstance = console,
+  message: string,
+  ...additionalMessages: LinesArg[]
+) => {
+  logLines(cmd, "", `[ERROR] 🙀 :${message}`);
+  if (additionalMessages?.length) {
+    logLines(cmd, ...additionalMessages.flat());
+  }
+  logLines(cmd, "");
+};
+
+/**
+ * Logs a warning message using the log method of the provided {@link LoggerCmdInstance}.
+ *
+ * Line items that are `null` or `undefined` are logged as empty lines.
+ *
+ * @example
+ * ```ts
+ * logWarning(cmd, "Message");
+ * logWarning(cmd, "Message", "Additional message");
+ * logWarning(cmd, "Message", "Additional message", "Another additional message line");
+ * logWarning(cmd, "Message", ...additionalMessages); // arrays can be spread into the function
+ * logWarning(cmd, "Message", additionalMessages); // but it will flatten the plain array for convenience
+ * ```
+ */
+export const logWarning = (
+  cmd: LoggerCmdInstance = console,
   message: string,
-  additionalMessage?: any
+  ...additionalMessages: LinesArg[]
 ) => {
-  cmd.log("");
-  cmd.log(`[ERROR] 🙀 :${message}`);
-  cmd.log("");
-  if (additionalMessage) {
-    cmd.log(additionalMessage);
-    cmd.log("");
+  logLines(cmd, "", `[WARNING] ⚠️ : ${message}`);
+  if (additionalMessages?.length) {
+    logLines(cmd, ...additionalMessages.flat());
   }
+  logLines(cmd, "");
 };

package/src/utils/portForwards.ts

@@ -0,0 +1,57 @@
+import { spawn } from "child-process-promise";
+import type { ReadStream } from "fs-extra";
+import open from "open";
+
+export type PortForward = Promise<unknown> & {
+  childProcess: {
+    stdout: ReadStream;
+    kill: () => any;
+  };
+};
+const portForwards = new Map<string, PortForward>();
+
+export const stopPortForward = async (name: string) => {
+  const old = portForwards.get(name);
+
+  if (old) {
+    try {
+      old.childProcess.kill();
+      await old;
+    } catch (e) {
+      //
+    }
+    portForwards.delete(name);
+  }
+};
+
+export const getAllRunningPortForwards = () => {
+  return Array.from(portForwards.keys());
+};
+
+const addPortForward = (name: string, portForward: PortForward) => {
+  portForwards.set(name, portForward);
+};
+
+export const startPortForwardCommand = async (
+  name: string,
+  command: string,
+) => {
+  // stop if already there
+  await stopPortForward(name);
+  const [cmd, ...args] = command.split(" ");
+
+  const promise = spawn(cmd, args, {
+    env: {
+      ...process.env,
+      DEBUG: "",
+    },
+  }) as PortForward;
+  addPortForward(name, promise);
+  // wait a moment so that is surley started, unfortunatly we don't know that
+  await new Promise((r) => setTimeout(r, 1000));
+};
+export const stopAllPortForwards = async () => {
+  getAllRunningPortForwards().forEach((name) => {
+    stopPortForward(name);
+  });
+};

package/src/utils/preferences.ts

@@ -0,0 +1,60 @@
+import { readFile, writeFile, mkdir, cp, chmod } from "fs/promises";
+import { existsSync } from "fs";
+import { join, dirname } from "path";
+import { homedir } from "os";
+import { parse, stringify } from "yaml";
+
+const legacyPreferencesPath = join(homedir(), ".catladder/preferences.yml");
+const preferencesPath = join(homedir(), ".config/catladder/preferences.yml");
+
+const ensurePreferencesFile = async () => {
+  if (existsSync(preferencesPath)) {
+    return;
+  }
+  await mkdir(dirname(preferencesPath), { recursive: true, mode: 0o700 });
+  if (!existsSync(legacyPreferencesPath)) {
+    return writeFile(preferencesPath, "---\n{}", {
+      encoding: "utf-8",
+      mode: 0o600,
+    });
+  }
+  await cp(legacyPreferencesPath, preferencesPath);
+  await chmod(preferencesPath, 0o600);
+  await cp(
+    preferencesPath,
+    join(dirname(preferencesPath), "old_preferences_backup.yml"),
+  );
+};
+
+const loadPreferences = async () => {
+  await ensurePreferencesFile();
+  return readFile(preferencesPath, { encoding: "utf-8" });
+};
+
+export const getPreferences = async () => {
+  const yamlContent = await loadPreferences();
+  return (parse(yamlContent) ?? {}) as Record<string, string>;
+};
+
+export const hasPreference = async (key: string) => {
+  const preferences = await getPreferences();
+  return key in preferences;
+};
+export const getPreference = async (key: string) => {
+  const preferences = await getPreferences();
+  return preferences[key];
+};
+
+export const setPreference = async (key: string, value: string | number) => {
+  const preferences = await getPreferences();
+
+  const newPreferences = {
+    ...preferences,
+    [key]: value,
+  };
+
+  await writeFile(preferencesPath, stringify(newPreferences), {
+    encoding: "utf-8",
+    mode: 0o600,
+  });
+};

package/src/utils/projects/index.ts

@@ -1,11 +1,16 @@
-import { $ } from "zx";
 import { join } from "path";
 import { getProjectConfig, parseChoice } from "../../config/getProjectConfig";
-import { getk8sApi } from "../../k8sApi";
-import { readFileOrError } from "../files";
 
-export const getGitRoot = async (): Promise<string> => {
-  return (await $`git rev-parse --show-toplevel`).stdout.trim();
+import { readFileOrError } from "../files";
+import { exec } from "child-process-promise";
+
+export const getGitRoot = async (): Promise<string | null> => {
+  try {
+    return (await exec(`git rev-parse --show-toplevel`)).stdout.trim();
+  } catch (e) {
+    // not a git repo
+    return null;
+  }
 };
 
 export const getRootGitlabCiFile = async () => {
@@ -29,24 +34,3 @@ export const getProjectNamespace = async (envComponent: string) => {
   const config = await getProjectConfig();
   return `${config.customerName}-${config.appName}-${env}`;
 };
-
-export const getProjectPods = async (envComponent: string) => {
-  const namespace = await getProjectNamespace(envComponent);
-  const k8sApi = getk8sApi();
-  const res = await k8sApi.listNamespacedPod(namespace);
-
-  return res.body.items;
-};
-
-export const getProjectPvcs = async (envComponent: string) => {
-  const namespace = await getProjectNamespace(envComponent);
-  const k8sApi = getk8sApi();
-  const res = await k8sApi.listNamespacedPersistentVolumeClaim(namespace);
-
-  return res.body.items;
-};
-
-export const getProjectPodNames = async (envComponent: string) => {
-  const pods = await getProjectPods(envComponent);
-  return pods.map((n) => n.metadata.name);
-};

package/src/utils/promise.ts

@@ -1,11 +1,13 @@
 export async function filter<T>(
   arr: T[],
-  iterator: (item: T) => Promise<boolean>
+  iterator: (item: T) => Promise<boolean>,
 ) {
   const fail = Symbol("fail");
-  return ((await Promise.all(
-    arr.map(async item => ((await iterator(item)) ? item : fail))
-  )).filter(i => i !== fail) as any) as Promise<T[]>;
+  return (
+    await Promise.all(
+      arr.map(async (item) => ((await iterator(item)) ? item : fail)),
+    )
+  ).filter((i) => i !== fail) as any as Promise<T[]>;
 }
 
-export const delay = (ms: number) => new Promise(res => setTimeout(res, ms));
+export const delay = (ms: number) => new Promise((res) => setTimeout(res, ms));

package/tsconfig.json

@@ -1,16 +1,20 @@
 {
   "compilerOptions": {
-    "lib": ["es2020", "DOM"],
+    "lib": ["es2022", "DOM"],
     "module": "CommonJS",
+    "downlevelIteration": true,
     "declaration": true,
-    "noImplicitAny": true,
+    "target": "ES2018",
     "moduleResolution": "node",
     "esModuleInterop": true,
     "sourceMap": true,
     "outDir": "dist",
     "strict": false,
     "incremental": true,
-    "resolveJsonModule": true
+    "resolveJsonModule": true,
+    "paths": {
+      "@catladder/pipeline": ["../pipeline/src/index.ts"]
+    }
   },
   "include": ["./src/**/*"]
 }

package/.nvmrc

@@ -1 +0,0 @@
-16