@catladder/cli 0.0.0-node16-ddfae617 → 0.0.0-perf-cli-reduce-bundle-size-with-tsconfig-optimization-3556aaa4

package/src/kubernetes/portForward.ts

@@ -0,0 +1,15 @@
+import { startPortForwardCommand } from "../utils/portForwards";
+
+export const startKubePortForward = async (
+  podname: string,
+  localPort: number,
+  remotePort: number,
+  namespace: string,
+) => {
+  const name = `kube/${namespace}/${podname}/${localPort}:${remotePort}`;
+
+  await startPortForwardCommand(
+    name,
+    `kubectl port-forward ${podname} ${localPort}:${remotePort} -n ${namespace}`,
+  );
+};

package/src/packageInfos.ts

@@ -1,4 +1,10 @@
+// we can't import the package.json directly, because the bundling process will inline it
+// but the version is only set during deployement, when its already bundled
+// instead we need to use require.main.path to get the path of the main module, which is actually the file in the bin
+// this will point to the root package.json
+
+const path = require.main?.path + "/../package.json";
 // eslint-disable-next-line @typescript-eslint/no-var-requires
-const ownPkg = require("../package.json");
+const ownPkg = require(path);
 
 export default ownPkg;

package/src/types/child-process-promise.d.ts

@@ -1 +1,37 @@
-declare module 'child-process-promise'
+import type { ReadStream } from "fs-extra";
+
+declare module "child-process-promise" {
+  function exec(
+    cmd: string,
+    options?: {
+      env?: Record<string, string>;
+    },
+  ): Promise<{
+    stdout: string;
+  }>;
+
+  type Stdio = "inherit" | "pipe";
+  type SpawnOptions = {
+    stdio?: Stdio | Stdio[];
+    shell?: boolean | string;
+    env?: Record<string, string>;
+  };
+
+  function spawn(
+    cmd: string,
+    options?: SpawnOptions,
+  ): Promise<{
+    stdout: string;
+  }>;
+
+  function spawn(
+    cmd: string,
+    args: string[],
+    options?: SpawnOptions,
+  ): Promise<unknown> & {
+    childProcess: {
+      stdout: ReadStream;
+      kill: () => void;
+    };
+  };
+}

package/src/utils/boxWithAscii.ts

@@ -0,0 +1,9 @@
+// thx copilot
+export const boxWithAsci = (str: string) => {
+  const length = str.length;
+  const box = "╔" + "═".repeat(length + 3) + "╗";
+  const empty = "║ " + " ".repeat(length + 1) + " ║";
+  const text = "║ " + str + " ║";
+  const bottom = "╚" + "═".repeat(length + 3) + "╝";
+  return [box, empty, text, empty, bottom].join("\n");
+};

package/src/utils/cluster.ts

@@ -1,12 +1,12 @@
-import { $ } from "zx";
+import { exec } from "child-process-promise";
 
 export const getCurrentContext = async () =>
-  (await $`kubectl config current-context`).stdout.trim();
+  (await exec(`kubectl config current-context`)).stdout.trim();
 
 export const getCurrentConnectedClusterName = async () => {
   return await getCurrentContext();
 };
 
 export const connectToCluster = async (fullname: string) => {
-  await $`kubectl config use-context ${fullname}`;
+  await exec(`kubectl config use-context ${fullname}`);
 };

package/src/utils/editAsFile.ts

@@ -1,12 +1,12 @@
 import { withFile } from "tmp-promise";
-import { dump, load } from "js-yaml";
+import { parse, stringify } from "yaml";
 
 import { readFile, writeFile } from "fs-extra";
 import getEditor from "./getEditor";
 
 export const editAsFile = async <T>(
   inObject: T,
-  preamble?: string
+  preamble?: string,
 ): Promise<T> => {
   const fullPreamble = preamble
     ? `#
@@ -15,16 +15,17 @@ export const editAsFile = async <T>(
 
 `
     : "\n";
-  const asString = fullPreamble + dump(inObject, { noRefs: true });
+  const asString =
+    fullPreamble + stringify(inObject, { aliasDuplicateObjects: false });
   let newContent: T;
 
   await withFile(
     async ({ path: tmpFilePath }) => {
       await writeFile(tmpFilePath, asString);
       await (await getEditor()).open(tmpFilePath);
-      newContent = load((await readFile(tmpFilePath)).toString("utf-8")) as T;
+      newContent = parse((await readFile(tmpFilePath)).toString("utf-8")) as T;
     },
-    { postfix: ".yml" }
+    { postfix: ".yml" },
   );
 
   return newContent;

package/src/utils/files.ts

@@ -1,5 +1,4 @@
 import fs from "fs-extra";
-import yaml from "js-yaml";
 
 const readFile = async (file: string) =>
   fs.readFile(file, { encoding: "utf-8" });
@@ -12,11 +11,3 @@ export const readFileOrError = async (filePath: string) => {
     return [e];
   }
 };
-
-export const readYaml = async <T = Record<string, any>>(filename: string) => {
-  try {
-    return yaml.load(await readFile(filename)) as T;
-  } catch (e) {
-    return null;
-  }
-};

package/src/utils/getEditor.ts

@@ -1,11 +1,13 @@
-/* eslint-disable @typescript-eslint/no-var-requires */
 import { spawn } from "child-process-promise";
 import commandExists from "command-exists-promise";
 
 let cmd: string;
 export default async () => {
   if (!cmd) {
-    cmd = (await commandExists("code")) ? "code --wait" : "vim";
+    cmd =
+      process.env.VISUAL ??
+      process.env.EDITOR ??
+      ((await commandExists("code")) ? "code --wait" : "vim");
   }
 
   return {

package/src/utils/gitlab.ts

@@ -1,10 +1,10 @@
-import { exec } from "child-process-promise";
 import { getSecretVarName } from "@catladder/pipeline";
-import { isObject } from "lodash";
+import { exec } from "child-process-promise";
+import { has, isObject } from "lodash";
 import memoizee from "memoizee";
 import fetch from "node-fetch";
 import open from "open";
-import { CommandInstance } from "vorpal";
+import type { CommandInstance } from "vorpal";
 import { getPreference, hasPreference, setPreference } from "./preferences";
 
 const TOKEN_KEY = "gitlab-personal-access-token";
@@ -38,11 +38,11 @@ export const setupGitlabToken = async (vorpal: CommandInstance) => {
     await setPreference(TOKEN_KEY, personalToken);
   }
 };
-export const getGitlabToken = async (vorpal: CommandInstance) => {
+export const getGitlabToken = async (vorpal: CommandInstance | null) => {
   if (!(await hasGitlabToken())) {
     if (!vorpal) {
       console.error(
-        "⚠️ gitlab token missing, please run catladder to set it up"
+        "⚠️ gitlab token missing, please run catladder to set it up",
       );
       process.exit(1);
     }
@@ -51,15 +51,16 @@ export const getGitlabToken = async (vorpal: CommandInstance) => {
   return getPreference(TOKEN_KEY);
 };
 
+type Method = "GET" | "PUT" | "POST" | "DELETE";
 export const doGitlabRequest = async <T = any>(
-  vorpal: CommandInstance,
+  vorpal: CommandInstance | null,
   path: string,
   data: any = undefined,
-  update?: boolean
+  method: Method = "GET",
 ): Promise<T> => {
   const rootToken = await getGitlabToken(vorpal);
 
-  const method = data ? (update ? "PUT" : "POST") : "GET";
+  //const method = data ? (update ? "PUT" : "POST") : "GET";
 
   const result = await fetch(`https://git.panter.ch/api/v4/${path}`, {
     method,
@@ -67,11 +68,14 @@ export const doGitlabRequest = async <T = any>(
       "Content-Type": "application/json",
       "Private-Token": rootToken,
     },
-    body: JSON.stringify(data),
+    body: data ? JSON.stringify(data) : undefined,
   });
 
   if (result.status >= 200 && result.status < 400) {
-    return result.json();
+    if (result.headers.get("content-type") === "application/json") {
+      return result.json();
+    }
+    return null;
   }
   if (result.status === 404) {
     throw new Error("not found");
@@ -80,22 +84,23 @@ export const doGitlabRequest = async <T = any>(
   throw new Error(
     `Could not send request to gitlab api ${path}: ${result.status} "${
       result.statusText
-    }".\nResponse: ${JSON.stringify(await result.json(), null, 2)}`
+    }".\nResponse: ${JSON.stringify(await result.json(), null, 2)}`,
   );
 };
 
 export const getProjectInfo = async (
-  vorpal: CommandInstance
+  vorpal: CommandInstance | null,
 ): Promise<{ id: string; web_url: string }> => {
   const gitRemoteOriginUrl = (
     await exec("git config --get remote.origin.url")
   ).stdout.trim();
-  const projectPath = /(https:\/\/|git@)git\.panter\.ch[:/](.*)\.git/g.exec(
-    gitRemoteOriginUrl
-  );
+  const projectPath =
+    /(https:\/\/|git@)git\.panter\.ch[:/]([^.]*)(\.git)?/g.exec(
+      gitRemoteOriginUrl,
+    );
   const project = await doGitlabRequest(
     vorpal,
-    `projects/${encodeURIComponent(projectPath[2])}`
+    `projects/${encodeURIComponent(projectPath[2])}`,
   );
   return project;
 };
@@ -109,28 +114,44 @@ type GitlabVariable = {
   environment_scope: string;
 };
 export const getAllVariables = memoizee(
-  async (vorpal: CommandInstance): Promise<Array<GitlabVariable>> => {
+  async (
+    vorpal: CommandInstance | null,
+    n = 5, // how many requests to do in parallel, 5 seems a good value for many projects, ideally we would do one request in parallel per component
+  ): Promise<Array<GitlabVariable>> => {
     const { id } = await getProjectInfo(vorpal);
+
     let all: Array<GitlabVariable> = [];
-    let result: Array<GitlabVariable>;
+    let result: Array<Array<GitlabVariable>> = [];
     let page = 1;
+
     do {
-      result = await doGitlabRequest(
-        vorpal,
-        // 100 is max page size
-        `projects/${id}/variables?per_page=100&page=${page}`
-      );
-      page++;
-      all = [...all, ...result];
-    } while (result?.length > 0);
+      // Create an array of promises for N pages
+      const promises = Array.from({ length: n }, (_, i) => {
+        return doGitlabRequest(
+          vorpal,
+          `projects/${id}/variables?per_page=100&page=${page + i}`,
+        );
+      });
+
+      // Wait for all promises to resolve
+      result = await Promise.all(promises);
+
+      // Increment the page by N
+      page += n;
+
+      // Flatten the result array and add it to 'all'
+      all = [...all, ...result.flat()];
+      // Continue only if the last page had results
+    } while (result.length > 0 && result[result.length - 1].length > 0);
+
     return all;
   },
-  { promise: true }
+  { promise: true },
 );
 
 export const getVariableValueByRawName = async (
   vorpal: CommandInstance,
-  rawName: string
+  rawName: string,
 ) => {
   const allVariables = await getAllVariables(vorpal);
   return allVariables.find((v) => v.key === rawName)?.value;
@@ -143,48 +164,154 @@ const createVariable = async (
   vorpal: CommandInstance,
   projectId: string,
   key: string,
-  value: string
+  value: string,
+  masked = true,
+  environment_scope = "*",
 ) => {
-  return await doGitlabRequest(vorpal, `projects/${projectId}/variables`, {
-    key,
-    value,
-    masked: isMaskable(value),
-  });
+  return await doGitlabRequest(
+    vorpal,
+    `projects/${projectId}/variables`,
+    {
+      key,
+      value,
+      masked: masked && isMaskable(value),
+      environment_scope,
+    },
+    "POST",
+  );
 };
 
 const updateVariable = async (
   vorpal: CommandInstance,
   projectId: string,
   key: string,
-  value: string
+  value: string,
+  masked = true,
 ) => {
   return await doGitlabRequest(
     vorpal,
     `projects/${projectId}/variables/${key}`,
     {
       value,
-      masked: isMaskable(value),
+      masked: masked && isMaskable(value),
     },
-    true
+    "PUT",
   );
 };
+
+const deleteVariable = async (
+  vorpal: CommandInstance,
+  projectId: string,
+  key: string,
+) => {
+  return await doGitlabRequest(
+    vorpal,
+    `projects/${projectId}/variables/${key}`,
+    undefined,
+    "DELETE",
+  );
+};
+
+const getAllCatladderEnvVarsInGitlab = async (vorpal: CommandInstance) => {
+  const allVariables = await getAllVariables(vorpal).then((v) =>
+    v.reduce<{
+      [key: string]: {
+        value?: string;
+        backups: number[];
+      };
+    }>((acc, variable) => {
+      const { key } = variable;
+
+      if (key.startsWith("CL_")) {
+        const matchBackup = key.match(/(CL_.*)_backup_([0-9]+)/);
+
+        if (matchBackup) {
+          const key = matchBackup[1];
+          const timestamp = Number(matchBackup[2]);
+          const backups = [...(acc[key]?.backups ?? []), timestamp];
+          return {
+            ...acc,
+            [key]: {
+              ...(acc[key] ?? {}), // add value
+              backups,
+            },
+          };
+        }
+
+        return {
+          ...acc,
+          [key]: {
+            backups: [],
+            ...(acc[key] ?? {}), // may add backups
+            value: variable.value,
+          },
+        };
+      }
+
+      return acc;
+    }, {}),
+  );
+  return allVariables;
+};
+
+const getBackupKey = (fullKey: string, timestamp: number) =>
+  `${fullKey}_backup_${timestamp}`;
+export const clearBackups = async (vorpal: CommandInstance, keep: number) => {
+  const existingVariables = await getAllCatladderEnvVarsInGitlab(vorpal);
+  const { id } = await getProjectInfo(vorpal);
+  for (const [key, { backups }] of Object.entries(existingVariables)) {
+    const backupsSorted = backups.sort((a, b) => b - a);
+    //const toKeep = backupsSorted.slice(0, keep);
+    const toDelete = backupsSorted.slice(keep);
+
+    for (const timestamp of toDelete) {
+      await deleteVariable(vorpal, id, getBackupKey(key, timestamp));
+    }
+  }
+};
+
 export const upsertAllVariables = async (
   vorpal: CommandInstance,
   variables: Record<string, any>,
   env: string,
-  componentName: string
+  componentName: string,
+  backup = true,
+  masked = true, // FIXME: would be better to have this per variable
 ): Promise<void> => {
   const { id } = await getProjectInfo(vorpal);
 
+  // we list all existing variables. We also gather backup versions, but its currently unused
+  const existingVariables = await getAllCatladderEnvVarsInGitlab(vorpal);
   for (const [key, value] of Object.entries(variables ?? {})) {
     const fullKey = getSecretVarName(env, componentName, key);
     const valueSanitized = isObject(value) ? JSON.stringify(value) : `${value}`;
-    try {
-      await updateVariable(vorpal, id, fullKey, valueSanitized);
-    } catch (e) {
-      await createVariable(vorpal, id, fullKey, valueSanitized);
-    } finally {
-      getAllVariables.clear();
+
+    const exists = has(existingVariables, fullKey);
+    const oldValue = existingVariables[fullKey]?.value;
+    const changed = oldValue !== valueSanitized;
+    if (changed) {
+      if (exists) {
+        vorpal.log(`changed: ${key}`);
+
+        await updateVariable(vorpal, id, fullKey, valueSanitized, masked);
+        // write backup
+        if (backup) {
+          await createVariable(
+            vorpal,
+            id,
+            getBackupKey(fullKey, new Date().getTime()),
+            oldValue,
+            masked,
+            "_backup",
+          );
+        }
+      } else {
+        vorpal.log(`new    : ${key}`);
+        await createVariable(vorpal, id, fullKey, valueSanitized, masked);
+      }
+    } else {
+      vorpal.log(`skip   : ${key}`);
     }
   }
+  getAllVariables.clear();
 };

package/src/utils/log.ts

@@ -1,7 +1,7 @@
 export const logError = (
   cmd: any,
   message: string,
-  additionalMessage?: any
+  additionalMessage?: any,
 ) => {
   cmd.log("");
   cmd.log(`[ERROR] 🙀 :${message}`);

package/src/utils/{portForward.ts → portForwards.ts}

@@ -1,6 +1,14 @@
 import { spawn } from "child-process-promise";
+import type { ReadStream } from "fs-extra";
+import open from "open";
 
-const portForwards = new Map();
+export type PortForward = Promise<unknown> & {
+  childProcess: {
+    stdout: ReadStream;
+    kill: () => any;
+  };
+};
+const portForwards = new Map<string, PortForward>();
 
 export const stopPortForward = async (name: string) => {
   const old = portForwards.get(name);
@@ -19,32 +27,29 @@ export const stopPortForward = async (name: string) => {
 export const getAllRunningPortForwards = () => {
   return Array.from(portForwards.keys());
 };
-export const startPortForward = async (
-  podname: string,
-  localPort: number,
-  remotePort: number,
-  namespace: string
+
+const addPortForward = (name: string, portForward: PortForward) => {
+  portForwards.set(name, portForward);
+};
+
+export const startPortForwardCommand = async (
+  name: string,
+  command: string,
 ) => {
-  const name = `${namespace}/${podname}/${localPort}:${remotePort}`;
   // stop if already there
-  if (portForwards.has(name)) {
-    stopPortForward(name);
-  }
-  const promise = spawn(
-    "kubectl",
-    ["port-forward", podname, `${localPort}:${remotePort}`, "-n", namespace],
-    {
-      env: {
-        ...process.env,
-        DEBUG: "",
-      },
-    }
-  );
-  portForwards.set(name, promise);
+  await stopPortForward(name);
+  const [cmd, ...args] = command.split(" ");
+
+  const promise = spawn(cmd, args, {
+    env: {
+      ...process.env,
+      DEBUG: "",
+    },
+  }) as PortForward;
+  addPortForward(name, promise);
   // wait a moment so that is surley started, unfortunatly we don't know that
   await new Promise((r) => setTimeout(r, 1000));
 };
-
 export const stopAllPortForwards = async () => {
   getAllRunningPortForwards().forEach((name) => {
     stopPortForward(name);

package/src/utils/preferences/index.ts

@@ -1,7 +1,7 @@
 import fs from "fs-extra";
 import { homedir } from "os";
 
-import { load, dump } from "js-yaml";
+import { parse, stringify } from "yaml";
 const directory = `${homedir()}/.catladder`;
 const file = `${directory}/preferences.yml`;
 
@@ -9,10 +9,8 @@ const getPreferences = async () => {
   if (!(await fs.pathExists(file))) {
     await fs.createFile(file);
   }
-  return (load(await fs.readFile(file, { encoding: "utf-8" })) ?? {}) as Record<
-    string,
-    string
-  >;
+  return (parse(await fs.readFile(file, { encoding: "utf-8" })) ??
+    {}) as Record<string, string>;
 };
 
 export const hasPreference = async (key: string) => {
@@ -32,5 +30,5 @@ export const setPreference = async (key: string, value: string | number) => {
     [key]: value,
   };
 
-  await fs.writeFile(file, dump(newPreferences));
+  await fs.writeFile(file, stringify(newPreferences));
 };

package/src/utils/projects/index.ts

@@ -1,11 +1,16 @@
-import { $ } from "zx";
 import { join } from "path";
 import { getProjectConfig, parseChoice } from "../../config/getProjectConfig";
-import { getk8sApi } from "../../k8sApi";
-import { readFileOrError } from "../files";
 
-export const getGitRoot = async (): Promise<string> => {
-  return (await $`git rev-parse --show-toplevel`).stdout.trim();
+import { readFileOrError } from "../files";
+import { exec } from "child-process-promise";
+
+export const getGitRoot = async (): Promise<string | null> => {
+  try {
+    return (await exec(`git rev-parse --show-toplevel`)).stdout.trim();
+  } catch (e) {
+    // not a git repo
+    return null;
+  }
 };
 
 export const getRootGitlabCiFile = async () => {
@@ -29,24 +34,3 @@ export const getProjectNamespace = async (envComponent: string) => {
   const config = await getProjectConfig();
   return `${config.customerName}-${config.appName}-${env}`;
 };
-
-export const getProjectPods = async (envComponent: string) => {
-  const namespace = await getProjectNamespace(envComponent);
-  const k8sApi = getk8sApi();
-  const res = await k8sApi.listNamespacedPod(namespace);
-
-  return res.body.items;
-};
-
-export const getProjectPvcs = async (envComponent: string) => {
-  const namespace = await getProjectNamespace(envComponent);
-  const k8sApi = getk8sApi();
-  const res = await k8sApi.listNamespacedPersistentVolumeClaim(namespace);
-
-  return res.body.items;
-};
-
-export const getProjectPodNames = async (envComponent: string) => {
-  const pods = await getProjectPods(envComponent);
-  return pods.map((n) => n.metadata.name);
-};

package/src/utils/promise.ts

@@ -1,11 +1,13 @@
 export async function filter<T>(
   arr: T[],
-  iterator: (item: T) => Promise<boolean>
+  iterator: (item: T) => Promise<boolean>,
 ) {
   const fail = Symbol("fail");
-  return ((await Promise.all(
-    arr.map(async item => ((await iterator(item)) ? item : fail))
-  )).filter(i => i !== fail) as any) as Promise<T[]>;
+  return (
+    await Promise.all(
+      arr.map(async (item) => ((await iterator(item)) ? item : fail)),
+    )
+  ).filter((i) => i !== fail) as any as Promise<T[]>;
 }
 
-export const delay = (ms: number) => new Promise(res => setTimeout(res, ms));
+export const delay = (ms: number) => new Promise((res) => setTimeout(res, ms));