@catladder/cli 0.0.0-refactor-cli-add-improved-git-utilities-224e6ee7 → 0.0.0-refactor-cli-add-improved-git-utilities-916545cc

package/package.json

@@ -53,7 +53,7 @@
     }
   ],
   "license": "MIT",
-  "version": "0.0.0-refactor-cli-add-improved-git-utilities-224e6ee7",
+  "version": "0.0.0-refactor-cli-add-improved-git-utilities-916545cc",
   "scripts": {
     "lint": "eslint \"src/**/*.ts\"",
     "lint:fix": "eslint \"src/**/*.ts\" --fix",

package/src/apps/cli/commands/project/setup/setupAccessTokens.ts

@@ -1,10 +1,20 @@
 import type { CommandInstance } from "vorpal";
 import dayjs from "dayjs";
 import { doGitlabRequest, getProjectInfo } from "../../../../../utils/gitlab";
+import {
+  getGitlabProjectId,
+  gitlabApiClient,
+  type GitlabRestClient,
+} from "../../../../../gitlab";
 
 const TOKEN_NAME = "semantic-release";
 
-export const setupAccessTokens = async (instance: CommandInstance) => {
+export const setupAccessTokens = async (
+  instance: CommandInstance,
+  preinit?: [number, GitlabRestClient],
+) => {
+  const [id, gitlabApi] =
+    preinit ?? (await Promise.all([getGitlabProjectId(), gitlabApiClient()]));
   const { id: projectId } = await getProjectInfo(instance);
 
   const projectTokens = await doGitlabRequest(

package/src/apps/cli/commands/project/setup/setupCloudRun.ts

@@ -78,7 +78,9 @@ export const setupCloudRun = async (
     },
     context.env,
     context.name,
-    false, // backup
-    false, // masked
+    {
+      backup: false,
+      masked: false,
+    },
   );
 };

package/src/apps/cli/commands/project/setup/setupKubernetes.ts

@@ -6,27 +6,30 @@ import {
 import type { CommandInstance } from "vorpal";
 import { exec } from "child-process-promise";
 import { connectToCluster } from "../../../../../utils/cluster";
-import {
-  doGitlabRequest,
-  getProjectInfo,
-  upsertAllVariables,
-} from "../../../../../utils/gitlab";
+import { upsertAllVariables } from "../../../../../utils/gitlab";
 import ensureNamespace from "../utils/ensureNamespace";
+import type { GitlabRestClient } from "../../../../../gitlab";
+import { getGitlabProjectId, gitlabApiClient } from "../../../../../gitlab";
 
 export const setupKubernetes = async (
   instance: CommandInstance,
   context: ComponentContext,
+  preinit?: [number, GitlabRestClient],
 ) => {
   const deployConfig = context.deploy?.config;
   if (!isOfDeployType(deployConfig, "kubernetes")) {
     throw new Error("cannot run setupKubernetes on non-kubernetes deployments");
   }
+  const [id, gitlabApi] =
+    preinit ?? (await Promise.all([getGitlabProjectId(), gitlabApiClient()]));
 
-  const { id: projectId } = await getProjectInfo(instance);
-  const deploy_tokens = await doGitlabRequest(
-    instance,
-    `projects/${projectId}/deploy_tokens`,
-  );
+  const deploy_tokens = await gitlabApi.DeployTokens.all({ projectId: id });
+
+  // const { id: projectId } = await getProjectInfo(instance);
+  // const deploy_tokens = await doGitlabRequest(
+  //   instance,
+  //   `projects/${projectId}/deploy_tokens`,
+  // );
 
   if (
     !deploy_tokens.find(
@@ -36,17 +39,21 @@ export const setupKubernetes = async (
     instance.log(
       "I will setup the 'GitLab Deploy Token', so Kubernetes can pull images from this project.",
     );
-
-    await doGitlabRequest(
-      instance,
-      `projects/${projectId}/deploy_tokens`,
-      {
-        id: projectId,
-        name: "gitlab-deploy-token",
-        scopes: ["read_registry"],
-      },
-      "POST",
+    await gitlabApi.DeployTokens.create(
+      "gitlab-deploy-token",
+      ["read_registry"],
+      { projectId: id },
     );
+    // await doGitlabRequest(
+    //   instance,
+    //   `projects/${projectId}/deploy_tokens`,
+    //   {
+    //     id: projectId,
+    //     name: "gitlab-deploy-token",
+    //     scopes: ["read_registry"],
+    //   },
+    //   "POST",
+    // );
   }
 
   const fullName = getFullKubernetesClusterName(deployConfig.cluster);
@@ -160,12 +167,8 @@ EOF
   instance.log("");
   instance.log("pusing secrets to gitlab...");
 
-  await upsertAllVariables(
-    instance,
-    vars,
-    context.env,
-    context.name,
-    false, // no backup
-  );
+  await upsertAllVariables(instance, vars, context.env, context.name, {
+    backup: false,
+  });
   instance.log("done!");
 };

package/src/apps/cli/commands/project/setup/setupTopic.ts

@@ -1,17 +1,12 @@
 import type { CommandInstance } from "vorpal";
-import { doGitlabRequest, getProjectInfo } from "../../../../../utils/gitlab";
+import { getGitlabProjectInfo, gitlabApiClient } from "../../../../../gitlab";
 
 const catladderTopic = "catladder";
 
 export const setupTopic = async (instance: CommandInstance) => {
-  const { id: projectId } = await getProjectInfo(instance);
-
-  const { topics } = await doGitlabRequest(instance, `projects/${projectId}`);
-
-  await doGitlabRequest(
-    instance,
-    `projects/${projectId}`,
-    { topics: [catladderTopic, ...topics] },
-    "PUT",
-  );
+  const api = await gitlabApiClient();
+  const info = await getGitlabProjectInfo(api);
+  await api.Projects.edit(info.id, {
+    topics: [catladderTopic, ...info.topics],
+  });
 };

package/src/gitlab/apiClient.ts

@@ -1,11 +1,13 @@
 import { Gitlab } from "@gitbeaker/rest";
 import { getGitlabToken } from "./gitlabToken";
 
+export type GitlabRestClient = InstanceType<typeof Gitlab<false>>;
+
 export const gitlabApiClient = async ({
   gitlabHost,
 }: {
   gitlabHost?: string;
-} = {}) => {
+} = {}): Promise<GitlabRestClient> => {
   const { token, gitRemoteHost } = await getGitlabToken(gitlabHost);
   if (typeof token !== "string") {
     throw new Error("Gitlab personal access token is missing!");
@@ -15,6 +17,7 @@ export const gitlabApiClient = async ({
   }
   return new Gitlab({
     token,
-    host: gitRemoteHost,
+    host: `https://${gitRemoteHost}`,
+    camelize: false,
   });
 };

package/src/gitlab/getGitlabProjectId.ts

@@ -0,0 +1,32 @@
+import type { ProjectSchema } from "@gitbeaker/rest";
+import { getGitRemoteHostAndPath } from "../git/getGitRemoteHostAndPath";
+import type { GitlabRestClient } from "./apiClient";
+import { gitlabApiClient } from "./apiClient";
+import {
+  upsertProjectPerferences,
+  getProjectPreferences,
+} from "../utils/projectPreferences";
+
+export const getGitlabProjectInfo = async (
+  existingInstance?: GitlabRestClient,
+): Promise<ProjectSchema> => {
+  const api = existingInstance ?? (await gitlabApiClient());
+  const { gitRemotePath } = await getGitRemoteHostAndPath();
+  const projectInfo = await api.Projects.show(gitRemotePath, {
+    statistics: false,
+    withCustomAttributes: false,
+    license: false,
+  });
+  // write unawaited intentionally
+  upsertProjectPerferences({ gitlabProjectId: projectInfo.id });
+  return projectInfo;
+};
+
+export const getGitlabProjectId = async (): Promise<number> => {
+  const { gitlabProjectId } = await getProjectPreferences();
+  if (gitlabProjectId) {
+    return gitlabProjectId;
+  }
+  const projectInfo = await getGitlabProjectInfo();
+  return projectInfo.id;
+};

package/src/gitlab/index.ts

@@ -0,0 +1,3 @@
+export * from "./apiClient";
+export * from "./getGitlabProjectId";
+export * from "./gitlabToken";

package/src/utils/gitlab.ts

@@ -5,7 +5,16 @@ import memoizee from "memoizee";
 import fetch from "node-fetch";
 import open from "open";
 import type { CommandInstance } from "vorpal";
-import { getPreference, hasPreference, setPreference } from "./preferences";
+import { hasPreference } from "./preferences";
+import {
+  setGitlabToken,
+  getGitlabToken as getGitlabTokenFromPerferences,
+  getGitlabProjectId,
+  getGitlabProjectInfo,
+  gitlabApiClient,
+  type GitlabRestClient,
+} from "../gitlab";
+import type { Camelize, ProjectVariableSchema } from "@gitbeaker/rest";
 
 export const hasGitlabToken = async () =>
   await hasPreference("gitlab-personal-access-token");
@@ -23,6 +32,10 @@ export const setupGitlabToken = async (vorpal: CommandInstance) => {
     name: "shouldContinue",
     type: "prompt",
   });
+  if (!shouldContinue) {
+    console.info("Aborted by user");
+    process.exit(0);
+  }
 
   open("https://git.panter.ch/-/profile/personal_access_tokens");
 
@@ -35,13 +48,14 @@ export const setupGitlabToken = async (vorpal: CommandInstance) => {
     message: "Your personal access token ",
   });
   if (typeof personalToken === "string" && personalToken.length) {
-    await setPreference("gitlab-personal-access-token", personalToken);
+    await setGitlabToken(personalToken);
   } else {
     console.error("⚠️ no gitlab personal access token set");
   }
 };
 export const getGitlabToken = async (vorpal: CommandInstance | null) => {
-  if (!(await hasGitlabToken())) {
+  const { token } = await getGitlabTokenFromPerferences();
+  if (!token?.length) {
     if (!vorpal) {
       console.error(
         "⚠️ gitlab token missing, please run catladder to set it up",
@@ -50,7 +64,7 @@ export const getGitlabToken = async (vorpal: CommandInstance | null) => {
     }
     await setupGitlabToken(vorpal);
   }
-  return getPreference("gitlab-personal-access-token");
+  return token;
 };
 
 type Method = "GET" | "PUT" | "POST" | "DELETE";
@@ -93,22 +107,8 @@ export const doGitlabRequest = async <T = any>(
   );
 };
 
-export const getProjectInfo = async (
-  vorpal: CommandInstance | null,
-): Promise<{ id: string; web_url: string }> => {
-  const gitRemoteOriginUrl = (
-    await exec("git config --get remote.origin.url")
-  ).stdout.trim();
-  const projectPath =
-    /(https:\/\/|git@)git\.panter\.ch[:/]([^.]*)(\.git)?/g.exec(
-      gitRemoteOriginUrl,
-    );
-  const project = await doGitlabRequest(
-    vorpal,
-    `projects/${encodeURIComponent(projectPath[2])}`,
-  );
-  return project;
-};
+export const getProjectInfo = async (vorpal: CommandInstance | null) =>
+  getGitlabProjectInfo();
 
 export type GitlabVariable = {
   variable_type: string;
@@ -121,25 +121,36 @@ export type GitlabVariable = {
 export const getAllVariables = memoizee(
   async (
     vorpal: CommandInstance | null,
-    n = 5, // how many requests to do in parallel, 5 seems a good value for many projects, ideally we would do one request in parallel per component
-  ): Promise<Array<GitlabVariable>> => {
-    const { id } = await getProjectInfo(vorpal);
-
-    let all: Array<GitlabVariable> = [];
-    let result: Array<Array<GitlabVariable>> = [];
+    preinit?: [number, GitlabRestClient],
+  ): Promise<(ProjectVariableSchema | Camelize<ProjectVariableSchema>)[]> => {
+    const [id, gitlabApi] =
+      preinit ?? (await Promise.all([getGitlabProjectId(), gitlabApiClient()]));
+    // how many requests to do in parallel, 5 seems a good value for many projects, ideally we would do one request in parallel per component
+    const n = 5;
+    let all: (ProjectVariableSchema | Camelize<ProjectVariableSchema>)[] = [];
+    let result: (ProjectVariableSchema | Camelize<ProjectVariableSchema>)[][] =
+      [];
     let page = 1;
 
     do {
       // Create an array of promises for N pages
       const promises = Array.from({ length: n }, (_, i) => {
-        return doGitlabRequest(
-          vorpal,
-          `projects/${id}/variables?per_page=100&page=${page + i}`,
-        );
+        return gitlabApi.ProjectVariables.all(id, {
+          page: page + i,
+          perPage: 100,
+        });
       });
 
       // Wait for all promises to resolve
-      result = await Promise.all(promises);
+      result = await Promise.all(
+        Array.from({ length: n }, async (_, i) => {
+          const res = await gitlabApi.ProjectVariables.all(id, {
+            page: page + i,
+            perPage: 100,
+          });
+          return res;
+        }),
+      );
 
       // Increment the page by N
       page += n;
@@ -165,60 +176,11 @@ export const getVariableValueByRawName = async (
 const maskableRegex = new RegExp("^[a-zA-Z0-9_+=/@:.~-]{8,}$"); // SEE https://gitlab.com/gitlab-org/gitlab-foss/-/blob/master/spec/frontend/ci_variable_list/components/ci_variable_modal_spec.js#L20
 const isMaskable = (value: string): boolean => maskableRegex.test(value);
 
-const createVariable = async (
-  vorpal: CommandInstance,
-  projectId: string,
-  key: string,
-  value: string,
-  masked = true,
-  environment_scope = "*",
-) => {
-  return await doGitlabRequest(
-    vorpal,
-    `projects/${projectId}/variables`,
-    {
-      key,
-      value,
-      masked: masked && isMaskable(value),
-      environment_scope,
-    },
-    "POST",
-  );
-};
-
-const updateVariable = async (
+const getAllCatladderEnvVarsInGitlab = async (
   vorpal: CommandInstance,
-  projectId: string,
-  key: string,
-  value: string,
-  masked = true,
+  preinit?: [number, GitlabRestClient],
 ) => {
-  return await doGitlabRequest(
-    vorpal,
-    `projects/${projectId}/variables/${key}`,
-    {
-      value,
-      masked: masked && isMaskable(value),
-    },
-    "PUT",
-  );
-};
-
-const deleteVariable = async (
-  vorpal: CommandInstance,
-  projectId: string,
-  key: string,
-) => {
-  return await doGitlabRequest(
-    vorpal,
-    `projects/${projectId}/variables/${key}`,
-    undefined,
-    "DELETE",
-  );
-};
-
-const getAllCatladderEnvVarsInGitlab = async (vorpal: CommandInstance) => {
-  const allVariables = await getAllVariables(vorpal).then((v) =>
+  const allVariables = await getAllVariables(vorpal, preinit).then((v) =>
     v.reduce<{
       [key: string]: {
         value?: string;
@@ -263,15 +225,23 @@ const getBackupKey = (fullKey: string, timestamp: number) =>
   `${fullKey}_backup_${timestamp}`;
 
 export const clearBackups = async (vorpal: CommandInstance, keep: number) => {
-  const existingVariables = await getAllCatladderEnvVarsInGitlab(vorpal);
-  const { id } = await getProjectInfo(vorpal);
+  const [id, gitlabApi] = await Promise.all([
+    getGitlabProjectId(),
+    gitlabApiClient(),
+  ]);
+  const existingVariables = await getAllCatladderEnvVarsInGitlab(vorpal, [
+    id,
+    gitlabApi,
+  ]);
+  // const { id } = await getProjectInfo(vorpal);
   for (const [key, { backups }] of Object.entries(existingVariables)) {
     const backupsSorted = backups.sort((a, b) => b - a);
     //const toKeep = backupsSorted.slice(0, keep);
     const toDelete = backupsSorted.slice(keep);
 
     for (const timestamp of toDelete) {
-      await deleteVariable(vorpal, id, getBackupKey(key, timestamp));
+      await gitlabApi.ProjectVariables.remove(id, getBackupKey(key, timestamp));
+      // await deleteVariable(vorpal, id, getBackupKey(key, timestamp));
     }
   }
 };
@@ -281,13 +251,26 @@ export const upsertAllVariables = async (
   variables: Record<string, any>,
   env: string,
   componentName: string,
-  backup = true,
-  masked = true, // FIXME: would be better to have this per variable
+  {
+    backup = true,
+    masked = true,
+    preinit,
+  }: {
+    backup?: boolean;
+    // FIXME: would be better to have this per variable
+    masked?: boolean;
+    preinit?: [number, GitlabRestClient];
+  } = {},
 ): Promise<void> => {
-  const { id } = await getProjectInfo(vorpal);
+  const [id, gitlabApi] =
+    preinit ?? (await Promise.all([getGitlabProjectId(), gitlabApiClient()]));
+  // const { id } = await getProjectInfo(vorpal);
 
   // we list all existing variables. We also gather backup versions, but its currently unused
-  const existingVariables = await getAllCatladderEnvVarsInGitlab(vorpal);
+  const existingVariables = await getAllCatladderEnvVarsInGitlab(vorpal, [
+    id,
+    gitlabApi,
+  ]);
   for (const [key, value] of Object.entries(variables ?? {})) {
     const fullKey = getSecretVarName(env, componentName, key);
     const valueSanitized = isObject(value) ? JSON.stringify(value) : `${value}`;
@@ -299,21 +282,30 @@ export const upsertAllVariables = async (
       if (exists) {
         vorpal.log(`changed: ${key}`);
 
-        await updateVariable(vorpal, id, fullKey, valueSanitized, masked);
+        await gitlabApi.ProjectVariables.edit(id, fullKey, valueSanitized, {
+          filter: { environment_scope: "All (default)" },
+          masked,
+        });
+        // await updateVariable(vorpal, id, fullKey, valueSanitized, masked);
         // write backup
         if (backup) {
-          await createVariable(
-            vorpal,
+          await gitlabApi.ProjectVariables.create(
             id,
             getBackupKey(fullKey, new Date().getTime()),
             oldValue,
-            masked,
-            "_backup",
+            {
+              environmentScope: "_backup",
+              masked,
+            },
           );
         }
       } else {
         vorpal.log(`new    : ${key}`);
-        await createVariable(vorpal, id, fullKey, valueSanitized, masked);
+        await gitlabApi.ProjectVariables.create(id, fullKey, valueSanitized, {
+          environmentScope: "All (default)",
+          masked,
+        });
+        // await createVariable(vorpal, id, fullKey, valueSanitized, masked);
       }
     } else {
       vorpal.log(`skip   : ${key}`);

package/src/utils/preferences.ts

@@ -1,9 +1,11 @@
-import { createFile, pathExists, readFile, writeFile } from "fs-extra";
+import { writeFile, readFile, mkdir } from "fs/promises";
+import { existsSync } from "fs";
 import { homedir } from "os";
-
 import { parse, stringify } from "yaml";
-const directory = `${homedir()}/.catladder`;
-const file = `${directory}/preferences.yml`;
+import { join } from "path";
+
+const directory = join(homedir(), ".catladder");
+const file = join(directory, "preferences.yml");
 
 export type CatladderCliPreferences = {
   "gitlab-personal-access-token": string | null;
@@ -15,7 +17,7 @@ const perfKeys = [
   "gitlab-personal-access-token",
   "googleAuthUserNumber",
   "gitlabTokens",
-] as const;
+] as const satisfies readonly PreferenceKey[];
 
 const fallbackValues = {
   "gitlab-personal-access-token": null,
@@ -23,23 +25,47 @@ const fallbackValues = {
   googleAuthUserNumber: null,
 } satisfies CatladderCliPreferences;
 
-export const getPreferences = async (): Promise<CatladderCliPreferences> => {
-  if (!(await pathExists(file))) {
-    await createFile(file);
-  }
-  const fileString = await readFile(file, { encoding: "utf-8" });
-  const fileParsed = parse(fileString || "---\nnull") ?? {};
-  return perfKeys.reduce(
+const perferencesWithParsed = (
+  parsed: Partial<CatladderCliPreferences>,
+): CatladderCliPreferences =>
+  perfKeys.reduce(
     (acc, key) =>
-      Object.assign(acc, { [key]: fileParsed[key] ?? fallbackValues[key] }),
+      Object.assign(acc, { [key]: parsed[key] ?? fallbackValues[key] }),
     {} as CatladderCliPreferences,
   );
+
+const encoding = "utf-8";
+
+const readPreferences = () => readFile(file, { encoding });
+
+const writePreferences = (preferences: CatladderCliPreferences) =>
+  writeFile(file, stringify(preferences), { encoding, mode: 0o600 });
+
+const ensureDir = async () =>
+  existsSync(directory)
+    ? directory
+    : mkdir(directory, { recursive: true, mode: 0o700 });
+
+const ensureFile = async () => {
+  if (existsSync(file)) {
+    return file;
+  }
+  await ensureDir();
+  return writePreferences(perferencesWithParsed({}));
+};
+
+export const getPreferences = async (): Promise<CatladderCliPreferences> => {
+  await ensureFile();
+  const fileString = await readPreferences();
+  const fileParsed = parse(fileString || "---\nnull") ?? {};
+  return perferencesWithParsed(fileParsed);
 };
 
 export const hasPreference = async (key: PreferenceKey) => {
   const preferences = await getPreferences();
   return preferences[key] !== null && preferences[key] !== undefined;
 };
+
 export const getPreference = async <K extends PreferenceKey>(
   key: K,
 ): Promise<CatladderCliPreferences[K] | null> => {
@@ -55,11 +81,5 @@ export const setPreference = async (
   value: CatladderCliPreferences[typeof key],
 ) => {
   const preferences = await getPreferences();
-
-  const newPreferences = {
-    ...preferences,
-    [key]: value,
-  };
-
-  await writeFile(file, stringify(newPreferences));
+  return writePreferences({ ...preferences, [key]: value });
 };

package/src/utils/projectPreferences.ts

@@ -0,0 +1,78 @@
+import { writeFile, readFile, mkdir } from "fs/promises";
+import { existsSync } from "fs";
+import { join } from "path";
+import { getProjectRootPath } from "../git";
+
+const directory = ".catladder";
+const file = "preferences.json";
+
+export type CatladderCliProjectPreferences = {
+  gitlabProjectId?: number | null;
+};
+type ProjectPreferenceKey = keyof CatladderCliProjectPreferences;
+const perfKeys = [
+  "gitlabProjectId",
+] as const satisfies readonly ProjectPreferenceKey[];
+const fallbackValues = {
+  gitlabProjectId: null,
+} satisfies CatladderCliProjectPreferences;
+
+const perferencesWithParsed = (
+  parsed: Partial<CatladderCliProjectPreferences>,
+): CatladderCliProjectPreferences =>
+  perfKeys.reduce(
+    (acc, key) =>
+      Object.assign(acc, { [key]: parsed[key] ?? fallbackValues[key] ?? null }),
+    {} as CatladderCliProjectPreferences,
+  );
+
+const encoding = "utf-8";
+
+const readPreferences = (filePath: string) => readFile(filePath, { encoding });
+
+const writePreferences = (
+  filePath: string,
+  preferences: CatladderCliProjectPreferences,
+) =>
+  writeFile(filePath, JSON.stringify(preferences, null, 2), {
+    encoding,
+    mode: 0o600,
+  });
+
+const ensureDir = async (dirPath: string) =>
+  existsSync(dirPath)
+    ? dirPath
+    : mkdir(dirPath, { recursive: true, mode: 0o700 });
+
+const ensureFile = async (givenFilePath?: string) => {
+  if (givenFilePath?.length && existsSync(givenFilePath)) {
+    return givenFilePath;
+  }
+  const rootPath = await getProjectRootPath();
+  const dirPath = join(rootPath, directory);
+  const filePath = join(dirPath, file);
+  if (existsSync(filePath)) {
+    return filePath;
+  }
+  await ensureDir(dirPath);
+  await writePreferences(filePath, perferencesWithParsed({}));
+  return filePath;
+};
+
+export const getProjectPreferences = async (
+  givenFilePath?: string,
+): Promise<CatladderCliProjectPreferences> => {
+  const filePath = await ensureFile(givenFilePath);
+  const fileString = await readPreferences(filePath);
+  const fileParsed = JSON.parse(fileString || "null") ?? {};
+  return perferencesWithParsed(fileParsed);
+};
+
+export const upsertProjectPerferences = async (
+  newPerfs: Partial<CatladderCliProjectPreferences> = {},
+  givenFilePath?: string,
+) => {
+  const filePath = await ensureFile(givenFilePath);
+  const currentPerfs = await getProjectPreferences(filePath);
+  return writePreferences(filePath, { ...currentPerfs, ...newPerfs });
+};