@catladder/cli 0.0.0-refactor-untangle-gitlab-api-requests-from-vorpal-57cf3cf2 → 0.0.0-reference-database-url-1befbbee

package/src/config/getProjectConfig.ts

@@ -1,5 +1,8 @@
-import { watch } from "fs";
-import type { Config, EnvironmentEnvVars } from "@catladder/pipeline";
+import type {
+  Config,
+  EnvironmentEnvVars,
+  VariableValue,
+} from "@catladder/pipeline";
 import {
   readConfig,
   getAllEnvs,
@@ -9,14 +12,11 @@ import {
 } from "@catladder/pipeline";
 
 import type { CommandInstance } from "vorpal";
-import { getGitlabToken, getVariableValueByRawName } from "../utils/gitlab";
+import { getAllVariables, getVariableValueByRawName } from "../utils/gitlab";
 
 import { getGitRoot } from "../utils/projects";
-import {
-  getGitLabHostPathToken,
-  gitLabVariable,
-  type GitLabApiDataOptional,
-} from "../utils/gitlab-without-vorpal";
+
+import { watch } from "fs";
 
 export { parseChoice } from "./parseChoice";
 
@@ -94,10 +94,21 @@ export const getAllComponentsWithAllEnvsHierarchical = async (): Promise<{
   );
 };
 
-export const getAllPipelineContexts = async () => {
+export const getAllPipelineContexts = async (
+  onlyComponent?: string | string[],
+) => {
+  const onlyComponentsArray = onlyComponent
+    ? Array.isArray(onlyComponent)
+      ? onlyComponent
+      : [onlyComponent]
+    : null;
   return Promise.all(
     (await getAllComponentsWithAllEnvsFlat())
       .filter((c) => c.env !== "local")
+      .filter(
+        (c) =>
+          !onlyComponentsArray || onlyComponentsArray.includes(c.componentName),
+      )
       .map(({ env, componentName }) =>
         getPipelineContextByChoice(env, componentName),
       ),
@@ -123,20 +134,8 @@ export const getGitlabVar = async (
 const resolveSecrets = async (
   vorpal: CommandInstance | null,
   varSets: EnvironmentEnvVars[],
-  existing: GitLabApiDataOptional = {},
 ): Promise<Record<string, string>> => {
-  const {
-    host,
-    path,
-    token: inputToken,
-  } = await getGitLabHostPathToken(existing, vorpal);
-  const token = inputToken || (await getGitlabToken(vorpal, host));
-  const allVariablesInGitlab = await gitLabVariable.getAll({
-    logger: vorpal,
-    token,
-    host,
-    path,
-  });
+  const allVariablesInGitlab = await getAllVariables(vorpal);
 
   return Object.fromEntries(
     varSets.flatMap(({ envVars, secretEnvVarKeys }) =>

package/src/git/gitProjectInformation.ts

@@ -12,16 +12,10 @@ export const getProjectRootPath = async (): Promise<string> => {
 /**
  * Get the git remote hostname and project path from the git config.
  */
-export const getGitRemoteHostAndPath = async ({
-  host,
-  path,
-}: { host?: string; path?: string } = {}): Promise<{
+export const getGitRemoteHostAndPath = async (): Promise<{
   gitRemoteHost: string;
   gitRemotePath: string;
 }> => {
-  if (host?.length && path?.length) {
-    return { gitRemoteHost: host, gitRemotePath: path };
-  }
   const remoteUrl = await gitConfigGet("remote.origin.url");
   const remoteReg = /(https:\/\/|git@)([^:/]+)[:/]([^.]*)(\.git)?/;
   const match = remoteUrl.match(remoteReg) ?? [];
@@ -44,27 +38,3 @@ export const gitProjectInformation = async () => {
     projectRootPath,
   };
 };
-
-/**
- * Get the git remote hostname.
- * Will only run a shell execution when hostnamePresent is nil or empty.
- */
-export const getGitRemoteHost = async (hostnamePresent?: string) => {
-  if (hostnamePresent?.length) {
-    return hostnamePresent;
-  }
-  const { gitRemoteHost } = await getGitRemoteHostAndPath();
-  return gitRemoteHost;
-};
-
-/**
- * Get the git remote path.
- * Will only run a shell execution when pathPresent is nil or empty.
- */
-export const getGitRemotePath = async (pathPresent?: string) => {
-  if (pathPresent?.length) {
-    return pathPresent;
-  }
-  const { gitRemotePath } = await getGitRemoteHostAndPath();
-  return gitRemotePath;
-};

package/src/utils/gitlab.ts

@@ -1,186 +1,272 @@
+import { getSecretVarName } from "@catladder/pipeline";
 import { has, isObject } from "lodash";
+import memoizee from "memoizee";
+import fetch from "node-fetch";
 import open from "open";
 import type { CommandInstance } from "vorpal";
+import { getPreference, hasPreference, setPreference } from "./preferences";
+import { getGitRemoteHostAndPath } from "../git/gitProjectInformation";
 
-import { getSecretVarName } from "@catladder/pipeline";
-import { getGitRemoteHost } from "../git/gitProjectInformation";
-import { logError, logNotice, logLines } from "./log";
-import {
-  type FetchMethod,
-  type GitLabApiDataOptional,
-  fetchGitLabApi,
-  GitLabTokenMissingError,
-  loadGitLabToken,
-  storeGitLabToken,
-  gitLabVariable,
-  getGitLabHostPathToken,
-  getGitLabProjectId,
-} from "./gitlab-without-vorpal";
-
-export const setupGitlabToken = async (
-  vorpal: CommandInstance,
-  host?: string,
-) => {
-  logNotice(
-    vorpal,
-    "In order to access the API, you need your personal access token.",
-    "It's best to create a specific token for CatLadder.",
-  );
-
-  const gitRemoteHost = await getGitRemoteHost(host);
-  const accessTokenSettingsUrl = `https://${gitRemoteHost}/-/user_settings/personal_access_tokens`;
-  logNotice(
-    vorpal,
-    "We'll open up the GitLab settings page in your browser for you!",
-    accessTokenSettingsUrl,
-  );
+const TOKEN_KEY = "gitlab-personal-access-token";
 
-  const { shouldContinue } = await vorpal.prompt({
-    default: true,
-    message: "Ok",
-    name: "shouldContinue",
-    type: "prompt",
-  });
+export const hasGitlabToken = async () => await hasPreference(TOKEN_KEY);
+export const setupGitlabToken = async (vorpal: CommandInstance) => {
+  vorpal.log("");
+  vorpal.log("☝ in order to access the api, we need a personal access token");
+  vorpal.log("Its best to create one specifically for catladder");
+  vorpal.log("Scopes needed: api");
+  vorpal.log("");
+  vorpal.log("☝ we open up the settings page for you!");
+  vorpal.log("");
+  const [{ shouldContinue }, { gitRemoteHost }] = await Promise.all([
+    vorpal.prompt({
+      default: true,
+      message: "Ok",
+      name: "shouldContinue",
+      type: "prompt",
+    }),
+    getGitRemoteHostAndPath(),
+  ]);
 
-  open(accessTokenSettingsUrl);
+  open(`https://${gitRemoteHost}/-/user_settings/personal_access_tokens`);
 
-  logLines(vorpal, "Please enter GitLab's personal access token.");
+  vorpal.log("Please type in gitlab's personal access token");
 
-  const { personalToken: token } = await vorpal.prompt({
+  const { personalToken } = await vorpal.prompt({
     type: "string",
     name: "personalToken",
     default: "",
     message: "Your personal access token ",
   });
-  try {
-    return storeGitLabToken({ token, host }, vorpal);
-  } catch (error) {
-    if (error instanceof GitLabTokenMissingError) {
-      logError(vorpal, "Token missing", "Please run catladder again");
+  if (personalToken) {
+    await setPreference(TOKEN_KEY, personalToken);
+  }
+};
+export const getGitlabToken = async (vorpal: CommandInstance | null) => {
+  if (!(await hasGitlabToken())) {
+    if (!vorpal) {
+      console.error(
+        "⚠️ gitlab token missing, please run catladder to set it up",
+      );
       process.exit(1);
-    } else {
-      throw error;
     }
+    await setupGitlabToken(vorpal);
   }
+  return getPreference(TOKEN_KEY);
 };
 
-export const getGitlabToken = async (
+type Method = "GET" | "PUT" | "POST" | "DELETE";
+export const doGitlabRequest = async <T = any>(
   vorpal: CommandInstance | null,
-  host?: string,
-) => {
-  const gitRemoteHost = await getGitRemoteHost(host);
-  try {
-    return loadGitLabToken(gitRemoteHost, vorpal);
-  } catch (error) {
-    if (!(error instanceof GitLabTokenMissingError)) throw error;
-    if (vorpal) {
-      return setupGitlabToken(vorpal, gitRemoteHost);
+  path: string,
+  data: any = undefined,
+  method: Method = "GET",
+): Promise<T> => {
+  const [rootToken, { gitRemoteHost }] = await Promise.all([
+    getGitlabToken(vorpal),
+    getGitRemoteHostAndPath(),
+  ]);
+
+  //const method = data ? (update ? "PUT" : "POST") : "GET";
+
+  const result = await fetch(`https://${gitRemoteHost}/api/v4/${path}`, {
+    method,
+    headers: {
+      "Content-Type": "application/json",
+      "Private-Token": rootToken,
+    },
+    body: data ? JSON.stringify(data) : undefined,
+  });
+
+  if (result.status >= 200 && result.status < 400) {
+    if (result.headers.get("content-type") === "application/json") {
+      return result.json();
     }
-    logError(
-      console,
-      "GitLab token missing!",
-      "Please run catladder to set it up",
-    );
-    process.exit(1);
+    return null;
+  }
+  if (result.status === 404) {
+    throw new Error("not found");
   }
+
+  throw new Error(
+    `Could not send request to gitlab api ${path}: ${result.status} "${
+      result.statusText
+    }".\nResponse: ${JSON.stringify(await result.json(), null, 2)}`,
+  );
 };
 
-export const doGitlabRequest = async <T = any>(
+export const getProjectInfo = async (
   vorpal: CommandInstance | null,
-  {
-    path: resourcePath,
-    ...args
-  }: {
-    path: string;
-    data?: any;
-    method?: FetchMethod;
-    host?: string;
-    token?: string;
+): Promise<{ id: string; web_url: string }> => {
+  const { gitRemotePath } = await getGitRemoteHostAndPath();
+  const project = await doGitlabRequest(
+    vorpal,
+    `projects/${encodeURIComponent(gitRemotePath)}`,
+  );
+  return project;
+};
+
+type GitlabVariable = {
+  variable_type: string;
+  key: string;
+  value: string;
+  protected: boolean;
+  masked: boolean;
+  environment_scope: string;
+};
+export const getAllVariables = memoizee(
+  async (
+    vorpal: CommandInstance | null,
+    n = 5, // how many requests to do in parallel, 5 seems a good value for many projects, ideally we would do one request in parallel per component
+  ): Promise<Array<GitlabVariable>> => {
+    const { id } = await getProjectInfo(vorpal);
+
+    let all: Array<GitlabVariable> = [];
+    let result: Array<Array<GitlabVariable>> = [];
+    let page = 1;
+
+    do {
+      // Create an array of promises for N pages
+      const promises = Array.from({ length: n }, (_, i) => {
+        return doGitlabRequest(
+          vorpal,
+          `projects/${id}/variables?per_page=100&page=${page + i}`,
+        );
+      });
+
+      // Wait for all promises to resolve
+      result = await Promise.all(promises);
+
+      // Increment the page by N
+      page += n;
+
+      // Flatten the result array and add it to 'all'
+      all = [...all, ...result.flat()];
+      // Continue only if the last page had results
+    } while (result.length > 0 && result[result.length - 1].length > 0);
+
+    return all;
   },
-): Promise<T> => fetchGitLabApi<T>({ resourcePath, ...args }, vorpal);
+  { promise: true },
+);
 
 export const getVariableValueByRawName = async (
   vorpal: CommandInstance,
   rawName: string,
-  {
-    projectId,
-    ...existing
-  }: GitLabApiDataOptional & { projectId?: number | string } = {},
 ) => {
-  const {
-    host,
-    path,
-    token: inputToken,
-  } = await getGitLabHostPathToken(existing, vorpal);
-  const token = inputToken || (await getGitlabToken(vorpal, host));
-  const id =
-    projectId || (await getGitLabProjectId({ host, path, token }, vorpal));
-  const allVariables = await gitLabVariable.getAll({
-    logger: vorpal,
-    host,
-    path,
-    token,
-    projectId: id,
-  });
+  const allVariables = await getAllVariables(vorpal);
   return allVariables.find((v) => v.key === rawName)?.value;
 };
 
-const getAllCatladderEnvVarsInGitlab = async (
+const maskableRegex = new RegExp("^[a-zA-Z0-9_+=/@:.~-]{8,}$"); // SEE https://gitlab.com/gitlab-org/gitlab-foss/-/blob/master/spec/frontend/ci_variable_list/components/ci_variable_modal_spec.js#L20
+const isMaskable = (value: string): boolean => maskableRegex.test(value);
+
+const createVariable = async (
   vorpal: CommandInstance,
-  {
-    projectId,
-    ...existing
-  }: GitLabApiDataOptional & { projectId?: string | number } = {},
+  projectId: string,
+  key: string,
+  value: string,
+  masked = true,
+  environment_scope = "*",
 ) => {
-  const {
-    host,
-    path,
-    token: inputToken,
-  } = await getGitLabHostPathToken(existing, vorpal);
-  const token = inputToken || (await getGitlabToken(vorpal, host));
-  const id =
-    projectId || (await getGitLabProjectId({ host, path, token }, vorpal));
-  return gitLabVariable.getAllCatladder({
-    logger: vorpal,
-    host,
-    path,
-    token,
-    projectId: id,
-  });
+  return await doGitlabRequest(
+    vorpal,
+    `projects/${projectId}/variables`,
+    {
+      key,
+      value,
+      masked: masked && isMaskable(value),
+      environment_scope,
+    },
+    "POST",
+  );
 };
 
-const getBackupKey = (fullKey: string, timestamp: number) =>
-  `${fullKey}_backup_${timestamp}`;
+const updateVariable = async (
+  vorpal: CommandInstance,
+  projectId: string,
+  key: string,
+  value: string,
+  masked = true,
+) => {
+  return await doGitlabRequest(
+    vorpal,
+    `projects/${projectId}/variables/${key}`,
+    {
+      value,
+      masked: masked && isMaskable(value),
+    },
+    "PUT",
+  );
+};
 
-export const clearBackups = async (
+const deleteVariable = async (
   vorpal: CommandInstance,
-  keep: number,
-  existing: GitLabApiDataOptional = {},
+  projectId: string,
+  key: string,
 ) => {
-  const {
-    host,
-    path,
-    token: inputToken,
-  } = await getGitLabHostPathToken(existing, vorpal);
-  const token = inputToken || (await getGitlabToken(vorpal, host));
-  const id = await getGitLabProjectId({ host, path, token }, vorpal);
-
-  const existingVariables = await getAllCatladderEnvVarsInGitlab(vorpal, {
-    projectId: id,
-    host,
-    path,
-    token,
-  });
+  return await doGitlabRequest(
+    vorpal,
+    `projects/${projectId}/variables/${key}`,
+    undefined,
+    "DELETE",
+  );
+};
+
+const getAllCatladderEnvVarsInGitlab = async (vorpal: CommandInstance) => {
+  const allVariables = await getAllVariables(vorpal).then((v) =>
+    v.reduce<{
+      [key: string]: {
+        value?: string;
+        backups: number[];
+      };
+    }>((acc, variable) => {
+      const { key } = variable;
+
+      if (key.startsWith("CL_")) {
+        const matchBackup = key.match(/(CL_.*)_backup_([0-9]+)/);
+
+        if (matchBackup) {
+          const key = matchBackup[1];
+          const timestamp = Number(matchBackup[2]);
+          const backups = [...(acc[key]?.backups ?? []), timestamp];
+          return {
+            ...acc,
+            [key]: {
+              ...(acc[key] ?? {}), // add value
+              backups,
+            },
+          };
+        }
+
+        return {
+          ...acc,
+          [key]: {
+            backups: [],
+            ...(acc[key] ?? {}), // may add backups
+            value: variable.value,
+          },
+        };
+      }
+
+      return acc;
+    }, {}),
+  );
+  return allVariables;
+};
+
+const getBackupKey = (fullKey: string, timestamp: number) =>
+  `${fullKey}_backup_${timestamp}`;
+export const clearBackups = async (vorpal: CommandInstance, keep: number) => {
+  const existingVariables = await getAllCatladderEnvVarsInGitlab(vorpal);
+  const { id } = await getProjectInfo(vorpal);
   for (const [key, { backups }] of Object.entries(existingVariables)) {
     const backupsSorted = backups.sort((a, b) => b - a);
     //const toKeep = backupsSorted.slice(0, keep);
     const toDelete = backupsSorted.slice(keep);
 
     for (const timestamp of toDelete) {
-      await gitLabVariable.delete(
-        { projectId: id, key: getBackupKey(key, timestamp), token, host },
-        vorpal,
-      );
+      await deleteVariable(vorpal, id, getBackupKey(key, timestamp));
     }
   }
 };
@@ -192,27 +278,11 @@ export const upsertAllVariables = async (
   componentName: string,
   backup = true,
   masked = true, // FIXME: would be better to have this per variable
-  {
-    projectId,
-    ...existing
-  }: GitLabApiDataOptional & { projectId?: number | string } = {},
 ): Promise<void> => {
-  const {
-    host,
-    path,
-    token: inputToken,
-  } = await getGitLabHostPathToken(existing, vorpal);
-  const token = inputToken || (await getGitlabToken(vorpal, host));
-  const id =
-    projectId || (await getGitLabProjectId({ host, path, token }, vorpal));
+  const { id } = await getProjectInfo(vorpal);
 
   // we list all existing variables. We also gather backup versions, but its currently unused
-  const existingVariables = await getAllCatladderEnvVarsInGitlab(vorpal, {
-    projectId: id,
-    host,
-    path,
-    token,
-  });
+  const existingVariables = await getAllCatladderEnvVarsInGitlab(vorpal);
   for (const [key, value] of Object.entries(variables ?? {})) {
     const fullKey = getSecretVarName(env, componentName, key);
     const valueSanitized = isObject(value) ? JSON.stringify(value) : `${value}`;
@@ -220,52 +290,29 @@ export const upsertAllVariables = async (
     const exists = has(existingVariables, fullKey);
     const oldValue = existingVariables[fullKey]?.value;
     const changed = oldValue !== valueSanitized;
-    if (!changed) {
+    if (changed) {
+      if (exists) {
+        vorpal.log(`changed: ${key}`);
+
+        await updateVariable(vorpal, id, fullKey, valueSanitized, masked);
+        // write backup
+        if (backup) {
+          await createVariable(
+            vorpal,
+            id,
+            getBackupKey(fullKey, new Date().getTime()),
+            oldValue,
+            masked,
+            "_backup",
+          );
+        }
+      } else {
+        vorpal.log(`new    : ${key}`);
+        await createVariable(vorpal, id, fullKey, valueSanitized, masked);
+      }
+    } else {
       vorpal.log(`skip   : ${key}`);
-      continue;
-    }
-    if (!exists) {
-      vorpal.log(`new    : ${key}`);
-      await gitLabVariable.create(
-        {
-          projectId: id,
-          key: fullKey,
-          value: valueSanitized,
-          masked,
-          host,
-          token,
-        },
-        vorpal,
-      );
-      continue;
-    }
-    vorpal.log(`changed: ${key}`);
-
-    await gitLabVariable.update(
-      {
-        projectId: id,
-        key: fullKey,
-        value: valueSanitized,
-        masked,
-        host,
-        token,
-      },
-      vorpal,
-    );
-    // write backup
-    if (backup) {
-      await gitLabVariable.create(
-        {
-          projectId: id,
-          key: getBackupKey(fullKey, new Date().getTime()),
-          value: oldValue,
-          masked,
-          host,
-          token,
-        },
-        vorpal,
-      );
     }
   }
-  gitLabVariable.getAll.clear();
+  getAllVariables.clear();
 };

package/src/utils/log.ts

@@ -111,29 +111,3 @@ export const logWarning = (
   }
   logLines(cmd, "");
 };
-
-/**
- * Logs a notification message using the log method of the provided {@link LoggerCmdInstance}.
- *
- * Line items that are `null` or `undefined` are logged as empty lines.
- *
- * @example
- * ```ts
- * logNotice(cmd, "Message");
- * logNotice(cmd, "Message", "Additional message");
- * logNotice(cmd, "Message", "Additional message", "Another additional message line");
- * logNotice(cmd, "Message", ...additionalMessages); // arrays can be spread into the function
- * logNotice(cmd, "Message", additionalMessages); // but it will flatten the plain array for convenience
- * ```
- */
-export const logNotice = (
-  cmd: LoggerCmdInstance = console,
-  message: string,
-  ...additionalMessages: LinesArg[]
-) => {
-  logLines(cmd, "", `[NOTICE] ☝ : ${message}`);
-  if (additionalMessages?.length) {
-    logLines(cmd, ...additionalMessages.flat());
-  }
-  logLines(cmd, "");
-};