@catladder/cli 0.0.0-node16-ddfae617 → 0.0.0-perf-cli-reduce-bundle-size-with-tsconfig-optimization-3556aaa4

package/src/config/getProjectConfig.ts

@@ -1,25 +1,30 @@
+import type { Config, EnvironmentEnvVars } from "@catladder/pipeline";
 import {
-  readConfigSync,
+  readConfig,
   getAllEnvs,
   getEnvironment as _getEnvironment,
-  createContext,
+  createComponentContext,
   getSecretVarName,
-  Config,
 } from "@catladder/pipeline";
 
-import { CommandInstance } from "vorpal";
+import type { CommandInstance } from "vorpal";
 import { getAllVariables, getVariableValueByRawName } from "../utils/gitlab";
 
 import { getGitRoot } from "../utils/projects";
-import { readYaml } from "../utils/files";
+
 import { watch } from "fs";
 
-let currentConfig: Config = null;
+export { parseChoice } from "./parseChoice";
+
+let currentConfig: Config | null = null;
 
 // reload the config on change
 const reloadConfigAndObserve = async () => {
   const gitRoot = await getGitRoot();
-  const result = readConfigSync(gitRoot);
+  if (!gitRoot) {
+    return;
+  }
+  const result = await readConfig(gitRoot);
   if (!result) {
     // can't do anything, there is no config
     return;
@@ -37,20 +42,7 @@ export const getProjectConfig = async () => {
     // initially
     await reloadConfigAndObserve();
   }
-  return currentConfig;
-};
-
-export const getGitlabCiFilePath = async () => {
-  const gitRoot = await getGitRoot();
-  return gitRoot + "/.gitlab-ci.yml";
-};
-export const getGitlabCi = async <T = Record<string, any>>() => {
-  try {
-    return readYaml(await getGitlabCiFilePath()) as Promise<T>;
-  } catch (e) {
-    // ignore
-    return null;
-  }
+  return currentConfig as Config;
 };
 
 export const getProjectComponents = async () => {
@@ -59,17 +51,16 @@ export const getProjectComponents = async () => {
   return Object.keys(config.components);
 };
 
-export const parseChoice = (envComponent: string) => {
-  const [env, componentName] = envComponent.split(":");
-  return { env, componentName };
-};
-
 export const getPipelineContextByChoice = async (
   env: string,
-  componentName: string
+  componentName: string,
 ) => {
   const config = await getProjectConfig();
-  return createContext(config, componentName, env);
+  return await createComponentContext({
+    config,
+    componentName,
+    env,
+  });
 };
 export const getAllComponentsWithAllEnvsFlat = async (): Promise<
   Array<{ env: string; componentName: string }>
@@ -79,7 +70,7 @@ export const getAllComponentsWithAllEnvsFlat = async (): Promise<
     return [];
   }
   return Object.keys(config.components).flatMap((componentName) =>
-    getAllEnvs(config, componentName).map((env) => ({ env, componentName }))
+    getAllEnvs(config, componentName).map((env) => ({ env, componentName })),
   );
 };
 
@@ -95,7 +86,7 @@ export const getAllComponentsWithAllEnvsHierarchical = async (): Promise<{
     Object.keys(config.components).map((componentName) => [
       componentName,
       getAllEnvs(config, componentName),
-    ])
+    ]),
   );
 };
 
@@ -104,57 +95,96 @@ export const getAllPipelineContexts = async () => {
     (await getAllComponentsWithAllEnvsFlat())
       .filter((c) => c.env !== "local")
       .map(({ env, componentName }) =>
-        getPipelineContextByChoice(env, componentName)
-      )
+        getPipelineContextByChoice(env, componentName),
+      ),
   );
 };
 
 export const getEnvironment = async (env: string, componentName: string) => {
   const config = await getProjectConfig();
 
-  return _getEnvironment(config, componentName, env);
+  return _getEnvironment({ config, componentName, env });
 };
 
 export const getGitlabVar = async (
   vorpal: CommandInstance,
   env: string,
   componentName: string,
-  variableName: string
+  variableName: string,
 ) => {
   const rawVariableName = getSecretVarName(env, componentName, variableName);
   return await getVariableValueByRawName(vorpal, rawVariableName);
 };
 
 const resolveSecrets = async (
-  vorpal: CommandInstance,
-  allEnvVars: Record<string, string>
-) => {
+  vorpal: CommandInstance | null,
+  varSets: EnvironmentEnvVars[],
+): Promise<Record<string, string>> => {
   const allVariablesInGitlab = await getAllVariables(vorpal);
 
   return Object.fromEntries(
-    Object.entries(allEnvVars).map(([key, value]) => {
-      const containsSecret = String(value)?.includes?.("$CL_");
-      if (containsSecret) {
-        for (const variable of allVariablesInGitlab) {
-          value = value.replace(
-            new RegExp("\\$" + variable.key, "g"),
-            variable.value
-          );
-        }
-        return [key, value];
-      }
-      return [key, value];
-    })
+    varSets.flatMap(({ envVars, secretEnvVarKeys }) =>
+      Object.entries(envVars)
+        .filter(
+          ([key, value]) =>
+            value !== undefined &&
+            value !== null &&
+            !secretEnvVarKeys.find((k) => k.key === key)?.hidden,
+        )
+        .map(([key, value]) => [
+          key,
+          allVariablesInGitlab.reduce(
+            (acc, curr) =>
+              acc.replace(new RegExp("\\$" + curr.key, "g"), curr.value),
+            `${value}`,
+          ),
+        ]),
+    ),
   );
 };
 
-export const getEnvVars = async (
+export const getEnvVarsResolved = async (
+  vorpal: CommandInstance | null,
+  env: string,
+  componentName: string | null,
+) => {
+  if (!componentName) {
+    return {};
+  }
+  try {
+    const envionment = await getEnvironment(env, componentName);
+
+    // in the pipeline the secrets alreadyy exists  and bash will expand them
+    // but here we need to manually load them
+    return resolveSecrets(vorpal, [
+      {
+        envVars: envionment.envVars,
+        secretEnvVarKeys: envionment.secretEnvVarKeys,
+      },
+    ]);
+  } catch (e) {
+    // env is disabled
+    return {};
+  }
+};
+
+/**
+ *
+ * is used to get job only vars that should also be editable locally with catladder.
+ */
+export const getJobOnlyEnvVarsResolved = async (
   vorpal: CommandInstance,
   env: string,
-  componentName: string
+  componentName: string,
 ) => {
-  const envionment = await getEnvironment(env, componentName);
-  // in the pipeline the secrets alreadyy exists  and bash will expand them
-  // but here we need to manually load them
-  return resolveSecrets(vorpal, envionment.envVars);
+  try {
+    const envionment = await getEnvironment(env, componentName);
+    return resolveSecrets(vorpal, [
+      envionment.jobOnlyVars.build,
+      envionment.jobOnlyVars.deploy,
+    ]);
+  } catch (e) {
+    // env is disabled
+    return {};
+  }
 };

package/src/config/parseChoice.ts

@@ -0,0 +1,4 @@
+export const parseChoice = (envComponent: string) => {
+  const [env, componentName] = envComponent.split(":").map((x) => x || null);
+  return { env: env as string, componentName };
+};

package/src/gcloud/artifactsRegistry.ts

@@ -0,0 +1,16 @@
+import type { Gcloudregion } from "@catladder/pipeline";
+import { exec } from "child-process-promise";
+
+export const upsertGcloudArtifactsRegistry = async (config: {
+  projectId: string;
+  region: Gcloudregion;
+}) => {
+  try {
+    await exec(
+      `gcloud artifacts repositories create catladder-deploy --project="${config.projectId}" --repository-format=docker --location=${config.region}`,
+    );
+  } catch (e) {
+    // probably already exists
+    //
+  }
+};

package/src/gcloud/cloudRun/getCloudRunDomainSuffix.ts

@@ -0,0 +1,35 @@
+import type { DeployConfigCloudRun } from "@catladder/pipeline";
+import { exec } from "child-process-promise";
+
+const getSuffixFromService = (service: any) => {
+  const fullUrl = service.status.url;
+  const name = service.metadata.name;
+
+  return fullUrl.replace("https://" + name + "-", "");
+};
+export const getCloudRunDomainSuffix = async (config: DeployConfigCloudRun) => {
+  /* google cloud run domains are partially predictable, they have a fixed suffix that depends on the region and project, but we don't know it beforehand
+
+  So what we do is either:
+   - get one service and extract its suffix
+   - create a dummy project, extract the suffix and save that as a gitlab variable
+  
+  
+  */
+
+  const dummyServiceName = "cl-dummy-service-delete-me";
+
+  const existingServices = await exec(
+    `gcloud run services list --format=json  --project="${config.projectId}"  --region=${config.region} --limit=1`,
+  ).then((r) => JSON.parse(r.stdout));
+
+  if (existingServices.length > 0) {
+    return getSuffixFromService(existingServices[0]);
+  }
+
+  const result = await exec(
+    `gcloud run deploy ${dummyServiceName} --region=${config.region} --allow-unauthenticated --project ${config.projectId} --image=us-docker.pkg.dev/cloudrun/container/hello --format=json`,
+  ).then((r) => JSON.parse(r.stdout));
+
+  return await getSuffixFromService(result);
+};

package/src/gcloud/cloudRun/openCloudRunDashboards.ts

@@ -0,0 +1,22 @@
+import type { ComponentContext } from "@catladder/pipeline";
+import { isOfDeployType } from "@catladder/pipeline";
+import type { CommandInstance } from "vorpal";
+import { openGoogleCloudDashboard } from "../openDashboard";
+
+export const openGoogleCloudRunDashboard = async (
+  instance: CommandInstance,
+  context: ComponentContext,
+) => {
+  if (!isOfDeployType(context.deploy?.config, "google-cloudrun")) {
+    throw new Error("deploy type is not google-cloudrun ");
+  }
+  const { fullName } = context.environment;
+  const { region, projectId } = context.deploy.config;
+  await openGoogleCloudDashboard(
+    instance,
+    `run/detail/${region}/${fullName}/metrics`,
+    {
+      project: projectId,
+    },
+  );
+};

package/src/gcloud/cloudSql/copyDb.ts

@@ -0,0 +1,57 @@
+import { spawn } from "child-process-promise";
+type Opts = {
+  targetPassword: string;
+  targetPort: number;
+  targetUsername: string;
+  sourceUsername: string;
+  sourcePassword: string;
+  sourcePort: number;
+  sourceDbName: string;
+  targetDbName: string;
+};
+export const spawnCopyDb = async (opts: Opts) => {
+  const script = createCopyDbScript(opts);
+
+  return await spawn(script, [], {
+    shell: "bash",
+    stdio: "inherit",
+  });
+};
+const createCopyDbScript = ({
+  targetPassword,
+  targetPort,
+  targetUsername,
+  sourceUsername,
+  sourcePassword,
+  sourcePort,
+  sourceDbName,
+  targetDbName,
+}: Opts) => {
+  const targetPSQL = (command: string) =>
+    `PGPASSWORD=${targetPassword} psql -p ${targetPort} --host=localhost --user=${targetUsername} -q ${command}`;
+
+  const copyDBScript = `
+      set -e
+     
+    
+
+      dumptmp=$(mktemp /tmp/dump.XXXXXX)
+
+      echo "Dumping file to $dumptmp"
+      pg_dump --dbname=postgres://${sourceUsername}:${sourcePassword}@localhost:${sourcePort}/${sourceDbName} --no-owner --no-privileges > $dumptmp
+      echo "dump done"
+      ${targetPSQL(
+        `-c 'drop database "${targetDbName}" WITH (FORCE)' 1> /dev/null || true`,
+      )}
+      ${targetPSQL(`-c 'create database "${targetDbName}"' 1> /dev/null`)}
+          echo "Restoring dump..."
+      ${targetPSQL(`"${targetDbName}" < $dumptmp 1> /dev/null`)}
+  
+
+      echo "Clean up..."
+      set +e
+      rm $dumptmp
+      echo "\n🐱 Done!"
+      `;
+  return copyDBScript;
+};

package/src/gcloud/cloudSql/startProxy.ts

@@ -0,0 +1,74 @@
+import { spawn } from "child-process-promise";
+import commandExists from "command-exists-promise";
+
+export const ERROR_NOT_INSTALLED = "cloud-sql-proxy not installed";
+
+export type CloudSqlBackgroundProxy = {
+  stop: () => void;
+};
+
+export type CloudSqlProxyOptions = {
+  instanceName: string;
+  localPort: number;
+};
+
+const getProxyCommandSpawnArgs = async ({
+  localPort,
+  instanceName,
+}: CloudSqlProxyOptions) => {
+  const commandString = (await commandExists("cloud-sql-proxy"))
+    ? `cloud-sql-proxy --port ${localPort} ${instanceName}`
+    : (await commandExists(
+          "cloud_sql_proxy", // v1
+        ))
+      ? `cloud_sql_proxy -instances ${instanceName}=tcp:${localPort}`
+      : null;
+  if (!commandString) {
+    throw new Error(ERROR_NOT_INSTALLED);
+  }
+
+  const [cmd, ...args] = commandString.split(" ");
+  return { cmd, args };
+};
+
+export const startCloudSqlProxyInCurrentShell = async (
+  opts: CloudSqlProxyOptions,
+) => {
+  const { cmd, args } = await getProxyCommandSpawnArgs(opts);
+
+  await spawn(cmd, args, {
+    stdio: "inherit",
+    shell: true,
+  });
+};
+
+export const startCloudSqlProxyInBackground = async (
+  opts: CloudSqlProxyOptions,
+): Promise<CloudSqlBackgroundProxy> => {
+  const { cmd, args } = await getProxyCommandSpawnArgs(opts);
+
+  const proxyPromise = spawn(cmd, args, { shell: "bash" });
+
+  // wait until it starts
+  await spawn(
+    `echo -n "Waiting for proxy"
+      until echo > /dev/tcp/localhost/${opts.localPort}; do
+        sleep 0.2
+        echo -n "."
+      done 2>/dev/null`,
+    [],
+    { shell: "bash" },
+  );
+  const stop = () => {
+    proxyPromise.catch(() => {
+      // ignore
+    });
+    proxyPromise.childProcess.kill();
+  };
+  // stop if catladder i stopped
+  process.on("beforeExit", stop);
+
+  return {
+    stop,
+  };
+};

package/src/gcloud/enableServices.ts

@@ -0,0 +1,14 @@
+import { exec } from "child-process-promise";
+
+export const enableGCloudServices = async (
+  services: string[],
+  config: {
+    projectId: string;
+  },
+) => {
+  for (const service of services) {
+    await exec(
+      `gcloud services enable ${service} --project=${config.projectId}`,
+    );
+  }
+};

package/src/gcloud/openDashboard.ts

@@ -0,0 +1,19 @@
+import type { CommandInstance } from "vorpal";
+import { getGoogleAuthUserNumber } from "../apps/cli/utils/getGoogleAuthUserNumber";
+import open from "open";
+export const openGoogleCloudDashboard = async (
+  instance: CommandInstance,
+  path: string,
+  params: Record<string, string>,
+) => {
+  const url = new URL("https://console.cloud.google.com/");
+  url.pathname = path;
+  for (const [key, value] of Object.entries(params)) {
+    url.searchParams.set(key, value);
+  }
+
+  const googleAuthUserNumber = await getGoogleAuthUserNumber.call(instance);
+  url.searchParams.set("authuser", googleAuthUserNumber);
+
+  open(url.toString());
+};

package/src/gcloud/serviceAccounts.ts

@@ -0,0 +1,120 @@
+import type { ComponentContext } from "@catladder/pipeline";
+
+import { exec } from "child-process-promise";
+import { createHmac } from "crypto";
+import type { CommandInstance } from "vorpal";
+import { upsertAllVariables } from "../utils/gitlab";
+
+export const accountExists = async (fullIdentifier: string) => {
+  try {
+    await exec(`gcloud iam service-accounts describe ${fullIdentifier}`);
+    return true;
+  } catch {
+    return false;
+  }
+};
+
+type ServiceAccount = {
+  projectId: string;
+  name: string;
+  displayName: string;
+  roles: string[];
+  description: string;
+};
+const upsertGcloudServiceAccount = async (
+  context: ComponentContext,
+  account: ServiceAccount,
+): Promise<string> => {
+  const { projectId, name, displayName, roles, description } = account;
+
+  // name has limit of 30
+  const namePrefix = `${name}`;
+  const nameSuffixRaw = `${context.env}-${context.name}`;
+  const nameMiddleRaw = `${context.fullConfig.customerName}-${context.fullConfig.appName}`;
+  const MAX_LENGTH = 30;
+  const NUM_SEPARATORS = 2;
+
+  // we want to first hash middle, then suffix
+  // if for middle we have at least 1 char left, its ok, so we don't hash nameSuffix, otherwise we need to hash that as well
+
+  const middleMaxLength =
+    MAX_LENGTH - namePrefix.length - nameSuffixRaw.length - NUM_SEPARATORS;
+
+  let nameMiddle: string;
+  let nameSuffix: string;
+  if (middleMaxLength < 1) {
+    nameMiddle = hashIfNessecary(nameMiddleRaw, 1);
+    nameSuffix = hashIfNessecary(
+      nameSuffixRaw,
+      MAX_LENGTH - namePrefix.length - 1 - NUM_SEPARATORS,
+    );
+  } else {
+    nameMiddle = hashIfNessecary(nameMiddleRaw, middleMaxLength);
+    nameSuffix = nameSuffixRaw;
+  }
+
+  const fullName = `${namePrefix}-${nameMiddle}-${nameSuffix}`;
+
+  const fullDisplayName = `${context.fullConfig.customerName}-${context.fullConfig.appName} ${context.env}:${context.name} | ${displayName}`;
+
+  const fullIdentifier = `${fullName}@${projectId}.iam.gserviceaccount.com`;
+
+  const existing = await accountExists(fullIdentifier);
+
+  if (!existing) {
+    await exec(
+      `gcloud iam service-accounts create ${fullName} --display-name="${fullDisplayName}" --project="${projectId}"  --description="${description}"`,
+    );
+  }
+  const memberName = `serviceAccount:${fullIdentifier}`;
+  for (const role of roles) {
+    await exec(
+      `gcloud projects add-iam-policy-binding ${projectId} --member=${memberName} --role=${role} `,
+    );
+  }
+
+  // create key
+
+  // delete first all keys
+  const keys = await exec(
+    `gcloud iam service-accounts keys list --iam-account=${fullIdentifier} --managed-by=user --format=json`,
+  ).then((o) => JSON.parse(o.stdout));
+
+  for (const key of keys) {
+    await exec(
+      `gcloud iam service-accounts keys delete ${key.name} --quiet --iam-account=${fullIdentifier}`,
+    );
+  }
+
+  return await exec(
+    // on some platforms /dev/stdout is not available without the pipe
+    `gcloud iam service-accounts keys create /dev/stdout --iam-account=${fullIdentifier} | cat`,
+  ).then((o) => o.stdout);
+};
+
+export const upsertGcloudServiceAccountAndSaveSecret = async (
+  instance: CommandInstance,
+  context: ComponentContext,
+  account: ServiceAccount,
+  secretName: string,
+): Promise<void> => {
+  instance.log("upserting service account " + account.name + "...");
+  const key = await upsertGcloudServiceAccount(context, account);
+
+  await upsertAllVariables(
+    instance,
+    {
+      [secretName]: key,
+    },
+    context.env,
+    context.name,
+  );
+  instance.log("done!");
+};
+
+const hashIfNessecary = (str: string, maxLength: number) =>
+  str.length > maxLength ? hash(str, maxLength) : str;
+
+const hash = (str: string, length: number) => {
+  return createHmac("sha256", str).digest("hex").substring(0, length);
+};

package/src/index.ts

@@ -0,0 +1 @@
+export type * from "@catladder/pipeline"; // reexport for convenience

package/src/kubernetes/index.ts

@@ -0,0 +1,27 @@
+import { parseChoice } from "../config/parseChoice";
+import { getk8sApi } from "../k8sApi";
+import { getProjectNamespace } from "../utils/projects";
+
+export const getProjectPods = async (envComponent: string) => {
+  const namespace = await getProjectNamespace(envComponent);
+  const k8sApi = getk8sApi();
+  const res = await k8sApi.listNamespacedPod(namespace);
+
+  const { componentName } = parseChoice(envComponent);
+  return res.body.items.filter((item) =>
+    componentName ? item.metadata?.name?.includes(componentName + "-") : true,
+  );
+};
+
+export const getProjectPvcs = async (envComponent: string) => {
+  const namespace = await getProjectNamespace(envComponent);
+  const k8sApi = getk8sApi();
+  const res = await k8sApi.listNamespacedPersistentVolumeClaim(namespace);
+
+  return res.body.items;
+};
+
+export const getProjectPodNames = async (envComponent: string) => {
+  const pods = await getProjectPods(envComponent);
+  return pods.map((n) => n.metadata.name);
+};

package/src/kubernetes/openKubernetesDashboards.ts

@@ -0,0 +1,52 @@
+import type { ComponentContext } from "@catladder/pipeline";
+import { isOfDeployType } from "@catladder/pipeline";
+import type { CommandInstance } from "vorpal";
+import { openGoogleCloudDashboard } from "../gcloud/openDashboard";
+
+export const openGoogleCloudLogs = async (
+  instance: CommandInstance,
+  context: ComponentContext,
+) => {
+  const deployConfig = context.deploy?.config;
+  if (!isOfDeployType(deployConfig, "kubernetes")) {
+    throw new Error("context is not of type kubernetes");
+  }
+
+  if (!deployConfig.cluster || deployConfig.cluster.type !== "gcloud") {
+    throw new Error("no gcloud custer configured");
+  }
+
+  const namespace = context.environment.envVars.KUBE_NAMESPACE;
+  const cluster = deployConfig.cluster;
+
+  const resource = `k8s_container/cluster_name/${cluster.name}${
+    namespace ? `/namespace_name/${namespace}` : ""
+  }`;
+
+  await openGoogleCloudDashboard(instance, "logs/viewer", {
+    project: cluster.projectId,
+    resource: resource,
+  });
+};
+
+export const openGoogleCloudKubernetesDashboard = async (
+  instance: CommandInstance,
+  context: ComponentContext,
+) => {
+  const deployConfig = context.deploy?.config;
+  if (!isOfDeployType(deployConfig, "kubernetes")) {
+    throw new Error("context is not of type kubernetes");
+  }
+
+  if (!deployConfig.cluster || deployConfig.cluster.type !== "gcloud") {
+    throw new Error("no gcloud custer configured");
+  }
+  const namespace = context.environment.envVars.KUBE_NAMESPACE;
+  const cluster = deployConfig.cluster;
+  const pageState = `("savedViews":("c":["gke/${cluster.region}/${cluster.name}"],"n":["${namespace}"],"i":"4e42e0b9cd6147f8a4fba7516752ec48"))`;
+
+  await openGoogleCloudDashboard(instance, "kubernetes/workload", {
+    project: cluster.projectId,
+    pageState: pageState,
+  });
+};