@catladder/cli 0.0.0-refactore-catenv-command-validates-argument-78d05852 → 0.0.0-reference-database-url-1befbbee

package/src/config/getProjectConfig.ts

@@ -1,9 +1,13 @@
-import type { Config, EnvironmentEnvVars } from "@catladder/pipeline";
+import type {
+  Config,
+  EnvironmentEnvVars,
+  VariableValue,
+} from "@catladder/pipeline";
 import {
-  readConfigSync,
+  readConfig,
   getAllEnvs,
   getEnvironment as _getEnvironment,
-  createContext,
+  createComponentContext,
   getSecretVarName,
 } from "@catladder/pipeline";
 
@@ -11,7 +15,7 @@ import type { CommandInstance } from "vorpal";
 import { getAllVariables, getVariableValueByRawName } from "../utils/gitlab";
 
 import { getGitRoot } from "../utils/projects";
-import { readYaml } from "../utils/files";
+
 import { watch } from "fs";
 
 export { parseChoice } from "./parseChoice";
@@ -21,7 +25,10 @@ let currentConfig: Config | null = null;
 // reload the config on change
 const reloadConfigAndObserve = async () => {
   const gitRoot = await getGitRoot();
-  const result = readConfigSync(gitRoot);
+  if (!gitRoot) {
+    return;
+  }
+  const result = await readConfig(gitRoot);
   if (!result) {
     // can't do anything, there is no config
     return;
@@ -42,19 +49,6 @@ export const getProjectConfig = async () => {
   return currentConfig as Config;
 };
 
-export const getGitlabCiFilePath = async () => {
-  const gitRoot = await getGitRoot();
-  return gitRoot + "/.gitlab-ci.yml";
-};
-export const getGitlabCi = async <T = Record<string, any>>() => {
-  try {
-    return readYaml(await getGitlabCiFilePath()) as Promise<T>;
-  } catch (e) {
-    // ignore
-    return null;
-  }
-};
-
 export const getProjectComponents = async () => {
   const config = await getProjectConfig();
   if (!config) return [];
@@ -63,10 +57,14 @@ export const getProjectComponents = async () => {
 
 export const getPipelineContextByChoice = async (
   env: string,
-  componentName: string
+  componentName: string,
 ) => {
   const config = await getProjectConfig();
-  return await createContext(config, componentName, env);
+  return await createComponentContext({
+    config,
+    componentName,
+    env,
+  });
 };
 export const getAllComponentsWithAllEnvsFlat = async (): Promise<
   Array<{ env: string; componentName: string }>
@@ -76,7 +74,7 @@ export const getAllComponentsWithAllEnvsFlat = async (): Promise<
     return [];
   }
   return Object.keys(config.components).flatMap((componentName) =>
-    getAllEnvs(config, componentName).map((env) => ({ env, componentName }))
+    getAllEnvs(config, componentName).map((env) => ({ env, componentName })),
   );
 };
 
@@ -92,31 +90,42 @@ export const getAllComponentsWithAllEnvsHierarchical = async (): Promise<{
     Object.keys(config.components).map((componentName) => [
       componentName,
       getAllEnvs(config, componentName),
-    ])
+    ]),
   );
 };
 
-export const getAllPipelineContexts = async () => {
+export const getAllPipelineContexts = async (
+  onlyComponent?: string | string[],
+) => {
+  const onlyComponentsArray = onlyComponent
+    ? Array.isArray(onlyComponent)
+      ? onlyComponent
+      : [onlyComponent]
+    : null;
   return Promise.all(
     (await getAllComponentsWithAllEnvsFlat())
       .filter((c) => c.env !== "local")
-      .map(({ env, componentName }) =>
-        getPipelineContextByChoice(env, componentName)
+      .filter(
+        (c) =>
+          !onlyComponentsArray || onlyComponentsArray.includes(c.componentName),
       )
+      .map(({ env, componentName }) =>
+        getPipelineContextByChoice(env, componentName),
+      ),
   );
 };
 
 export const getEnvironment = async (env: string, componentName: string) => {
   const config = await getProjectConfig();
 
-  return _getEnvironment(config, componentName, env);
+  return _getEnvironment({ config, componentName, env });
 };
 
 export const getGitlabVar = async (
   vorpal: CommandInstance,
   env: string,
   componentName: string,
-  variableName: string
+  variableName: string,
 ) => {
   const rawVariableName = getSecretVarName(env, componentName, variableName);
   return await getVariableValueByRawName(vorpal, rawVariableName);
@@ -124,56 +133,50 @@ export const getGitlabVar = async (
 
 const resolveSecrets = async (
   vorpal: CommandInstance | null,
-  varSets: EnvironmentEnvVars[]
+  varSets: EnvironmentEnvVars[],
 ): Promise<Record<string, string>> => {
   const allVariablesInGitlab = await getAllVariables(vorpal);
 
   return Object.fromEntries(
-    varSets.flatMap((set) =>
-      Object.entries(set.envVars)
-        .map(([key, value]) => {
-          const secretKey = set.secretEnvVarKeys.find((k) => k.key === key);
-
-          if (secretKey?.hidden || value === undefined) {
-            return null;
-          }
-
-          for (const variable of allVariablesInGitlab) {
-            value = value.replace(
-              new RegExp("\\$" + variable.key, "g"),
-              variable.value
-            );
-          }
-          return [key, value];
-        })
-        .filter(Boolean)
-    )
+    varSets.flatMap(({ envVars, secretEnvVarKeys }) =>
+      Object.entries(envVars)
+        .filter(
+          ([key, value]) =>
+            value !== undefined &&
+            value !== null &&
+            !secretEnvVarKeys.find((k) => k.key === key)?.hidden,
+        )
+        .map(([key, value]) => [
+          key,
+          allVariablesInGitlab.reduce(
+            (acc, curr) =>
+              acc.replace(new RegExp("\\$" + curr.key, "g"), curr.value),
+            `${value}`,
+          ),
+        ]),
+    ),
   );
 };
 
 export const getEnvVarsResolved = async (
   vorpal: CommandInstance | null,
   env: string,
-  componentName: string | null
+  componentName: string | null,
 ) => {
   if (!componentName) {
     return {};
   }
-  try {
-    const envionment = await getEnvironment(env, componentName);
-
-    // in the pipeline the secrets alreadyy exists  and bash will expand them
-    // but here we need to manually load them
-    return resolveSecrets(vorpal, [
-      {
-        envVars: envionment.envVars,
-        secretEnvVarKeys: envionment.secretEnvVarKeys,
-      },
-    ]);
-  } catch (e) {
-    // env is disabled
-    return {};
-  }
+
+  const envionment = await getEnvironment(env, componentName);
+
+  // in the pipeline the secrets alreadyy exists  and bash will expand them
+  // but here we need to manually load them
+  return resolveSecrets(vorpal, [
+    {
+      envVars: envionment.envVars,
+      secretEnvVarKeys: envionment.secretEnvVarKeys,
+    },
+  ]);
 };
 
 /**
@@ -183,16 +186,11 @@ export const getEnvVarsResolved = async (
 export const getJobOnlyEnvVarsResolved = async (
   vorpal: CommandInstance,
   env: string,
-  componentName: string
+  componentName: string,
 ) => {
-  try {
-    const envionment = await getEnvironment(env, componentName);
-    return resolveSecrets(vorpal, [
-      envionment.jobOnlyVars.build,
-      envionment.jobOnlyVars.deploy,
-    ]);
-  } catch (e) {
-    // env is disabled
-    return {};
-  }
+  const envionment = await getEnvironment(env, componentName);
+  return resolveSecrets(vorpal, [
+    envionment.jobOnlyVars.build,
+    envionment.jobOnlyVars.deploy,
+  ]);
 };

package/src/gcloud/artifactsRegistry.ts

@@ -7,7 +7,7 @@ export const upsertGcloudArtifactsRegistry = async (config: {
 }) => {
   try {
     await exec(
-      `gcloud artifacts repositories create catladder-deploy --project="${config.projectId}" --repository-format=docker --location=${config.region}`
+      `gcloud artifacts repositories create catladder-deploy --project="${config.projectId}" --repository-format=docker --location=${config.region}`,
     );
   } catch (e) {
     // probably already exists

package/src/gcloud/cloudRun/getCloudRunDomainSuffix.ts

@@ -20,7 +20,7 @@ export const getCloudRunDomainSuffix = async (config: DeployConfigCloudRun) => {
   const dummyServiceName = "cl-dummy-service-delete-me";
 
   const existingServices = await exec(
-    `gcloud run services list --format=json  --project="${config.projectId}"  --region=${config.region} --limit=1`
+    `gcloud run services list --format=json  --project="${config.projectId}"  --region=${config.region} --limit=1`,
   ).then((r) => JSON.parse(r.stdout));
 
   if (existingServices.length > 0) {
@@ -28,7 +28,7 @@ export const getCloudRunDomainSuffix = async (config: DeployConfigCloudRun) => {
   }
 
   const result = await exec(
-    `gcloud run deploy ${dummyServiceName} --region=${config.region} --allow-unauthenticated --project ${config.projectId} --image=us-docker.pkg.dev/cloudrun/container/hello --format=json`
+    `gcloud run deploy ${dummyServiceName} --region=${config.region} --allow-unauthenticated --project ${config.projectId} --image=us-docker.pkg.dev/cloudrun/container/hello --format=json`,
   ).then((r) => JSON.parse(r.stdout));
 
   return await getSuffixFromService(result);

package/src/gcloud/cloudRun/openCloudRunDashboards.ts

@@ -1,22 +1,22 @@
-import type { Context } from "@catladder/pipeline";
+import type { ComponentContext } from "@catladder/pipeline";
 import { isOfDeployType } from "@catladder/pipeline";
 import type { CommandInstance } from "vorpal";
 import { openGoogleCloudDashboard } from "../openDashboard";
 
 export const openGoogleCloudRunDashboard = async (
   instance: CommandInstance,
-  context: Context
+  context: ComponentContext,
 ) => {
-  if (!isOfDeployType(context.componentConfig.deploy, "google-cloudrun")) {
+  if (!isOfDeployType(context.deploy?.config, "google-cloudrun")) {
     throw new Error("deploy type is not google-cloudrun ");
   }
   const { fullName } = context.environment;
-  const { region, projectId } = context.componentConfig.deploy;
+  const { region, projectId } = context.deploy.config;
   await openGoogleCloudDashboard(
     instance,
     `run/detail/${region}/${fullName}/metrics`,
     {
       project: projectId,
-    }
+    },
   );
 };

package/src/gcloud/cloudSql/copyDb.ts

@@ -41,7 +41,7 @@ const createCopyDbScript = ({
       pg_dump --dbname=postgres://${sourceUsername}:${sourcePassword}@localhost:${sourcePort}/${sourceDbName} --no-owner --no-privileges > $dumptmp
       echo "dump done"
       ${targetPSQL(
-        `-c 'drop database "${targetDbName}" WITH (FORCE)' 1> /dev/null || true`
+        `-c 'drop database "${targetDbName}" WITH (FORCE)' 1> /dev/null || true`,
       )}
       ${targetPSQL(`-c 'create database "${targetDbName}"' 1> /dev/null`)}
           echo "Restoring dump..."

package/src/gcloud/cloudSql/startProxy.ts

@@ -19,10 +19,10 @@ const getProxyCommandSpawnArgs = async ({
   const commandString = (await commandExists("cloud-sql-proxy"))
     ? `cloud-sql-proxy --port ${localPort} ${instanceName}`
     : (await commandExists(
-        "cloud_sql_proxy" // v1
-      ))
-    ? `cloud_sql_proxy -instances ${instanceName}=tcp:${localPort}`
-    : null;
+          "cloud_sql_proxy", // v1
+        ))
+      ? `cloud_sql_proxy -instances ${instanceName}=tcp:${localPort}`
+      : null;
   if (!commandString) {
     throw new Error(ERROR_NOT_INSTALLED);
   }
@@ -32,7 +32,7 @@ const getProxyCommandSpawnArgs = async ({
 };
 
 export const startCloudSqlProxyInCurrentShell = async (
-  opts: CloudSqlProxyOptions
+  opts: CloudSqlProxyOptions,
 ) => {
   const { cmd, args } = await getProxyCommandSpawnArgs(opts);
 
@@ -43,7 +43,7 @@ export const startCloudSqlProxyInCurrentShell = async (
 };
 
 export const startCloudSqlProxyInBackground = async (
-  opts: CloudSqlProxyOptions
+  opts: CloudSqlProxyOptions,
 ): Promise<CloudSqlBackgroundProxy> => {
   const { cmd, args } = await getProxyCommandSpawnArgs(opts);
 
@@ -57,7 +57,7 @@ export const startCloudSqlProxyInBackground = async (
         echo -n "."
       done 2>/dev/null`,
     [],
-    { shell: "bash" }
+    { shell: "bash" },
   );
   const stop = () => {
     proxyPromise.catch(() => {

package/src/gcloud/enableServices.ts

@@ -4,11 +4,11 @@ export const enableGCloudServices = async (
   services: string[],
   config: {
     projectId: string;
-  }
+  },
 ) => {
   for (const service of services) {
     await exec(
-      `gcloud services enable ${service} --project=${config.projectId}`
+      `gcloud services enable ${service} --project=${config.projectId}`,
     );
   }
 };

package/src/gcloud/openDashboard.ts

@@ -4,7 +4,7 @@ import open from "open";
 export const openGoogleCloudDashboard = async (
   instance: CommandInstance,
   path: string,
-  params: Record<string, string>
+  params: Record<string, string>,
 ) => {
   const url = new URL("https://console.cloud.google.com/");
   url.pathname = path;

package/src/gcloud/serviceAccounts.ts

@@ -1,4 +1,4 @@
-import type { Context } from "@catladder/pipeline";
+import type { ComponentContext } from "@catladder/pipeline";
 
 import { exec } from "child-process-promise";
 import { createHmac } from "crypto";
@@ -22,22 +22,40 @@ type ServiceAccount = {
   description: string;
 };
 const upsertGcloudServiceAccount = async (
-  context: Context,
-  account: ServiceAccount
+  context: ComponentContext,
+  account: ServiceAccount,
 ): Promise<string> => {
   const { projectId, name, displayName, roles, description } = account;
 
   // name has limit of 30
-  const namePrefix = `${name}-`;
-  const nameSuffix = `-${context.environment.shortName}-${context.componentName}`;
-  const nameMiddleLength = 30 - namePrefix.length - nameSuffix.length;
-  const nameMiddle = `${context.fullConfig.customerName}-${context.fullConfig.appName}`;
-
-  const middle = hashIfNessecary(nameMiddle, nameMiddleLength);
+  const namePrefix = `${name}`;
+  const nameSuffixRaw = `${context.env}-${context.name}`;
+  const nameMiddleRaw = `${context.fullConfig.customerName}-${context.fullConfig.appName}`;
+  const MAX_LENGTH = 30;
+  const NUM_SEPARATORS = 2;
+
+  // we want to first hash middle, then suffix
+  // if for middle we have at least 1 char left, its ok, so we don't hash nameSuffix, otherwise we need to hash that as well
+
+  const middleMaxLength =
+    MAX_LENGTH - namePrefix.length - nameSuffixRaw.length - NUM_SEPARATORS;
+
+  let nameMiddle: string;
+  let nameSuffix: string;
+  if (middleMaxLength < 1) {
+    nameMiddle = hashIfNessecary(nameMiddleRaw, 1);
+    nameSuffix = hashIfNessecary(
+      nameSuffixRaw,
+      MAX_LENGTH - namePrefix.length - 1 - NUM_SEPARATORS,
+    );
+  } else {
+    nameMiddle = hashIfNessecary(nameMiddleRaw, middleMaxLength);
+    nameSuffix = nameSuffixRaw;
+  }
 
-  const fullName = `${namePrefix}${middle}${nameSuffix}`;
+  const fullName = `${namePrefix}-${nameMiddle}-${nameSuffix}`;
 
-  const fullDisplayName = `${context.fullConfig.customerName}-${context.fullConfig.appName} ${context.environment.shortName}:${context.componentName} | ${displayName}`;
+  const fullDisplayName = `${context.fullConfig.customerName}-${context.fullConfig.appName} ${context.env}:${context.name} | ${displayName}`;
 
   const fullIdentifier = `${fullName}@${projectId}.iam.gserviceaccount.com`;
 
@@ -45,13 +63,13 @@ const upsertGcloudServiceAccount = async (
 
   if (!existing) {
     await exec(
-      `gcloud iam service-accounts create ${fullName} --display-name="${fullDisplayName}" --project="${projectId}"  --description="${description}"`
+      `gcloud iam service-accounts create ${fullName} --display-name="${fullDisplayName}" --project="${projectId}"  --description="${description}"`,
     );
   }
   const memberName = `serviceAccount:${fullIdentifier}`;
   for (const role of roles) {
     await exec(
-      `gcloud projects add-iam-policy-binding ${projectId} --member=${memberName} --role=${role} `
+      `gcloud projects add-iam-policy-binding ${projectId} --member=${memberName} --role=${role} --condition=None`,
     );
   }
 
@@ -59,26 +77,26 @@ const upsertGcloudServiceAccount = async (
 
   // delete first all keys
   const keys = await exec(
-    `gcloud iam service-accounts keys list --iam-account=${fullIdentifier} --managed-by=user --format=json`
+    `gcloud iam service-accounts keys list --iam-account=${fullIdentifier} --managed-by=user --format=json`,
   ).then((o) => JSON.parse(o.stdout));
 
   for (const key of keys) {
     await exec(
-      `gcloud iam service-accounts keys delete ${key.name} --quiet --iam-account=${fullIdentifier}`
+      `gcloud iam service-accounts keys delete ${key.name} --quiet --iam-account=${fullIdentifier}`,
     );
   }
 
   return await exec(
     // on some platforms /dev/stdout is not available without the pipe
-    `gcloud iam service-accounts keys create /dev/stdout --iam-account=${fullIdentifier} | cat`
+    `gcloud iam service-accounts keys create /dev/stdout --iam-account=${fullIdentifier} | cat`,
   ).then((o) => o.stdout);
 };
 
 export const upsertGcloudServiceAccountAndSaveSecret = async (
   instance: CommandInstance,
-  context: Context,
+  context: ComponentContext,
   account: ServiceAccount,
-  secretName: string
+  secretName: string,
 ): Promise<void> => {
   instance.log("upserting service account " + account.name + "...");
   const key = await upsertGcloudServiceAccount(context, account);
@@ -88,8 +106,8 @@ export const upsertGcloudServiceAccountAndSaveSecret = async (
     {
       [secretName]: key,
     },
-    context.environment.shortName,
-    context.componentName
+    context.env,
+    context.name,
   );
   instance.log("done!");
 };

package/src/git/gitConfig.ts

@@ -0,0 +1,32 @@
+import { exec } from "child-process-promise";
+
+export type GitLocationScope = "global" | "local" | "system" | "worktree";
+export type GitConfigOptions = {
+  location?: GitLocationScope;
+  comment?: string;
+};
+
+const argsJoin = (args: (string | undefined)[]): string =>
+  args.filter(Boolean).join(" ");
+
+export async function gitConfigGet(
+  key: string,
+  { location }: Pick<GitConfigOptions, "location"> = {},
+) {
+  const { stdout } = await exec(
+    `git config --get ${argsJoin([location ? `--${location}` : undefined])} ${key}`,
+  );
+  return stdout.trim();
+}
+
+export const gitConfigSet = (
+  key: string,
+  value: string,
+  { location, comment }: GitConfigOptions = {},
+) =>
+  exec(
+    `git config --set ${argsJoin([
+      location ? `--${location}` : undefined,
+      comment ? `--comment=${comment}` : undefined,
+    ])} ${key} ${value}`,
+  );

package/src/git/gitProjectInformation.ts

@@ -0,0 +1,40 @@
+import { gitConfigGet } from "./gitConfig";
+import { exec } from "child-process-promise";
+
+/**
+ * Find the absolute path of the current git project root.
+ */
+export const getProjectRootPath = async (): Promise<string> => {
+  const { stdout } = await exec(`git rev-parse --show-toplevel`);
+  return stdout.trim();
+};
+
+/**
+ * Get the git remote hostname and project path from the git config.
+ */
+export const getGitRemoteHostAndPath = async (): Promise<{
+  gitRemoteHost: string;
+  gitRemotePath: string;
+}> => {
+  const remoteUrl = await gitConfigGet("remote.origin.url");
+  const remoteReg = /(https:\/\/|git@)([^:/]+)[:/]([^.]*)(\.git)?/;
+  const match = remoteUrl.match(remoteReg) ?? [];
+  const [, , gitRemoteHost, gitRemotePath] = match;
+  if (!gitRemoteHost?.length || !gitRemotePath?.length) {
+    throw new Error(
+      `Failed to parse git remote hostname and path from git configs remote.origin.url! ${remoteUrl}`,
+    );
+  }
+  return { gitRemoteHost, gitRemotePath };
+};
+
+export const gitProjectInformation = async () => {
+  const [{ gitRemoteHost, gitRemotePath }, projectRootPath] = await Promise.all(
+    [getGitRemoteHostAndPath(), getProjectRootPath()],
+  );
+  return {
+    gitRemoteHost,
+    gitRemotePath,
+    projectRootPath,
+  };
+};

package/src/git/index.ts

@@ -0,0 +1,2 @@
+export * from "./gitConfig";
+export * from "./gitProjectInformation";

package/src/index.ts

@@ -0,0 +1 @@
+export type * from "@catladder/pipeline"; // reexport for convenience

package/src/kubernetes/index.ts

@@ -9,7 +9,7 @@ export const getProjectPods = async (envComponent: string) => {
 
   const { componentName } = parseChoice(envComponent);
   return res.body.items.filter((item) =>
-    componentName ? item.metadata?.name?.includes(componentName + "-") : true
+    componentName ? item.metadata?.name?.includes(componentName + "-") : true,
   );
 };
 

package/src/kubernetes/openKubernetesDashboards.ts

@@ -1,13 +1,13 @@
-import type { Context } from "@catladder/pipeline";
+import type { ComponentContext } from "@catladder/pipeline";
 import { isOfDeployType } from "@catladder/pipeline";
 import type { CommandInstance } from "vorpal";
 import { openGoogleCloudDashboard } from "../gcloud/openDashboard";
 
 export const openGoogleCloudLogs = async (
   instance: CommandInstance,
-  context: Context
+  context: ComponentContext,
 ) => {
-  const deployConfig = context.componentConfig.deploy;
+  const deployConfig = context.deploy?.config;
   if (!isOfDeployType(deployConfig, "kubernetes")) {
     throw new Error("context is not of type kubernetes");
   }
@@ -31,9 +31,9 @@ export const openGoogleCloudLogs = async (
 
 export const openGoogleCloudKubernetesDashboard = async (
   instance: CommandInstance,
-  context: Context
+  context: ComponentContext,
 ) => {
-  const deployConfig = context.componentConfig.deploy;
+  const deployConfig = context.deploy?.config;
   if (!isOfDeployType(deployConfig, "kubernetes")) {
     throw new Error("context is not of type kubernetes");
   }

package/src/kubernetes/portForward.ts

@@ -4,12 +4,12 @@ export const startKubePortForward = async (
   podname: string,
   localPort: number,
   remotePort: number,
-  namespace: string
+  namespace: string,
 ) => {
   const name = `kube/${namespace}/${podname}/${localPort}:${remotePort}`;
 
   await startPortForwardCommand(
     name,
-    `kubectl port-forward ${podname} ${localPort}:${remotePort} -n ${namespace}`
+    `kubectl port-forward ${podname} ${localPort}:${remotePort} -n ${namespace}`,
   );
 };

package/src/packageInfos.ts

@@ -1,4 +1,9 @@
-// eslint-disable-next-line @typescript-eslint/no-var-requires
-const ownPkg = require("../package.json");
+// we can't import the package.json directly, because the bundling process will inline it
+// but the version is only set during deployement, when its already bundled
+// instead we need to use require.main.path to get the path of the main module, which is actually the file in the bin
+// this will point to the root package.json
+
+const path = require.main?.path + "/../package.json";
+const ownPkg = require(path);
 
 export default ownPkg;

package/src/types/child-process-promise.d.ts

@@ -5,7 +5,7 @@ declare module "child-process-promise" {
     cmd: string,
     options?: {
       env?: Record<string, string>;
-    }
+    },
   ): Promise<{
     stdout: string;
   }>;
@@ -19,7 +19,7 @@ declare module "child-process-promise" {
 
   function spawn(
     cmd: string,
-    options?: SpawnOptions
+    options?: SpawnOptions,
   ): Promise<{
     stdout: string;
   }>;
@@ -27,7 +27,7 @@ declare module "child-process-promise" {
   function spawn(
     cmd: string,
     args: string[],
-    options?: SpawnOptions
+    options?: SpawnOptions,
   ): Promise<unknown> & {
     childProcess: {
       stdout: ReadStream;