@catladder/pipeline 1.54.0 → 1.55.1

package/src/context/getEnvironment.ts

@@ -1,187 +1,39 @@
-import { isObject, merge } from "lodash";
-import type { EnvVarContext } from "../deploy";
-import { DEPLOY_TYPES } from "../deploy";
-import type { Config, DevLocalEnvConfig } from "../types/config";
-import { isKnowEnvType } from "../types/config";
-import type { CommitInfo, Context, Environment } from "../types/context";
-import { mergeWithMergingArrays } from "../utils";
-import {
-  resolveReferences,
-  translateLegacyFromComponents,
-} from "./resolveReferences";
+import type { Config } from "../types/config";
+
+import type { CommitInfo, Environment } from "../types/context";
+import { getEnvironmentContext } from "./getEnvironmentContext";
+import { getEnvironmentVariables } from "./getEnvironmentVariables";
 
 export const getEnvironment = async (
   config: Config,
   componentName: string,
   env: string,
-  commitInfo?: CommitInfo,
-  alreadyVisited: Record<string, Record<string, boolean>> = {} // to prevent endless loop
+  commitInfo?: CommitInfo
 ): Promise<Environment> => {
-  const envConfig = getEnvConfig(config, componentName, env);
-  // env type: if its set manually, use that, otherwise use the known env types
-  const envType = envConfig?.type ?? (isKnowEnvType(env) ? env : null);
-
-  if (!envType) {
-    throw new Error(
-      "Missing type in environment " + env + " in component " + componentName
-    );
-  }
-
-  const basePredefinedVariables = {
-    ENV_SHORT: env,
-    APP_DIR: envConfig.dir,
-    ENV_TYPE: envType,
-  };
+  const { envVars, secretEnvVarKeys, host, url } =
+    await getEnvironmentVariables(config, componentName, env, commitInfo);
 
-  const gitlabEnvironmentName =
-    envType === "review" && commitInfo
-      ? `${env}/${commitInfo.refName}/${componentName}`
-      : `${env}/${componentName}`;
-
-  const environmentSlug =
-    envType === "review" && commitInfo
-      ? `${env}-${commitInfo.reviewSlug}-${componentName}`
-      : `${env}-${componentName}`;
-
-  let predefinedVariables: Record<string, string>;
-  let host: string;
-  let url: string;
-  const fullName = `${config.customerName}-${config.appName}-${environmentSlug}`;
-
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  const envVarContext: EnvVarContext<any> = {
-    deployConfig: envConfig.deploy,
-    fullName,
-    envType,
-    commitInfo,
-    componentName,
+  const envContext = getEnvironmentContext(
+    config,
     env,
-    fullConfig: config,
-  };
-
-  if (envType === "local") {
-    const devLocalConfig: DevLocalEnvConfig = envConfig;
-    const port = devLocalConfig.port ?? 3000;
-    host = "localhost:" + port;
-    url = "http://" + host;
-    predefinedVariables = {
-      ENV_SHORT: "local",
-      ROOT_URL: url,
-      PORT: port.toString(),
-    };
-  } else {
-    const additionalEnvVars = envConfig.deploy
-      ? DEPLOY_TYPES[envConfig.deploy.type].getAdditionalEnvVars(envVarContext)
-      : {};
-
-    host =
-      envConfig?.host ??
-      additionalEnvVars.HOST_CANONICAL ??
-      "unknown-host.example.com";
-    url = `https://${host}`;
-
-    predefinedVariables = {
-      ...basePredefinedVariables,
-      HOST: host,
-      ROOT_URL: url,
-      ...additionalEnvVars,
-    };
-  }
-  const publicEnvVarsRaw = envConfig.vars?.public ?? {};
-
-  const additionalSecretKeys = envConfig.deploy
-    ? DEPLOY_TYPES[envConfig.deploy.type].additionalSecretKeys(envVarContext)
-    : [];
-
-  const secretEnvVarKeys = [
-    ...(envConfig.vars?.secret ?? []),
-    ...additionalSecretKeys,
-  ];
-  const secretEnvVars = Object.fromEntries(
-    secretEnvVarKeys.map((key) => [
-      key,
-      `$${getSecretVarName(env, componentName, key)}`,
-    ])
-  );
-
-  // this is deprecated, we now support: $componentname:FOO
-  const legacyFromComponents = envConfig.vars?.fromComponents ?? {};
-  const publicEnvVarsRawWithLegasyFromComponents = merge(
-    {},
-    translateLegacyFromComponents(legacyFromComponents),
-    publicEnvVarsRaw
-  );
-
-  const publicEnvVarsRawSanitized = Object.fromEntries(
-    Object.entries(publicEnvVarsRawWithLegasyFromComponents).map(
-      ([key, value]) => [
-        key,
-        isObject(value) ? JSON.stringify(value) : `${value}`,
-      ]
-    )
+    componentName,
+    commitInfo
   );
 
-  const envVarsRaw = addIndexVar({
-    ...predefinedVariables,
-    ...secretEnvVars,
-    ...publicEnvVarsRawSanitized,
-  });
+  const envType = envContext.envType;
 
-  const envVars = await resolveReferences(
-    envVarsRaw,
-    async (componentName, variableName, alreadyVisited) => {
-      const { envVars } = await getEnvironment(
-        config,
-        componentName,
-        env,
-        commitInfo,
-        alreadyVisited
-      );
-      return envVars[variableName];
-    },
-    alreadyVisited
-  );
   return {
     envType,
     host,
     url,
     gitlabEnvironment: {
-      name: gitlabEnvironmentName,
+      name: envContext.gitlabEnvironmentName,
       url,
     },
-    fullName,
-    slug: environmentSlug,
+    fullName: envContext.fullName,
+    slug: envContext.environmentSlug,
     shortName: env,
     envVars,
     secretEnvVarKeys,
   };
 };
-
-const sanitizeForEnVar = (s: string) => s.replace(/-/g, "_");
-export const getSecretVarName = (
-  env: string,
-  componentName: string,
-  key: string
-) => `CL_${sanitizeForEnVar(env)}_${sanitizeForEnVar(componentName)}_${key}`; // remove dash from component name
-
-const addIndexVar = (vars: Record<string, unknown>) => ({
-  ...vars,
-  _ALL_ENV_VAR_KEYS: JSON.stringify(Object.keys(vars)),
-});
-
-export const getSecretVarNameForContext = (context: Context, key: string) =>
-  getSecretVarName(context.environment.shortName, context.componentName, key);
-
-const getEnvConfig = (config: Config, componentName: string, env: string) => {
-  const defaultConfig = config.components[componentName];
-  if (!defaultConfig) {
-    throw new Error("unknown component " + componentName);
-  }
-
-  const envCustomizations = defaultConfig.env?.[env] ?? {};
-  if (envCustomizations === false) {
-    throw new Error("env is disabled: " + env);
-  }
-
-  return mergeWithMergingArrays(defaultConfig, envCustomizations);
-};

package/src/context/getEnvironmentContext.ts

@@ -0,0 +1,57 @@
+import type { Config, EnvConfigWithComponent } from "../types/config";
+
+import type { CommitInfo } from "../types/context";
+import type { EnvironmentContext } from "../types/environmentContext";
+import { getEnvConfig } from "./getEnvConfig";
+import { getEnvType } from "./getEnvType";
+
+const getEnvironmentSlug = (
+  envConfig: EnvConfigWithComponent,
+  env: string,
+  componentName: string,
+  commitInfo?: CommitInfo
+) => {
+  const envType = getEnvType(env, envConfig);
+
+  return envType === "review" && commitInfo
+    ? `${env}-${commitInfo.reviewSlug}-${componentName}`
+    : `${env}-${componentName}`;
+};
+
+export const getEnvironmentContext = (
+  config: Config,
+  env: string,
+  componentName: string,
+  commitInfo?: CommitInfo
+): EnvironmentContext<any, any> => {
+  const envConfigRaw = getEnvConfig(config, componentName, env);
+  const envType = getEnvType(env, envConfigRaw);
+
+  const environmentSlug = getEnvironmentSlug(
+    envConfigRaw,
+    env,
+    componentName,
+    commitInfo
+  );
+
+  const gitlabEnvironmentName =
+    envType === "review" && commitInfo
+      ? `${env}/${commitInfo.refName}/${componentName}`
+      : `${env}/${componentName}`;
+
+  const fullName = `${config.customerName}-${config.appName}-${environmentSlug}`;
+
+  return {
+    envConfigRaw,
+    deployConfigRaw: envConfigRaw.deploy,
+    buildConfigRaw: envConfigRaw.build,
+    environmentSlug,
+    gitlabEnvironmentName,
+    fullName,
+    envType,
+    commitInfo,
+    componentName,
+    env,
+    fullConfig: config,
+  };
+};

package/src/context/getEnvironmentVariables.ts

@@ -0,0 +1,152 @@
+import { isObject, merge } from "lodash";
+import { DEPLOY_TYPES } from "../deploy";
+import type { CommitInfo, Context } from "../types";
+import type { Config, DevLocalEnvConfig } from "../types/config";
+
+import { getEnvironmentContext } from "./getEnvironmentContext";
+import {
+  resolveReferences,
+  translateLegacyFromComponents,
+} from "./resolveReferences";
+
+export const getEnvironmentVariables = async (
+  config: Config,
+  componentName: string,
+  env: string,
+  commitInfo?: CommitInfo,
+  alreadyVisited: Record<string, Record<string, boolean>> = {} // to prevent endless loop
+): Promise<{
+  envVars: Record<string, string>;
+  secretEnvVarKeys: string[];
+  host: string;
+  url: string;
+}> => {
+  const environmentContext = getEnvironmentContext(
+    config,
+    env,
+    componentName,
+    commitInfo
+  );
+
+  const { envConfigRaw, deployConfigRaw, envType } = environmentContext;
+
+  const basePredefinedVariables = {
+    ENV_SHORT: env,
+    APP_DIR: envConfigRaw.dir,
+    ENV_TYPE: envType,
+  };
+
+  let predefinedVariables: Record<string, string>;
+  let host: string;
+  let url: string;
+
+  if (envType === "local") {
+    const devLocalConfig: DevLocalEnvConfig = envConfigRaw;
+    const port = devLocalConfig.port ?? 3000;
+    host = "localhost:" + port;
+    url = "http://" + host;
+    predefinedVariables = {
+      ENV_SHORT: "local",
+      ROOT_URL: url,
+      PORT: port.toString(),
+    };
+  } else {
+    const additionalEnvVars = deployConfigRaw
+      ? DEPLOY_TYPES[deployConfigRaw.type].getAdditionalEnvVars(
+          environmentContext
+        )
+      : {};
+
+    host =
+      envConfigRaw?.host ??
+      additionalEnvVars.HOST_CANONICAL ??
+      "unknown-host.example.com";
+    url = `https://${host}`;
+
+    predefinedVariables = {
+      ...basePredefinedVariables,
+      HOST: host,
+      ROOT_URL: url,
+      ...additionalEnvVars,
+    };
+  }
+  const publicEnvVarsRaw = envConfigRaw.vars?.public ?? {};
+
+  const additionalSecretKeys = deployConfigRaw
+    ? DEPLOY_TYPES[deployConfigRaw.type].additionalSecretKeys(
+        environmentContext
+      )
+    : [];
+
+  const secretEnvVarKeys = [
+    ...(envConfigRaw.vars?.secret ?? []),
+    ...additionalSecretKeys,
+  ];
+  const secretEnvVars = Object.fromEntries(
+    secretEnvVarKeys.map((key) => [
+      key,
+      `$${getSecretVarName(env, componentName, key)}`,
+    ])
+  );
+
+  // this is deprecated, we now support: $componentname:FOO
+  const legacyFromComponents = envConfigRaw.vars?.fromComponents ?? {};
+  const publicEnvVarsRawWithLegasyFromComponents = merge(
+    {},
+    translateLegacyFromComponents(legacyFromComponents),
+    publicEnvVarsRaw
+  );
+
+  const publicEnvVarsRawSanitized = Object.fromEntries(
+    Object.entries(publicEnvVarsRawWithLegasyFromComponents).map(
+      ([key, value]) => [
+        key,
+        isObject(value) ? JSON.stringify(value) : `${value}`,
+      ]
+    )
+  );
+
+  const envVarsRaw = addIndexVar({
+    ...predefinedVariables,
+    ...secretEnvVars,
+    ...publicEnvVarsRawSanitized,
+  });
+
+  const envVars = await resolveReferences(
+    envVarsRaw,
+    async (otherComponentName, variableName, alreadyVisited) => {
+      const { envVars: otherEnvVars } = await getEnvironmentVariables(
+        config,
+        otherComponentName,
+        env,
+        commitInfo,
+        alreadyVisited
+      );
+      return otherEnvVars[variableName];
+    },
+    alreadyVisited
+  );
+
+  return {
+    envVars,
+    secretEnvVarKeys,
+    host,
+    url,
+  };
+};
+
+const sanitizeForEnVar = (s: string) => s.replace(/-/g, "_");
+
+export const getSecretVarName = (
+  env: string,
+  componentName: string,
+  key: string
+) => `CL_${sanitizeForEnVar(env)}_${sanitizeForEnVar(componentName)}_${key}`; // remove dash from component name
+
+const addIndexVar = (vars: Record<string, unknown>) => ({
+  ...vars,
+  _ALL_ENV_VAR_KEYS: JSON.stringify(Object.keys(vars)),
+});
+
+export const getSecretVarNameForContext = (context: Context, key: string) =>
+  getSecretVarName(context.environment.shortName, context.componentName, key);

package/src/context/index.ts

@@ -1,13 +1,15 @@
 import { BUILD_TYPES } from "../build";
-import type { BuildConfig } from "../build/types";
+import type { BuildConfig, BuildConfigType } from "../build/types";
 import { DEPLOY_TYPES } from "../deploy";
-import type { DeployConfig } from "../deploy/types";
+import type { DeployConfig, DeployConfigType } from "../deploy/types";
 import type { Config } from "../types/config";
 import type { CommitInfo, Context, PackageManagerInfo } from "../types/context";
 import { mergeWithMergingArrays } from "../utils";
 import { getEnvironment } from "./getEnvironment";
+import { getEnvironmentContext } from "./getEnvironmentContext";
 
 export * from "./getEnvironment";
+export * from "./getEnvironmentVariables";
 
 export const createContext = async (
   config: Config,
@@ -21,27 +23,28 @@ export const createContext = async (
       "componentName may only contain lower case letters, numbers and -"
     );
   }
-  const rawConfig = config.components[componentName];
-  if (!rawConfig) {
-    throw new Error("unknown component " + componentName);
-  }
-  // envs can override the config
-  const envConfig = rawConfig.env?.[env] ?? {};
-  const componentConfigWithoutDefaults = mergeWithMergingArrays(
-    rawConfig,
-    envConfig
+
+  const envContext = getEnvironmentContext(
+    config,
+    env,
+    componentName,
+    commitInfo
   );
 
-  // fill in defaults of build and deploy
+  const componentConfigWithoutDefaults = envContext.envConfigRaw;
   const defaults: {
     build: Partial<BuildConfig>;
     deploy: Partial<DeployConfig>;
   } = componentConfigWithoutDefaults.deploy
     ? {
         build:
-          BUILD_TYPES[componentConfigWithoutDefaults.build.type].defaults(),
+          BUILD_TYPES[
+            componentConfigWithoutDefaults.build.type as BuildConfigType
+          ].defaults(envContext),
         deploy:
-          DEPLOY_TYPES[componentConfigWithoutDefaults.deploy.type].defaults(),
+          DEPLOY_TYPES[
+            componentConfigWithoutDefaults.deploy.type as DeployConfigType
+          ].defaults(envContext),
       }
     : {
         build: {},

package/src/deploy/cloudRun/deployJob.ts

@@ -7,6 +7,10 @@ import { getRunnerImage } from "../../runner";
 import type { Context } from "../../types/context";
 import type { CatladderJob } from "../../types/jobs";
 import { getBaseDeploymentJob, getBaseDeploymentStopJob } from "../base";
+import type {
+  DeployConfigCloudRunJob,
+  DeployConfigCloudRunService,
+} from "../types";
 import { isOfDeployType } from "../types";
 import { gcloudServiceAccountLoginCommands } from "./utils/gcloudServiceAccountLoginCommands";
 
@@ -17,7 +21,7 @@ export const createGoogleCloudRunDeployJobs = (
   if (deployConfig === false) {
     return [];
   }
-  if (!isOfDeployType(deployConfig, "google-cloudrun", "google-cloudrun-job")) {
+  if (!isOfDeployType(deployConfig, "google-cloudrun")) {
     // should not happen
     throw new Error("deploy config is wrong");
   }
@@ -46,46 +50,87 @@ export const createGoogleCloudRunDeployJobs = (
     .join(",");
 
   const commonArgs = `--project ${deployConfig.projectId} --region=${deployConfig.region}`;
-  const getDeployCommand = () => {
-    const name = context.environment.fullName.toLowerCase();
+
+  const commonDeployArgs = `--image ${gcloudImageName} ${commonArgs} --labels ${labelsString}`;
+  const serviceName = context.environment.fullName.toLowerCase();
+
+  const getServiceDeployScript = (
+    service?: DeployConfigCloudRunService | true
+  ) => {
     const command =
-      deployConfig.command ?? context.componentConfig.build.startCommand;
+      service !== true
+        ? service?.command ?? context.componentConfig.build.startCommand
+        : undefined;
+
     const commandArg = command
       ? `--command="${command.split(" ").join(",")}"`
       : "";
 
-    const commonDeployArgs = `${commandArg} --image ${gcloudImageName} ${commonArgs} --env-vars-file=____envvars.yaml --labels ${labelsString}`;
-    if (deployConfig.type === "google-cloudrun") {
-      return `gcloud run deploy ${name} ${commonDeployArgs} --allow-unauthenticated`;
-    }
-    if (deployConfig.type === "google-cloudrun-job") {
-      return `gcloud beta run jobs create ${name} ${commonDeployArgs}`;
-    }
+    return `gcloud run deploy ${serviceName} ${commandArg} ${commonDeployArgs} --env-vars-file=____envvars.yaml --allow-unauthenticated`;
+  };
+
+  const getJobDeployScripts = (
+    jobName: string,
+    job: DeployConfigCloudRunJob
+  ) => {
+    // due to some oversight from google, jobs create does not accept `--env-vars-file` 🤦
+    // lucky, update on the other hand accepts it... so let's just imediatly update it
+
+    // also we cannot upsert a job, so we have to create it and catch the error and then update
+    const args = `${jobName} --command="${job.command
+      .split(" ")
+      .join(",")}" ${commonDeployArgs}`;
+    return [
+      "set +e", // disable fail job on error
+      `gcloud beta run jobs create ${args}`,
+      "set -e", // reenable
+      `gcloud beta run jobs update ${args} --env-vars-file=____envvars.yaml`,
+    ];
   };
 
-  const getStopCommand = () => {
-    const name = context.environment.fullName.toLowerCase();
-    const commonStopArgs = `${commonArgs} --quiet`;
-    if (deployConfig.type === "google-cloudrun") {
-      return `gcloud run services delete ${name} ${commonStopArgs}`;
-    }
-    if (deployConfig.type === "google-cloudrun-job") {
-      return `gcloud beta run jobs delete ${name} ${commonStopArgs}`;
-    }
+  const getJobRunScript = (jobName: string) => {
+    return `gcloud beta run jobs execute ${jobName} ${commonArgs}`;
   };
 
-  const cloudRunDeploy = [
+  const getFullJobName = (name: string) =>
+    context.environment.fullName.toLowerCase() + "-" + name;
+
+  const jobsWithNames = Object.entries(deployConfig.jobs ?? {}).map(
+    ([name, job]) => [getFullJobName(name), job] as const
+  );
+  const cloudRunDeployScripts = [
     `echo "$ENV_VARS" > ____envvars.yaml`, // TODO: split secrets out
-    getDeployCommand(),
+
+    ...jobsWithNames
+      .map(([name, job]) => getJobDeployScripts(name, job))
+      .flat(),
+
+    ...(deployConfig.service !== false
+      ? [getServiceDeployScript(deployConfig.service)]
+      : []),
+
+    ...jobsWithNames
+      .filter(([name, job]) => job.when === "postDeploy")
+      .map(([name, job]) => getJobRunScript(name)),
     `docker image rm ${gcloudImageName}`,
   ];
 
+  const cloudRunStopScripts = [
+    ...(deployConfig.service !== false
+      ? [`gcloud run services delete ${serviceName}`]
+      : []),
+    ...Object.entries(deployConfig.jobs ?? {}).map(
+      ([jobName, job]) => `gcloud beta run jobs delete ${jobName}`
+    ),
+  ];
+
   const baseStopJob = getBaseDeploymentStopJob(context);
 
   return [
     merge({}, baseDeploymentJob, getDockerJobBaseProps(context), {
       artifacts: { paths: ["____envvars.yaml"] },
       variables: {
+        CLOUDSDK_CORE_DISABLE_PROMPTS: "1",
         ENV_VARS: dump(allEnvVars, {
           lineWidth: -1,
           quotingType: "'",
@@ -93,12 +138,18 @@ export const createGoogleCloudRunDeployJobs = (
         }),
       },
       image: getRunnerImage("gcloud"),
-      script: [...pushImageToArtifactsRegistry, ...cloudRunDeploy],
+      script: [...pushImageToArtifactsRegistry, ...cloudRunDeployScripts],
     }),
 
     merge({}, baseStopJob, {
       image: getRunnerImage("gcloud"),
-      script: [...gcloudServiceAccountLoginCommands(context), getStopCommand()],
+      variables: {
+        CLOUDSDK_CORE_DISABLE_PROMPTS: "1",
+      },
+      script: [
+        ...gcloudServiceAccountLoginCommands(context),
+        ...cloudRunStopScripts,
+      ],
     }),
   ];
 };

package/src/deploy/cloudRun/index.ts

@@ -11,7 +11,7 @@ export const GCLOUD_RUN_DEPLOY_TYPE: DeployTypeDefinition<"google-cloudrun"> = {
   jobs: createGoogleCloudRunDeployJobs,
   defaults: () => ({}),
   additionalSecretKeys: () => [GCLOUD_DEPLOY_CREDENTIALS_KEY],
-  getAdditionalEnvVars: ({ fullName, env, componentName }) => {
+  getAdditionalEnvVars: ({ fullName, env, componentName, deployConfigRaw }) => {
     const HOST_CANONICAL =
       fullName.toLowerCase() +
       "-" +
@@ -19,30 +19,19 @@ export const GCLOUD_RUN_DEPLOY_TYPE: DeployTypeDefinition<"google-cloudrun"> = {
         getSecretVarName(env, componentName, GCLOUD_RUN_CANONICAL_HOST_SUFFIX)
       ];
 
+    const jobTriggers =
+      deployConfigRaw && deployConfigRaw.jobs
+        ? Object.fromEntries(
+            Object.entries(deployConfigRaw.jobs).map(([name, job]) => [
+              "CLOUD_RUN_JOB_TRIGGER_URL_" + name,
+              `https://${deployConfigRaw.region}-run.googleapis.com/apis/run.googleapis.com/v1/namespaces/${deployConfigRaw.projectId}/jobs/${name}:run`,
+            ])
+          )
+        : {};
+
     return {
       HOST_CANONICAL,
+      ...jobTriggers,
     };
   },
 };
-
-export const GCLOUD_RUN_JOB_DEPLOY_TYPE: DeployTypeDefinition<"google-cloudrun-job"> =
-  {
-    jobs: createGoogleCloudRunDeployJobs,
-    defaults: () => ({}),
-    additionalSecretKeys: () => [GCLOUD_DEPLOY_CREDENTIALS_KEY],
-    getAdditionalEnvVars: ({ deployConfig, fullName }) => {
-      if (!deployConfig) {
-        return {};
-      }
-
-      const jobName = fullName.toLowerCase();
-      const CLOUD_RUN_JOB_TRIGGER_URL = `https://${deployConfig.region}-run.googleapis.com/apis/run.googleapis.com/v1/namespaces/${deployConfig.projectId}/jobs/${jobName}:run`;
-
-      return { CLOUD_RUN_JOB_TRIGGER_URL };
-    },
-  };
-
-export const GCLOUD_RUN_DEPLOY_TYPES = {
-  "google-cloudrun": GCLOUD_RUN_DEPLOY_TYPE,
-  "google-cloudrun-job": GCLOUD_RUN_JOB_DEPLOY_TYPE,
-};