@catladder/pipeline 0.0.0-fix-command-710f75f1 → 0.0.0-fix-error-sql-db-delete-b3878c2e

package/src/context/getEnvironmentContext.ts

@@ -5,17 +5,16 @@ import type { EnvironmentContext } from "../types/environmentContext";
 import { getEnvConfig } from "./getEnvConfig";
 import { getEnvType } from "./getEnvType";
 
-const getEnvironmentSlug = (
+const getEnvironmentSlugPrefix = (
   envConfig: EnvConfigWithComponent,
   env: string,
-  componentName: string,
   commitInfo?: CommitInfo
 ) => {
   const envType = getEnvType(env, envConfig);
 
   return envType === "review" && commitInfo
-    ? `${env}-${commitInfo.reviewSlug}-${componentName}`
-    : `${env}-${componentName}`;
+    ? `${env}-${commitInfo.reviewSlug}`
+    : `${env}`;
 };
 
 export const getEnvironmentContext = (
@@ -27,13 +26,13 @@ export const getEnvironmentContext = (
   const envConfigRaw = getEnvConfig(config, componentName, env);
   const envType = getEnvType(env, envConfigRaw);
 
-  const environmentSlug = getEnvironmentSlug(
+  const environmentSlugPrefix = getEnvironmentSlugPrefix(
     envConfigRaw,
     env,
-    componentName,
     commitInfo
   );
 
+  const environmentSlug = `${environmentSlugPrefix}-${componentName}`;
   const gitlabEnvironmentName =
     envType === "review" && commitInfo
       ? `${env}/${commitInfo.refName}/${componentName}`
@@ -45,6 +44,7 @@ export const getEnvironmentContext = (
     envConfigRaw,
     deployConfigRaw: envConfigRaw.deploy,
     buildConfigRaw: envConfigRaw.build,
+    environmentSlugPrefix,
     environmentSlug,
     gitlabEnvironmentName,
     fullName,

package/src/context/getEnvironmentVariables.ts

@@ -9,6 +9,11 @@ import {
   translateLegacyFromComponents,
 } from "./resolveReferences";
 
+export type SecretEnvVar = {
+  key: string;
+  // hidden env vars are not shown in config-secrets
+  hidden?: boolean;
+};
 export const getEnvironmentVariables = async (
   config: Config,
   componentName: string,
@@ -17,7 +22,7 @@ export const getEnvironmentVariables = async (
   alreadyVisited: Record<string, Record<string, boolean>> = {} // to prevent endless loop
 ): Promise<{
   envVars: Record<string, string>;
-  secretEnvVarKeys: string[];
+  secretEnvVarKeys: SecretEnvVar[];
   host: string;
   url: string;
 }> => {
@@ -81,12 +86,12 @@ export const getEnvironmentVariables = async (
       )
     : [];
 
-  const secretEnvVarKeys = [
-    ...(envConfigRaw.vars?.secret ?? []),
+  const secretEnvVarKeys: SecretEnvVar[] = [
+    ...(envConfigRaw.vars?.secret ?? []).map((key) => ({ key })),
     ...additionalSecretKeys,
   ];
   const secretEnvVars = Object.fromEntries(
-    secretEnvVarKeys.map((key) => [
+    secretEnvVarKeys.map(({ key }) => [
       key,
       `$${getSecretVarName(env, componentName, key)}`,
     ])

package/src/deploy/base.ts

@@ -44,7 +44,7 @@ export const getBaseDeploymentJob = (context: Context): JobWithoutScript => {
     name: DEPLOY_JOB_NAME,
     envMode: "stagePerEnv", // makes it easier to run manual tasks er env
 
-    needsOtherComponent: context.componentConfig.deploy
+    needs: context.componentConfig.deploy
       ? context.componentConfig.deploy.waitFor?.map((c) => ({
           componentName: c,
           job: DEPLOY_JOB_NAME,

package/src/deploy/cloudRun/deployJob.ts

@@ -1,5 +1,5 @@
 import { dump } from "js-yaml";
-import { isNil, merge, omit } from "lodash";
+import { merge, omit } from "lodash";
 import { GCLOUD_DEPLOY_CREDENTIALS_KEY } from ".";
 import { getDockerJobBaseProps, gitlabDockerLogin } from "../../build/docker";
 import { getLabels } from "../../context/getLabels";

package/src/deploy/cloudRun/index.ts

@@ -12,7 +12,7 @@ export const GCLOUD_RUN_CANONICAL_HOST_SUFFIX =
 
 const getCloudSqlVariables = ({
   deployConfigRaw,
-  environmentSlug,
+  environmentSlugPrefix,
   env,
   componentName,
   fullConfig,
@@ -21,7 +21,8 @@ const getCloudSqlVariables = ({
     const DB_NAME = [
       deployConfigRaw.cloudSql.dbNamePrefix ??
         `${fullConfig.customerName}-${fullConfig.appName}`,
-      environmentSlug,
+      environmentSlugPrefix,
+      deployConfigRaw.cloudSql.dbBaseName ?? componentName,
     ]
       .filter(Boolean)
       .join("-");
@@ -57,9 +58,16 @@ export const GCLOUD_RUN_DEPLOY_TYPE: DeployTypeDefinition<"google-cloudrun"> = {
     return {};
   },
   additionalSecretKeys: (ctx) => [
-    GCLOUD_DEPLOY_CREDENTIALS_KEY,
+    {
+      key: GCLOUD_DEPLOY_CREDENTIALS_KEY,
+      hidden: true,
+    },
+    {
+      key: GCLOUD_RUN_CANONICAL_HOST_SUFFIX,
+      hidden: true,
+    },
     ...(ctx.deployConfigRaw && ctx.deployConfigRaw.cloudSql
-      ? ["DB_PASSWORD"]
+      ? [{ key: "DB_PASSWORD" }]
       : []),
   ],
   getAdditionalEnvVars: (ctx) => {

package/src/deploy/custom/deployJob.ts

@@ -4,7 +4,7 @@ import { getYarnInstall } from "../../build/node/yarn";
 import { getRunnerImage } from "../../runner";
 import type { Context } from "../../types/context";
 import type { CatladderJob } from "../../types/jobs";
-import { getBaseDeploymentJob } from "../base";
+import { getBaseDeploymentJob, getBaseDeploymentStopJob } from "../base";
 import { isOfDeployType } from "../types";
 
 export const createCustomDeployJobs = (context: Context): CatladderJob[] => {
@@ -29,5 +29,13 @@ export const createCustomDeployJobs = (context: Context): CatladderJob[] => {
         ...deployConfig.script,
       ],
     }),
+    ...(deployConfig.stopScript
+      ? [
+          merge({}, getBaseDeploymentStopJob(context), {
+            image: deployConfig.jobImage ?? getRunnerImage("jobs-default"),
+            script: deployConfig.stopScript,
+          }),
+        ]
+      : []),
   ];
 };

package/src/deploy/index.ts

@@ -1,3 +1,4 @@
+import type { SecretEnvVar } from "..";
 import type { Context } from "../types/context";
 import type { EnvironmentContext } from "../types/environmentContext";
 import type { CatladderJob } from "../types/jobs";
@@ -17,7 +18,9 @@ export type DeployTypeDefinition<T extends DeployConfigType> = {
   defaults: (
     envContext: EnvironmentContext<any, T>
   ) => PartialDeep<DeployConfigGeneric<T>>;
-  additionalSecretKeys: (envContext: EnvironmentContext<any, T>) => string[];
+  additionalSecretKeys: (
+    envContext: EnvironmentContext<any, T>
+  ) => SecretEnvVar[];
   getAdditionalEnvVars: (
     envContext: EnvironmentContext<any, T>
   ) => Record<string, string | undefined | null>;

package/src/deploy/kubernetes/additionalSecretKeys.ts

@@ -19,5 +19,5 @@ export const additionalKubernetesSecretKeys = ({
     keys.push("cloudsqlProxyCredentials");
   }
 
-  return keys;
+  return keys.map((key) => ({ key }));
 };

package/src/deploy/kubernetes/deployJob.ts

@@ -40,7 +40,8 @@ export const createKubernetesDeployJobs = (
         quotingType: "'",
         forceQuotes: true,
       }),
-      HELM_GITLAB_CHART_NAME: "the-panter-chart",
+      HELM_GITLAB_CHART_NAME:
+        deployConfig.chartName ?? "/helm-charts/the-panter-chart",
       HELM_ARGS: [
         ...(deployConfig.debug ? ["--debug"] : []),
         ...(deployConfig.additionalHelmArgs ?? []),

package/src/deploy/kubernetes/kubeValues.ts

@@ -20,27 +20,29 @@ const createAppConfig = (
     };
   }
 
+  const { healthRoute, command, ...rest } = application ?? {};
+
   return mergeWithMergingArrays(
     {
       host: context.environment.host,
-      command: context.componentConfig.build.startCommand,
+      command: command ?? context.componentConfig.build.startCommand,
       livenessProbe: {
         httpGet: {
-          path: application?.healthRoute ?? "__health",
+          path: healthRoute ?? "__health",
         },
       },
       readinessProbe: {
         httpGet: {
-          path: application?.healthRoute ?? "__health",
+          path: healthRoute ?? "__health",
         },
       },
       startupProbe: {
         httpGet: {
-          path: application?.healthRoute ?? "__health",
+          path: healthRoute ?? "__health",
         },
       },
     }, // default
-    application // merge rest in
+    rest // merge rest in
   );
 };
 

package/src/deploy/types/custom.ts

@@ -1,3 +1,4 @@
+import type { GitlabJobImage } from "../../types";
 import type { CatladderJob } from "../../types/jobs";
 import type { DeployConfigBase } from "./base";
 
@@ -26,7 +27,7 @@ export type DeployConfigCustom = {
   /**
    * image to use
    */
-  jobImage?: string;
+  jobImage?: GitlabJobImage;
 
   /**
    * customize cache for the job

package/src/deploy/types/googleCloudRun.ts

@@ -46,11 +46,16 @@ export type DeployConfigCloudRunCloudSql = {
   dbUser?: string;
 
   /**
-   * the prefix of the database, the full db name is this plus the environment slug
+   * the prefix of the database, the full db name is this plus the environment slug prefix plus the componentName
    *
    * defaults to customerName-appName
    */
   dbNamePrefix?: string | false;
+
+  /**
+   * the base name of the db, defaults to the componentName
+   */
+  dbBaseName?: string;
   /**
    * whether to delete the database if the environment is stopped
    * defaults to true for review envs, to false for every other environment

package/src/deploy/types/kubernetes.ts

@@ -192,6 +192,11 @@ export type DeployConfigKubernetesValues = AllowUnknownProps<{
   application?:
     | false
     | AllowUnknownProps<{
+        /**
+         * command to start, defaults to build's startCommand
+         */
+        command?: string;
+
         /**
          * enable, disable app deployment, defaults to true
          */
@@ -240,7 +245,7 @@ export type DeployConfigKubernetesValues = AllowUnknownProps<{
          */
         jobDefaults?: {
           resources: KubernetesResourcesDef;
-        }
+        };
       }>;
 
   /**
@@ -288,4 +293,9 @@ export type DeployConfigKubernetes = {
    * values to configure the app
    */
   values?: DeployConfigKubernetesValues;
+  /**
+   * Custom Helm chart location.
+   * Not recommended to use.
+   */
+  chartName?: string;
 } & DeployConfigBase;

package/src/pipeline/createAllJobs.ts

@@ -0,0 +1,38 @@
+import { getAllEnvsByTrigger } from "../config";
+import type { Config, PipelineTrigger } from "../types";
+import type { CatladderJob } from "../types/jobs";
+import { createJobsForComponent } from "./createJobsForComponent";
+
+export type AllCatladderJobs = {
+  [componentName: string]: {
+    [env: string]: Array<CatladderJob>;
+  };
+};
+export const createAllJobs = async (
+  config: Config,
+  trigger: PipelineTrigger
+): Promise<AllCatladderJobs> => {
+  return Object.fromEntries(
+    await Promise.all(
+      Object.keys(config.components).map(async (componentName) => {
+        const envs = getAllEnvsByTrigger(config, componentName, trigger);
+        return [
+          componentName,
+          Object.fromEntries(
+            await Promise.all(
+              envs.map(async (env) => [
+                env,
+                await createJobsForComponent(
+                  config,
+                  componentName,
+                  env,
+                  trigger
+                ),
+              ])
+            )
+          ),
+        ];
+      })
+    )
+  );
+};

package/src/pipeline/createChildPipeline.ts

@@ -1,34 +1,18 @@
-import { getAllEnvsByTrigger, getAllEnvsInAllComponents } from "../config";
+import { getAllEnvsInAllComponents } from "../config";
 import { RULES_ALWAYS } from "../rules";
 import { getRunnerImage } from "../runner";
-import type {
-  GitlabPipeline,
-  Pipeline,
-  PipelineJob,
-  PipelineType,
-} from "../types";
+import type { GitlabPipeline, Pipeline, PipelineType } from "../types";
 import type { Config, PipelineTrigger } from "../types/config";
 import { BASE_STAGES } from "../types/jobs";
-import { createJobs } from "./createJobs";
+import { createAllJobs } from "./createAllJobs";
+import { createGitlabJobs } from "./gitlab/createGitlabJobs";
 
 export const createChildPipeline = async <T extends PipelineType>(
   type: T,
   trigger: PipelineTrigger,
   config: Config
 ): Promise<Pipeline<T>> => {
-  const components = Object.keys(config.components);
-
-  // 2. write the triggering pipeline
-  const jobs = await components.reduce<Promise<Record<string, PipelineJob<T>>>>(
-    async (acc, componentName) => {
-      const envs = getAllEnvsByTrigger(config, componentName, trigger);
-      return {
-        ...(await acc),
-        ...(await createJobs(type, envs, config, componentName, trigger)),
-      };
-    },
-    Promise.resolve({})
-  );
+  const jobs = await createAllJobs(config, trigger);
 
   // while technically not required, we group different envs in its own stage
   // each job from `createJobs` that is defined as `envMode: "stagePerEnv"` will have `deploy dev`, etc. instead of just `deploy`
@@ -53,7 +37,7 @@ export const createChildPipeline = async <T extends PipelineType>(
         rules: RULES_ALWAYS,
       },
       stages,
-      jobs,
+      jobs: await createGitlabJobs(jobs),
     };
     return pipeline as Pipeline<T>;
   }

package/src/pipeline/createJobsForComponent.ts

@@ -0,0 +1,64 @@
+import { isFunction } from "lodash";
+import { BUILD_TYPES } from "../build";
+import { createContext } from "../context";
+import { DEPLOY_TYPES } from "../deploy";
+import type { Config, PipelineTrigger } from "../types/config";
+import type { CommitInfo, Context } from "../types/context";
+import type { CatladderJob } from "../types/jobs";
+import { getBaseCommitInfo } from "./commitInfo/getCommitInfo";
+import { getPackageManagerInfo } from "./packageManager";
+
+const injectDefaultVarsInCustomJobs = (
+  context: Context,
+  jobs: CatladderJob[]
+) =>
+  jobs.map(({ variables, ...job }) => ({
+    variables: {
+      ...(context.environment.envVars ?? {}),
+      ...(variables ?? {}),
+    },
+    ...job,
+  }));
+const getCustomJobs = (context: Context) => {
+  if (!context.componentConfig.customJobs) {
+    return [];
+  }
+  const rawJobs = isFunction(context.componentConfig.customJobs)
+    ? context.componentConfig.customJobs(context)
+    : context.componentConfig.customJobs;
+  return injectDefaultVarsInCustomJobs(context, rawJobs);
+};
+const createRawJobs = (context: Context): CatladderJob[] => {
+  if (context.componentConfig.deploy === false) {
+    return [];
+  }
+  const buildJobs =
+    BUILD_TYPES[context.componentConfig.build.type].jobs(context);
+  const deployJobs =
+    DEPLOY_TYPES[context.componentConfig.deploy.type].jobs(context);
+
+  const customJobs = getCustomJobs(context);
+  return [...buildJobs, ...deployJobs, ...customJobs];
+};
+export const createJobsForComponent = async (
+  config: Config,
+  componentName: string,
+  env: string,
+  trigger: PipelineTrigger
+): Promise<Array<CatladderJob>> => {
+  const commitInfo: CommitInfo = {
+    ...(await getBaseCommitInfo()),
+    trigger,
+  };
+
+  const packageManagerInfo = await getPackageManagerInfo(config, componentName);
+
+  const context = await createContext(
+    config,
+    componentName,
+    env,
+    commitInfo,
+    packageManagerInfo
+  );
+  return createRawJobs(context);
+};

package/src/pipeline/gitlab/createGitlabJobs.ts

@@ -0,0 +1,149 @@
+import { isObject } from "lodash";
+import { BASE_RETRY } from "../../defaults";
+import type { GitlabJobDef } from "../../types";
+import type { CatladderJob, CatladderJobNeed } from "../../types/jobs";
+import type { AllCatladderJobs } from "../createAllJobs";
+
+type AllGitlabJobs = Record<string, GitlabJobDef>;
+
+const getFullJobName = (
+  name: string,
+  componentName: string,
+  env?: string | null
+) => {
+  if (env) {
+    return `${componentName} ${name} | ${env} `;
+  }
+  return `${componentName} ${name}`;
+};
+
+const getFullReferencedJobName = (
+  referencedJobName: string,
+  componentName: string,
+  env: string,
+  allJobs: AllCatladderJobs
+) => {
+  const referencedJob = allJobs[componentName]?.[env]?.find(
+    (j) => j.name === referencedJobName
+  );
+  if (!referencedJob) {
+    throw new Error(
+      `unknown job referenced: '${referencedJobName}' from '${env}:${componentName}'`
+    );
+  }
+  const envToSet = referencedJob.envMode !== "none" ? env : null;
+  return getFullJobName(referencedJobName, componentName, envToSet);
+};
+
+const getJobName = (need: CatladderJobNeed) =>
+  isObject(need) ? need.job : need;
+
+export const makeGitlabJob = (
+  componentName: string,
+  env: string,
+  {
+    envMode,
+    needsStages,
+    needsOtherComponent,
+    name,
+    needs,
+    ...job
+  }: CatladderJob<string>,
+  allJobs: AllCatladderJobs
+): [fullName: string, job: GitlabJobDef] => {
+  const stage = envMode === "stagePerEnv" ? `${job.stage} ${env}` : job.stage;
+
+  const needsFromStages: CatladderJob["needs"] = needsStages?.flatMap((n) => {
+    const referencedComponentName = componentName;
+    const allJobNamesFromThatStage =
+      allJobs[referencedComponentName]?.[env]
+        ?.filter((j) => j.stage === n.stage)
+        ?.map((j) => j.name) ?? [];
+
+    return allJobNamesFromThatStage.map((job) => ({
+      job,
+      artifacts: n.artifacts ?? false,
+      componentName: referencedComponentName,
+    }));
+  });
+  const cleanedNeeds: CatladderJob["needs"] = [
+    ...(needs ?? []),
+    // pull in legacy needs from other component, which is now identical to needs
+    ...(needsOtherComponent ?? []),
+    ...(needsFromStages ?? []),
+  ];
+
+  const gitlabNeeds: GitlabJobDef["needs"] = cleanedNeeds
+    ?.map((n) =>
+      isObject(n)
+        ? {
+            job: getFullReferencedJobName(
+              n.job,
+              n.componentName ?? componentName,
+              env,
+              allJobs
+            ),
+            artifacts: n.artifacts,
+          }
+        : getFullReferencedJobName(n, componentName, env, allJobs)
+    ) // sort in a predictable manner for snapshot tests
+    .sort((a, b) => getJobName(a).localeCompare(getJobName(b)));
+
+  const fullJobName = getFullJobName(
+    name,
+    componentName,
+    envMode !== "none" ? env : undefined
+  );
+
+  const gitlabJob = {
+    ...job,
+    stage,
+    environment: job.environment?.on_stop
+      ? {
+          ...job.environment,
+          on_stop: getFullReferencedJobName(
+            job.environment.on_stop,
+            componentName,
+            env,
+            allJobs
+          ),
+        }
+      : job.environment,
+    // sort in a predictable manner for snapshot tests
+    needs: gitlabNeeds,
+    retry: BASE_RETRY,
+    interruptible: true,
+  };
+
+  return [fullJobName, gitlabJob];
+};
+
+export const createGitlabJobs = async (
+  allJobs: AllCatladderJobs
+): Promise<AllGitlabJobs> => {
+  return Object.keys(allJobs).reduce((accForComponents, componentName) => {
+    const componentJobs = allJobs[componentName];
+    return {
+      ...accForComponents,
+      ...Object.keys(componentJobs).reduce((accForEnvs, env) => {
+        const jobs = componentJobs[env];
+
+        return {
+          ...accForEnvs,
+          ...jobs.reduce((accForJobs, job) => {
+            const [fullJobName, gitlabJob] = makeGitlabJob(
+              componentName,
+              env,
+              job,
+              allJobs
+            );
+            return {
+              ...accForJobs,
+              [fullJobName]: gitlabJob,
+            };
+          }, {} as AllGitlabJobs),
+        };
+      }, {} as AllGitlabJobs),
+    };
+  }, {} as AllGitlabJobs);
+};

package/src/pipeline/index.ts

@@ -1,2 +1,2 @@
 export * from "./createChildPipeline";
-export * from "./createJobs";
+export * from "./createJobsForComponent";

package/src/types/context.ts

@@ -1,3 +1,4 @@
+import type { SecretEnvVar } from "../context";
 import type {
   PipelineTrigger,
   ComponentConfig,
@@ -18,13 +19,14 @@ export type Environment = {
     url: string;
   };
   shortName: string;
+  slugPrefix: string;
   slug: string;
   /**
    * env vars contain all build-time env vars. secrets have to be resolved (they are stored in gitlab)
    */
   envVars: Record<string, string>;
   envType: EnvType;
-  secretEnvVarKeys: string[];
+  secretEnvVarKeys: SecretEnvVar[];
 };
 
 export type CommitInfo = {

package/src/types/environmentContext.ts

@@ -16,6 +16,13 @@ export type EnvironmentContext<
   envType: EnvType;
   componentName: string;
   fullName: string;
+  /**
+   * the environment slug without component name.
+   */
+  environmentSlugPrefix: string;
+  /**
+   * the full environment slug, including the componentName
+   */
   environmentSlug: string;
   gitlabEnvironmentName: string;
 

package/src/types/gitlab-types.ts

@@ -5,7 +5,7 @@ export type Artifacts = {
   reports?: ArtifactReports;
 };
 // Reports won't show up on MRs until https://gitlab.com/groups/gitlab-org/-/epics/8205
-export type ArtifactReports = 
+export type ArtifactReports =
   | { accessibility: string | string[] }
   | { coverage_report: string | string[] }
   | { codequality: string | string[] }
@@ -14,7 +14,7 @@ export type ArtifactReports =
   | { junit: string | string[] }
   | { sast: string | string[] }
   | { secret_detection: string | string[] }
-  | { terraform: string | string[] }
+  | { terraform: string | string[] };
 export type GitlabJobCache = {
   key: GitlabJobCacheKey;
   policy?: string;
@@ -43,6 +43,13 @@ export type GitlabJobService = {
   alias?: string;
 };
 
+export type GitlabJobImage =
+  | string
+  | {
+      name: string;
+      entrypoint?: string[];
+    };
+
 export type GitlabEnvironment = {
   url: string;
   name: string;
@@ -64,7 +71,7 @@ export type GitlabJobDef = {
   artifacts?: Artifacts;
   retry?: Retry;
   services?: GitlabJobService[];
-  image?: string;
+  image?: GitlabJobImage;
   variables?: GitlabVariables;
   dependencies?: string[];
   environment?: GitlabEnvironment;