@catladder/pipeline 1.144.0 → 1.145.0

package/src/deploy/kubernetes/kubeValues.ts

@@ -70,7 +70,6 @@ export const createKubeValues = (context: Context) => {
 
   // we remove the application config because it can be just the value `false` which is a convenience feature, but not supported in the helm chart
   // we only merge the rest of the values in
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
   const { application, jobs, cronjobs, ...rest } = values ?? {};
 
   const env = createKubeEnv(context);

package/src/deploy/kubernetes/processSecretsAsFiles.ts

@@ -1,10 +1,11 @@
 import { omit } from "lodash";
 import type { DeployConfigKubernetesValues } from "../types";
+import type { StringOrBashExpression } from "../../bash/BashExpression";
 
 export const processSecretsAsFiles = <
   T extends {
     env: {
-      secret: Record<string, string>;
+      secret: Record<string, StringOrBashExpression>;
     };
   } & DeployConfigKubernetesValues
 >(

package/src/deploy/types/base.ts

@@ -31,6 +31,12 @@ export type DeployConfigBase = {
    * @deprecated use deploy.jobVars for deploy job specific variables
    */
   extraVars?: Record<string, string>;
+  /**
+   * additional vars only for the runner.
+   * Also if you use services: that require env vars, you need to set them here.
+   *
+   */
+  runnerVariables?: Record<string, string>;
 };
 
 export type AllowUnknownProps<T extends Record<string, unknown>> = T &

package/src/deploy/types/googleCloudRun.ts

@@ -256,6 +256,8 @@ export type DeployConfigCloudRun = {
    * add cloudSql
    */
   cloudSql?: DeployConfigCloudRunCloudSql | false;
+
+  debug?: boolean;
 } & DeployConfigBase;
 
 export type DeployConfigCloudRunVolumes = Record<

package/src/index.ts

@@ -1,4 +1,4 @@
-export * from "./pipeline/";
+export * from "./pipeline";
 export * from "./types";
 export * from "./rules";
 export * from "./runner";
@@ -6,3 +6,4 @@ export * from "./config";
 export * from "./context";
 export * from "./build";
 export * from "./deploy";
+export * from "./utils/writeFiles";

package/src/packageInfos.ts

@@ -0,0 +1,10 @@
+// we can't import the package.json directly, because the bundling process will inline it
+// but the version is only set during deployement, when its already bundled
+// instead we need to use require.main.path to get the path of the main module, which is actually the file in the bin
+// this will point to the root package.json
+
+const path = require.main?.path + "/../package.json";
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const ownPkg = require(path);
+
+export default ownPkg;

package/src/pipeline/createAllJobs.ts

@@ -1,17 +1,29 @@
 import { getAllEnvsByTrigger } from "../config";
-import type { Config, PipelineTrigger } from "../types";
-import type { CatladderJob } from "../types/jobs";
+import type {
+  CatladderJobWithContext,
+  Config,
+  PipelineTrigger,
+  PipelineType,
+} from "../types";
 import { createJobsForComponent } from "./createJobsForComponent";
 
 export type AllCatladderJobs = {
   [componentName: string]: {
-    [env: string]: Array<CatladderJob>;
+    [env: string]: Array<CatladderJobWithContext>;
   };
 };
-export const createAllJobs = async (
-  config: Config,
-  trigger: PipelineTrigger
-): Promise<AllCatladderJobs> => {
+
+export type AllJobsContext = {
+  config: Config;
+  trigger: PipelineTrigger;
+  pipelineType: PipelineType;
+};
+
+export const createAllJobs = async ({
+  config,
+  trigger,
+  pipelineType,
+}: AllJobsContext): Promise<AllCatladderJobs> => {
   return Object.fromEntries(
     await Promise.all(
       Object.keys(config.components).map(async (componentName) => {
@@ -22,12 +34,14 @@ export const createAllJobs = async (
             await Promise.all(
               envs.map(async (env) => [
                 env,
-                await createJobsForComponent(
+
+                await createJobsForComponent({
                   config,
                   componentName,
                   env,
-                  trigger
-                ),
+                  trigger,
+                  pipelineType,
+                }),
               ])
             )
           ),

package/src/pipeline/createChildPipeline.ts

@@ -1,24 +1,35 @@
+import { RULES_ALWAYS } from "../rules";
+
 import type { Pipeline, PipelineType } from "../types";
 import type { Config, PipelineTrigger } from "../types/config";
 import { createAllJobs } from "./createAllJobs";
 import { getPipelineStages } from "./getPipelineStages";
 import { createGitlabJobs } from "./gitlab/createGitlabJobs";
-import { createGitlabPipelineFromStagesAndJobs } from "./gitlab/createGitlabPipeline";
+import { createGitlabPipelineWithDefaults } from "./gitlab/createGitlabPipeline";
 
 export const createChildPipeline = async <T extends PipelineType>(
-  type: T,
+  pipelineType: T,
   trigger: PipelineTrigger,
   config: Config
 ): Promise<Pipeline<T>> => {
-  const jobs = await createAllJobs(config, trigger);
+  const jobs = await createAllJobs({
+    config,
+    trigger,
+    pipelineType,
+  });
   const stages = getPipelineStages(config);
 
-  if (type === "gitlab") {
+  if (pipelineType === "gitlab") {
     const gitlabJobs = await createGitlabJobs(jobs);
-    return createGitlabPipelineFromStagesAndJobs(
+    return createGitlabPipelineWithDefaults({
+      workflow: {
+        rules: RULES_ALWAYS,
+      },
       stages,
-      gitlabJobs
-    ) as Pipeline<T>;
+      jobs: Object.fromEntries(
+        gitlabJobs.map(({ gitlabJob, name }) => [name, gitlabJob])
+      ),
+    }) as Pipeline<T>;
   }
-  throw new Error(`${type} is not supported`);
+  throw new Error(`${pipelineType} is not supported`);
 };

package/src/pipeline/createJobsForComponent.ts

@@ -1,11 +1,10 @@
 import { isFunction } from "lodash";
 import { BUILD_TYPES } from "../build";
+import type { CreateContextContext } from "../context";
 import { createContext } from "../context";
 import { DEPLOY_TYPES } from "../deploy";
-import type { Config, PipelineTrigger } from "../types/config";
-import type { CommitInfo, Context } from "../types/context";
+import type { CatladderJobWithContext, Context } from "../types/context";
 import type { CatladderJob } from "../types/jobs";
-import { getBaseCommitInfo } from "./commitInfo/getCommitInfo";
 import { getPackageManagerInfo } from "./packageManager";
 
 const injectDefaultVarsInCustomJobs = (
@@ -39,25 +38,18 @@ const createRawJobs = (context: Context): CatladderJob[] => {
   const customJobs = getCustomJobs(context);
   return [...buildJobs, ...deployJobs, ...customJobs];
 };
-export const createJobsForComponent = async (
-  config: Config,
-  componentName: string,
-  env: string,
-  trigger: PipelineTrigger
-): Promise<Array<CatladderJob>> => {
-  const commitInfo: CommitInfo = {
-    ...(await getBaseCommitInfo()),
-    trigger,
-  };
-
-  const packageManagerInfo = await getPackageManagerInfo(config, componentName);
 
-  const context = await createContext(
-    config,
-    componentName,
-    env,
-    commitInfo,
-    packageManagerInfo
+export const createJobsForComponent = async (
+  contextContext: Omit<CreateContextContext, "packageManagerInfo">
+): Promise<Array<CatladderJobWithContext>> => {
+  const packageManagerInfo = await getPackageManagerInfo(
+    contextContext.config,
+    contextContext.componentName
   );
-  return createRawJobs(context);
+
+  const context = await createContext({
+    ...contextContext,
+    packageManagerInfo,
+  });
+  return createRawJobs(context).map((job) => ({ ...job, context }));
 };

package/src/pipeline/createMainPipeline.ts

@@ -0,0 +1,88 @@
+import {
+  RULE_IS_MAIN_BRANCH,
+  RULE_IS_MERGE_REQUEST,
+  RULE_IS_TAGGED_RELEASE,
+  RULE_NEVER_ON_RELEASE_COMMIT,
+} from "../rules";
+import type {
+  GitlabJobDef,
+  GitlabRule,
+  Pipeline,
+  PipelineTrigger,
+  PipelineType,
+} from "../types";
+import { ALL_PIPELINE_TRIGGERS, type Config } from "../types/config";
+import { createAllJobs } from "./createAllJobs";
+import { getPipelineStages } from "./getPipelineStages";
+import { createGitlabJobs } from "./gitlab/createGitlabJobs";
+import { createGitlabPipelineWithDefaults } from "./gitlab/createGitlabPipeline";
+import { getGitlabReleaseJobs } from "./gitlab/gitlabReleaseJobs";
+
+export const createMainPipeline = async <T extends PipelineType>(
+  pipelineType: T,
+  config: Config
+): Promise<Pipeline<T>> => {
+  const stages = getPipelineStages(config);
+
+  if (pipelineType === "gitlab") {
+    // for all triggers create jobs and add base rules
+
+    const allJobsPerTrigger = await Promise.all(
+      ALL_PIPELINE_TRIGGERS.map(
+        async (trigger) =>
+          await createGitlabJobs(
+            await createAllJobs({ config, trigger, pipelineType }),
+            getGitlabRulesForTrigger(trigger)
+          )
+      )
+    );
+
+    const allJobs = allJobsPerTrigger
+      .flat()
+      // sort by componentName in the same order as they appear in the config
+      // this is purely for better readability in git diffs when you add new components
+      .sort((a, b) => {
+        const componentNames = Object.keys(config.components);
+        const aIndex = componentNames.findIndex((c) => c === a.componentName);
+        const bIndex = componentNames.findIndex((c) => c === b.componentName);
+        return aIndex - bIndex;
+      })
+
+      .reduce((acc, { gitlabJob, name }) => {
+        // merge jobs, if a job is already there, merge the rules
+        // this is currently needed because of envMode: "none", which creates the same job for all triggers, so it can appear multiple times
+        if (acc[name]) {
+          acc[name].rules = [
+            ...(acc[name].rules ?? []),
+            ...(gitlabJob.rules ?? []),
+          ];
+        } else {
+          acc[name] = gitlabJob;
+        }
+
+        return acc;
+      }, {} as { [key: string]: GitlabJobDef });
+
+    return createGitlabPipelineWithDefaults({
+      stages: [...stages, "release"],
+      jobs: {
+        ...allJobs,
+        ...getGitlabReleaseJobs(),
+      },
+    }) as Pipeline<T>;
+  }
+  throw new Error(`${pipelineType} is not supported`);
+};
+function getGitlabRulesForTrigger(trigger: PipelineTrigger): GitlabRule[] {
+  // mainBranch: on push to main branch
+  // mr: on merge request
+  // taggedRelease: on tag
+  switch (trigger) {
+    case "mainBranch":
+      return [RULE_NEVER_ON_RELEASE_COMMIT, RULE_IS_MAIN_BRANCH];
+    case "mr":
+      return [RULE_IS_MERGE_REQUEST];
+    case "taggedRelease":
+      return [RULE_IS_TAGGED_RELEASE];
+  }
+}

package/src/pipeline/generatePipelineFiles.ts

@@ -0,0 +1,29 @@
+import type { Config, PipelineMode, PipelineType } from "../types";
+import { writeYamlfile } from "../utils/writeFiles";
+import { createChildPipeline } from "./createChildPipeline";
+import { createMainPipeline } from "./createMainPipeline";
+import { getPipelineTriggerForGitlabChildPipeline } from "./gitlab/getPipelineTriggerForGitlabChildPipeline";
+export async function generatePipelineFiles<T extends PipelineType>(
+  config: Config,
+  pipelineType: T,
+  mode: PipelineMode<T>
+) {
+  if (mode === "childpipeline") {
+    const trigger = getPipelineTriggerForGitlabChildPipeline();
+
+    const { jobs, ...mainPipeline } = await createChildPipeline(
+      pipelineType,
+      trigger,
+      config
+    );
+    // need to spread out the jobs, forgot why
+    await writeYamlfile(`__pipeline.yml`, { ...jobs, ...mainPipeline });
+  } else {
+    const { jobs, ...mainPipeline } = await createMainPipeline(
+      pipelineType,
+      config
+    );
+    // need to spread out the jobs, forgot why
+    await writeYamlfile(`.gitlab-ci.yml`, { ...jobs, ...mainPipeline });
+  }
+}

package/src/pipeline/gitlab/createGitlabJobs.ts

@@ -1,16 +1,29 @@
-import { isObject } from "lodash";
+import { isObject, merge } from "lodash";
+import { getInjectVarsScript } from "../../bash/getInjectVarsScript";
 import { BASE_RETRY } from "../../defaults";
-import type { GitlabJobDef } from "../../types";
+import type {
+  CatladderJobWithContext,
+  Context,
+  GitlabJobDef,
+  GitlabRule,
+} from "../../types";
 import type { CatladderJob, CatladderJobNeed } from "../../types/jobs";
-import type { AllCatladderJobs } from "../createAllJobs";
 import { notNil } from "../../utils";
+import { collapseableSection } from "../../utils/gitlab";
+import type { AllCatladderJobs } from "../createAllJobs";
 
-export type AllGitlabJobs = Record<string, GitlabJobDef>;
+export type AllGitlabJobs = {
+  name: string;
+  gitlabJob: GitlabJobDef;
+  componentName: string;
+  env: string;
+}[];
 
-const removeUndefined = <T extends Record<string, unknown>>(obj: T): T =>
+export const GITLAB_ENVIRONMENT_URL_VARIABLE = "CL_GITLAB_ENVIRONMENT_URL";
+const removeUndefined = (obj: GitlabJobDef): GitlabJobDef =>
   Object.fromEntries(
     Object.entries(obj).filter(([, value]) => value !== undefined)
-  ) as T;
+  ) as GitlabJobDef;
 const getFullJobName = (
   name: string,
   componentName: string,
@@ -47,6 +60,7 @@ export const makeGitlabJob = (
   componentName: string,
   env: string,
   {
+    environment,
     envMode,
     needsStages,
     needsOtherComponent,
@@ -54,9 +68,14 @@ export const makeGitlabJob = (
     needs,
     jobTags,
     script,
+    context,
+    variables,
+    runnerVariables,
+    when,
     ...job
-  }: CatladderJob<string>,
-  allJobs: AllCatladderJobs
+  }: CatladderJobWithContext<string>,
+  allJobs: AllCatladderJobs,
+  baseRules?: GitlabRule[]
 ): [fullName: string, job: GitlabJobDef] => {
   const stage = envMode === "stagePerEnv" ? `${job.stage} ${env}` : job.stage;
 
@@ -106,57 +125,161 @@ export const makeGitlabJob = (
     envMode !== "none" ? env : undefined
   );
 
-  const gitlabJob: GitlabJobDef = removeUndefined({
-    ...job,
-    script: script?.filter(notNil),
-    tags: jobTags,
-    stage,
-    environment: job.environment?.on_stop
-      ? {
-          ...job.environment,
-          on_stop: getFullReferencedJobName(
-            job.environment.on_stop,
-            componentName,
-            env,
-            allJobs
-          ),
-        }
-      : job.environment,
-    // sort in a predictable manner for snapshot tests
-    needs: deduplicatedGitlabNeeds,
-    retry: BASE_RETRY,
-    interruptible: true,
-  });
+  // backwards compatibility, some may still use KUBERNETES_CPU_REQUEST, KUBERNETES_MEMORY_REQUEST, etc. in variables.
+  // those should now be set in the runnerVariables as they don't work in the variables key of the catladder job, becuase those get injected
+  const PIPELINE_RUNNER_VARIABLES = [
+    "KUBERNETES_CPU_REQUEST",
+    "KUBERNETES_MEMORY_REQUEST",
+    "KUBERNETES_CPU_LIMIT",
+    "KUBERNETES_MEMORY_LIMIT",
+  ];
+  // remove those from variables and add them to runnerVariables
+
+  const varsInjectScripts = collapseableSection(
+    "injectvars",
+    "Injecting variables"
+  )([
+    ...getInjectVarsScript(
+      // remove legacy variables
+      Object.fromEntries(
+        Object.entries(variables ?? {}).filter(
+          ([key]) => !PIPELINE_RUNNER_VARIABLES.includes(key)
+        )
+      )
+    ),
+  ]);
+
+  const legacyRunnerVariables = Object.fromEntries(
+    Object.entries(variables ?? {}).filter(([key]) =>
+      PIPELINE_RUNNER_VARIABLES.includes(key)
+    )
+  );
+
+  if (Object.keys(legacyRunnerVariables).length > 0) {
+    console.warn(
+      `Legacy variables detected in ${fullJobName}: ${Object.keys(
+        legacyRunnerVariables
+      ).join(", ")}. Please move them to the runnerVariables key.`
+    );
+  }
+  const rules = [
+    ...(job.rules ?? []),
+    ...(baseRules
+      ? baseRules.map((rule) => ({
+          when: when,
+          ...rule,
+        }))
+      : when
+      ? [{ when }]
+      : []),
+  ];
+  const modified = addGitlabEnvironment(
+    context,
+    environment,
+    {
+      ...job,
+      rules: rules.length > 0 ? rules : undefined,
+      variables: {
+        ...legacyRunnerVariables,
+        ...runnerVariables,
+      },
+      script: [...varsInjectScripts, ...(script?.filter(notNil) ?? [])],
+      tags: jobTags,
+      stage,
+
+      // sort in a predictable manner for snapshot tests
+      needs: deduplicatedGitlabNeeds,
+      retry: BASE_RETRY,
+      interruptible: true,
+    },
+    componentName,
+    env,
+    allJobs
+  );
+
+  const gitlabJob: GitlabJobDef = removeUndefined(modified);
 
   return [fullJobName, gitlabJob];
 };
 
-export const createGitlabJobs = async (
+const addGitlabEnvironment = (
+  context: Context,
+  environment: CatladderJob["environment"],
+  job: GitlabJobDef,
+  componentName: string,
+  env: string, // TODO: we could actually pull this from contxt
   allJobs: AllCatladderJobs
-): Promise<AllGitlabJobs> => {
-  return Object.keys(allJobs).reduce((accForComponents, componentName) => {
-    const componentJobs = allJobs[componentName];
-    return {
-      ...accForComponents,
-      ...Object.keys(componentJobs).reduce((accForEnvs, env) => {
-        const jobs = componentJobs[env];
+): GitlabJobDef => {
+  if (!environment) {
+    return job;
+  }
+  const { url, envType } = context.environment;
+  const { on_stop, ...restEnvironment } = environment;
+  // those can be dynamic, so we therefore have to do this: https://docs.gitlab.com/ee/ci/environments/#set-a-dynamic-environment-url
 
-        return {
-          ...accForEnvs,
-          ...jobs.reduce((accForJobs, job) => {
-            const [fullJobName, gitlabJob] = makeGitlabJob(
+  const dotEnvFile = "gitlab_environment.env";
+
+  const scriptToAdd = [
+    `echo "${GITLAB_ENVIRONMENT_URL_VARIABLE}=${url}" >> ${dotEnvFile}`,
+  ];
+
+  // this is NOT a bashVariable since it NEEDS to be used as a string in gitlab
+  const gitlabEnvironmentName =
+    envType === "review"
+      ? `${env}/$CI_COMMIT_REF_NAME/${componentName}` // FIXME: should be replaced with mr name as well
+      : `${env}/${componentName}`;
+
+  return {
+    ...job,
+    environment: {
+      name: gitlabEnvironmentName,
+      url: `$${GITLAB_ENVIRONMENT_URL_VARIABLE}`,
+      ...(on_stop
+        ? {
+            on_stop: getFullReferencedJobName(
+              on_stop,
               componentName,
               env,
-              job,
               allJobs
-            );
-            return {
-              ...accForJobs,
-              [fullJobName]: gitlabJob,
-            };
-          }, {} as AllGitlabJobs),
+            ),
+          }
+        : {}),
+      ...restEnvironment,
+    },
+    artifacts: merge(job.artifacts ?? {}, {
+      reports: {
+        dotenv: `${dotEnvFile}`,
+      },
+    }),
+
+    script: [...(job.script ?? []), ...scriptToAdd],
+  };
+};
+
+export const createGitlabJobs = async (
+  allJobs: AllCatladderJobs,
+  baseRules?: GitlabRule[]
+): Promise<AllGitlabJobs> => {
+  return Object.keys(allJobs).flatMap((componentName) => {
+    const componentJobs = allJobs[componentName];
+    return Object.keys(componentJobs).flatMap((env) => {
+      const jobs = componentJobs[env];
+
+      return jobs.flatMap((job) => {
+        const [fullJobName, gitlabJob] = makeGitlabJob(
+          componentName,
+          env,
+          job,
+          allJobs,
+          baseRules
+        );
+        return {
+          name: fullJobName,
+          gitlabJob,
+          componentName,
+          env,
         };
-      }, {} as AllGitlabJobs),
-    };
-  }, {} as AllGitlabJobs);
+      });
+    });
+  });
 };

package/src/pipeline/gitlab/createGitlabPipeline.ts

@@ -1,22 +1,24 @@
-import { RULES_ALWAYS } from "../../rules";
 import { getRunnerImage } from "../../runner";
 import type { Pipeline } from "../../types";
-import type { AllGitlabJobs } from "./createGitlabJobs";
 
-export function createGitlabPipelineFromStagesAndJobs(
-  stages: string[],
-  gitlabJobs: AllGitlabJobs
-): Pipeline<"gitlab"> {
+type PickRequired<T, K extends keyof T> = Required<Pick<T, K>> & Omit<T, K>;
+
+export const createGitlabPipelineWithDefaults = ({
+  image,
+  variables,
+  ...config
+}: PickRequired<
+  Partial<Pipeline<"gitlab">>,
+  "stages" | "jobs"
+>): Pipeline<"gitlab"> => {
   return {
-    image: getRunnerImage("jobs-default"), // default image
+    image: image ?? getRunnerImage("jobs-default"), // default image
     variables: {
       FF_USE_FASTZIP: "true",
-      GIT_DEPTH: 1, // no need the full depth
-    },
-    workflow: {
-      rules: RULES_ALWAYS,
+      GIT_DEPTH: "1", // no need the full depth
+      ...(variables ?? {}),
     },
-    stages,
-    jobs: gitlabJobs,
+
+    ...config,
   };
-}
+};