@catladder/pipeline 1.144.1 → 1.146.0

package/src/build/docker.ts

@@ -13,7 +13,7 @@ import {
 } from "../deploy/cloudRun/artifactsRegistry";
 import { gcloudServiceAccountLoginCommands } from "../deploy/cloudRun/utils/gcloudServiceAccountLoginCommands";
 
-const DOCKER_RUNNER_BUILD_VARIABLES = {
+const DOCKER_BUILD_RUNNER_REQUESTS = {
   KUBERNETES_CPU_REQUEST: "0.5",
   KUBERNETES_MEMORY_REQUEST: "1Gi",
   KUBERNETES_MEMORY_LIMIT: "2Gi",
@@ -52,7 +52,8 @@ export const requiresDockerBuild = ({
   isOfDeployType(deploy, "kubernetes", "google-cloudrun", "dockerTag") ||
   (isOfDeployType(deploy, "custom") && deploy.requiresDocker);
 
-const getDockerBaseVariables = () => ({
+// those need to be runner variables
+const getDockerBuildRunnerVariables = () => ({
   DOCKER_HOST: "tcp://0.0.0.0:2375",
   DOCKER_TLS_CERTDIR: "",
   DOCKER_DRIVER: "overlay2",
@@ -61,15 +62,12 @@ const getDockerBaseVariables = () => ({
 
 export const getDockerBuildVariables = (context: Context) => {
   return {
-    ...DOCKER_RUNNER_BUILD_VARIABLES,
     DOCKERFILE_ADDITIONS:
       context.componentConfig.build.docker?.additionsBegin?.join("\n"),
     DOCKERFILE_ADDITIONS_END:
       context.componentConfig.build.docker?.additionsEnd?.join("\n"),
     APP_DIR: context.componentConfig.dir,
-
     DOCKER_DIR: ".", // relative to componentdir
-    ...getDockerBaseVariables(),
 
     ...getDockerImageVariables(context),
   };
@@ -77,7 +75,12 @@ export const getDockerBuildVariables = (context: Context) => {
 
 export const DOCKER_BUILD_JOB_NAME = "🔨 docker";
 
-export const getDockerJobBaseProps = (context: Context) => {
+export const getDockerJobBaseProps = (
+  context: Context,
+): Pick<
+  CatladderJob,
+  "image" | "services" | "variables" | "runnerVariables"
+> => {
   return {
     image: getRunnerImage("docker-build"),
     services: [
@@ -86,14 +89,15 @@ export const getDockerJobBaseProps = (context: Context) => {
         command: ["--tls=false"],
       },
     ],
-    variables: getDockerBaseVariables(),
+    variables: {},
+    runnerVariables: getDockerBuildRunnerVariables(),
   };
 };
 
 export type DockerBuildJobDefinition = Partial<CatladderJob>;
 export const createDockerBuildJobBase = (
   context: Context,
-  { script, ...def }: Partial<CatladderJob>
+  { script, ...def }: Partial<CatladderJob>,
 ): CatladderJob => {
   return merge(
     {
@@ -105,8 +109,12 @@ export const createDockerBuildJobBase = (
     },
     {
       variables: getDockerBuildVariables(context),
+      runnerVariables: {
+        ...DOCKER_BUILD_RUNNER_REQUESTS,
+        ...getDockerBuildRunnerVariables(),
+      },
     },
-    def
+    def,
   );
 };
 
@@ -122,23 +130,23 @@ export const gitlabDockerLogin = (context: Context) =>
 
 export const getDockerBuildDefaultScript = (
   context: Context,
-  ensureDockerFileScript?: string
+  ensureDockerFileScript?: string,
 ) =>
   [
     ensureDockerFileScript,
     ...collapseableSection(
       "docker-login",
-      "Docker Login"
+      "Docker Login",
     )(gitlabDockerLogin(context)),
     ...collapseableSection(
       "docker-build",
-      "Docker build"
+      "Docker build",
     )([
       "docker build --network host --cache-from $DOCKER_CACHE_IMAGE --tag $DOCKER_IMAGE:$DOCKER_IMAGE_TAG -f $APP_DIR/Dockerfile . --build-arg BUILDKIT_INLINE_CACHE=1", //BUILDKIT_INLINE_CACHE,  see https://testdriven.io/blog/faster-ci-builds-with-docker-cache/
     ]),
     ...collapseableSection(
       "docker-push",
-      "Docker push and tag"
+      "Docker push and tag",
     )([
       "docker push $DOCKER_IMAGE:$DOCKER_IMAGE_TAG",
       "docker tag $DOCKER_IMAGE:$DOCKER_IMAGE_TAG $DOCKER_CACHE_IMAGE",

package/src/build/index.ts

@@ -14,7 +14,7 @@ export type BuildTypes = {
   [type in BuildConfigType]: {
     jobs: (context: Context) => CatladderJob[];
     defaults: (
-      envContext: EnvironmentContext<BuildConfigType, any>
+      envContext: EnvironmentContext<BuildConfigType, any>,
     ) => Partial<Extract<BuildConfig, { type: type }>>;
   };
 };

package/src/build/node/buildJob.ts

@@ -25,9 +25,7 @@ export const createNodeBuildJobs = (context: Context): CatladderJob[] => {
       buildConfig.buildCommand !== null
         ? {
             image: buildConfig.jobImage ?? defaultImage,
-            variables: {
-              ...NODE_RUNNER_BUILD_VARIABLES,
-            },
+            runnerVariables: NODE_RUNNER_BUILD_VARIABLES,
             cache: [
               ...(ensureArray(buildConfig.jobCache) ?? []),
               ...getNodeCache(context),
@@ -48,7 +46,7 @@ export const createNodeBuildJobs = (context: Context): CatladderJob[] => {
                 join(dir, "dist"),
                 join(dir, ".next"),
                 ...(buildConfig.artifactsPaths?.map((path) =>
-                  join(dir, path)
+                  join(dir, path),
                 ) ?? []),
               ]),
               expire_in: "1 day",
@@ -63,7 +61,7 @@ export const createNodeBuildJobs = (context: Context): CatladderJob[] => {
         context,
         buildConfig.type === "node-static" || buildConfig.type === "storybook"
           ? "ensureNginxDockerfile"
-          : "ensureNodeDockerfile"
+          : "ensureNodeDockerfile",
       ),
       cache: [...getYarnCache(context, "pull")],
       variables: {

package/src/build/node/cache.ts

@@ -6,7 +6,7 @@ import { uniq } from "lodash";
 
 export const getYarnCache = (
   context: Context,
-  policy = "pull-push"
+  policy = "pull-push",
 ): GitlabJobCache[] => {
   const componentIsInWorkspace =
     context.packageManagerInfo?.componentIsInWorkspace;
@@ -27,7 +27,7 @@ export const getYarnCache = (
 
 export const getNodeModulesCache = (
   context: Context,
-  policy = "pull-push"
+  policy = "pull-push",
 ): GitlabJobCache[] => {
   const componentIsInWorkspace =
     context.packageManagerInfo?.componentIsInWorkspace;
@@ -46,7 +46,7 @@ export const getNodeModulesCache = (
           ? uniq([
               "node_modules",
               ...(context.packageManagerInfo?.workspaces.map((w) =>
-                join(w.location, "node_modules")
+                join(w.location, "node_modules"),
               ) ?? []),
             ])
           : [join(context.componentConfig.dir, "node_modules")]),
@@ -56,7 +56,7 @@ export const getNodeModulesCache = (
 };
 export const getNodeCache = (
   context: Context,
-  policy = "pull-push"
+  policy = "pull-push",
 ): GitlabJobCache[] => {
   return [
     ...getYarnCache(context, policy),

package/src/build/node/meteor.ts

@@ -18,7 +18,7 @@ const getMeteorCache = (context: Context): GitlabJobCache[] => [
     paths: [
       join(
         context.componentConfig.dir,
-        ".meteor/local/resolver-result-cache.json"
+        ".meteor/local/resolver-result-cache.json",
       ),
       join(context.componentConfig.dir, ".meteor/local/plugin-cache"),
       join(context.componentConfig.dir, ".meteor/local/isopacks"),

package/src/build/node/testJob.ts

@@ -9,7 +9,8 @@ import { createArtifactsConfig } from "../base/createArtifactsConfig";
 
 export const createNodeTestJobs = (context: Context): CatladderJob[] => {
   // don't run tests after release
-  if (context.commitInfo?.trigger === "taggedRelease") {
+  // TODO: this will be replaced by using rules
+  if (context.trigger === "taggedRelease") {
     return [];
   }
 
@@ -18,10 +19,10 @@ export const createNodeTestJobs = (context: Context): CatladderJob[] => {
   const base: Omit<CatladderJob, "script" | "name"> = {
     variables: {
       APP_PATH: context.componentConfig.dir,
-      ...NODE_RUNNER_BUILD_VARIABLES,
       ...context.environment.jobOnlyVars.build.envVars,
       ...(buildConfig.extraVars ?? {}),
     },
+    runnerVariables: NODE_RUNNER_BUILD_VARIABLES,
     stage: "test",
     needs: [],
     envMode: "none",
@@ -46,7 +47,7 @@ export const createNodeTestJobs = (context: Context): CatladderJob[] => {
           ...createArtifactsConfig(
             context.componentConfig.dir,
             buildConfig.audit?.artifactsReports,
-            buildConfig.audit?.artifacts
+            buildConfig.audit?.artifacts,
           ),
         }
       : null;
@@ -67,7 +68,7 @@ export const createNodeTestJobs = (context: Context): CatladderJob[] => {
           ...createArtifactsConfig(
             context.componentConfig.dir,
             buildConfig.lint?.artifactsReports,
-            buildConfig.lint?.artifacts
+            buildConfig.lint?.artifacts,
           ),
         }
       : null;
@@ -89,7 +90,7 @@ export const createNodeTestJobs = (context: Context): CatladderJob[] => {
           ...createArtifactsConfig(
             context.componentConfig.dir,
             buildConfig.test?.artifactsReports,
-            buildConfig.test?.artifacts
+            buildConfig.test?.artifacts,
           ),
         }
       : null;

package/src/build/node/yarn.ts

@@ -1,3 +1,4 @@
+import { BashExpression } from "../../bash/BashExpression";
 import type { Context } from "../../types";
 import { ensureArray } from "../../utils";
 import { collapseableSection } from "../../utils/gitlab";
@@ -18,7 +19,7 @@ const getYarnInstallCommand = (context: Context) => {
 export const ensureNodeVersion = (context: Context) =>
   collapseableSection(
     "nodeinstall",
-    "Ensure node version"
+    "Ensure node version",
   )([
     "if [ -f ~/.nvm/nvm.sh ];  then source ~/.nvm/nvm.sh; fi",
     "if command -v nvm &> /dev/null && [ -f ./.nvmrc ]; then nvm install; fi",
@@ -28,7 +29,7 @@ export const getYarnInstall = (
   context: Context,
   options?: {
     noCustomPostInstall: boolean;
-  }
+  },
 ) => {
   const postInstall =
     "postInstall" in context.componentConfig.build
@@ -38,12 +39,12 @@ export const getYarnInstall = (
     ...ensureNodeVersion(context),
     ...collapseableSection(
       "yarninstall",
-      "Yarn install"
+      "Yarn install",
     )([getYarnInstallCommand(context)]),
     ...(postInstall && !options?.noCustomPostInstall
       ? collapseableSection(
           "postinstall",
-          "Custom post install"
+          "Custom post install",
         )(ensureArray(postInstall) ?? [])
       : []),
   ];
@@ -53,17 +54,19 @@ const DOCKER_COPY_FILES = `COPY --chown=node:node $APP_DIR .`;
 
 export const getDockerAppCopyAndBuildScript = (context: Context) => {
   if (context.packageManagerInfo?.isClassic) {
-    return `
+    return new BashExpression(
+      `
 RUN ${YARN_INSTALL_CLASSIC} --production --ignore-scripts
 ${DOCKER_COPY_FILES}
 RUN ${YARN_INSTALL_CLASSIC} --production 
-    `.trim();
+    `.trim(),
+    );
   }
 
   // yarn >= 4 ships with build in plugins, see https://github.com/yarnpkg/berry/pull/4253
   // trying to import those fail on this version
   const doesNotShipWithBuiltInPlugins = ["2", "3"].some((v) =>
-    context.packageManagerInfo?.version.startsWith(v)
+    context.packageManagerInfo?.version.startsWith(v),
   );
   const maybeAddWorkspaceToolsCommand = doesNotShipWithBuiltInPlugins
     ? "RUN yarn plugin import workspace-tools"
@@ -71,10 +74,12 @@ RUN ${YARN_INSTALL_CLASSIC} --production
 
   // copy first everything and then install
   // rebuild first does not work as it will run postinstall and that might require files in the app
-  return `
+  return new BashExpression(
+    `
 ${DOCKER_COPY_FILES}
 ${maybeAddWorkspaceToolsCommand}
 RUN ${YARN_BERRY_PROD_REBUILD}
 
-    `.trim();
+    `.trim(),
+  );
 };

package/src/build/rails/test.ts

@@ -4,7 +4,8 @@ import { ensureArray, notNil } from "../../utils";
 
 export const createRailsTestJobs = (context: Context): CatladderJob[] => {
   // don't run tests after release
-  if (context.commitInfo?.trigger === "taggedRelease") {
+  // TODO: this will be replaced by using rules
+  if (context.trigger === "taggedRelease") {
     return [];
   }
 

package/src/build/types.ts

@@ -43,9 +43,16 @@ export type BuildConfigBase = {
   /**
    * additional env vars for the buid jobs
    *
-   * @deprecated use jobVars
+   * @deprecated use jobVars or runnerVariables
    */
   extraVars?: Record<string, string>;
+
+  /**
+   * additional vars only for the runner.
+   * Also if you use services: that require env vars, you need to set them here.
+   *
+   */
+  runnerVariables?: Record<string, string>;
   /**
    * define the build command
    */

package/src/catladder-gitlab.ts

@@ -1,59 +1,12 @@
-import { writeFileSync } from "fs";
-import { dump } from "js-yaml";
 import { readConfigSync } from "./config";
-import { PIPELINE_IMAGE_TAG } from "./constants";
-import { createChildPipeline } from "./pipeline";
-import type { PipelineTrigger } from "./types";
+import { generatePipelineFiles } from "./pipeline/generatePipelineFiles";
+import type { PipelineMode } from "./types";
 
-const {
-  CI_MERGE_REQUEST_ID,
-  CI_COMMIT_TAG,
-  CI_COMMIT_BRANCH,
-  CI_DEFAULT_BRANCH,
-} = process.env;
+const mode = process.argv[2] || "local";
 
-const isDefaultBranch =
-  Boolean(CI_DEFAULT_BRANCH) && CI_COMMIT_BRANCH === CI_DEFAULT_BRANCH;
-const isHotfixBranch = CI_COMMIT_BRANCH
-  ? /^[0-9]+\.([0-9]+|x)\.x$/.test(CI_COMMIT_BRANCH)
-  : false;
-const isMergeRequest = Boolean(CI_MERGE_REQUEST_ID);
-const isTaggedRelease = Boolean(CI_COMMIT_TAG);
-
-console.info(`catladder version ${PIPELINE_IMAGE_TAG}`);
-
-const trigger: PipelineTrigger | null =
-  isMergeRequest || isHotfixBranch
-    ? "mr"
-    : isDefaultBranch
-    ? "mainBranch"
-    : isTaggedRelease
-    ? "taggedRelease"
-    : null;
-if (trigger) {
-  const config = readConfigSync()?.config;
-  if (!config) {
-    throw new Error("no catladder config found");
-  }
-  createChildPipeline("gitlab", trigger, config).then(
-    ({ jobs, ...mainPipeline }) => {
-      // need to spread out the jobs
-      writeFileSync(`__pipeline.yml`, dump({ ...jobs, ...mainPipeline }), {
-        encoding: "utf-8",
-      });
-    }
-  );
-} else {
-  throw new Error(
-    "no matching trigger: " +
-      JSON.stringify(
-        {
-          isMergeRequest,
-          isDefaultBranch,
-          isTaggedRelease,
-        },
-        null,
-        2
-      )
-  );
+const config = readConfigSync()?.config;
+if (!config) {
+  throw new Error("no catladder config found");
 }
+
+generatePipelineFiles(config, "gitlab", mode as PipelineMode<"gitlab">);

package/src/config/configruedEnvs.ts

@@ -4,25 +4,25 @@ import { DEFAULT_ENV_TYPES, getEnvTypesByTrigger } from "../types";
 const getConfiguredAndDefaultEnvs = (
   config: Config,
   componentName: string,
-  envTypes: EnvType[]
+  envTypes: EnvType[],
 ) => {
   const configuredEnvs = config.components[componentName].env ?? {};
   // the default envs have the same name as the env types
   // these can be disabled with settimg them to `false`
   // this is the list of all not disabled envs. These are always returned
   const enabledDefaultEnvs = envTypes.filter(
-    (e) => configuredEnvs[e] !== false
+    (e) => configuredEnvs[e] !== false,
   );
 
   const configuredCustomEnvs = Object.entries(
-    config.components[componentName].env ?? {}
+    config.components[componentName].env ?? {},
   )
     .filter(
       ([, config]) =>
         config &&
         "type" in config &&
         config.type &&
-        envTypes.includes(config.type)
+        envTypes.includes(config.type),
     )
     .map(([envName]) => envName);
 
@@ -36,7 +36,7 @@ export const getAllEnvs = (config: Config, componentName: string) => {
 export const getAllEnvsInAllComponents = (config: Config) => {
   return [
     ...new Set(
-      Object.keys(config.components).flatMap((c) => getAllEnvs(config, c))
+      Object.keys(config.components).flatMap((c) => getAllEnvs(config, c)),
     ),
   ];
 };
@@ -44,7 +44,7 @@ export const getAllEnvsInAllComponents = (config: Config) => {
 export const getAllEnvsByTrigger = (
   config: Config,
   componentName: string,
-  trigger: PipelineTrigger
+  trigger: PipelineTrigger,
 ) => {
   const envTypesByTrigger = getEnvTypesByTrigger(trigger);
   return getConfiguredAndDefaultEnvs(config, componentName, envTypesByTrigger);

package/src/config/readConfig.ts

@@ -1,6 +1,6 @@
 import { existsSync, readFileSync } from "fs";
 import { register } from "ts-node";
-import { load } from "js-yaml";
+import { parse } from "yaml";
 import type { Config } from "../types";
 
 // allows us to load ts files
@@ -14,7 +14,7 @@ function requireUncached(module: string) {
 }
 
 export const readConfigSync = (
-  directory: string = process.cwd()
+  directory: string = process.cwd(),
 ): { config: Config; path: string; ext: string } | null => {
   register({
     cwd: directory,
@@ -25,7 +25,7 @@ export const readConfigSync = (
   });
 
   const found = ["ts", "js", "yml", "yaml"].find((extension) =>
-    existsSync(fullPath(directory, extension))
+    existsSync(fullPath(directory, extension)),
   );
   if (found) {
     const filePath = fullPath(directory, found);
@@ -33,14 +33,13 @@ export const readConfigSync = (
       return {
         path: filePath,
         ext: found,
-        // eslint-disable-next-line @typescript-eslint/no-var-requires
         config: requireUncached(filePath).default,
       };
     } else {
       return {
         path: filePath,
         ext: found,
-        config: load(readFileSync(filePath, { encoding: "utf-8" })) as Config,
+        config: parse(readFileSync(filePath, { encoding: "utf-8" })) as Config,
       };
     }
   }

package/src/constants.ts

@@ -1,2 +1,5 @@
-export const PIPELINE_IMAGE_TAG = process.env.INLINE_PIPELINE_IMAGE_TAG || "";
-export const DOCKER_REGISTRY = process.env.INLINE_DOCKER_REGISTRY || "";
+export const PIPELINE_IMAGE_TAG =
+  process.env.INLINE_PIPELINE_IMAGE_TAG || "latest";
+export const DOCKER_REGISTRY =
+  process.env.INLINE_DOCKER_REGISTRY ||
+  "git.panter.ch:5001/catladder/catladder";

package/src/context/__tests__/resolveReferences.test.ts

@@ -1,8 +1,13 @@
+import type { BashExpression } from "../../bash/BashExpression";
 import {
   resolveReferences,
   translateLegacyFromComponents,
 } from "../resolveReferences";
 
+const unpackBashExpressions = (obj: Record<string, string | BashExpression>) =>
+  Object.fromEntries(
+    Object.entries(obj).map(([key, value]) => [key, value.toString()]),
+  );
 describe("resolveReferences", () => {
   it("replaces occurences of ${componentName:VARIABLE_NAME}", async () => {
     const variables = {
@@ -21,7 +26,7 @@ describe("resolveReferences", () => {
       return otherVariables[componentName];
     });
 
-    expect(result).toEqual({
+    expect(unpackBashExpressions(result)).toEqual({
       a: "hello world",
       b: "a replaced value looks like this: 'foo from api', nice!",
     });
@@ -34,7 +39,7 @@ describe("resolveReferences", () => {
     };
     const result = await resolveReferences(variables);
 
-    expect(result).toEqual({
+    expect(unpackBashExpressions(result)).toEqual({
       FOO: "hello world",
       BAR: "this: hello world!",
     });
@@ -54,7 +59,7 @@ describe("resolveReferences", () => {
       return otherVariables[componentName];
     });
 
-    expect(result).toEqual({
+    expect(unpackBashExpressions(result)).toEqual({
       FOO: "hello from foo from api",
       BAR: "this: hello from foo from api!",
     });
@@ -70,7 +75,7 @@ describe("resolveReferences", () => {
       return otherVariables[componentName];
     });
 
-    expect(result).toEqual({
+    expect(unpackBashExpressions(result)).toEqual({
       a: "hello world",
       b: "a not found value looks like this: '${api:FOO}'",
     });
@@ -92,7 +97,7 @@ describe("resolveReferences", () => {
       return otherVariables[componentName];
     });
 
-    expect(result).toEqual({
+    expect(unpackBashExpressions(result)).toEqual({
       a: "value is hi, foo from api!",
     });
   });
@@ -113,7 +118,7 @@ describe("resolveReferences", () => {
       return otherVariables[componentName];
     });
 
-    expect(result).toEqual({
+    expect(unpackBashExpressions(result)).toEqual({
       a: "value is frontend api ${frontend:FOO}!",
     });
   });

package/src/context/getBuildInfoVariables.ts

@@ -0,0 +1,40 @@
+import { BashExpression } from "../bash/BashExpression";
+import type { BashExpressionPerPipelineType } from "../bash/bashExpressionPerPipelineType";
+import { getBashExpressionPerPipelineType } from "../bash/bashExpressionPerPipelineType";
+import type { EnvironmentContext } from "../types/environmentContext";
+
+const BUILD_TIME: BashExpressionPerPipelineType = {
+  default: "unknown-build-time",
+  gitlab: new BashExpression(`$CI_JOB_STARTED_AT`),
+};
+
+const BUILD_ID: BashExpressionPerPipelineType = {
+  default: new BashExpression(
+    `$(git describe --tags 2>/dev/null || git rev-parse HEAD)`,
+  ),
+};
+const CURRENT_VERSION: BashExpressionPerPipelineType = {
+  default: new BashExpression(
+    // because we do shallow fetch, we need to ask the origin
+    `$(tag=$(git ls-remote origin "refs/tags/v*[0-9]" 2>/dev/null | cut -f 2- | sort -V | tail -1 | sed 's/refs\\/tags\\///'); [ -z "$tag" ] && echo "v0.0.0" || echo "$tag")`,
+  ),
+};
+
+export const getBuildInfoVariables = (ctx: EnvironmentContext<any, any>) => {
+  const { pipelineType } = ctx;
+
+  return {
+    BUILD_INFO_BUILD_ID: getBashExpressionPerPipelineType(
+      BUILD_ID,
+      pipelineType,
+    ),
+    BUILD_INFO_BUILD_TIME: getBashExpressionPerPipelineType(
+      BUILD_TIME,
+      pipelineType,
+    ),
+    BUILD_INFO_CURRENT_VERSION: getBashExpressionPerPipelineType(
+      CURRENT_VERSION,
+      pipelineType,
+    ),
+  };
+};

package/src/context/getEnvConfig.ts

@@ -5,7 +5,7 @@ import { mergeWithMergingArrays } from "../utils";
 export const getEnvConfig = (
   config: Config,
   componentName: string,
-  env: string
+  env: string,
 ): EnvConfigWithComponent => {
   const defaultConfig = config.components[componentName];
   if (!defaultConfig) {

package/src/context/getEnvType.ts

@@ -5,7 +5,7 @@ export const getEnvType = (
   env: string,
   envConfig: {
     type?: EnvType;
-  }
+  },
 ): EnvType => {
   if (envConfig.type) return envConfig.type;