@catladder/pipeline 0.0.0-node16-ddfae617 → 0.0.0-prettier-cbceeed9

package/src/build/custom/buildJob.ts

@@ -0,0 +1,64 @@
+import { join } from "path";
+import type { Context } from "../../types/context";
+import { ensureArray } from "../../utils";
+import { APP_BUILD_JOB_NAME } from "../base/constants";
+import { createBuildJob } from "../base/createBuildJob";
+import { createDockerBuildJobDefault, requiresDockerBuild } from "../docker";
+import { isOfBuildType } from "../types";
+
+import type { CatladderJob } from "../../types/jobs";
+
+const RUNNER_BUILD_VARIABLES = {
+  KUBERNETES_CPU_REQUEST: "0.5",
+  KUBERNETES_CPU_LIMIT: "2",
+  KUBERNETES_MEMORY_REQUEST: "2Gi",
+  KUBERNETES_MEMORY_LIMIT: "4Gi",
+};
+
+export const createCustomBuildJobs = (context: Context): CatladderJob[] => {
+  const buildConfig = context.componentConfig.build;
+
+  if (!isOfBuildType(buildConfig, "custom")) {
+    throw new Error("deploy config is not custom");
+  }
+
+  const appBuildJob: CatladderJob | null =
+    buildConfig.buildCommand !== null
+      ? createBuildJob(context, {
+          image: buildConfig.jobImage,
+          variables: {
+            ...RUNNER_BUILD_VARIABLES,
+          },
+          cache: buildConfig.jobCache,
+          services: buildConfig.jobServices,
+          script: [...(ensureArray(buildConfig.buildCommand) ?? [])],
+          artifacts: {
+            paths: [
+              join(context.componentConfig.dir, "__build_info.json"),
+              join(context.componentConfig.dir, "dist"),
+              ...(buildConfig.artifactsPaths?.map((path) =>
+                join(context.componentConfig.dir, path)
+              ) ?? []),
+            ],
+          },
+        })
+      : null;
+  return [
+    ...(appBuildJob ? [appBuildJob] : []),
+    ...(requiresDockerBuild(context)
+      ? [
+          createDockerBuildJobDefault(context, {
+            script: [
+              buildConfig.docker?.type === "nginx"
+                ? "ensureNginxDockerfile"
+                : "", // e.g. custom,
+            ].filter(Boolean),
+
+            variables: {},
+
+            needs: appBuildJob ? [APP_BUILD_JOB_NAME] : [],
+          }),
+        ]
+      : []),
+  ];
+};

package/src/build/custom/index.ts

@@ -0,0 +1,7 @@
+import type { Context } from "../../types/context";
+import type { CatladderJob } from "../../types/jobs";
+import { createCustomBuildJobs } from "./buildJob";
+
+export const createCustomJobs = (context: Context): CatladderJob[] => {
+  return [...createCustomBuildJobs(context)];
+};

package/src/build/docker.ts

@@ -1,8 +1,8 @@
 import { merge } from "lodash";
 import { isOfDeployType } from "../deploy";
 import { getRunnerImage } from "../runner";
-import { Context } from "../types";
-import { CatladderJob } from "../types/jobs";
+import type { Context } from "../types";
+import type { CatladderJob } from "../types/jobs";
 
 const DOCKER_RUNNER_BUILD_VARIABLES = {
   KUBERNETES_CPU_REQUEST: "0.5",
@@ -11,73 +11,97 @@ const DOCKER_RUNNER_BUILD_VARIABLES = {
   KUBERNETES_MEMORY_LIMIT: "2Gi",
 };
 
+export const getDockerImageVariables = (context: Context) => {
+  return {
+    DOCKER_REGISTRY: "$CI_REGISTRY",
+    DOCKER_REGISTRY_IMAGE_PATH: "$CI_REGISTRY_IMAGE",
+    DOCKER_CACHE_IMAGE:
+      "$DOCKER_REGISTRY_IMAGE_PATH/caches/" + context.componentName,
+    DOCKER_IMAGE_NAME:
+      context.environment.shortName + "/" + context.componentName,
+    DOCKER_IMAGE: "$DOCKER_REGISTRY_IMAGE_PATH/$DOCKER_IMAGE_NAME",
+    DOCKER_IMAGE_TAG: "$CI_COMMIT_SHA",
+  };
+};
+
 export const requiresDockerBuild = (context: Context) => {
   const deployConfig = context.componentConfig.deploy;
   if (isOfDeployType(deployConfig, "kubernetes")) {
     return true;
   }
 
+  if (isOfDeployType(deployConfig, "google-cloudrun")) {
+    return true;
+  }
+
   if (isOfDeployType(deployConfig, "custom") && deployConfig.requiresDocker) {
     return true;
   }
 
+  if (isOfDeployType(deployConfig, "dockerTag")) {
+    return true;
+  }
+
   return false;
 };
 
+const getDockerBaseVariables = () => ({
+  DOCKER_HOST: "tcp://0.0.0.0:2375",
+  DOCKER_TLS_CERTDIR: "",
+  DOCKER_DRIVER: "overlay2",
+  DOCKER_BUILDKIT: "1", // see https://docs.docker.com/develop/develop-images/build_enhancements/
+});
+
 export const getDockerBuildVariables = (context: Context) => {
   return {
     ...DOCKER_RUNNER_BUILD_VARIABLES,
-    DOCKER_BUILDKIT: "1", // see https://docs.docker.com/develop/develop-images/build_enhancements/
     DOCKERFILE_ADDITIONS:
       context.componentConfig.build.docker?.additionsBegin?.join("\n"),
     DOCKERFILE_ADDITIONS_END:
       context.componentConfig.build.docker?.additionsEnd?.join("\n"),
     APP_DIR: context.componentConfig.dir,
-    DOCKER_HOST: "tcp://0.0.0.0:2375",
-    DOCKER_TLS_CERTDIR: "",
-    DOCKER_DIR: ".", // relative to componentdir
-    IMAGE_TAG: "$CI_COMMIT_SHA",
-    DOCKER_DRIVER: "overlay2",
 
-    IMAGE_NAME:
-      "$CI_REGISTRY_IMAGE/" +
-      context.environment.shortName +
-      "/" +
-      context.componentName,
+    DOCKER_DIR: ".", // relative to componentdir
+    ...getDockerBaseVariables(),
 
-    CACHE_IMAGE: "$CI_REGISTRY_IMAGE/caches/" + context.componentName,
+    ...getDockerImageVariables(context),
   };
 };
 
 export const DOCKER_BUILD_JOB_NAME = "🔨 docker";
-
+export const getDockerJobBaseProps = (context: Context) => {
+  return {
+    image: getRunnerImage("docker-build"),
+    services: [
+      {
+        name: "docker:20-dind", // see see https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27300#note_466755332
+        command: ["--tls=false"],
+      },
+    ],
+    variables: getDockerBaseVariables(),
+  };
+};
 export const createDockerBuildJobBase = (
   context: Context,
-  { script, variables, ...def }: Partial<CatladderJob>
+  { script, ...def }: Partial<CatladderJob>
 ): CatladderJob => {
   return merge(
     {
       name: DOCKER_BUILD_JOB_NAME,
       envMode: "jobPerEnv",
       stage: "build",
-      image: getRunnerImage("docker-build"),
-
-      services: [
-        {
-          name: "docker:20-dind", // see see https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27300#note_466755332
-          command: ["--tls=false"],
-        },
-      ],
-      variables: {
-        ...getDockerBuildVariables(context),
-        ...(variables ?? {}),
-      },
+      ...getDockerJobBaseProps(context),
       script: script || [],
     },
+    {
+      variables: getDockerBuildVariables(context),
+    },
     def
   );
 };
 
+export const gitlabDockerLogin =
+  "docker login --username gitlab-ci-token --password $CI_JOB_TOKEN $CI_REGISTRY";
 export const createDockerBuildJobDefault = (
   context: Context,
   { script, ...def }: Partial<CatladderJob>
@@ -85,11 +109,12 @@ export const createDockerBuildJobDefault = (
   return createDockerBuildJobBase(context, {
     script: [
       ...(script || []),
-      "docker login --username gitlab-ci-token --password $CI_JOB_TOKEN $CI_REGISTRY",
-      "docker build --network host --cache-from $CACHE_IMAGE --tag $IMAGE_NAME:$IMAGE_TAG -f $APP_DIR/Dockerfile . --build-arg BUILDKIT_INLINE_CACHE=1", //BUILDKIT_INLINE_CACHE,  see https://testdriven.io/blog/faster-ci-builds-with-docker-cache/
-      "docker push $IMAGE_NAME:$IMAGE_TAG",
-      "docker tag $IMAGE_NAME:$IMAGE_TAG $CACHE_IMAGE",
-      "docker push $CACHE_IMAGE",
+      gitlabDockerLogin,
+      "docker version",
+      "docker build --network host --cache-from $DOCKER_CACHE_IMAGE --tag $DOCKER_IMAGE:$DOCKER_IMAGE_TAG -f $APP_DIR/Dockerfile . --build-arg BUILDKIT_INLINE_CACHE=1", //BUILDKIT_INLINE_CACHE,  see https://testdriven.io/blog/faster-ci-builds-with-docker-cache/
+      "docker push $DOCKER_IMAGE:$DOCKER_IMAGE_TAG",
+      "docker tag $DOCKER_IMAGE:$DOCKER_IMAGE_TAG $DOCKER_CACHE_IMAGE",
+      "docker push $DOCKER_CACHE_IMAGE",
     ],
     ...def,
   });

package/src/build/index.ts

@@ -1,16 +1,21 @@
-import { Context } from "../types/context";
+import type { Context } from "../types/context";
+import type { EnvironmentContext } from "../types/environmentContext";
 
-import { CatladderJob } from "../types/jobs";
+import type { CatladderJob } from "../types/jobs";
+import { createCustomJobs } from "./custom";
 import { createNodeJobs, createStorybookJobs, createMeteorJobs } from "./node";
+import { createRailsJobs } from "./rails";
 
-import { BuildConfig } from "./types";
+import type { BuildConfig, BuildConfigType } from "./types";
 export * from "./types";
 export * from "./node";
 
 export type BuildTypes = {
-  [type in BuildConfig["type"]]: {
+  [type in BuildConfigType]: {
     jobs: (context: Context) => CatladderJob[];
-    defaults: () => Partial<Extract<BuildConfig, { type: type }>>;
+    defaults: (
+      envContext: EnvironmentContext<BuildConfigType, any>
+    ) => Partial<Extract<BuildConfig, { type: type }>>;
   };
 };
 
@@ -40,4 +45,18 @@ export const BUILD_TYPES: BuildTypes = {
       startCommand: "node main.js",
     }),
   },
+  custom: {
+    jobs: createCustomJobs,
+    defaults: () => ({}),
+  },
+  rails: {
+    jobs: createRailsJobs,
+    defaults: () => ({
+      startCommand: "/cnb/process/web",
+      cnbBuilder: {
+        image: "heroku/buildpacks:20",
+        packVersion: "0.28.0",
+      },
+    }),
+  },
 };

package/src/build/node/buildJob.ts

@@ -1,4 +1,4 @@
-import { Context } from "../../types/context";
+import type { Context } from "../../types/context";
 import { ensureArray } from "../../utils";
 import { APP_BUILD_JOB_NAME } from "../base/constants";
 import { createBuildJob } from "../base/createBuildJob";
@@ -8,7 +8,7 @@ import { isOfBuildType } from "../types";
 import { getNextCache, getNodeCache, getYarnCache } from "./cache";
 import { NODE_RUNNER_BUILD_VARIABLES } from "./constants";
 import { getYarnInstall, getYarnInstallCommand } from "./yarn";
-import { CatladderJob } from "../../types/jobs";
+import type { CatladderJob } from "../../types/jobs";
 
 export const createNodeBuildJobs = (context: Context): CatladderJob[] => {
   const buildConfig = context.componentConfig.build;

package/src/build/node/cache.ts

@@ -1,17 +1,26 @@
 import { join } from "path";
-import { Context } from "../../types/context";
-import { GitlabJobCache } from "../../types/gitlab-types";
+import slugify from "slugify";
+import type { Context } from "../../types/context";
+import type { GitlabJobCache } from "../../types/gitlab-types";
 
 export const getYarnCache = (
   context: Context,
   policy = "pull-push"
 ): GitlabJobCache[] => {
+  const componentIsInWorkspace =
+    context.packageManagerInfo?.componentIsInWorkspace;
   return [
-    {
-      key: "yarn",
-      policy,
-      paths: [".yarn"],
-    },
+    componentIsInWorkspace
+      ? {
+          key: "yarn",
+          policy,
+          paths: [".yarn"],
+        }
+      : {
+          key: slugify(context.componentConfig.dir) + "-yarn",
+          policy,
+          paths: [join(context.componentConfig.dir, ".yarn")],
+        },
   ];
 };
 
@@ -22,12 +31,14 @@ export const getNodeModulesCache = (
   const componentIsInWorkspace =
     context.packageManagerInfo?.componentIsInWorkspace;
 
+  // We intentionally do not use the contents of yarn.lock as a cache key, as yarn install should always guarantee that the files are updated, but it can still use part of the cache if not all packages are up-to-date.
+  // It would slow down all pipelines whenever one adds a new dependency as it will need to download all node_modules again.
   return [
     {
       // if component is in a shared workspace, use workspace cache. use individual cache else
       key: componentIsInWorkspace
         ? "node-modules-workspace"
-        : context.componentConfig.dir + "-node-modules", // we use the dirname, not the component name, because in certain cases we have two apps in the same directory and want to share the cache, e.g. when having storybook in the same package.json
+        : slugify(context.componentConfig.dir) + "-node-modules", // we use the dirname, not the component name, because in certain cases we have two apps in the same directory and want to share the cache, e.g. when having storybook in the same package.json
       policy,
       paths: [
         ...(componentIsInWorkspace

package/src/build/node/index.ts

@@ -1,5 +1,5 @@
 import type { Context } from "../../types/context";
-import { CatladderJob } from "../../types/jobs";
+import type { CatladderJob } from "../../types/jobs";
 import { createNodeBuildJobs } from "./buildJob";
 import { createMeteorBuildJobs } from "./meteor";
 import { createNodeTestJobs } from "./testJob";

package/src/build/node/meteor.ts

@@ -1,9 +1,9 @@
 import { join } from "path";
 import { getRunnerImage } from "../../runner";
-import { GitlabJobCache } from "../../types";
+import type { GitlabJobCache } from "../../types";
 import type { Context } from "../../types/context";
 
-import { CatladderJob } from "../../types/jobs";
+import type { CatladderJob } from "../../types/jobs";
 
 import { APP_BUILD_JOB_NAME } from "../base/constants";
 import { createBuildJob } from "../base/createBuildJob";
@@ -40,6 +40,9 @@ export const createMeteorBuildJobs = (context: Context): CatladderJob[] => {
       ? createBuildJob(context, {
           cache: [...getNodeCache(context), ...getMeteorCache(context)],
           image: getRunnerImage("jobs-meteor"),
+          variables: {
+            METEOR_DISABLE_OPTIMISTIC_CACHING: "1", // see https://forums.meteor.com/t/veeery-long-building-time-inside-docker-container/58673/17?u=macrozone
+          },
           script: [
             ...yarnInstall,
 
@@ -63,6 +66,9 @@ export const createMeteorBuildJobs = (context: Context): CatladderJob[] => {
     ...(appBuildJob ? [appBuildJob] : []),
     createDockerBuildJobDefault(context, {
       script: ["ensureMeteorDockerfile"],
+      variables: {
+        METEOR_INSTALL_SCRIPTS: buildConfig.installScripts ? "true" : "",
+      },
       needs: appBuildJob ? [APP_BUILD_JOB_NAME] : [],
     }),
   ];

package/src/build/node/testJob.ts

@@ -1,5 +1,6 @@
-import { Context } from "../../types/context";
-import { CatladderJob } from "../../types/jobs";
+import { getRunnerImage } from "../../runner";
+import type { Context } from "../../types/context";
+import type { CatladderJob } from "../../types/jobs";
 import { ensureArray, notNil } from "../../utils";
 import { getNodeCache } from "./cache";
 import { NODE_RUNNER_BUILD_VARIABLES } from "./constants";
@@ -63,6 +64,7 @@ export const createNodeTestJobs = (context: Context): CatladderJob[] => {
           name: "🧪 test",
 
           ...base,
+          image: getRunnerImage("jobs-testing-chrome"),
           cache: getNodeCache(context),
           script: [
             `cd ${context.componentConfig.dir}`,

package/src/build/node/yarn.ts

@@ -1,4 +1,4 @@
-import { Context } from "../../types";
+import type { Context } from "../../types";
 
 export const getYarnInstallCommand = (
   context: Context,
@@ -6,18 +6,29 @@ export const getYarnInstallCommand = (
     prodOnly?: boolean;
     noScripts?: boolean;
   }
-) =>
-  context.packageManagerInfo?.isClassic
-    ? `yarn install --frozen-lockfile ${
-        options?.prodOnly ? "--production" : ""
-      } ${options?.noScripts ? "--ignore-scripts" : ""}`
-    : options?.prodOnly
-    ? `${
-        options?.noScripts ? "YARN_ENABLE_SCRIPTS=false " : ""
-      }yarn plugin import workspace-tools && yarn workspaces focus --production` // needs yarn plugin import workspace-tools
+) => {
+  if (context.packageManagerInfo?.isClassic) {
+    return `yarn install --frozen-lockfile ${
+      options?.prodOnly ? "--production" : ""
+    } ${options?.noScripts ? "--ignore-scripts" : ""}`;
+  }
+  // is modern
+
+  // yarn >= 4 ships with build in plugins, see https://github.com/yarnpkg/berry/pull/4253
+  // trying to import those fail on this version
+  const doesNotShipWithBuiltInPlugins = ["2", "3"].some((v) =>
+    context.packageManagerInfo?.version.startsWith(v)
+  );
+  return options?.prodOnly
+    ? `${options?.noScripts ? "YARN_ENABLE_SCRIPTS=false " : ""}${
+        doesNotShipWithBuiltInPlugins
+          ? "yarn plugin import workspace-tools && "
+          : " "
+      }yarn workspaces focus --production && yarn rebuild` // needs yarn plugin import workspace-tools
     : `${
         options?.noScripts ? "YARN_ENABLE_SCRIPTS=false " : ""
       }yarn install --immutable`;
+};
 
 export const ensureNodeVersion = (context: Context) => [
   "if [ -f ./.nvmrc ]; then source /root/.nvm/nvm.sh && nvm install <<< .nvmrc; fi",

package/src/build/rails/build.ts

@@ -0,0 +1,38 @@
+import type { Context } from "../..";
+import { isOfBuildType } from "../types";
+import type { CatladderJob } from "../../types/jobs";
+import { createDockerBuildJobBase, gitlabDockerLogin } from "../docker";
+
+export const createRailsBuildJobs = (context: Context): CatladderJob[] => {
+  const buildConfig = context.componentConfig.build;
+  if (!isOfBuildType(buildConfig, "rails")) {
+    // should not happen
+    throw new Error("build type is not rails");
+  }
+
+  const cnbConf = buildConfig.cnbBuilder;
+
+  const packEnvArgs = Object.entries(cnbConf?.buildVars ?? {})
+    .map(([k, v]) => `--env '${k}${v ? `=${v}` : ""}'`)
+    .join(" ");
+
+  return [
+    createDockerBuildJobBase(context, {
+      variables: context.componentConfig.build.extraVars,
+      // custom script
+      script: [
+        gitlabDockerLogin,
+        `docker pull $DOCKER_CACHE_IMAGE || true`,
+        `wget --output-document=- https://github.com/buildpacks/pack/releases/download/v${cnbConf?.packVersion}/pack-v${cnbConf?.packVersion}-linux.tgz | tar -zx --directory /usr/local/bin pack`,
+        `chmod +x /usr/local/bin/pack`,
+        //  replace private git ssh gem sources with https to make bundler with credentials via env var work
+        `sed --in-place 's|git@\\([^:]*\\):|https://\\1/|g' Gemfile Gemfile.lock`,
+        `pack build "$DOCKER_IMAGE:$DOCKER_IMAGE_TAG" --builder '${
+          cnbConf?.image
+        }' --publish --cache-image "$DOCKER_CACHE_IMAGE" ${packEnvArgs} ${
+          cnbConf?.packExtraArgs?.join(" ") ?? ""
+        }`,
+      ],
+    }),
+  ];
+};

package/src/build/rails/index.ts

@@ -0,0 +1,8 @@
+import type { Context } from "../../types";
+import type { CatladderJob } from "../../types/jobs";
+import { createRailsBuildJobs } from "./build";
+import { createRailsTestJobs } from "./test";
+
+export const createRailsJobs = (context: Context): CatladderJob[] => {
+  return [...createRailsTestJobs(context), ...createRailsBuildJobs(context)];
+};

package/src/build/rails/test.ts

@@ -0,0 +1,86 @@
+import type { Context } from "../..";
+import type { CatladderJob } from "../../types/jobs";
+import { ensureArray, notNil } from "../../utils";
+
+export const createRailsTestJobs = (context: Context): CatladderJob[] => {
+  // don't run tests after release
+  if (context.commitInfo?.trigger === "taggedRelease") {
+    return [];
+  }
+
+  const buildConfig = context.componentConfig.build;
+
+  const base: Omit<CatladderJob, "script" | "name"> = {
+    variables: buildConfig.extraVars ?? {},
+    stage: "test",
+    needs: [],
+    envMode: "none",
+  };
+  const defaultImage = "docker.io/ruby";
+  const bundlerCacheDir = "tmp/cache";
+  const bundlerInstall = [
+    `bundle config set path '${bundlerCacheDir}'`,
+    "bundle install -j $(nproc)",
+  ];
+  const bundlerCache = {
+    key: {
+      files: ["Gemfile.lock"],
+      prefix: "$CI_JOB_IMAGE", // a changed image might have different OS libraries which no longer work with the cached gems
+    },
+    paths: [bundlerCacheDir],
+  };
+  const auditJob: CatladderJob | null =
+    buildConfig.audit !== false
+      ? {
+          name: "🛡 audit",
+
+          ...base,
+          cache: undefined, // audit does not need bundle install and no cache
+          image: buildConfig.audit?.jobImage ?? defaultImage,
+          script: [
+            `cd ${context.componentConfig.dir}`,
+            ...(ensureArray(buildConfig.audit?.command) ?? [
+              "gem install bundler-audit",
+              "bundle audit check",
+            ]),
+          ],
+          allow_failure: true,
+        }
+      : null;
+
+  const lintJob: CatladderJob | null =
+    buildConfig.lint !== false
+      ? {
+          name: "👮 lint",
+
+          ...base,
+          cache: bundlerCache,
+          image: buildConfig.lint?.jobImage ?? defaultImage,
+          script: [
+            `cd ${context.componentConfig.dir}`,
+            ...bundlerInstall,
+            ...(ensureArray(buildConfig.lint?.command) ?? [
+              "bundle exec rubocop",
+            ]),
+          ],
+        }
+      : null;
+  const testJob: CatladderJob | null =
+    buildConfig.test !== false
+      ? {
+          name: "🧪 test",
+
+          ...base,
+          cache: bundlerCache,
+          image: buildConfig.test?.jobImage ?? defaultImage,
+          script: [
+            `cd ${context.componentConfig.dir}`,
+            ...bundlerInstall,
+            ...(ensureArray(buildConfig.test?.command) ?? [
+              "bundle exec rspec",
+            ]),
+          ],
+        }
+      : null;
+  return [auditJob, lintJob, testJob].filter(notNil);
+};