@catladder/pipeline 1.12.0 → 1.13.0

package/dist/types/jobs.d.ts

@@ -0,0 +1,75 @@
+import { Artifacts, GitlabEnvironment, GitlabJobCache, GitlabRule, GitlabVariables, Service } from "./gitlab-types";
+export declare const BASE_STAGES: readonly ["setup", "test", "build", "deploy", "verify", "stop"];
+export declare type BaseStage = typeof BASE_STAGES[number];
+export declare type CatladderJob<S = BaseStage> = {
+  /**
+   * the name of the job (without any env or app prefix and suffix)
+   */
+  name: string;
+  /**
+   * envMode sets the behavior of the job regarding multiple envs:
+   * - none: the job does not run per env, but once for all envs
+   * - jobPerEnv: the job runs once per env
+   * - stagePerEnv: the job runs once per env and is organized in its own stage. This mproves usability in gitlab, but works the same as `jobPerEnv`
+   */
+
+  envMode: "jobPerEnv" | "stagePerEnv" | "none";
+  /**
+   * the stage of the job
+   */
+
+  stage: S;
+  /**
+   * does this require another stage?
+   */
+
+  /**
+   * script to run
+   */
+
+  script: (string | undefined)[];
+  needsStages?: {
+    stage: S;
+    artifacts?: boolean;
+  }[];
+  /**
+   * does this require another job?
+   */
+
+  needs?: Array<string | {
+    job: string;
+    artifacts: boolean;
+  }>;
+  /**
+   * cache config, we use here the same shape as gitlab itself
+   */
+
+  cache?: GitlabJobCache | GitlabJobCache[];
+  /**
+   * job artifacts, we also use gitlab shape here
+   */
+
+  artifacts?: Artifacts;
+  /**
+   * additional services, mainly used for docker
+   */
+
+  services?: Service[];
+  /**
+   * image to use
+   */
+
+  image?: string;
+  /**
+   * variables to pass
+   */
+
+  variables: GitlabVariables;
+  /**
+   * whether failures are allowed
+   */
+
+  allow_failure?: boolean;
+  environment?: GitlabEnvironment;
+  rules?: GitlabRule[];
+};

package/dist/types/jobs.js

@@ -0,0 +1,5 @@
+"use strict";
+
+exports.__esModule = true;
+exports.BASE_STAGES = void 0;
+exports.BASE_STAGES = ["setup", "test", "build", "deploy", "verify", "stop"];

package/dist/types/pipeline.d.ts

@@ -0,0 +1,4 @@
+import { GitlabJobDef, GitlabPipeline } from "./gitlab-types";
+export declare type PipelineType = "gitlab";
+export declare type PipelineJob<T extends PipelineType> = T extends "gitlab" ? GitlabJobDef : never;
+export declare type Pipeline<T extends PipelineType> = T extends "gitlab" ? GitlabPipeline : never;

package/dist/types/pipeline.js

@@ -0,0 +1,3 @@
+"use strict";
+
+exports.__esModule = true;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@catladder/pipeline",
-  "version": "1.12.0",
+  "version": "1.13.0",
   "scripts": {
     "build:tsc": "yarn tsc",
     "build": "yarn tsc && yarn build:inline-variables",

package/src/build/base/createBuildJob.ts

@@ -1,6 +1,6 @@
 import { merge } from "lodash";
-import { Context, getRunnerImage, GitlabJob, GitlabJobDef } from "../..";
-import { BASE_RETRY } from "../../defaults";
+import { Context, getRunnerImage } from "../..";
+import { CatladderJob } from "../../types/jobs";
 import { ensureArray } from "../../utils";
 import {
   APP_BUILD_JOB_NAME,
@@ -10,36 +10,32 @@ import { getBuildInfo } from "./getBuildInfo";
 
 export const createBuildJob = (
   context: Context,
-  { script, variables, ...def }: Partial<GitlabJobDef>
-): GitlabJob => {
-  return {
-    name: APP_BUILD_JOB_NAME,
-    envMode: "jobPerEnv",
-    job: merge(
-      {
-        stage: "build",
-        image: getRunnerImage("jobs-default"),
-        needs: [],
-        cache: [],
-        variables: {
-          ...RUNNER_BUILD_RESOURCE_VARIABLES,
-          ...(variables ?? {}),
-          ...context.environment.envVars,
-          ...(context.componentConfig.build.extraVars ?? {}),
-        },
-        retry: BASE_RETRY,
-        interruptible: true,
+  { script, variables, ...def }: Partial<CatladderJob>
+): CatladderJob => {
+  return merge(
+    {
+      name: APP_BUILD_JOB_NAME,
+      envMode: "jobPerEnv",
+      stage: "build",
+      image: getRunnerImage("jobs-default"),
+      needs: [],
+      cache: [],
+      variables: {
+        ...RUNNER_BUILD_RESOURCE_VARIABLES,
+        ...(variables ?? {}),
+        ...context.environment.envVars,
+        ...(context.componentConfig.build.extraVars ?? {}),
+      },
 
-        script: [
-          ...getBuildInfo(context),
-          `cd ${context.componentConfig.dir}`,
-          ...(ensureArray(script) ?? []),
-        ],
-        artifacts: {
-          paths: [context.componentConfig.dir + "/__build_info.json"],
-        },
+      script: [
+        ...getBuildInfo(context),
+        `cd ${context.componentConfig.dir}`,
+        ...(ensureArray(script) ?? []),
+      ],
+      artifacts: {
+        paths: [context.componentConfig.dir + "/__build_info.json"],
       },
-      def
-    ),
-  };
+    },
+    def
+  );
 };

package/src/build/docker.ts

@@ -1,7 +1,7 @@
-import { getRunnerImage } from "../runner";
-import { GitlabJobDef, Context, GitlabJob } from "../types";
-
 import { merge } from "lodash";
+import { getRunnerImage } from "../runner";
+import { Context } from "../types";
+import { CatladderJob } from "../types/jobs";
 
 const DOCKER_RUNNER_BUILD_VARIABLES = {
   KUBERNETES_CPU_REQUEST: "0.5",
@@ -14,55 +14,53 @@ export const DOCKER_BUILD_JOB_NAME = "🔨 docker";
 
 export const createDockerBuildJob = (
   context: Context,
-  { script, variables, ...def }: Partial<GitlabJobDef>
-): GitlabJob => {
-  return {
-    name: DOCKER_BUILD_JOB_NAME,
-    envMode: "jobPerEnv",
-    job: merge(
-      {
-        stage: "build",
-        image: getRunnerImage("docker-build"),
-        interruptible: true,
-        services: [
-          {
-            name: "docker:20-dind", // see see https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27300#note_466755332
-            command: ["--tls=false"],
-          },
-        ],
-        variables: {
-          ...DOCKER_RUNNER_BUILD_VARIABLES,
-          DOCKER_BUILDKIT: "1", // see https://docs.docker.com/develop/develop-images/build_enhancements/
-          DOCKERFILE_ADDITIONS:
-            context.componentConfig.build.docker?.additionsBegin?.join("\n"),
-          DOCKERFILE_ADDITIONS_END:
-            context.componentConfig.build.docker?.additionsEnd?.join("\n"),
-          APP_DIR: context.componentConfig.dir,
-          DOCKER_HOST: "tcp://0.0.0.0:2375",
-          DOCKER_TLS_CERTDIR: "",
-          DOCKER_DIR: ".", // relative to componentdir
-          IMAGE_TAG: "$CI_COMMIT_SHA",
-          DOCKER_DRIVER: "overlay2",
+  { script, variables, ...def }: Partial<CatladderJob>
+): CatladderJob => {
+  return merge(
+    {
+      name: DOCKER_BUILD_JOB_NAME,
+      envMode: "jobPerEnv",
+      stage: "build",
+      image: getRunnerImage("docker-build"),
 
-          IMAGE_NAME:
-            "$CI_REGISTRY_IMAGE/" +
-            context.environment.shortName +
-            "/" +
-            context.componentName,
-
-          CACHE_IMAGE: "$CI_REGISTRY_IMAGE/caches/" + context.componentName,
-          ...(variables ?? {}),
+      services: [
+        {
+          name: "docker:20-dind", // see see https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27300#note_466755332
+          command: ["--tls=false"],
         },
-        script: [
-          ...(script || []),
-          "docker login --username gitlab-ci-token --password $CI_JOB_TOKEN $CI_REGISTRY",
-          "docker build --network host --cache-from $CACHE_IMAGE --tag $IMAGE_NAME:$IMAGE_TAG -f $APP_DIR/Dockerfile . --build-arg BUILDKIT_INLINE_CACHE=1", //BUILDKIT_INLINE_CACHE,  see https://testdriven.io/blog/faster-ci-builds-with-docker-cache/
-          "docker push $IMAGE_NAME:$IMAGE_TAG",
-          "docker tag $IMAGE_NAME:$IMAGE_TAG $CACHE_IMAGE",
-          "docker push $CACHE_IMAGE",
-        ],
+      ],
+      variables: {
+        ...DOCKER_RUNNER_BUILD_VARIABLES,
+        DOCKER_BUILDKIT: "1", // see https://docs.docker.com/develop/develop-images/build_enhancements/
+        DOCKERFILE_ADDITIONS:
+          context.componentConfig.build.docker?.additionsBegin?.join("\n"),
+        DOCKERFILE_ADDITIONS_END:
+          context.componentConfig.build.docker?.additionsEnd?.join("\n"),
+        APP_DIR: context.componentConfig.dir,
+        DOCKER_HOST: "tcp://0.0.0.0:2375",
+        DOCKER_TLS_CERTDIR: "",
+        DOCKER_DIR: ".", // relative to componentdir
+        IMAGE_TAG: "$CI_COMMIT_SHA",
+        DOCKER_DRIVER: "overlay2",
+
+        IMAGE_NAME:
+          "$CI_REGISTRY_IMAGE/" +
+          context.environment.shortName +
+          "/" +
+          context.componentName,
+
+        CACHE_IMAGE: "$CI_REGISTRY_IMAGE/caches/" + context.componentName,
+        ...(variables ?? {}),
       },
-      def
-    ),
-  };
+      script: [
+        ...(script || []),
+        "docker login --username gitlab-ci-token --password $CI_JOB_TOKEN $CI_REGISTRY",
+        "docker build --network host --cache-from $CACHE_IMAGE --tag $IMAGE_NAME:$IMAGE_TAG -f $APP_DIR/Dockerfile . --build-arg BUILDKIT_INLINE_CACHE=1", //BUILDKIT_INLINE_CACHE,  see https://testdriven.io/blog/faster-ci-builds-with-docker-cache/
+        "docker push $IMAGE_NAME:$IMAGE_TAG",
+        "docker tag $IMAGE_NAME:$IMAGE_TAG $CACHE_IMAGE",
+        "docker push $CACHE_IMAGE",
+      ],
+    },
+    def
+  );
 };

package/src/build/index.ts

@@ -1,5 +1,6 @@
 import { Context } from "../types/context";
-import { GitlabJobs } from "../types/gitlab-types";
+
+import { CatladderJob } from "../types/jobs";
 import { createNodeJobs, createStorybookJobs, createMeteorJobs } from "./node";
 
 import { BuildConfig } from "./types";
@@ -8,7 +9,7 @@ export * from "./node";
 
 export type BuildTypes = {
   [type in BuildConfig["type"]]: {
-    jobs: (context: Context) => GitlabJobs;
+    jobs: (context: Context) => CatladderJob[];
     defaults: () => Partial<Extract<BuildConfig, { type: type }>>;
   };
 };

package/src/build/node/buildJob.ts

@@ -1,5 +1,4 @@
 import { Context } from "../../types/context";
-import { GitlabJob, GitlabJobs } from "../../types/gitlab-types";
 import { ensureArray } from "../../utils";
 import { APP_BUILD_JOB_NAME } from "../base/constants";
 import { createBuildJob } from "../base/createBuildJob";
@@ -9,8 +8,9 @@ import { isOfBuildType } from "../types";
 import { getNextCache, getNodeCache, getYarnCache } from "./cache";
 import { NODE_RUNNER_BUILD_VARIABLES } from "./constants";
 import { getYarnInstall, getYarnInstallCommand } from "./yarn";
+import { CatladderJob } from "../../types/jobs";
 
-export const createNodeBuildJobs = (context: Context): GitlabJobs => {
+export const createNodeBuildJobs = (context: Context): CatladderJob[] => {
   const buildConfig = context.componentConfig.build;
 
   if (!isOfBuildType(buildConfig, "node", "node-static", "storybook")) {
@@ -18,7 +18,7 @@ export const createNodeBuildJobs = (context: Context): GitlabJobs => {
   }
 
   const yarnInstall = getYarnInstall(context);
-  const appBuildJob: GitlabJob | null =
+  const appBuildJob: CatladderJob | null =
     buildConfig.buildCommand !== null
       ? createBuildJob(context, {
           variables: {

package/src/build/node/index.ts

@@ -1,17 +1,17 @@
 import type { Context } from "../../types/context";
-import type { GitlabJobs } from "../../types/gitlab-types";
+import { CatladderJob } from "../../types/jobs";
 import { createNodeBuildJobs } from "./buildJob";
 import { createMeteorBuildJobs } from "./meteor";
 import { createNodeTestJobs } from "./testJob";
 
-export const createNodeJobs = (context: Context): GitlabJobs => {
+export const createNodeJobs = (context: Context): CatladderJob[] => {
   return [...createNodeTestJobs(context), ...createNodeBuildJobs(context)];
 };
 
-export const createStorybookJobs = (context: Context): GitlabJobs => {
+export const createStorybookJobs = (context: Context): CatladderJob[] => {
   return [...createNodeBuildJobs(context)];
 };
 
-export const createMeteorJobs = (context: Context): GitlabJobs => {
+export const createMeteorJobs = (context: Context): CatladderJob[] => {
   return [...createNodeTestJobs(context), ...createMeteorBuildJobs(context)];
 };

package/src/build/node/meteor.ts

@@ -1,11 +1,9 @@
 import { join } from "path";
 import { getRunnerImage } from "../../runner";
+import { GitlabJobCache } from "../../types";
 import type { Context } from "../../types/context";
-import type {
-  GitlabJob,
-  GitlabJobCache,
-  GitlabJobs,
-} from "../../types/gitlab-types";
+
+import { CatladderJob } from "../../types/jobs";
 
 import { APP_BUILD_JOB_NAME } from "../base/constants";
 import { createBuildJob } from "../base/createBuildJob";
@@ -29,7 +27,7 @@ const getMeteorCache = (context: Context): GitlabJobCache[] => [
     ],
   },
 ];
-export const createMeteorBuildJobs = (context: Context): GitlabJobs => {
+export const createMeteorBuildJobs = (context: Context): CatladderJob[] => {
   const buildConfig = context.componentConfig.build;
 
   if (!isOfBuildType(buildConfig, "meteor")) {
@@ -37,7 +35,7 @@ export const createMeteorBuildJobs = (context: Context): GitlabJobs => {
   }
 
   const yarnInstall = getYarnInstall(context);
-  const appBuildJob: GitlabJob | null =
+  const appBuildJob: CatladderJob | null =
     buildConfig.buildCommand !== null
       ? createBuildJob(context, {
           cache: [...getNodeCache(context), ...getMeteorCache(context)],

package/src/build/node/testJob.ts

@@ -1,12 +1,11 @@
-import { BASE_RETRY } from "../../defaults";
 import { Context } from "../../types/context";
-import { GitlabJob, GitlabJobDef, GitlabJobs } from "../../types/gitlab-types";
+import { CatladderJob } from "../../types/jobs";
 import { ensureArray, notNil } from "../../utils";
 import { getNodeCache } from "./cache";
 import { NODE_RUNNER_BUILD_VARIABLES } from "./constants";
 import { getYarnInstall } from "./yarn";
 
-export const createNodeTestJobs = (context: Context): GitlabJobs => {
+export const createNodeTestJobs = (context: Context): CatladderJob[] => {
   // don't run tests after release
   if (context.commitInfo?.trigger === "taggedRelease") {
     return [];
@@ -14,66 +13,58 @@ export const createNodeTestJobs = (context: Context): GitlabJobs => {
 
   const buildConfig = context.componentConfig.build;
 
-  const base: Omit<GitlabJobDef, "script"> = {
+  const base: Omit<CatladderJob, "script" | "name"> = {
     variables: {
       APP_PATH: context.componentConfig.dir,
       ...NODE_RUNNER_BUILD_VARIABLES,
       ...(buildConfig.extraVars ?? {}),
     },
-
     stage: "test",
-    interruptible: true,
     needs: [],
-    retry: BASE_RETRY,
+    envMode: "none",
   };
   const yarnInstall = getYarnInstall(context);
-  const auditJob: GitlabJob | null =
+  const auditJob: CatladderJob | null =
     buildConfig.audit !== false
       ? {
           name: "🛡 audit",
-          envMode: "none",
-          job: {
-            ...base,
-            cache: undefined, // audit does not need yarn install and no cache
-            script: [
-              `cd ${context.componentConfig.dir}`,
-              ...(ensureArray(buildConfig.audit?.command) ?? ["yarn audit"]),
-            ],
-            allow_failure: true,
-          },
+
+          ...base,
+          cache: undefined, // audit does not need yarn install and no cache
+          script: [
+            `cd ${context.componentConfig.dir}`,
+            ...(ensureArray(buildConfig.audit?.command) ?? ["yarn audit"]),
+          ],
+          allow_failure: true,
         }
       : null;
 
-  const lintJob: GitlabJob | null =
+  const lintJob: CatladderJob | null =
     buildConfig.lint !== false
       ? {
           name: "👮 lint",
-          envMode: "none",
-          job: {
-            ...base,
-            cache: getNodeCache(context),
-            script: [
-              `cd ${context.componentConfig.dir}`,
-              ...yarnInstall,
-              ...(ensureArray(buildConfig.lint?.command) ?? ["yarn lint"]),
-            ],
-          },
+
+          ...base,
+          cache: getNodeCache(context),
+          script: [
+            `cd ${context.componentConfig.dir}`,
+            ...yarnInstall,
+            ...(ensureArray(buildConfig.lint?.command) ?? ["yarn lint"]),
+          ],
         }
       : null;
-  const testJob: GitlabJob | null =
+  const testJob: CatladderJob | null =
     buildConfig.test !== false
       ? {
           name: "🧪 test",
-          envMode: "none",
-          job: {
-            ...base,
-            cache: getNodeCache(context),
-            script: [
-              `cd ${context.componentConfig.dir}`,
-              ...yarnInstall,
-              ...(ensureArray(buildConfig.test?.command) ?? ["yarn test"]),
-            ],
-          },
+
+          ...base,
+          cache: getNodeCache(context),
+          script: [
+            `cd ${context.componentConfig.dir}`,
+            ...yarnInstall,
+            ...(ensureArray(buildConfig.test?.command) ?? ["yarn test"]),
+          ],
         }
       : null;
   return [auditJob, lintJob, testJob].filter(notNil);

package/src/build/tests/storybook.test.ts

@@ -26,7 +26,8 @@ describe("storybook build", () => {
     };
 
     it("creates a pipeline for storybook that does not contain test stages", async () => {
-      const { image, stages, workflow, ...jobs } = await createChildPipeline(
+      const { jobs } = await createChildPipeline(
+        "gitlab",
         "mainBranch",
         config
       );
@@ -40,7 +41,8 @@ describe("storybook build", () => {
     });
 
     it("runs build and renames folder", async () => {
-      const { image, stages, workflow, ...jobs } = await createChildPipeline(
+      const { jobs } = await createChildPipeline(
+        "gitlab",
         "mainBranch",
         config
       );

package/src/catladder-gitlab.ts

@@ -1,4 +1,5 @@
 import { writeFileSync } from "fs";
+import { dump } from "js-yaml";
 import { readConfigSync } from "./config";
 import { PIPELINE_IMAGE_TAG } from "./constants";
 import { createChildPipeline } from "./pipeline";
@@ -32,11 +33,14 @@ if (trigger) {
   if (!config) {
     throw new Error("no catladder config found");
   }
-  createChildPipeline(trigger, config).then((mainPipeline) => {
-    writeFileSync(`__pipeline.yml`, JSON.stringify(mainPipeline, null, 2), {
-      encoding: "utf-8",
-    });
-  });
+  createChildPipeline("gitlab", trigger, config).then(
+    ({ jobs, ...mainPipeline }) => {
+      // need to spread out the jobs
+      writeFileSync(`__pipeline.yml`, dump({ ...jobs, ...mainPipeline }), {
+        encoding: "utf-8",
+      });
+    }
+  );
 } else {
   throw new Error(
     "no matching trigger: " +

package/src/deploy/base.ts

@@ -1,4 +1,5 @@
-import { GitlabJob, Context, GitlabJobDef } from "../types";
+import { Context } from "../types";
+import { CatladderJob } from "../types/jobs";
 
 export const DEPLOY_JOB_NAME = "🚀 Deploy";
 export const STOP_JOB_NAME = "🛑 Stop ⚠️";
@@ -9,9 +10,7 @@ const DEPLOY_RUNNER_VARIABLES = {
   KUBERNETES_MEMORY_REQUEST: "200Mi",
   KUBERNETES_MEMORY_LIMIT: "500Mi",
 };
-type JobWithoutScript = Omit<GitlabJob, "job"> & {
-  job: Omit<GitlabJobDef, "script">;
-};
+type JobWithoutScript = Omit<CatladderJob, "script">;
 export const getBaseDeploymentJob = (context: Context): JobWithoutScript => {
   const environment = {
     name: context.environment.fullName,
@@ -45,26 +44,24 @@ export const getBaseDeploymentJob = (context: Context): JobWithoutScript => {
         artifacts: false,
       },
     ], // workaround for https://gitlab.com/gitlab-org/gitlab/-/issues/220758
-    job: {
-      rules: [
-        autoDeploy
-          ? {
-              when: "on_success",
-            }
-          : {
-              when: "manual",
-            },
-      ],
-      stage: "deploy",
-      dependencies: [],
-      variables: {
-        ...DEPLOY_RUNNER_VARIABLES,
-      },
-      environment: {
-        ...environment,
-        on_stop: STOP_JOB_NAME,
-        auto_stop_in: autoStop,
-      },
+
+    rules: [
+      autoDeploy
+        ? {
+            when: "on_success",
+          }
+        : {
+            when: "manual",
+          },
+    ],
+    stage: "deploy",
+    variables: {
+      ...DEPLOY_RUNNER_VARIABLES,
+    },
+    environment: {
+      ...environment,
+      on_stop: STOP_JOB_NAME,
+      auto_stop_in: autoStop,
     },
   };
 };
@@ -79,29 +76,27 @@ export const getBaseDeploymentStopJob = (
   return {
     name: STOP_JOB_NAME,
     envMode: "stagePerEnv", // makes it easier to run manual tasks er env
-    job: {
-      needs: [DEPLOY_JOB_NAME],
-      rules: [
-        {
-          if: "$CI_COMMIT_BRANCH =~ /^[0-9]+\\.([0-9]+|x)\\.x$/", // automatic on hotfix branches
-          when: "on_success",
-          allow_failure: true,
-        },
-        {
-          when: "manual",
-          allow_failure: true,
-        },
-      ],
-      variables: {
-        ...DEPLOY_RUNNER_VARIABLES,
-        GIT_STRATEGY: "none",
+
+    needs: [DEPLOY_JOB_NAME],
+    rules: [
+      {
+        if: "$CI_COMMIT_BRANCH =~ /^[0-9]+\\.([0-9]+|x)\\.x$/", // automatic on hotfix branches
+        when: "on_success",
+        allow_failure: true,
       },
-      stage: "stop",
-      dependencies: [],
-      environment: {
-        ...environment,
-        action: "stop",
+      {
+        when: "manual",
+        allow_failure: true,
       },
+    ],
+    variables: {
+      ...DEPLOY_RUNNER_VARIABLES,
+      GIT_STRATEGY: "none",
+    },
+    stage: "stop",
+    environment: {
+      ...environment,
+      action: "stop",
     },
   };
 };