@catladder/pipeline 1.88.0 → 1.89.1

package/examples/kubernetes-application-customization.ts

@@ -0,0 +1,57 @@
+import type { Config } from "../src";
+import { createAllPipelines } from "./__utils__/helpers";
+const config: Config = {
+  appName: "test-app",
+  customerName: "pan",
+  components: {
+    api: {
+      dir: "api",
+      build: {
+        type: "node",
+      },
+      deploy: {
+        type: "kubernetes",
+        cluster: {
+          name: "some-cluster-name",
+          region: "europe-west6",
+          projectId: "some-project-id",
+          type: "gcloud",
+          domainCanonical: "panter.cloud",
+        },
+
+        values: {
+          application: {
+            command: "node main.js",
+            autoscale: {
+              minReplicas: 2,
+              maxReplicas: 5,
+              metrics: [
+                {
+                  type: "Resource",
+                  resource: {
+                    name: "cpu",
+
+                    target: {
+                      type: "Utilization",
+                      averageUtilization: 0.5,
+                    },
+                  },
+                },
+              ],
+            },
+            resources: {
+              limits: {
+                cpu: "1",
+                memory: "2048Mi",
+              },
+            },
+          },
+        },
+      },
+    },
+  },
+};
+
+it("matches snapshot", async () => {
+  expect(await createAllPipelines(config)).toMatchSnapshot();
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@catladder/pipeline",
-  "version": "1.88.0",
+  "version": "1.89.1",
   "scripts": {
     "build:tsc": "yarn tsc",
     "build": "yarn build:compile && yarn build:inline-variables && yarn build:bundle",

package/src/build/docker.ts

@@ -45,20 +45,24 @@ export const requiresDockerBuild = (context: Context) => {
   return false;
 };
 
+const getDockerBaseVariables = () => ({
+  DOCKER_HOST: "tcp://0.0.0.0:2375",
+  DOCKER_TLS_CERTDIR: "",
+  DOCKER_DRIVER: "overlay2",
+  DOCKER_BUILDKIT: "1", // see https://docs.docker.com/develop/develop-images/build_enhancements/
+});
+
 export const getDockerBuildVariables = (context: Context) => {
   return {
     ...DOCKER_RUNNER_BUILD_VARIABLES,
-    DOCKER_BUILDKIT: "1", // see https://docs.docker.com/develop/develop-images/build_enhancements/
     DOCKERFILE_ADDITIONS:
       context.componentConfig.build.docker?.additionsBegin?.join("\n"),
     DOCKERFILE_ADDITIONS_END:
       context.componentConfig.build.docker?.additionsEnd?.join("\n"),
     APP_DIR: context.componentConfig.dir,
-    DOCKER_HOST: "tcp://0.0.0.0:2375",
-    DOCKER_TLS_CERTDIR: "",
-    DOCKER_DIR: ".", // relative to componentdir
 
-    DOCKER_DRIVER: "overlay2",
+    DOCKER_DIR: ".", // relative to componentdir
+    ...getDockerBaseVariables(),
 
     ...getDockerImageVariables(context),
   };
@@ -74,7 +78,7 @@ export const getDockerJobBaseProps = (context: Context) => {
         command: ["--tls=false"],
       },
     ],
-    variables: getDockerBuildVariables(context),
+    variables: getDockerBaseVariables(),
   };
 };
 export const createDockerBuildJobBase = (
@@ -89,6 +93,9 @@ export const createDockerBuildJobBase = (
       ...getDockerJobBaseProps(context),
       script: script || [],
     },
+    {
+      variables: getDockerBuildVariables(context),
+    },
     def
   );
 };

package/src/build/rails/build.ts

@@ -12,10 +12,7 @@ export const createRailsBuildJobs = (context: Context): CatladderJob[] => {
 
   const cnbConf = buildConfig.cnbBuilder;
 
-  // backwards compatabilty with CNB_ENV_VARS
-  // TODO: remove when all projects are migrated
-  const packEnvArgs = buildConfig.extraVars?.CNB_ENV_VARS?.split(" ").map(v => `--env '${v}'`)
-    ?? Object.entries(cnbConf?.buildVars ?? {}).map(([k, v]) => `--env '${k}${v ? `=${v}` : ""}'`)
+  const packEnvArgs = Object.entries(cnbConf?.buildVars ?? {}).map(([k, v]) => `--env '${k}${v ? `=${v}` : ""}'`).join(" ")
 
   return [
     createDockerBuildJobBase(context, {
@@ -28,7 +25,7 @@ export const createRailsBuildJobs = (context: Context): CatladderJob[] => {
         `chmod +x /usr/local/bin/pack`,
         //  replace private git ssh gem sources with https to make bundler with credentials via env var work
         `sed --in-place 's|git@\\([^:]*\\):|https://\\1/|g' Gemfile Gemfile.lock`,
-        `pack build "$DOCKER_IMAGE:$DOCKER_IMAGE_TAG" --builder '${cnbConf?.image}' --publish --cache-image "$DOCKER_CACHE_IMAGE" ${packEnvArgs.join(" ")} ${cnbConf?.packExtraArgs?.join(" ") ?? ""}`
+        `pack build "$DOCKER_IMAGE:$DOCKER_IMAGE_TAG" --builder '${cnbConf?.image}' --publish --cache-image "$DOCKER_CACHE_IMAGE" ${packEnvArgs} ${cnbConf?.packExtraArgs?.join(" ") ?? ""}`
       ],
     }),
   ];

package/src/deploy/kubernetes/kubeValues.ts

@@ -20,27 +20,29 @@ const createAppConfig = (
     };
   }
 
+  const { healthRoute, command, ...rest } = application ?? {};
+
   return mergeWithMergingArrays(
     {
       host: context.environment.host,
-      command: context.componentConfig.build.startCommand,
+      command: command ?? context.componentConfig.build.startCommand,
       livenessProbe: {
         httpGet: {
-          path: application?.healthRoute ?? "__health",
+          path: healthRoute ?? "__health",
         },
       },
       readinessProbe: {
         httpGet: {
-          path: application?.healthRoute ?? "__health",
+          path: healthRoute ?? "__health",
         },
       },
       startupProbe: {
         httpGet: {
-          path: application?.healthRoute ?? "__health",
+          path: healthRoute ?? "__health",
         },
       },
     }, // default
-    application // merge rest in
+    rest // merge rest in
   );
 };
 

package/src/deploy/types/kubernetes.ts

@@ -192,6 +192,11 @@ export type DeployConfigKubernetesValues = AllowUnknownProps<{
   application?:
     | false
     | AllowUnknownProps<{
+        /**
+         * command to start, defaults to build's startCommand
+         */
+        command?: string;
+
         /**
          * enable, disable app deployment, defaults to true
          */
@@ -240,7 +245,7 @@ export type DeployConfigKubernetesValues = AllowUnknownProps<{
          */
         jobDefaults?: {
           resources: KubernetesResourcesDef;
-        }
+        };
       }>;
 
   /**
@@ -292,5 +297,5 @@ export type DeployConfigKubernetes = {
    * Custom Helm chart location.
    * Not recommended to use.
    */
-  chartName?: string
+  chartName?: string;
 } & DeployConfigBase;

package/src/pipeline/createAllJobs.ts

@@ -0,0 +1,38 @@
+import { getAllEnvsByTrigger } from "../config";
+import type { Config, PipelineTrigger } from "../types";
+import type { CatladderJob } from "../types/jobs";
+import { createJobsForComponent } from "./createJobsForComponent";
+
+export type AllCatladderJobs = {
+  [componentName: string]: {
+    [env: string]: Array<CatladderJob>;
+  };
+};
+export const createAllJobs = async (
+  config: Config,
+  trigger: PipelineTrigger
+): Promise<AllCatladderJobs> => {
+  return Object.fromEntries(
+    await Promise.all(
+      Object.keys(config.components).map(async (componentName) => {
+        const envs = getAllEnvsByTrigger(config, componentName, trigger);
+        return [
+          componentName,
+          Object.fromEntries(
+            await Promise.all(
+              envs.map(async (env) => [
+                env,
+                await createJobsForComponent(
+                  config,
+                  componentName,
+                  env,
+                  trigger
+                ),
+              ])
+            )
+          ),
+        ];
+      })
+    )
+  );
+};

package/src/pipeline/createChildPipeline.ts

@@ -1,34 +1,18 @@
-import { getAllEnvsByTrigger, getAllEnvsInAllComponents } from "../config";
+import { getAllEnvsInAllComponents } from "../config";
 import { RULES_ALWAYS } from "../rules";
 import { getRunnerImage } from "../runner";
-import type {
-  GitlabPipeline,
-  Pipeline,
-  PipelineJob,
-  PipelineType,
-} from "../types";
+import type { GitlabPipeline, Pipeline, PipelineType } from "../types";
 import type { Config, PipelineTrigger } from "../types/config";
 import { BASE_STAGES } from "../types/jobs";
-import { createJobs } from "./createJobs";
+import { createAllJobs } from "./createAllJobs";
+import { createGitlabJobs } from "./gitlab/createGitlabJobs";
 
 export const createChildPipeline = async <T extends PipelineType>(
   type: T,
   trigger: PipelineTrigger,
   config: Config
 ): Promise<Pipeline<T>> => {
-  const components = Object.keys(config.components);
-
-  // 2. write the triggering pipeline
-  const jobs = await components.reduce<Promise<Record<string, PipelineJob<T>>>>(
-    async (acc, componentName) => {
-      const envs = getAllEnvsByTrigger(config, componentName, trigger);
-      return {
-        ...(await acc),
-        ...(await createJobs(type, envs, config, componentName, trigger)),
-      };
-    },
-    Promise.resolve({})
-  );
+  const jobs = await createAllJobs(config, trigger);
 
   // while technically not required, we group different envs in its own stage
   // each job from `createJobs` that is defined as `envMode: "stagePerEnv"` will have `deploy dev`, etc. instead of just `deploy`
@@ -53,7 +37,7 @@ export const createChildPipeline = async <T extends PipelineType>(
         rules: RULES_ALWAYS,
       },
       stages,
-      jobs,
+      jobs: await createGitlabJobs(jobs),
     };
     return pipeline as Pipeline<T>;
   }

package/src/pipeline/createJobsForComponent.ts

@@ -0,0 +1,64 @@
+import { isFunction } from "lodash";
+import { BUILD_TYPES } from "../build";
+import { createContext } from "../context";
+import { DEPLOY_TYPES } from "../deploy";
+import type { Config, PipelineTrigger } from "../types/config";
+import type { CommitInfo, Context } from "../types/context";
+import type { CatladderJob } from "../types/jobs";
+import { getBaseCommitInfo } from "./commitInfo/getCommitInfo";
+import { getPackageManagerInfo } from "./packageManager";
+
+const injectDefaultVarsInCustomJobs = (
+  context: Context,
+  jobs: CatladderJob[]
+) =>
+  jobs.map(({ variables, ...job }) => ({
+    variables: {
+      ...(context.environment.envVars ?? {}),
+      ...(variables ?? {}),
+    },
+    ...job,
+  }));
+const getCustomJobs = (context: Context) => {
+  if (!context.componentConfig.customJobs) {
+    return [];
+  }
+  const rawJobs = isFunction(context.componentConfig.customJobs)
+    ? context.componentConfig.customJobs(context)
+    : context.componentConfig.customJobs;
+  return injectDefaultVarsInCustomJobs(context, rawJobs);
+};
+const createRawJobs = (context: Context): CatladderJob[] => {
+  if (context.componentConfig.deploy === false) {
+    return [];
+  }
+  const buildJobs =
+    BUILD_TYPES[context.componentConfig.build.type].jobs(context);
+  const deployJobs =
+    DEPLOY_TYPES[context.componentConfig.deploy.type].jobs(context);
+
+  const customJobs = getCustomJobs(context);
+  return [...buildJobs, ...deployJobs, ...customJobs];
+};
+export const createJobsForComponent = async (
+  config: Config,
+  componentName: string,
+  env: string,
+  trigger: PipelineTrigger
+): Promise<Array<CatladderJob>> => {
+  const commitInfo: CommitInfo = {
+    ...(await getBaseCommitInfo()),
+    trigger,
+  };
+
+  const packageManagerInfo = await getPackageManagerInfo(config, componentName);
+
+  const context = await createContext(
+    config,
+    componentName,
+    env,
+    commitInfo,
+    packageManagerInfo
+  );
+  return createRawJobs(context);
+};

package/src/pipeline/gitlab/createGitlabJobs.ts

@@ -0,0 +1,149 @@
+import { isObject } from "lodash";
+import { BASE_RETRY } from "../../defaults";
+import type { GitlabJobDef } from "../../types";
+import type { CatladderJob, CatladderJobNeed } from "../../types/jobs";
+import type { AllCatladderJobs } from "../createAllJobs";
+
+type AllGitlabJobs = Record<string, GitlabJobDef>;
+
+const getFullJobName = (
+  name: string,
+  componentName: string,
+  env?: string | null
+) => {
+  if (env) {
+    return `${componentName} ${name} | ${env} `;
+  }
+  return `${componentName} ${name}`;
+};
+
+const getFullReferencedJobName = (
+  referencedJobName: string,
+  componentName: string,
+  env: string,
+  allJobs: AllCatladderJobs
+) => {
+  const referencedJob = allJobs[componentName]?.[env]?.find(
+    (j) => j.name === referencedJobName
+  );
+  if (!referencedJob) {
+    throw new Error(
+      `unknown job referenced: '${referencedJobName}' from '${env}:${componentName}'`
+    );
+  }
+  const envToSet = referencedJob.envMode !== "none" ? env : null;
+  return getFullJobName(referencedJobName, componentName, envToSet);
+};
+
+const getJobName = (need: CatladderJobNeed) =>
+  isObject(need) ? need.job : need;
+
+export const makeGitlabJob = (
+  componentName: string,
+  env: string,
+  {
+    envMode,
+    needsStages,
+    needsOtherComponent,
+    name,
+    needs,
+    ...job
+  }: CatladderJob<string>,
+  allJobs: AllCatladderJobs
+): [fullName: string, job: GitlabJobDef] => {
+  const stage = envMode === "stagePerEnv" ? `${job.stage} ${env}` : job.stage;
+
+  const needsFromStages: CatladderJob["needs"] = needsStages?.flatMap((n) => {
+    const referencedComponentName = componentName;
+    const allJobNamesFromThatStage =
+      allJobs[referencedComponentName]?.[env]
+        ?.filter((j) => j.stage === n.stage)
+        ?.map((j) => j.name) ?? [];
+
+    return allJobNamesFromThatStage.map((job) => ({
+      job,
+      artifacts: n.artifacts ?? false,
+      componentName: referencedComponentName,
+    }));
+  });
+  const cleanedNeeds: CatladderJob["needs"] = [
+    ...(needs ?? []),
+    // pull in legacy needs from other component, which is now identical to needs
+    ...(needsOtherComponent ?? []),
+    ...(needsFromStages ?? []),
+  ];
+
+  const gitlabNeeds: GitlabJobDef["needs"] = cleanedNeeds
+    ?.map((n) =>
+      isObject(n)
+        ? {
+            job: getFullReferencedJobName(
+              n.job,
+              n.componentName ?? componentName,
+              env,
+              allJobs
+            ),
+            artifacts: n.artifacts,
+          }
+        : getFullReferencedJobName(n, componentName, env, allJobs)
+    ) // sort in a predictable manner for snapshot tests
+    .sort((a, b) => getJobName(a).localeCompare(getJobName(b)));
+
+  const fullJobName = getFullJobName(
+    name,
+    componentName,
+    envMode !== "none" ? env : undefined
+  );
+
+  const gitlabJob = {
+    ...job,
+    stage,
+    environment: job.environment?.on_stop
+      ? {
+          ...job.environment,
+          on_stop: getFullReferencedJobName(
+            job.environment.on_stop,
+            componentName,
+            env,
+            allJobs
+          ),
+        }
+      : job.environment,
+    // sort in a predictable manner for snapshot tests
+    needs: gitlabNeeds,
+    retry: BASE_RETRY,
+    interruptible: true,
+  };
+
+  return [fullJobName, gitlabJob];
+};
+
+export const createGitlabJobs = async (
+  allJobs: AllCatladderJobs
+): Promise<AllGitlabJobs> => {
+  return Object.keys(allJobs).reduce((accForComponents, componentName) => {
+    const componentJobs = allJobs[componentName];
+    return {
+      ...accForComponents,
+      ...Object.keys(componentJobs).reduce((accForEnvs, env) => {
+        const jobs = componentJobs[env];
+
+        return {
+          ...accForEnvs,
+          ...jobs.reduce((accForJobs, job) => {
+            const [fullJobName, gitlabJob] = makeGitlabJob(
+              componentName,
+              env,
+              job,
+              allJobs
+            );
+            return {
+              ...accForJobs,
+              [fullJobName]: gitlabJob,
+            };
+          }, {} as AllGitlabJobs),
+        };
+      }, {} as AllGitlabJobs),
+    };
+  }, {} as AllGitlabJobs);
+};

package/src/pipeline/index.ts

@@ -1,2 +1,2 @@
 export * from "./createChildPipeline";
-export * from "./createJobs";
+export * from "./createJobsForComponent";

package/dist/pipeline/createJobs.d.ts

@@ -1,3 +0,0 @@
-import type { PipelineJob } from "../types";
-import type { Config, PipelineTrigger } from "../types/config";
-export declare const createJobs: <T extends "gitlab">(type: T, envs: string[], config: Config, componentName: string, trigger: PipelineTrigger) => Promise<Record<string, PipelineJob<T>>>;

package/dist/pipeline/gitlab/makeGitlabJob.d.ts

@@ -1,10 +0,0 @@
-import type { GitlabJobDef } from "../../types";
-import type { CatladderJob } from "../../types/jobs";
-export declare const makeGitlabJob: ({
-  envMode,
-  needsStages,
-  needsOtherComponent,
-  name,
-  needs,
-  ...rest
-}: CatladderJob<string>) => GitlabJobDef;

package/dist/pipeline/gitlab/makeGitlabJob.js

@@ -1,75 +0,0 @@
-"use strict";
-
-var __assign = this && this.__assign || function () {
-  __assign = Object.assign || function (t) {
-    for (var s, i = 1, n = arguments.length; i < n; i++) {
-      s = arguments[i];
-      for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p)) t[p] = s[p];
-    }
-    return t;
-  };
-  return __assign.apply(this, arguments);
-};
-var __rest = this && this.__rest || function (s, e) {
-  var t = {};
-  for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0) t[p] = s[p];
-  if (s != null && typeof Object.getOwnPropertySymbols === "function") for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
-    if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i])) t[p[i]] = s[p[i]];
-  }
-  return t;
-};
-var __read = this && this.__read || function (o, n) {
-  var m = typeof Symbol === "function" && o[Symbol.iterator];
-  if (!m) return o;
-  var i = m.call(o),
-    r,
-    ar = [],
-    e;
-  try {
-    while ((n === void 0 || n-- > 0) && !(r = i.next()).done) ar.push(r.value);
-  } catch (error) {
-    e = {
-      error: error
-    };
-  } finally {
-    try {
-      if (r && !r.done && (m = i["return"])) m.call(i);
-    } finally {
-      if (e) throw e.error;
-    }
-  }
-  return ar;
-};
-var __spreadArray = this && this.__spreadArray || function (to, from, pack) {
-  if (pack || arguments.length === 2) for (var i = 0, l = from.length, ar; i < l; i++) {
-    if (ar || !(i in from)) {
-      if (!ar) ar = Array.prototype.slice.call(from, 0, i);
-      ar[i] = from[i];
-    }
-  }
-  return to.concat(ar || Array.prototype.slice.call(from));
-};
-exports.__esModule = true;
-exports.makeGitlabJob = void 0;
-var lodash_1 = require("lodash");
-var defaults_1 = require("../../defaults");
-var getJobName = function (need) {
-  return (0, lodash_1.isObject)(need) ? need.job : need;
-};
-var makeGitlabJob = function (_a) {
-  var envMode = _a.envMode,
-    needsStages = _a.needsStages,
-    needsOtherComponent = _a.needsOtherComponent,
-    name = _a.name,
-    needs = _a.needs,
-    rest = __rest(_a, ["envMode", "needsStages", "needsOtherComponent", "name", "needs"]);
-  return __assign(__assign({}, rest), {
-    // sort in a predictable manner for snapshot tests
-    needs: needs ? __spreadArray([], __read(needs), false).sort(function (a, b) {
-      return getJobName(a).localeCompare(getJobName(b));
-    }) : undefined,
-    retry: defaults_1.BASE_RETRY,
-    interruptible: true
-  });
-};
-exports.makeGitlabJob = makeGitlabJob;