@catladder/pipeline 1.144.1 → 1.145.0

package/examples/custom-deploy.ts

@@ -13,15 +13,15 @@ const config: Config = {
       deploy: {
         type: "custom",
         requiresDocker: true,
+
         jobVars: {
           secret: ["DEPLOY_API_KEY"],
-
-          public: {
-            // if your deploy job requires more memory and/or cpu, you can increase them with those variables:
-            KUBERNETES_CPU_REQUEST: "1",
-            KUBERNETES_MEMORY_REQUEST: "1024Mi",
-            KUBERNETES_MEMORY_LIMIT: "2048Mi",
-          },
+        },
+        runnerVariables: {
+          // if your deploy job requires more memory and/or cpu, you can increase them with those variables:
+          KUBERNETES_CPU_REQUEST: "1",
+          KUBERNETES_MEMORY_REQUEST: "1024Mi",
+          KUBERNETES_MEMORY_LIMIT: "2048Mi",
         },
         script: ["echo 'would deploy'"],
         stopScript: ["echo 'would stop'"],

package/examples/multiline-var.ts

@@ -0,0 +1,64 @@
+import type { Config } from "../src";
+import { createAllPipelines } from "./__utils__/helpers";
+
+const config: Config = {
+  appName: "test-app",
+  customerName: "pan",
+  components: {
+    api: {
+      dir: "api",
+      build: {
+        type: "node",
+      },
+      vars: {
+        public: {
+          foo: "foo-value",
+          multiline: `line1
+line2
+line3  
+
+single quote: '
+doouble quote: "
+`,
+        },
+      },
+      deploy: {
+        type: "google-cloudrun",
+        projectId: "asdf",
+        region: "asia-east1",
+      },
+    },
+    api2: {
+      dir: "api",
+      build: {
+        type: "node",
+      },
+      vars: {
+        public: {
+          multiline_from_api: "${api:multiline}",
+          multiline2: `yeah
+yeah2
+yeah3  
+
+single quote: '
+doouble quote: "
+`,
+        },
+      },
+      deploy: {
+        type: "kubernetes",
+        cluster: {
+          name: "some-cluster-name",
+          region: "europe-west6",
+          projectId: "some-project-id",
+          type: "gcloud",
+          domainCanonical: "panter.cloud",
+        },
+      },
+    },
+  },
+};
+
+it("matches snapshot", async () => {
+  expect(await createAllPipelines(config)).toMatchSnapshot();
+});

package/package.json

@@ -52,7 +52,7 @@
     }
   ],
   "license": "MIT",
-  "version": "1.144.1",
+  "version": "1.145.0",
   "scripts": {
     "build:tsc": "yarn tsc",
     "build": "yarn build:compile && yarn build:inline-variables && yarn build:bundle",
@@ -78,22 +78,19 @@
     "@jest/globals": "^29.3.1",
     "@types/child-process-promise": "^2.2.2",
     "@types/jest": "^27.4.0",
-    "@types/js-yaml": "^4.0.9",
     "@types/lodash": "^4.14.185",
-    "@types/node": "^17.0.10",
+    "@types/node": "^20.14.2",
     "babel-plugin-transform-inline-environment-variables": "^0.4.3",
     "child-process-promise": "^2.2.1",
-    "eslint": "^8.7.0",
-    "js-yaml": "^4.1.0",
     "json-schema-to-typescript": "^13.0.2",
     "lodash": "^4.17.21",
     "memoizee": "^0.4.15",
     "path-equal": "1.1.3",
-    "slugify": "^1.6.3"
+    "slugify": "^1.6.3",
+    "yaml": "^2.4.5"
   },
   "dependencies": {
-    "string-replace-async": "^2.0.0",
     "ts-node": "^10.4.0",
-    "typescript": "^4.5.4"
+    "typescript": "^5.4.5"
   }
 }

package/src/bash/BashExpression.ts

@@ -0,0 +1,81 @@
+/**
+ * this class represents a string that can be evaluated in bash scripts.
+ *
+ * you can do basic transforms like lowercase, but that does not return you a lowercase string, but instead an experssion represeting a lowercase string
+ */
+export class BashExpression {
+  constructor(private value: string | BashExpression) {}
+  public toJSON(): string {
+    return this.toString();
+  }
+  public toString(): string {
+    return this.value.toString();
+  }
+
+  /**
+   *
+   * @returns a bash expression to lowercase the string
+   */
+  public toLowerCase(): BashExpression {
+    return this.transformWithCommand("awk '{print tolower($0)}'");
+  }
+  /**
+   * concats a value to this one and returns a new BashExpression
+   * @param value
+   * @returns
+   */
+  public concat(...values: Array<string | BashExpression>): BashExpression {
+    return new BashExpression(
+      this.toString().concat(...values.map((v) => v.toString()))
+    );
+  }
+
+  public transformWithCommand(command: string): BashExpression {
+    return new BashExpression(
+      // see https://stackoverflow.com/a/2264537
+      `$(printf %s "${this.toString()}" | ${command})`
+    );
+  }
+}
+
+export type StringOrBashExpression = string | BashExpression;
+
+export const getBashVariable = (name: string) => new BashExpression(`$${name}`);
+
+/**
+ * joins bash expressions together with a joiner
+ * returns a bash expression if any of the parts is a bash expression
+ * returns a string otherwise
+ *
+ * @param parts
+ * @param joiner
+ * @returns
+ */
+export const joinBashExpressions = (
+  parts: Array<StringOrBashExpression>,
+  joiner = ""
+): StringOrBashExpression => {
+  const anyIsBashExpression = parts.some((p) => p instanceof BashExpression);
+  if (anyIsBashExpression) {
+    return new BashExpression(
+      parts
+        .map((p) => (p instanceof BashExpression ? p.toString() : p))
+        .join(joiner)
+    );
+  } else {
+    return parts.join(joiner);
+  }
+};
+
+/**
+ * escapes a string or bash expression for bash
+ * it mainly escapes double quotes
+ */
+export const bashEscape = (value: StringOrBashExpression | any) => {
+  if (value instanceof BashExpression) {
+    return value.toString(); // no need to escape bash expressions, as we want them to evaluate
+  }
+  // we wrap it in double quotes, so we need to escape them
+  if (typeof value === "string") return value.replace(/"/g, '\\"');
+  return value;
+};

package/src/bash/bashExpressionPerPipelineType.ts

@@ -0,0 +1,16 @@
+import type { PipelineType } from "../types/pipeline";
+import type { BashExpression } from "./BashExpression";
+
+export type BashExpressionPerPipelineType = {
+  [type in PipelineType]?: BashExpression | string;
+} & {
+  default: BashExpression | string;
+};
+
+export const getBashExpressionPerPipelineType = (
+  definition: BashExpressionPerPipelineType,
+  pipelineType?: PipelineType
+) => {
+  if (!pipelineType) return definition.default;
+  return definition[pipelineType] || definition.default;
+};

package/src/bash/bashYaml.ts

@@ -0,0 +1,46 @@
+import type { ScalarTag } from "yaml";
+import { stringify } from "yaml";
+import { BashExpression } from "./BashExpression";
+
+const bashExpressionType: ScalarTag = {
+  tag: "",
+  resolve: (str) => {
+    // not really needed,but let's make typescript happy
+    return new BashExpression(str);
+  },
+  stringify(node: any, ctx: any) {
+    // we create a BLOCK_LITERAL
+    // but because bash will interpret the value, it may resolve to a multiline string
+    // so we need to indent every line using sed
+    return "|-\n" + node.value.transformWithCommand(`sed 's/^/${ctx.indent}/'`);
+  },
+
+  identify: (v: any) => v instanceof BashExpression,
+};
+
+/***
+ * creates a yaml string that can be used in bash scripts
+ * it handles BashExpressions correctly so that they can be evaluated in bash
+ */
+export const yamlBashString = (value: any) => {
+  return stringify(value, {
+    defaultStringType: "BLOCK_LITERAL",
+    defaultKeyType: "PLAIN",
+    customTags: [bashExpressionType],
+    aliasDuplicateObjects: false,
+    lineWidth: 0,
+  });
+};
+
+/**
+ * creates a bash script that writes a yaml string to a file
+ * @param value the yaml value to write, it supports BashExpressions
+ */
+export const writeBashYamlToFileScript = (value: any, filename: string) => {
+  return [
+    `cat > ${filename} <<EOF
+${yamlBashString(value)}
+EOF
+`,
+  ];
+};

package/src/bash/getInjectVarsScript.ts

@@ -0,0 +1,12 @@
+import type { UnspecifiedEnvVars } from "../types";
+import { isNil } from "lodash";
+import { bashEscape } from "./BashExpression";
+
+export const getInjectVarsScript = (vars?: UnspecifiedEnvVars) => {
+  if (!vars) return [];
+  // filter out null and undefined values
+  return Object.entries(vars)
+    .filter(([, value]) => !isNil(value))
+
+    .map(([key, value]) => `export ${key}="${value ? bashEscape(value) : ""}"`);
+};

package/src/bash/replaceAsync.ts

@@ -0,0 +1,50 @@
+import { BashExpression } from "./BashExpression";
+
+// from https://github.com/dsblv/string-replace-async/blob/main/index.js
+// and adjusted a bit
+export default async function replaceAsync(
+  string: string | BashExpression,
+  searchValue: any,
+  replacer: (
+    substring: string,
+    ...args: any[]
+  ) => Promise<string | BashExpression>
+) {
+  const wasBashExpression = string instanceof BashExpression;
+  try {
+    // 1. Run fake pass of `replace`, collect values from `replacer` calls
+    // 2. Resolve them with `Promise.all`
+    // 3. Run `replace` with resolved values
+    const values: Array<Promise<string | BashExpression>> = [];
+    String.prototype.replace.call(
+      string.toString(),
+      searchValue,
+      function (...args) {
+        // eslint-disable-next-line prefer-spread
+        const result = replacer.apply(undefined, args);
+        values.push(result);
+        return "";
+      }
+    );
+    const resolvedValues = await Promise.all(values);
+
+    const containsBashExpression = resolvedValues.some(
+      (value) => value instanceof BashExpression
+    );
+
+    const result = String.prototype.replace.call(
+      string.toString(),
+      searchValue,
+      function () {
+        return resolvedValues.shift()?.toString() ?? "";
+      }
+    );
+    if (wasBashExpression || containsBashExpression) {
+      return new BashExpression(result);
+    } else {
+      return result;
+    }
+  } catch (error) {
+    return Promise.reject(error);
+  }
+}

package/src/build/base/createAppBuildJob.ts

@@ -15,7 +15,7 @@ import { writeDotEnv } from "./writeDotEnv";
 export type AppBuildJobDefinition = Partial<CatladderJob>;
 export const createAppBuildJob = (
   context: Context,
-  { script, variables, ...def }: AppBuildJobDefinition
+  { script, variables, runnerVariables, ...def }: AppBuildJobDefinition
 ): CatladderJob => {
   return merge(
     {
@@ -26,12 +26,16 @@ export const createAppBuildJob = (
       needs: [],
       cache: [],
       variables: {
-        ...RUNNER_BUILD_RESOURCE_VARIABLES,
         ...(variables ?? {}),
         ...context.environment.envVars,
         ...context.environment.jobOnlyVars.build.envVars,
         ...(context.componentConfig.build.extraVars ?? {}),
       },
+      runnerVariables: {
+        ...RUNNER_BUILD_RESOURCE_VARIABLES,
+        ...(runnerVariables ?? {}),
+        ...(context.componentConfig.build.runnerVariables ?? {}),
+      },
 
       script: [
         ...(context.componentConfig.dotEnv === true // don't build when set to `local`

package/src/build/base/writeBuildInfo.ts

@@ -2,6 +2,6 @@ import type { Context } from "../../types";
 
 export const writeBuildInfo = (context: Context) => {
   return [
-    `echo '{"id":"${context.commitInfo?.buildId}","time":"${context.commitInfo?.buildTime}"}' > ${context.componentConfig.dir}/__build_info.json`,
+    `echo '{"id":"${context.environment.envVars.BUILD_INFO_BUILD_ID}","time":"${context.environment.envVars.BUILD_INFO_BUILD_TIME}"}' > ${context.componentConfig.dir}/__build_info.json`,
   ];
 };

package/src/build/base/writeDotEnv.ts

@@ -1,3 +1,4 @@
+import { isNil } from "lodash";
 import type { Context } from "../../types";
 
 /**
@@ -10,7 +11,11 @@ export const writeDotEnv = (context: Context) => {
 
   // make key=value and sanitize multiline
   const keyValueString = Object.entries(envVars)
-    .map(([key, value]) => `${key}=${value.replaceAll("\n", "\\n")}`)
+    // filter out null and undefined values
+    .filter(([, value]) => !isNil(value))
+    .map(
+      ([key, value]) => `${key}=${value?.toString().replaceAll("\n", "\\n")}`
+    )
     .join("\n");
 
   return [

package/src/build/custom/buildJob.ts

@@ -1,7 +1,7 @@
 import { join } from "path";
 import type { Context } from "../../types/context";
 import { ensureArray } from "../../utils";
-import { getDockerBuildDefaultScript, requiresDockerBuild } from "../docker";
+import { getDockerBuildDefaultScript } from "../docker";
 import { isOfBuildType } from "../types";
 
 import type { CatladderJob } from "../../types/jobs";
@@ -25,9 +25,7 @@ export const createCustomBuildJobs = (context: Context): CatladderJob[] => {
       buildConfig.buildCommand !== null
         ? {
             image: buildConfig.jobImage,
-            variables: {
-              ...RUNNER_BUILD_VARIABLES,
-            },
+            runnerVariables: RUNNER_BUILD_VARIABLES,
             cache: buildConfig.jobCache,
             services: buildConfig.jobServices,
             script: [...(ensureArray(buildConfig.buildCommand) ?? [])],

package/src/build/custom/testJob.ts

@@ -13,7 +13,8 @@ const RUNNER_CUSTOM_TEST_VARIABLES = {
 
 export const createCustomTestJobs = (context: Context): CatladderJob[] => {
   // don't run tests after release
-  if (context.commitInfo?.trigger === "taggedRelease") {
+  // TODO: this will be replaced by using rules
+  if (context.trigger === "taggedRelease") {
     return [];
   }
 
@@ -26,10 +27,10 @@ export const createCustomTestJobs = (context: Context): CatladderJob[] => {
   const base: Omit<CatladderJob, "script" | "name"> = {
     variables: {
       APP_PATH: context.componentConfig.dir,
-      ...RUNNER_CUSTOM_TEST_VARIABLES,
       ...context.environment.jobOnlyVars.build.envVars,
       ...(buildConfig.extraVars ?? {}),
     },
+    runnerVariables: RUNNER_CUSTOM_TEST_VARIABLES,
     services: buildConfig.jobServices,
     cache: buildConfig.jobCache,
     stage: "test",

package/src/build/docker.ts

@@ -13,7 +13,7 @@ import {
 } from "../deploy/cloudRun/artifactsRegistry";
 import { gcloudServiceAccountLoginCommands } from "../deploy/cloudRun/utils/gcloudServiceAccountLoginCommands";
 
-const DOCKER_RUNNER_BUILD_VARIABLES = {
+const DOCKER_BUILD_RUNNER_REQUESTS = {
   KUBERNETES_CPU_REQUEST: "0.5",
   KUBERNETES_MEMORY_REQUEST: "1Gi",
   KUBERNETES_MEMORY_LIMIT: "2Gi",
@@ -52,7 +52,8 @@ export const requiresDockerBuild = ({
   isOfDeployType(deploy, "kubernetes", "google-cloudrun", "dockerTag") ||
   (isOfDeployType(deploy, "custom") && deploy.requiresDocker);
 
-const getDockerBaseVariables = () => ({
+// those need to be runner variables
+const getDockerBuildRunnerVariables = () => ({
   DOCKER_HOST: "tcp://0.0.0.0:2375",
   DOCKER_TLS_CERTDIR: "",
   DOCKER_DRIVER: "overlay2",
@@ -61,15 +62,12 @@ const getDockerBaseVariables = () => ({
 
 export const getDockerBuildVariables = (context: Context) => {
   return {
-    ...DOCKER_RUNNER_BUILD_VARIABLES,
     DOCKERFILE_ADDITIONS:
       context.componentConfig.build.docker?.additionsBegin?.join("\n"),
     DOCKERFILE_ADDITIONS_END:
       context.componentConfig.build.docker?.additionsEnd?.join("\n"),
     APP_DIR: context.componentConfig.dir,
-
     DOCKER_DIR: ".", // relative to componentdir
-    ...getDockerBaseVariables(),
 
     ...getDockerImageVariables(context),
   };
@@ -77,7 +75,12 @@ export const getDockerBuildVariables = (context: Context) => {
 
 export const DOCKER_BUILD_JOB_NAME = "🔨 docker";
 
-export const getDockerJobBaseProps = (context: Context) => {
+export const getDockerJobBaseProps = (
+  context: Context
+): Pick<
+  CatladderJob,
+  "image" | "services" | "variables" | "runnerVariables"
+> => {
   return {
     image: getRunnerImage("docker-build"),
     services: [
@@ -86,7 +89,8 @@ export const getDockerJobBaseProps = (context: Context) => {
         command: ["--tls=false"],
       },
     ],
-    variables: getDockerBaseVariables(),
+    variables: {},
+    runnerVariables: getDockerBuildRunnerVariables(),
   };
 };
 
@@ -105,6 +109,10 @@ export const createDockerBuildJobBase = (
     },
     {
       variables: getDockerBuildVariables(context),
+      runnerVariables: {
+        ...DOCKER_BUILD_RUNNER_REQUESTS,
+        ...getDockerBuildRunnerVariables(),
+      },
     },
     def
   );

package/src/build/node/buildJob.ts

@@ -25,9 +25,7 @@ export const createNodeBuildJobs = (context: Context): CatladderJob[] => {
       buildConfig.buildCommand !== null
         ? {
             image: buildConfig.jobImage ?? defaultImage,
-            variables: {
-              ...NODE_RUNNER_BUILD_VARIABLES,
-            },
+            runnerVariables: NODE_RUNNER_BUILD_VARIABLES,
             cache: [
               ...(ensureArray(buildConfig.jobCache) ?? []),
               ...getNodeCache(context),

package/src/build/node/testJob.ts

@@ -9,7 +9,8 @@ import { createArtifactsConfig } from "../base/createArtifactsConfig";
 
 export const createNodeTestJobs = (context: Context): CatladderJob[] => {
   // don't run tests after release
-  if (context.commitInfo?.trigger === "taggedRelease") {
+  // TODO: this will be replaced by using rules
+  if (context.trigger === "taggedRelease") {
     return [];
   }
 
@@ -18,10 +19,10 @@ export const createNodeTestJobs = (context: Context): CatladderJob[] => {
   const base: Omit<CatladderJob, "script" | "name"> = {
     variables: {
       APP_PATH: context.componentConfig.dir,
-      ...NODE_RUNNER_BUILD_VARIABLES,
       ...context.environment.jobOnlyVars.build.envVars,
       ...(buildConfig.extraVars ?? {}),
     },
+    runnerVariables: NODE_RUNNER_BUILD_VARIABLES,
     stage: "test",
     needs: [],
     envMode: "none",

package/src/build/node/yarn.ts

@@ -1,3 +1,4 @@
+import { BashExpression } from "../../bash/BashExpression";
 import type { Context } from "../../types";
 import { ensureArray } from "../../utils";
 import { collapseableSection } from "../../utils/gitlab";
@@ -53,11 +54,13 @@ const DOCKER_COPY_FILES = `COPY --chown=node:node $APP_DIR .`;
 
 export const getDockerAppCopyAndBuildScript = (context: Context) => {
   if (context.packageManagerInfo?.isClassic) {
-    return `
+    return new BashExpression(
+      `
 RUN ${YARN_INSTALL_CLASSIC} --production --ignore-scripts
 ${DOCKER_COPY_FILES}
 RUN ${YARN_INSTALL_CLASSIC} --production 
-    `.trim();
+    `.trim()
+    );
   }
 
   // yarn >= 4 ships with build in plugins, see https://github.com/yarnpkg/berry/pull/4253
@@ -71,10 +74,12 @@ RUN ${YARN_INSTALL_CLASSIC} --production
 
   // copy first everything and then install
   // rebuild first does not work as it will run postinstall and that might require files in the app
-  return `
+  return new BashExpression(
+    `
 ${DOCKER_COPY_FILES}
 ${maybeAddWorkspaceToolsCommand}
 RUN ${YARN_BERRY_PROD_REBUILD}
 
-    `.trim();
+    `.trim()
+  );
 };

package/src/build/rails/test.ts

@@ -4,7 +4,8 @@ import { ensureArray, notNil } from "../../utils";
 
 export const createRailsTestJobs = (context: Context): CatladderJob[] => {
   // don't run tests after release
-  if (context.commitInfo?.trigger === "taggedRelease") {
+  // TODO: this will be replaced by using rules
+  if (context.trigger === "taggedRelease") {
     return [];
   }
 

package/src/build/types.ts

@@ -43,9 +43,16 @@ export type BuildConfigBase = {
   /**
    * additional env vars for the buid jobs
    *
-   * @deprecated use jobVars
+   * @deprecated use jobVars or runnerVariables
    */
   extraVars?: Record<string, string>;
+
+  /**
+   * additional vars only for the runner.
+   * Also if you use services: that require env vars, you need to set them here.
+   *
+   */
+  runnerVariables?: Record<string, string>;
   /**
    * define the build command
    */

package/src/catladder-gitlab.ts

@@ -1,59 +1,12 @@
-import { writeFileSync } from "fs";
-import { dump } from "js-yaml";
 import { readConfigSync } from "./config";
-import { PIPELINE_IMAGE_TAG } from "./constants";
-import { createChildPipeline } from "./pipeline";
-import type { PipelineTrigger } from "./types";
+import { generatePipelineFiles } from "./pipeline/generatePipelineFiles";
+import type { PipelineMode } from "./types";
 
-const {
-  CI_MERGE_REQUEST_ID,
-  CI_COMMIT_TAG,
-  CI_COMMIT_BRANCH,
-  CI_DEFAULT_BRANCH,
-} = process.env;
+const mode = process.argv[2] || "local";
 
-const isDefaultBranch =
-  Boolean(CI_DEFAULT_BRANCH) && CI_COMMIT_BRANCH === CI_DEFAULT_BRANCH;
-const isHotfixBranch = CI_COMMIT_BRANCH
-  ? /^[0-9]+\.([0-9]+|x)\.x$/.test(CI_COMMIT_BRANCH)
-  : false;
-const isMergeRequest = Boolean(CI_MERGE_REQUEST_ID);
-const isTaggedRelease = Boolean(CI_COMMIT_TAG);
-
-console.info(`catladder version ${PIPELINE_IMAGE_TAG}`);
-
-const trigger: PipelineTrigger | null =
-  isMergeRequest || isHotfixBranch
-    ? "mr"
-    : isDefaultBranch
-    ? "mainBranch"
-    : isTaggedRelease
-    ? "taggedRelease"
-    : null;
-if (trigger) {
-  const config = readConfigSync()?.config;
-  if (!config) {
-    throw new Error("no catladder config found");
-  }
-  createChildPipeline("gitlab", trigger, config).then(
-    ({ jobs, ...mainPipeline }) => {
-      // need to spread out the jobs
-      writeFileSync(`__pipeline.yml`, dump({ ...jobs, ...mainPipeline }), {
-        encoding: "utf-8",
-      });
-    }
-  );
-} else {
-  throw new Error(
-    "no matching trigger: " +
-      JSON.stringify(
-        {
-          isMergeRequest,
-          isDefaultBranch,
-          isTaggedRelease,
-        },
-        null,
-        2
-      )
-  );
+const config = readConfigSync()?.config;
+if (!config) {
+  throw new Error("no catladder config found");
 }
+
+generatePipelineFiles(config, "gitlab", mode as PipelineMode<"gitlab">);