@catladder/pipeline 0.0.0-node16-ddfae617 → 0.0.0-prettier-cbceeed9

package/src/build/types.ts

@@ -1,3 +1,7 @@
+import type { GitlabJobImage, GitlabJobService } from "../types";
+
+import type { CatladderJob } from "../types/jobs";
+
 export type BuildConfigBase = {
   /**
    * command to run on the image to start the app (e.g. yarn start)
@@ -27,6 +31,7 @@ export type BuildConfigBase = {
     | false
     | {
         command?: string | string[];
+        jobImage?: GitlabJobImage;
       };
 
   /**
@@ -36,6 +41,7 @@ export type BuildConfigBase = {
     | false
     | {
         command?: string | string[];
+        jobImage?: GitlabJobImage;
       };
 
   /**
@@ -45,6 +51,7 @@ export type BuildConfigBase = {
     | false
     | {
         command?: string | string[];
+        jobImage?: GitlabJobImage;
       };
 
   /**
@@ -52,6 +59,20 @@ export type BuildConfigBase = {
    * by default "dist" and ".next" are allways included
    */
   artifactsPaths?: string[];
+
+  /**
+   * additional CI/CD artifacts reports,
+   * use to display information in merge requests, pipeline views and security dashboards.
+   */
+  artifactsReports?: {
+    /**
+     * The junit report collects JUnit report format XML files.
+     * The collected Unit test reports upload to GitLab as an artifact.
+     * Paths are prefixed with component's root folder.
+     * eg. `["dist/test-results/TEST-*.xml", "dist/rspec.xml", ...]`
+     */
+    junit?: string[];
+  };
 };
 
 export type BuildConfigNodeBase = BuildConfigBase;
@@ -67,6 +88,71 @@ export type BuildConfigNodeStatic = BuildConfigNodeBase & {
 
 export type BuildConfigMeteor = BuildConfigNodeBase & {
   type: "meteor";
+  /**
+   * whether to run yarn install inside the source folder in the docker image.
+   * This is only required if you have custom scripts in your image
+   */
+  installScripts?: boolean;
+};
+
+export type BuildConfigCustomDocker = BuildConfigBase["docker"] &
+  (
+    | {
+        /**
+         * use the built-in nginx image for simple static apps
+         */
+        type: "nginx";
+      }
+    | {
+        /**
+         * custom docker build, expect that a Dockerfile in your directory
+         */
+        type: "custom";
+      }
+  );
+
+export type BuildConfigCustom = BuildConfigBase & {
+  type: "custom";
+  jobImage: string;
+  jobServices?: GitlabJobService[];
+
+  docker: BuildConfigCustomDocker;
+
+  /**
+   * customize cache for the job
+   */
+  jobCache?: CatladderJob["cache"];
+};
+
+export type BuildConfigRails = BuildConfigBase & {
+  type: "rails";
+  cnbBuilder?: {
+    /**
+     * The Cloud Native Buildpacks builder image to use.
+     * See e.g. https://github.com/heroku/builder or others.
+     * Default: heroku/buildpacks:20
+     */
+    image?: string;
+    /**
+     * The version of the Cloud Native Buildpacks pack command to use.
+     * See https://buildpacks.io/docs/tools/pack/
+     * Default: 0.28.0
+     */
+    packVersion?: string;
+    /**
+     * Additional command arguments passed to the pack command.
+     * See https://buildpacks.io/docs/tools/pack/
+     */
+    packExtraArgs?: string[];
+    /**
+     * Variables needed for Bundler and Rails asset precompilation.
+     *
+     * Specify a key with the value `undefined` to inherit it from the EnvVars.
+     *
+     * See https://github.com/rails/rails/issues/32947
+     */
+    buildVars?: Record<string, string | undefined>;
+  };
 };
 
 export type BuildConfigStorybook = BuildConfigNodeBase & {
@@ -77,9 +163,18 @@ export type BuildConfig =
   | BuildConfigNode
   | BuildConfigNodeStatic
   | BuildConfigStorybook
-  | BuildConfigMeteor;
+  | BuildConfigMeteor
+  | BuildConfigCustom
+  | BuildConfigRails;
+
+export type BuildConfigType = BuildConfig["type"];
+
+export type BuildConfigGeneric<T extends BuildConfigType> = Extract<
+  BuildConfig,
+  { type: T }
+>;
 
-export const isOfBuildType = <T extends Array<BuildConfig["type"]>>(
+export const isOfBuildType = <T extends Array<BuildConfigType>>(
   t: BuildConfig,
   ...types: T
 ): t is Extract<BuildConfig, { type: T[number] }> => {

package/src/catladder-gitlab.ts

@@ -3,7 +3,7 @@ import { dump } from "js-yaml";
 import { readConfigSync } from "./config";
 import { PIPELINE_IMAGE_TAG } from "./constants";
 import { createChildPipeline } from "./pipeline";
-import { PipelineTrigger } from "./types";
+import type { PipelineTrigger } from "./types";
 
 const {
   CI_MERGE_REQUEST_ID,
@@ -14,7 +14,9 @@ const {
 
 const isDefaultBranch =
   Boolean(CI_DEFAULT_BRANCH) && CI_COMMIT_BRANCH === CI_DEFAULT_BRANCH;
-const isHotfixBranch = false; // TODO:  $CI_COMMIT_BRANCH =~ /^[0-9]+\.([0-9]+|x)\.x$/
+const isHotfixBranch = CI_COMMIT_BRANCH
+  ? /^[0-9]+\.([0-9]+|x)\.x$/.test(CI_COMMIT_BRANCH)
+  : false;
 const isMergeRequest = Boolean(CI_MERGE_REQUEST_ID);
 const isTaggedRelease = Boolean(CI_COMMIT_TAG);
 

package/src/config/{configruedEnvs.test.ts → __tests__/configruedEnvs.test.ts}

@@ -1,6 +1,6 @@
-import { getAllEnvsByTrigger } from ".";
-import { DeployConfigKubernetesCluster } from "..";
-import { Config } from "../types";
+import { getAllEnvsByTrigger } from "..";
+import type { DeployConfigKubernetesCluster } from "../..";
+import type { Config } from "../../types";
 
 describe("getAllEnvsByTrigger", () => {
   const cluster: DeployConfigKubernetesCluster = {

package/src/config/configruedEnvs.ts

@@ -1,10 +1,5 @@
-import {
-  Config,
-  DEFAULT_ENV_TYPES,
-  EnvType,
-  getEnvTypesByTrigger,
-  PipelineTrigger,
-} from "../types";
+import type { Config, EnvType, PipelineTrigger } from "../types";
+import { DEFAULT_ENV_TYPES, getEnvTypesByTrigger } from "../types";
 
 const getConfiguredAndDefaultEnvs = (
   config: Config,

package/src/config/readConfig.ts

@@ -1,7 +1,7 @@
 import { existsSync, readFileSync } from "fs";
 import { register } from "ts-node";
 import { load } from "js-yaml";
-import { Config } from "../types";
+import type { Config } from "../types";
 
 // allows us to load ts files
 

package/src/context/{resolveReferences.test.ts → __tests__/resolveReferences.test.ts}

@@ -1,10 +1,10 @@
 import {
   resolveReferences,
   translateLegacyFromComponents,
-} from "./resolveReferences";
+} from "../resolveReferences";
 
 describe("resolveReferences", () => {
-  it("replaces occurences of ${componentName:VARIABLE_NAME}", () => {
+  it("replaces occurences of ${componentName:VARIABLE_NAME}", async () => {
     const variables = {
       a: "hello world",
       b: "a replaced value looks like this: '${api:FOO}', nice!",
@@ -17,12 +17,9 @@ describe("resolveReferences", () => {
         FOO: "foo from frontend",
       },
     };
-    const result = resolveReferences(
-      variables,
-      (componentName, variableName) => {
-        return otherVariables[componentName]?.[variableName];
-      }
-    );
+    const result = await resolveReferences(variables, async (componentName) => {
+      return otherVariables[componentName];
+    });
 
     expect(result).toEqual({
       a: "hello world",
@@ -30,12 +27,12 @@ describe("resolveReferences", () => {
     });
   });
 
-  it("replaces self references with structure ${VARIABLE_NAME}", () => {
+  it("replaces self references with structure ${VARIABLE_NAME}", async () => {
     const variables = {
       FOO: "hello world",
       BAR: "this: ${FOO}!",
     };
-    const result = resolveReferences(variables);
+    const result = await resolveReferences(variables);
 
     expect(result).toEqual({
       FOO: "hello world",
@@ -43,7 +40,7 @@ describe("resolveReferences", () => {
     });
   });
 
-  it("replaces self references mixed with other vars", () => {
+  it("replaces self references mixed with other vars", async () => {
     const variables = {
       FOO: "hello from ${api:FOO}",
       BAR: "this: ${FOO}!",
@@ -53,12 +50,9 @@ describe("resolveReferences", () => {
         FOO: "foo from api",
       },
     };
-    const result = resolveReferences(
-      variables,
-      (componentName, variableName) => {
-        return otherVariables[componentName]?.[variableName];
-      }
-    );
+    const result = await resolveReferences(variables, async (componentName) => {
+      return otherVariables[componentName];
+    });
 
     expect(result).toEqual({
       FOO: "hello from foo from api",
@@ -66,18 +60,15 @@ describe("resolveReferences", () => {
     });
   });
 
-  it("keeps variables as is if not found (when null is returned)", () => {
+  it("keeps variables as is if not found (when null is returned)", async () => {
     const variables = {
       a: "hello world",
       b: "a not found value looks like this: '${api:FOO}'",
     };
     const otherVariables: Record<string, Record<string, string>> = {};
-    const result = resolveReferences(
-      variables,
-      (componentName, variableName) => {
-        return otherVariables[componentName]?.[variableName];
-      }
-    );
+    const result = await resolveReferences(variables, async (componentName) => {
+      return otherVariables[componentName];
+    });
 
     expect(result).toEqual({
       a: "hello world",
@@ -85,7 +76,7 @@ describe("resolveReferences", () => {
     });
   });
 
-  it("replaces mulitple levels deep", () => {
+  it("replaces mulitple levels deep", async () => {
     const variables = {
       a: "value is ${frontend:FOO}!",
     };
@@ -97,19 +88,16 @@ describe("resolveReferences", () => {
         FOO: "hi, ${api:FOO}",
       },
     };
-    const result = resolveReferences(
-      variables,
-      (componentName, variableName) => {
-        return otherVariables[componentName]?.[variableName] ?? "";
-      }
-    );
+    const result = await resolveReferences(variables, async (componentName) => {
+      return otherVariables[componentName];
+    });
 
     expect(result).toEqual({
       a: "value is hi, foo from api!",
     });
   });
 
-  it("prevents endless loops", () => {
+  it("prevents endless loops", async () => {
     const variables = {
       a: "value is ${frontend:FOO}!",
     };
@@ -121,12 +109,9 @@ describe("resolveReferences", () => {
         FOO: "frontend ${api:FOO}",
       },
     };
-    const result = resolveReferences(
-      variables,
-      (componentName, variableName) => {
-        return otherVariables[componentName]?.[variableName] ?? "";
-      }
-    );
+    const result = await resolveReferences(variables, async (componentName) => {
+      return otherVariables[componentName];
+    });
 
     expect(result).toEqual({
       a: "value is frontend api ${frontend:FOO}!",

package/src/context/getEnvConfig.ts

@@ -0,0 +1,21 @@
+import type { Config, EnvConfigWithComponent } from "../types/config";
+
+import { mergeWithMergingArrays } from "../utils";
+
+export const getEnvConfig = (
+  config: Config,
+  componentName: string,
+  env: string
+): EnvConfigWithComponent => {
+  const defaultConfig = config.components[componentName];
+  if (!defaultConfig) {
+    throw new Error("unknown component " + componentName);
+  }
+
+  const envCustomizations = defaultConfig.env?.[env] ?? {};
+  if (envCustomizations === false) {
+    throw new Error("env is disabled: " + env + ":" + componentName);
+  }
+
+  return mergeWithMergingArrays(defaultConfig, envCustomizations);
+};

package/src/context/getEnvType.ts

@@ -0,0 +1,17 @@
+import type { EnvType } from "../types";
+import { isKnowEnvType } from "../types";
+
+export const getEnvType = (
+  env: string,
+  envConfig: {
+    type?: EnvType;
+  }
+): EnvType => {
+  if (envConfig.type) return envConfig.type;
+
+  if (isKnowEnvType(env)) {
+    return env;
+  }
+
+  throw new Error("unknown env type: " + env);
+};

package/src/context/getEnvironment.ts

@@ -0,0 +1,40 @@
+import type { Config } from "../types/config";
+
+import type { CommitInfo, Environment } from "../types/context";
+import { getEnvironmentContext } from "./getEnvironmentContext";
+import { getEnvironmentVariables } from "./getEnvironmentVariables";
+
+export const getEnvironment = async (
+  config: Config,
+  componentName: string,
+  env: string,
+  commitInfo?: CommitInfo
+): Promise<Environment> => {
+  const { envVars, secretEnvVarKeys, host, url } =
+    await getEnvironmentVariables(config, componentName, env, commitInfo);
+
+  const envContext = getEnvironmentContext(
+    config,
+    env,
+    componentName,
+    commitInfo
+  );
+
+  const envType = envContext.envType;
+
+  return {
+    envType,
+    host,
+    url,
+    gitlabEnvironment: {
+      name: envContext.gitlabEnvironmentName,
+      url,
+    },
+    fullName: envContext.fullName,
+    slugPrefix: envContext.environmentSlugPrefix,
+    slug: envContext.environmentSlug,
+    shortName: env,
+    envVars,
+    secretEnvVarKeys,
+  };
+};

package/src/context/getEnvironmentContext.ts

@@ -0,0 +1,57 @@
+import type { Config, EnvConfigWithComponent } from "../types/config";
+
+import type { CommitInfo } from "../types/context";
+import type { EnvironmentContext } from "../types/environmentContext";
+import { getEnvConfig } from "./getEnvConfig";
+import { getEnvType } from "./getEnvType";
+
+const getEnvironmentSlugPrefix = (
+  envConfig: EnvConfigWithComponent,
+  env: string,
+  commitInfo?: CommitInfo
+) => {
+  const envType = getEnvType(env, envConfig);
+
+  return envType === "review" && commitInfo
+    ? `${env}-${commitInfo.reviewSlug}`
+    : `${env}`;
+};
+
+export const getEnvironmentContext = (
+  config: Config,
+  env: string,
+  componentName: string,
+  commitInfo?: CommitInfo
+): EnvironmentContext<any, any> => {
+  const envConfigRaw = getEnvConfig(config, componentName, env);
+  const envType = getEnvType(env, envConfigRaw);
+
+  const environmentSlugPrefix = getEnvironmentSlugPrefix(
+    envConfigRaw,
+    env,
+    commitInfo
+  );
+
+  const environmentSlug = `${environmentSlugPrefix}-${componentName}`;
+  const gitlabEnvironmentName =
+    envType === "review" && commitInfo
+      ? `${env}/${commitInfo.refName}/${componentName}`
+      : `${env}/${componentName}`;
+
+  const fullName = `${config.customerName}-${config.appName}-${environmentSlug}`;
+
+  return {
+    envConfigRaw,
+    deployConfigRaw: envConfigRaw.deploy,
+    buildConfigRaw: envConfigRaw.build,
+    environmentSlugPrefix,
+    environmentSlug,
+    gitlabEnvironmentName,
+    fullName,
+    envType,
+    commitInfo,
+    componentName,
+    env,
+    fullConfig: config,
+  };
+};

package/src/context/getEnvironmentVariables.ts

@@ -0,0 +1,155 @@
+import { isObject, merge } from "lodash";
+import { DEPLOY_TYPES } from "../deploy";
+import type { CommitInfo, Context } from "../types";
+import type { Config, DevLocalEnvConfig } from "../types/config";
+
+import { getEnvironmentContext } from "./getEnvironmentContext";
+import {
+  resolveReferences,
+  translateLegacyFromComponents,
+} from "./resolveReferences";
+
+export const getEnvironmentVariables = async (
+  config: Config,
+  componentName: string,
+  env: string,
+  commitInfo?: CommitInfo,
+  alreadyVisited: Record<string, Record<string, boolean>> = {} // to prevent endless loop
+): Promise<{
+  envVars: Record<string, string>;
+  secretEnvVarKeys: string[];
+  host: string;
+  url: string;
+}> => {
+  const environmentContext = getEnvironmentContext(
+    config,
+    env,
+    componentName,
+    commitInfo
+  );
+
+  const { envConfigRaw, deployConfigRaw, envType } = environmentContext;
+
+  const basePredefinedVariables = {
+    ENV_SHORT: env,
+    APP_DIR: envConfigRaw.dir,
+    ENV_TYPE: envType,
+  };
+
+  let predefinedVariables: Record<string, string>;
+  let host: string;
+  let url: string;
+
+  if (envType === "local") {
+    const devLocalConfig: DevLocalEnvConfig = envConfigRaw;
+    const port = devLocalConfig.port ?? 3000;
+    host = "localhost:" + port;
+    url = "http://" + host;
+    predefinedVariables = {
+      ENV_SHORT: "local",
+      ROOT_URL: url,
+      PORT: port.toString(),
+    };
+  } else {
+    const additionalEnvVars = deployConfigRaw
+      ? DEPLOY_TYPES[deployConfigRaw.type].getAdditionalEnvVars(
+          environmentContext as never
+        )
+      : {};
+
+    host =
+      envConfigRaw?.host ??
+      additionalEnvVars.HOST_CANONICAL ??
+      "unknown-host.example.com";
+    url = `https://${host}`;
+
+    predefinedVariables = {
+      ...basePredefinedVariables,
+      // Rails before 6.1 (mis)uses the `HOST` environment variable to specify the IP to bind to
+      ...(config.components[componentName].build.type === "rails"
+        ? {}
+        : { HOST: host }),
+      ROOT_URL: url,
+      ...additionalEnvVars,
+    };
+  }
+  const publicEnvVarsRaw = envConfigRaw.vars?.public ?? {};
+
+  const additionalSecretKeys = deployConfigRaw
+    ? DEPLOY_TYPES[deployConfigRaw.type].additionalSecretKeys(
+        environmentContext as never
+      )
+    : [];
+
+  const secretEnvVarKeys = [
+    ...(envConfigRaw.vars?.secret ?? []),
+    ...additionalSecretKeys,
+  ];
+  const secretEnvVars = Object.fromEntries(
+    secretEnvVarKeys.map((key) => [
+      key,
+      `$${getSecretVarName(env, componentName, key)}`,
+    ])
+  );
+
+  // this is deprecated, we now support: $componentname:FOO
+  const legacyFromComponents = envConfigRaw.vars?.fromComponents ?? {};
+  const publicEnvVarsRawWithLegasyFromComponents = merge(
+    {},
+    translateLegacyFromComponents(legacyFromComponents),
+    publicEnvVarsRaw
+  );
+
+  const publicEnvVarsRawSanitized = Object.fromEntries(
+    Object.entries(publicEnvVarsRawWithLegasyFromComponents).map(
+      ([key, value]) => [
+        key,
+        isObject(value) ? JSON.stringify(value) : `${value}`,
+      ]
+    )
+  );
+
+  const envVarsRaw = addIndexVar({
+    ...predefinedVariables,
+    ...secretEnvVars,
+    ...publicEnvVarsRawSanitized,
+  });
+
+  const envVars = await resolveReferences(
+    envVarsRaw,
+    async (otherComponentName, alreadyVisited) => {
+      const { envVars: otherEnvVars } = await getEnvironmentVariables(
+        config,
+        otherComponentName,
+        env,
+        commitInfo,
+        alreadyVisited
+      );
+      return otherEnvVars;
+    },
+    alreadyVisited
+  );
+
+  return {
+    envVars,
+    secretEnvVarKeys,
+    host,
+    url,
+  };
+};
+
+const sanitizeForEnVar = (s: string) => s.replace(/-/g, "_");
+
+export const getSecretVarName = (
+  env: string,
+  componentName: string,
+  key: string
+) => `CL_${sanitizeForEnVar(env)}_${sanitizeForEnVar(componentName)}_${key}`; // remove dash from component name
+
+const addIndexVar = (vars: Record<string, unknown>) => ({
+  ...vars,
+  _ALL_ENV_VAR_KEYS: JSON.stringify(Object.keys(vars)),
+});
+
+export const getSecretVarNameForContext = (context: Context, key: string) =>
+  getSecretVarName(context.environment.shortName, context.componentName, key);

package/src/context/getLabels.ts

@@ -0,0 +1,20 @@
+import slugify from "slugify";
+import type { Context } from "../types";
+
+const sanitize = (value?: string) => {
+  if (!value) return value;
+  //"The value can only contain lowercase letters, numeric characters, underscores and dashes. The value can be at most 63 characters long. International characters are allowed."
+  // slugify should do the job
+  return slugify(value).toLowerCase();
+};
+export const getLabels = (context: Context) => {
+  const labels = {
+    "customer-name": sanitize(context.fullConfig.customerName),
+    "app-name": sanitize(context.fullConfig.appName),
+    "env-type": sanitize(context.environment.envType),
+    "env-name": sanitize(context.environment.shortName),
+    "build-type": sanitize(context.componentConfig.build?.type),
+    ...(context.fullConfig.meta?.labels ?? {}),
+  };
+  return labels;
+};