@catladder/pipeline 0.0.7 → 1.0.2

package/src/build/types.ts

@@ -0,0 +1,75 @@
+export type BuildConfigBase = {
+  startCommand?: string;
+  extraVars?: Record<string, string>;
+  /**
+   * define the build command
+   */
+  buildCommand?: string | string[] | null;
+
+  /**
+   * customize docker build
+   */
+  docker?: {
+    additionsBegin?: string[];
+    additionsEnd?: string[];
+  };
+
+  /**
+   * customize lint, set false to disable
+   */
+  lint?:
+    | false
+    | {
+        command?: string | string[];
+      };
+
+  /**
+   * customize test, set false to disable
+   */
+  test?:
+    | false
+    | {
+        command?: string | string[];
+      };
+
+  /**
+   * customize audit, set false to disable
+   */
+  audit?:
+    | false
+    | {
+        command?: string | string[];
+      };
+};
+
+export type BuildConfigNodeBase = BuildConfigBase;
+
+export type BuildConfigNode = {
+  type: "node";
+} & BuildConfigNodeBase;
+
+export type BuildConfigNodeStatic = BuildConfigNodeBase & {
+  type: "node-static";
+  startCommand?: never;
+};
+
+export type BuildConfigMeteor = BuildConfigNodeBase & {
+  type: "meteor";
+};
+
+export type BuildConfigStorybook = BuildConfigNodeBase & {
+  type: "storybook";
+  startCommand?: never;
+};
+export type BuildConfig =
+  | BuildConfigNode
+  | BuildConfigNodeStatic
+  | BuildConfigStorybook
+  | BuildConfigMeteor;
+
+export const isOfBuildType = <T extends Array<BuildConfig["type"]>>(
+  t: BuildConfig,
+  ...types: T
+): t is Extract<BuildConfig, { type: T[number] }> => {
+  return types.includes(t.type);
+};

package/src/catladder-gitlab.ts

@@ -0,0 +1,52 @@
+import { writeFileSync } from "fs";
+import { readConfigSync } from "./config";
+import { PIPELINE_IMAGE_TAG } from "./constants";
+import { createChildPipeline } from "./pipeline";
+import { PipelineTrigger } from "./types";
+
+const {
+  CI_MERGE_REQUEST_ID,
+  CI_COMMIT_TAG,
+  CI_COMMIT_BRANCH,
+  CI_DEFAULT_BRANCH,
+} = process.env;
+
+const isDefaultBranch = CI_COMMIT_BRANCH == CI_DEFAULT_BRANCH;
+const isHotfixBranch = false; // TODO:  $CI_COMMIT_BRANCH =~ /^[0-9]+\.([0-9]+|x)\.x$/
+const isMergeRequest = Boolean(CI_MERGE_REQUEST_ID);
+const isTaggedRelease = Boolean(CI_COMMIT_TAG);
+
+console.info(`catladder version ${PIPELINE_IMAGE_TAG}`);
+
+const trigger: PipelineTrigger | null =
+  isMergeRequest || isHotfixBranch
+    ? "mr"
+    : isDefaultBranch
+    ? "mainBranch"
+    : isTaggedRelease
+    ? "taggedRelease"
+    : null;
+if (trigger) {
+  const config = readConfigSync();
+  if (!config) {
+    throw new Error("no catladder config found");
+  }
+  createChildPipeline(trigger, config).then((mainPipeline) => {
+    writeFileSync(`__pipeline.yml`, JSON.stringify(mainPipeline, null, 2), {
+      encoding: "utf-8",
+    });
+  });
+} else {
+  throw new Error(
+    "no matching trigger: " +
+      JSON.stringify(
+        {
+          isMergeRequest,
+          isDefaultBranch,
+          isTaggedRelease,
+        },
+        null,
+        2
+      )
+  );
+}

package/src/config/configruedEnvs.test.ts

@@ -0,0 +1,97 @@
+import { getAllEnvsByTrigger } from ".";
+import { Config } from "../types";
+
+describe("getAllEnvsByTrigger", () => {
+  const SIMPLE_CONFIG: Config = {
+    appName: "my-app",
+    customerName: "pan",
+    components: {
+      app1: {
+        dir: "dir1",
+        build: {
+          type: "node",
+        },
+        deploy: { type: "kubernetes" },
+      },
+      app2: {
+        dir: "dir2",
+        build: {
+          type: "node",
+        },
+        deploy: { type: "kubernetes" },
+        env: {
+          dev2: {
+            type: "dev",
+          },
+          review2: {
+            type: "review",
+          },
+          stage2: {
+            type: "stage",
+          },
+          prod2: {
+            type: "prod",
+          },
+        },
+      },
+      app3: {
+        dir: "dir2",
+        build: {
+          type: "node",
+        },
+        deploy: { type: "kubernetes" },
+        env: {
+          dev: false,
+          review: false,
+          stage: false,
+          prod: false,
+        },
+      },
+    },
+  };
+
+  describe("mainBranch", () => {
+    it("always includes dev if not disabled", () => {
+      const envs = getAllEnvsByTrigger(SIMPLE_CONFIG, "app1", "mainBranch");
+      expect(envs).toEqual(["dev"]);
+    });
+    it("returns additional dev environments", () => {
+      const envs = getAllEnvsByTrigger(SIMPLE_CONFIG, "app2", "mainBranch");
+      expect(envs).toEqual(["dev", "dev2"]);
+    });
+    it("does not include disabled envs", () => {
+      const envs = getAllEnvsByTrigger(SIMPLE_CONFIG, "app3", "mainBranch");
+      expect(envs).toEqual([]);
+    });
+  });
+
+  describe("mr", () => {
+    it("always includes review if not disabled", () => {
+      const envs = getAllEnvsByTrigger(SIMPLE_CONFIG, "app1", "mr");
+      expect(envs).toEqual(["review"]);
+    });
+    it("returns additional dev environments", () => {
+      const envs = getAllEnvsByTrigger(SIMPLE_CONFIG, "app2", "mr");
+      expect(envs).toEqual(["review", "review2"]);
+    });
+    it("does not include disabled envs", () => {
+      const envs = getAllEnvsByTrigger(SIMPLE_CONFIG, "app3", "mr");
+      expect(envs).toEqual([]);
+    });
+  });
+
+  describe("taggedRelease", () => {
+    it("always includes review if not disabled", () => {
+      const envs = getAllEnvsByTrigger(SIMPLE_CONFIG, "app1", "taggedRelease");
+      expect(envs).toEqual(["stage", "prod"]);
+    });
+    it("returns additional dev environments", () => {
+      const envs = getAllEnvsByTrigger(SIMPLE_CONFIG, "app2", "taggedRelease");
+      expect(envs).toEqual(["stage", "prod", "stage2", "prod2"]);
+    });
+    it("does not include disabled envs", () => {
+      const envs = getAllEnvsByTrigger(SIMPLE_CONFIG, "app3", "taggedRelease");
+      expect(envs).toEqual([]);
+    });
+  });
+});

package/src/config/configruedEnvs.ts

@@ -0,0 +1,56 @@
+import {
+  Config,
+  DEFAULT_ENV_TYPES,
+  EnvType,
+  getEnvTypesByTrigger,
+  PipelineTrigger,
+} from "../types";
+
+const getConfiguredAndDefaultEnvs = (
+  config: Config,
+  componentName: string,
+  envTypes: EnvType[]
+) => {
+  const configuredEnvs = config.components[componentName].env ?? {};
+  // the default envs have the same name as the env types
+  // these can be disabled with settimg them to `false`
+  // this is the list of all not disabled envs. These are always returned
+  const enabledDefaultEnvs = envTypes.filter(
+    (e) => configuredEnvs[e] !== false
+  );
+
+  const configuredCustomEnvs = Object.entries(
+    config.components[componentName].env ?? {}
+  )
+    .filter(
+      ([, config]) =>
+        config &&
+        "type" in config &&
+        config.type &&
+        envTypes.includes(config.type)
+    )
+    .map(([envName]) => envName);
+
+  return [...new Set([...enabledDefaultEnvs, ...configuredCustomEnvs])];
+};
+
+export const getAllEnvs = (config: Config, componentName: string) => {
+  return getConfiguredAndDefaultEnvs(config, componentName, DEFAULT_ENV_TYPES);
+};
+
+export const getAllEnvsInAllComponents = (config: Config) => {
+  return [
+    ...new Set(
+      Object.keys(config.components).flatMap((c) => getAllEnvs(config, c))
+    ),
+  ];
+};
+
+export const getAllEnvsByTrigger = (
+  config: Config,
+  componentName: string,
+  trigger: PipelineTrigger
+) => {
+  const envTypesByTrigger = getEnvTypesByTrigger(trigger);
+  return getConfiguredAndDefaultEnvs(config, componentName, envTypesByTrigger);
+};

package/src/config/index.ts

@@ -0,0 +1,2 @@
+export * from "./configruedEnvs";
+export * from "./readConfig";

package/src/config/readConfig.ts

@@ -0,0 +1,36 @@
+import { existsSync, readFileSync } from "fs";
+import { register } from "ts-node";
+import { load } from "js-yaml";
+import { Config } from "../types";
+
+// allows us to load ts files
+
+const fullPath = (directory: string, ext: string) =>
+  directory + "/catladder." + ext;
+
+export const readConfigSync = (
+  directory: string = process.cwd()
+): Config | null => {
+  register({
+    cwd: directory,
+    transpileOnly: true,
+    compilerOptions: {
+      module: "commonjs",
+    },
+  });
+
+  const found = ["ts", "js", "yml", "yaml"].find((extension) =>
+    existsSync(fullPath(directory, extension))
+  );
+  if (found) {
+    if (found === "ts" || found === "js") {
+      // eslint-disable-next-line @typescript-eslint/no-var-requires
+      return require(fullPath(directory, found)).default;
+    } else {
+      return load(
+        readFileSync(fullPath(directory, found), { encoding: "utf-8" })
+      ) as Config;
+    }
+  }
+  return null;
+};

package/src/constants.ts

@@ -0,0 +1,2 @@
+export const PIPELINE_IMAGE_TAG = process.env.INLINE_PIPELINE_IMAGE_TAG || "";
+export const DOCKER_REGISTRY = process.env.INLINE_DOCKER_REGISTRY || "";

package/src/context/index.ts

@@ -0,0 +1,194 @@
+import { merge } from "lodash";
+import slugify from "slugify";
+import { BUILD_TYPES } from "../build";
+import { BuildConfig } from "../build/types";
+import { DEPLOY_TYPES } from "../deploy";
+import { DeployConfig } from "../deploy/types";
+import { Config, isKnowEnvType, DevLocalEnvConfig } from "../types/config";
+import { CommitInfo, Context, Environment, YarnInfo } from "../types/context";
+
+const sanitizeForEnVar = (s: string) => s.replace(/-/g, "_");
+export const getSecretVarName = (
+  env: string,
+  componentName: string,
+  key: string
+) => `CL_${sanitizeForEnVar(env)}_${sanitizeForEnVar(componentName)}_${key}`; // remove dash from component name
+export const getEnvironment = (
+  config: Config,
+  componentName: string,
+  env: string,
+  commitInfo?: CommitInfo,
+  skipReferences = false // to prevent infinit loop
+): Environment => {
+  const componentConfig = config.components[componentName];
+  if (!componentConfig) {
+    throw new Error("unknown component " + componentName);
+  }
+
+  const defaultConfig = componentConfig;
+
+  const envConfig = componentConfig.env?.[env] ?? {};
+  if (envConfig === false) {
+    throw new Error("env is disabled: " + env);
+  }
+
+  const mergedConfig = merge({}, defaultConfig, envConfig);
+
+  // env type: if its set manually, use that, otherwise use the known env types
+
+  const basePredefinedVariables = {
+    ENV_SHORT: env,
+    APP_DIR: componentConfig.dir,
+  };
+  const envType = envConfig?.type ?? (isKnowEnvType(env) ? env : null);
+  if (!envType) {
+    throw new Error(
+      "Missing type in environment " + env + " in component " + componentName
+    );
+  }
+  const environmentName =
+    envType === "review" && commitInfo
+      ? `${env}/${commitInfo.refName}/${componentName}`
+      : `${env}/${componentName}`;
+
+  const environmentSlug =
+    envType === "review" && commitInfo
+      ? `${env}-${commitInfo.refSlug}-${componentName}`
+      : `${env}-${componentName}`;
+
+  let predefinedVariables: Record<string, string>;
+  let hostname: string;
+  let url: string;
+  if (envType === "local") {
+    const devLocalConfig: DevLocalEnvConfig = mergedConfig;
+    const port = devLocalConfig.port ?? 3000;
+    hostname = "localhost:" + port;
+    url = "http://" + hostname;
+    predefinedVariables = {
+      ROOT_URL: url,
+      PORT: port.toString(),
+    };
+  } else {
+    const KUBE_NAMESPACE = `${config.customerName}-${config.appName}-${env}`;
+    const RELEASE_NAME = `${config.customerName}-${config.appName}-${environmentSlug}`;
+
+    const componentSlug = slugify(componentName);
+    const envInUrl =
+      envType === "review" && commitInfo ? `${commitInfo.refSlug}.${env}` : env;
+
+    const HOST_CANONICAL = `${config.appName}-${componentSlug}.${envInUrl}.${config.customerName}.panter.cloud`;
+
+    hostname = mergedConfig?.hostname ?? HOST_CANONICAL;
+    const url = `https://${hostname}`;
+
+    const KUBE_APP_NAME =
+      envType === "review" && commitInfo
+        ? `${componentName}-${commitInfo.refSlug}`
+        : componentName;
+
+    predefinedVariables = {
+      ...basePredefinedVariables,
+      HOST_CANONICAL,
+      ROOT_URL: url,
+      KUBE_NAMESPACE,
+      KUBE_APP_NAME,
+      RELEASE_NAME,
+    };
+  }
+  const publicEnvVars = mergedConfig.vars?.public ?? {};
+  const secretEnvVarKeys = mergedConfig.vars?.secret ?? [];
+  const secretEnvVars = Object.fromEntries(
+    secretEnvVarKeys.map((key) => [
+      key,
+      `$${getSecretVarName(env, componentName, key)}`,
+    ])
+  );
+  const referencedRaw = mergedConfig.vars?.fromComponents ?? {};
+
+  const referenced = skipReferences
+    ? {}
+    : Object.entries(referencedRaw).reduce((acc, [otherApp, mapping]) => {
+        const { envVars } = getEnvironment(
+          config,
+          otherApp,
+          env,
+          commitInfo,
+          true // prevent infinit loop
+        );
+
+        return {
+          ...acc,
+          ...Object.fromEntries(
+            Object.entries(mapping).map(([ourKey, otherKey]) => [
+              ourKey,
+              envVars[otherKey],
+            ])
+          ),
+        };
+      }, {});
+
+  const envVars = {
+    ...predefinedVariables,
+    ...secretEnvVars,
+    ...referenced,
+    ...publicEnvVars,
+  };
+
+  return {
+    envType,
+    hostname,
+    fullName: environmentName,
+    slug: environmentSlug,
+    shortName: env,
+    url: predefinedVariables.ROOT_URL,
+    envVars,
+    secretEnvVarKeys,
+  };
+};
+
+export const createContext = (
+  config: Config,
+  componentName: string,
+  env: string,
+  commitInfo?: CommitInfo,
+  yarnInfo?: YarnInfo
+): Context => {
+  if (!/^[a-z0-9-]+$/.test(componentName)) {
+    throw new Error(
+      "componentName may only contain lower case letters, numbers and -"
+    );
+  }
+  const rawConfig = config.components[componentName];
+  if (!rawConfig) {
+    throw new Error("unknown component " + componentName);
+  }
+  // envs can override the config
+  const envConfig = rawConfig.env?.[env] ?? {};
+  const componentConfigWithoutDefaults = merge({}, rawConfig, envConfig);
+
+  // fill in defaults of build and deploy
+  const defaults: {
+    build: Partial<BuildConfig>;
+    deploy: Partial<DeployConfig>;
+  } = componentConfigWithoutDefaults.deploy
+    ? {
+        build:
+          BUILD_TYPES[componentConfigWithoutDefaults.build.type].defaults(),
+        deploy:
+          DEPLOY_TYPES[componentConfigWithoutDefaults.deploy.type].defaults(),
+      }
+    : {
+        build: {},
+        deploy: {},
+      };
+  const componentConfig = merge({}, defaults, componentConfigWithoutDefaults);
+
+  return {
+    fullConfig: config,
+    componentConfig,
+    componentName,
+    environment: getEnvironment(config, componentName, env, commitInfo),
+    commitInfo,
+    yarnInfo,
+  };
+};

package/src/defaults/index.ts

@@ -0,0 +1,6 @@
+import { Retry } from "../types";
+
+export const BASE_RETRY: Retry = {
+  max: 2,
+  when: ["runner_system_failure", "stuck_or_timeout_failure"],
+};

package/src/deploy/base.ts

@@ -0,0 +1,101 @@
+import { GitlabJob, Context, GitlabJobDef } from "../types";
+
+export const DEPLOY_JOB_NAME = "🚀 Deploy";
+export const STOP_JOB_NAME = "🛑 Stop ⚠️";
+
+const DEPLOY_RUNNER_VARIABLES = {
+  KUBERNETES_CPU_REQUEST: "0.5",
+  KUBERNETES_CPU_LIMIT: "1",
+  KUBERNETES_MEMORY_REQUEST: "200Mi",
+  KUBERNETES_MEMORY_LIMIT: "500Mi",
+};
+type JobWithoutScript = Omit<GitlabJob, "job"> & {
+  job: Omit<GitlabJobDef, "script">;
+};
+export const getBaseDeploymentJob = (context: Context): JobWithoutScript => {
+  const environment = {
+    name: context.environment.fullName,
+    url: context.environment.url,
+  };
+
+  const autoStop =
+    context.environment.envType === "review"
+      ? "2 weeks"
+      : context.environment.envType === "dev"
+      ? "3 weeks"
+      : undefined;
+
+  return {
+    name: DEPLOY_JOB_NAME,
+    envMode: "stagePerEnv", // makes it easier to run manual tasks er env
+    // we don't want to deploy when there is a broken test
+    needsStages: [
+      {
+        stage: "build",
+        artifacts: false,
+      },
+      {
+        stage: "test",
+        artifacts: false,
+      },
+    ], // workaround for https://gitlab.com/gitlab-org/gitlab/-/issues/220758
+    job: {
+      rules: [
+        context.environment.envType === "prod"
+          ? {
+              when: "manual",
+            }
+          : {
+              when: "on_success",
+            },
+      ],
+      stage: "deploy",
+      dependencies: [],
+      variables: {
+        ...DEPLOY_RUNNER_VARIABLES,
+      },
+      environment: {
+        ...environment,
+        on_stop: STOP_JOB_NAME,
+        auto_stop_in: autoStop,
+      },
+    },
+  };
+};
+
+export const getBaseDeploymentStopJob = (
+  context: Context
+): JobWithoutScript => {
+  const environment = {
+    name: context.environment.fullName,
+    url: context.environment.url,
+  };
+  return {
+    name: STOP_JOB_NAME,
+    envMode: "stagePerEnv", // makes it easier to run manual tasks er env
+    job: {
+      needs: [DEPLOY_JOB_NAME],
+      rules: [
+        {
+          if: "$CI_COMMIT_BRANCH =~ /^[0-9]+\\.([0-9]+|x)\\.x$/", // automatic on hotfix branches
+          when: "on_success",
+          allow_failure: true,
+        },
+        {
+          when: "manual",
+          allow_failure: true,
+        },
+      ],
+      variables: {
+        ...DEPLOY_RUNNER_VARIABLES,
+        GIT_STRATEGY: "none",
+      },
+      stage: "stop",
+      dependencies: [],
+      environment: {
+        ...environment,
+        action: "stop",
+      },
+    },
+  };
+};

package/src/deploy/index.ts

@@ -0,0 +1,19 @@
+import { GitlabJobs } from "../types/gitlab-types";
+import { Context } from "../types/context";
+import { createKubernetesDeployJobs } from "./kubernetes";
+import { DeployConfig } from "./types";
+
+export type DeployTypes = {
+  [type in DeployConfig["type"]]: {
+    jobs: (context: Context) => GitlabJobs;
+    defaults: () => Partial<Extract<DeployConfig, { type: type }>>;
+  };
+};
+
+export * from "./types";
+export const DEPLOY_TYPES: DeployTypes = {
+  kubernetes: {
+    jobs: createKubernetesDeployJobs,
+    defaults: () => ({}),
+  },
+};

package/src/deploy/kubernetes/cloudsql.ts

@@ -0,0 +1,13 @@
+import { Context, getSecretVarName } from "../..";
+
+export const createCloudsqlBaseConfig = (context: Context) => {
+  return {
+    cloudsql: {
+      proxyCredentials: `$${getSecretVarName(
+        context.environment.shortName,
+        context.componentName,
+        "cloudsqlProxyCredentials"
+      )}`,
+    },
+  };
+};