@catladder/pipeline 0.0.1

package/scripts/magic/createChildPipeline/index.ts

@@ -0,0 +1,163 @@
+import { merge } from "lodash";
+import slugify from "slugify";
+import { GitlabJobDef, GitlabJobs } from "../types/gitlab-types";
+import { buildJobCreators } from "../types/build";
+import { Config } from "../types/config";
+import { Context, Environment } from "../types/context";
+import { deployJobCreators } from "../types/deploy";
+
+const createRawJobs = (context: Context): GitlabJobs => {
+  const buildJobs =
+    buildJobCreators[context.componentConfig.build.type](context);
+  const deployJobs =
+    deployJobCreators[context.componentConfig.deploy.type](context);
+
+  return [...buildJobs, ...deployJobs];
+};
+
+const getEnvironment = (
+  config: Config,
+  componentName: string,
+  env: string
+): Environment => {
+  const componentConfig = config.components[componentName];
+  if (!componentConfig) {
+    throw new Error("unknown component " + componentName);
+  }
+
+  const defaultConfig = componentConfig;
+  const envConfig = componentConfig.env?.[env] ?? {};
+
+  const mergedConfig = merge({}, defaultConfig, envConfig);
+
+  const publicEnvVars = mergedConfig.vars?.public ?? {};
+  const secretEnvVarsRaw = mergedConfig.vars?.secret ?? {}; // TODO, map to gl secrets?
+  const secretEnvVars = {}; // TODO
+  const referencedRaw = mergedConfig.vars?.fromComponents ?? {};
+
+  const referenced = Object.entries(referencedRaw).reduce(
+    (acc, [otherApp, mapping]) => {
+      // TODO: prevent infinit looop
+      const { variables } = getEnvironment(config, otherApp, env);
+
+      return Object.fromEntries(
+        Object.entries(mapping).map(([ourKey, otherKey]) => [
+          ourKey,
+          variables[otherKey],
+        ])
+      );
+    },
+    {}
+  );
+
+  const envType = componentConfig.env?.[env]?.type ?? env;
+  const environmentName =
+    envType === "review"
+      ? `${env}-${componentName}/$CI_COMMIT_REF_NAME`
+      : `${env}-${componentName}`;
+
+  const KUBE_APP_NAME =
+    envType === "review"
+      ? `${componentName}-$CI_COMMIT_REF_SLUG`
+      : componentName;
+
+  const KUBE_NAMESPACE = `${config.customerName}-${config.appName}-${env}`;
+
+  const APP_SLUG = slugify(KUBE_APP_NAME);
+
+  const CANONICAL_HOST = `${config.appName}-${APP_SLUG}.${config.customerName}.panter.cloud`;
+
+  const hostname = mergedConfig.hostname ?? CANONICAL_HOST;
+  const url = `https://${hostname}`;
+
+  const predefinedVariables = {
+    CANONICAL_URL: url,
+    ROOT_URL: url,
+    KUBE_NAMESPACE,
+  };
+  const variables = {
+    ...predefinedVariables,
+    ...publicEnvVars,
+    ...secretEnvVars,
+    ...referenced,
+  };
+
+  return {
+    fullName: environmentName,
+    shortName: env,
+    url: url,
+    variables,
+  };
+};
+
+export const createContext = (
+  componentName: string,
+  config: Config
+): Context => {
+  const componentConfig = config.components[componentName];
+  if (!componentConfig) {
+    throw new Error("unknown component " + componentName);
+  }
+  return {
+    fullConfig: config,
+    componentConfig,
+    componentName,
+  };
+};
+
+const getFullJobName = (name: string, componentName: string, env?: string) => {
+  if (env) {
+    return `${env} ${componentName} ${name}`;
+  }
+  return `${componentName} ${name}`;
+};
+
+const replaceReferences = (
+  def: GitlabJobDef,
+  componentName: string,
+  env?: string
+): GitlabJobDef => {
+  return {
+    ...def,
+    needs: def.needs?.map((n) => getFullJobName(n, componentName, env)),
+    dependencies: def.dependencies?.map((n) =>
+      getFullJobName(n, componentName, env)
+    ),
+  };
+};
+
+export const createJobs = (
+  envs: string[],
+  config: Config,
+  componentName: string
+): Record<string, GitlabJobDef> => {
+  const context = createContext(componentName, config);
+  const jobs = createRawJobs(context);
+  const environments = envs.map((e) =>
+    getEnvironment(config, componentName, e)
+  );
+  return jobs.reduce((acc, job) => {
+    if (!job.env) {
+      const def = job.job();
+
+      return {
+        ...acc,
+        [getFullJobName(job.name, componentName)]: replaceReferences(
+          def,
+          componentName
+        ),
+      };
+    }
+    return {
+      ...acc,
+      ...environments.reduce((envacc, environment) => {
+        const def = job.job(environment);
+        return {
+          ...acc,
+          [getFullJobName(job.name, componentName, environment.shortName)]:
+            replaceReferences(def, componentName, environment.shortName),
+        };
+      }, {}),
+    };
+  }, {});
+};

package/scripts/magic/createPipeline.ts

@@ -0,0 +1,48 @@
+import { createJobs } from "./createChildPipeline";
+import { Config, ENV_TYPES, PipelineTrigger } from "./types/config";
+
+export const createPipeline = async (
+  trigger: PipelineTrigger,
+  config: Config
+) => {
+  const envs = Object.keys(ENV_TYPES).filter(
+    (e) => ENV_TYPES[e as keyof typeof ENV_TYPES].trigger === trigger
+  );
+  const components = Object.keys(config.components);
+
+  // 2. write the triggering pipeline
+
+  const jobs = components.reduce((acc, componentName) => {
+    return {
+      ...acc,
+      ...createJobs(envs, config, componentName),
+    };
+  }, {});
+
+  const rules = [
+    // same as rules "always", but with `changes` to only trigger changed branches
+    { if: "$CI_COMMIT_TAG" },
+    {
+      if: "$CI_COMMIT_MESSAGE =~ /^chore(release).*/",
+      when: "never",
+    },
+    { if: "$CI_COMMIT_BRANCH =~ /^[0-9]+.([0-9]+|x).x$/" },
+    {
+      if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH",
+    },
+
+    { if: "$CI_MERGE_REQUEST_ID" },
+  ];
+
+  const mainPipeline = {
+    image:
+      "git.panter.ch:5001/catladder/gitlab-ci/pipeline:$PIPELINE_IMAGE_TAG",
+    workflow: {
+      rules,
+    },
+    stages: ["setup", "test", "build", "deploy", "verify", "actions"],
+    ...jobs,
+  };
+
+  return mainPipeline;
+};

package/scripts/magic/index.ts

@@ -0,0 +1,2 @@
+export { createPipeline } from "./createPipeline";
+export * from "./types";

package/scripts/magic/sample-config.ts

@@ -0,0 +1,78 @@
+import { Config } from "./types/config";
+
+const sampleConfig: Config = {
+  customerName: "pan",
+  appName: "fonsi",
+  components: {
+    web: {
+      dir: "web",
+      build: {
+        type: "node-static",
+      },
+      deploy: {
+        type: "kubernetes",
+        values: {
+          application: {
+            resources: {
+              limits: {
+                cpu: "100m",
+              },
+            },
+          },
+        },
+      },
+      vars: {
+        public: {
+          GOOGLE_TAG_MANAGER_ID: "1234",
+        },
+        fromComponents: {
+          api: {
+            API_URL: "ROOT_URL",
+          },
+        },
+      },
+      env: {
+        prod: {
+          hostname: "fudilo.ch",
+          vars: {},
+          deploy: {
+            type: "kubernetes",
+            values: {
+              application: {
+                resources: {
+                  limits: {
+                    cpu: "1000m",
+                  },
+                },
+              },
+            },
+          },
+        },
+        myCustomProd: {
+          type: "prod",
+          hostname: "fudilo2.ch",
+        },
+      },
+    },
+    api: {
+      env: {
+        prod: {
+          hostname: "api.fudilo.ch",
+        },
+        myCustomProd: {
+          type: "prod",
+          hostname: "api.fudilo2.ch",
+        },
+      },
+      build: {
+        type: "node",
+      },
+      deploy: {
+        type: "kubernetes",
+      },
+      dir: "api",
+    },
+  },
+};
+
+export default sampleConfig;

package/scripts/magic/types/build/docker.ts

@@ -0,0 +1,36 @@
+import { GitlabJobDef } from "../gitlab-types";
+
+export const createDockerBuildJob = ({
+  script,
+}: Pick<GitlabJobDef, "script">): GitlabJobDef => {
+  const base: Omit<GitlabJobDef, "script"> = {
+    stage: "build",
+    image:
+      "git.panter.ch:5001/catladder/gitlab-ci/docker-build:$PIPELINE_IMAGE_TAG",
+    interruptible: true,
+    services: [
+      {
+        name: "docker:20-dind", // see see https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27300#note_466755332
+        command: ["--tls=false"],
+      },
+    ],
+    variables: {
+      DOCKER_DRIVER: "overlay2",
+      IMAGE_NAME: "$CI_REGISTRY_IMAGE/$COMPONENT_NAME",
+      CACHE_IMAGE: "$CI_REGISTRY_IMAGE/$COMPONENT_NAME/cache:cache",
+    },
+  };
+
+  return {
+    ...base,
+    script: [
+      ...script,
+      "docker login --username gitlab-ci-token --password $CI_JOB_TOKEN $CI_REGISTRY",
+      "docker pull $CACHE_IMAGE || true",
+      "docker build --network host --cache-from $CACHE_IMAGE --tag $IMAGE_NAME:$IMAGE_TAG $DOCKER_DIR",
+      "docker push $IMAGE_NAME:$IMAGE_TAG",
+      "docker tag $IMAGE_NAME:$IMAGE_TAG $CACHE_IMAGE",
+      "docker push $CACHE_IMAGE",
+    ],
+  };
+};

package/scripts/magic/types/build/index.ts

@@ -0,0 +1,11 @@
+import { GitlabJobs } from "../gitlab-types";
+import { Context } from "../context";
+import { createNodeJobs } from "./node";
+import { BuildConfig } from "./types";
+
+export const buildJobCreators: {
+  [type in BuildConfig["type"]]: (context: Context) => GitlabJobs;
+} = {
+  node: createNodeJobs,
+  "node-static": createNodeJobs, // TODO
+};

package/scripts/magic/types/build/node.ts

@@ -0,0 +1,107 @@
+import {
+  GitlabJobDef,
+  GitlabJobCache,
+  GitlabJobs,
+  Retry,
+} from "../gitlab-types";
+import { Context } from "../context";
+import { createDockerBuildJob } from "./docker";
+
+const yarnInstall = [
+  "cd $APP_PATH",
+  "if [ -f ./.nvmrc ]; then source /root/.nvm/nvm.sh && nvm install <<< .nvmrc; fi",
+  "yarn install --frozen-lockfile",
+];
+
+const baseRetry: Retry = {
+  max: 2,
+  when: ["runner_system_failure", "stuck_or_timeout_failure"],
+};
+
+const createNodeTestJobs = (context: Context): GitlabJobs => {
+  const cache: GitlabJobCache[] = [
+    {
+      key: "node-modules",
+      policy: "pull-push",
+      paths: ["node_modules", "**/node_modules/"],
+    },
+  ];
+  const base: Omit<GitlabJobDef, "script"> = {
+    variables: {
+      APP_PATH: context.componentConfig.dir,
+    },
+    cache,
+    stage: "test",
+    interruptible: true,
+    needs: [],
+    retry: baseRetry,
+  };
+  return [
+    {
+      name: "test",
+      env: false,
+      job: () => ({
+        ...base,
+        script: [...yarnInstall, "yarn test"],
+      }),
+    },
+    {
+      name: "lint",
+      env: false,
+      job: () => ({
+        ...base,
+        script: [...yarnInstall, "yarn lint"],
+      }),
+    },
+    {
+      name: "audit",
+      env: false,
+      job: () => ({
+        ...base,
+        script: [...yarnInstall, "yarn audit"],
+      }),
+    },
+  ];
+};
+
+const createNodeBuildJobs = (context: Context): GitlabJobs => {
+  const buildInfo = [
+    ". getCommitInfo", // TODO: inline
+    `echo '{"id":"'$BUILD_ID'","commit":"'$BUILD_COMMIT'","tag":"'$BUILD_TAG'","time":"'$BUILD_TIME'"}' > $APP_DIR/__build_info.json`,
+  ];
+
+  return [
+    {
+      name: "app-build",
+      env: true,
+      job: (environment) => ({
+        variables: environment.variables,
+        retry: baseRetry,
+        interruptible: true,
+        stage: "build",
+        script: [...buildInfo, ...yarnInstall, "yarn build"],
+        artifacts: {
+          paths: [
+            context.componentConfig.dir + "/__build_info.json",
+            context.componentConfig.dir + "/dist",
+            context.componentConfig.dir + "/.next",
+          ],
+        },
+      }),
+    },
+    {
+      name: "docker-build",
+      env: true,
+      job: () => ({
+        ...createDockerBuildJob({
+          script: ["ensureNodeDockerfile"], // TOOD: inline
+        }),
+        needs: ["app-build"],
+      }),
+    },
+  ];
+};
+
+export const createNodeJobs = (context: Context): GitlabJobs => {
+  return [...createNodeBuildJobs(context), ...createNodeTestJobs(context)];
+};

package/scripts/magic/types/build/types.ts

@@ -0,0 +1,9 @@
+export type BuildConfigNode = {
+  type: "node";
+};
+
+export type BuildConfigNodeStatic = {
+  type: "node-static";
+};
+
+export type BuildConfig = BuildConfigNode | BuildConfigNodeStatic;

package/scripts/magic/types/config.ts

@@ -0,0 +1,54 @@
+import { BuildConfig } from "./build/types";
+import { DeployConfig } from "./deploy/types";
+
+export type PipelineTrigger = "mainBranch" | "mr" | "taggedRelease";
+
+export const ENV_TYPES = {
+  dev: {
+    trigger: "mainBranch",
+  },
+  review: {
+    trigger: "mr",
+  },
+  prod: {
+    trigger: "taggedRelease",
+  },
+  stage: {
+    trigger: "taggedRelease",
+  },
+} as const;
+type EnvType = keyof typeof ENV_TYPES;
+
+type DefaultEnvConfig = {
+  deploy: DeployConfig;
+  build: BuildConfig;
+  vars?: {
+    public?: Record<string, any>;
+    secret?: Record<string, string>;
+    fromComponents?: {
+      [otherApp: string]: Record<string, string>;
+    };
+  };
+};
+type EnvConfig<E extends EnvType = EnvType> = {
+  type?: E;
+  hostname?: string;
+} & Partial<DefaultEnvConfig>;
+
+export type Env = {
+  dev?: EnvConfig<"dev">;
+  stage?: EnvConfig<"stage">;
+  review?: EnvConfig<"review">;
+  prod?: EnvConfig<"prod">;
+} & Record<string, EnvConfig>;
+
+export type ComponentConfig = {
+  env?: Env;
+  dir: string;
+} & DefaultEnvConfig;
+
+export type Config = {
+  customerName: string;
+  appName: string;
+  components: Record<string, ComponentConfig>;
+};

package/scripts/magic/types/context.ts

@@ -0,0 +1,13 @@
+import { ComponentConfig, Config, PipelineTrigger } from "./config";
+
+export type Environment = {
+  fullName: string;
+  shortName: string;
+  url: string;
+  variables: Record<string, string>;
+};
+export type Context = {
+  componentConfig: ComponentConfig;
+  componentName: string;
+  fullConfig: Config;
+};

package/scripts/magic/types/deploy/index.ts

@@ -0,0 +1,11 @@
+import { GitlabJobs } from "../gitlab-types";
+import { ComponentConfig } from "../config";
+import { Context } from "../context";
+import { createKubernetesDeployJobs } from "./kubernetes";
+
+export const deployJobCreators: {
+  [type in ComponentConfig["deploy"]["type"]]: (context: Context) => GitlabJobs;
+} = {
+  kubernetes: createKubernetesDeployJobs,
+  serverless: () => [],
+};

package/scripts/magic/types/deploy/kubernetes.ts

@@ -0,0 +1,38 @@
+import { GitlabJobs } from "../gitlab-types";
+import { Context } from "../context";
+import { isOfType } from "./types";
+
+export const createKubernetesDeployJobs = (context: Context): GitlabJobs => {
+  const deployConfig = context.componentConfig.deploy;
+  if (!isOfType(deployConfig, "kubernetes")) {
+    // should not happen
+    throw new Error("deploy config is not kubernetes");
+  }
+
+  return [
+    {
+      name: "deploy-to-kubernetes",
+      env: true,
+      job: (environment) => ({
+        variables: {
+          ...environment.variables,
+          RELEASE_NAME: "${CUSTOMER_NAME}-${APP_NAME}-${CI_ENVIRONMENT_SLUG}",
+        },
+        stage: "deploy",
+        dependencies: [],
+        script: [
+          "kubernetesEnsureNamespace",
+          "kubernetesCreateSecret",
+          "kubernetesDeploy",
+        ],
+        environment: {
+          name: environment.fullName,
+          url: environment.url,
+          kubernetes: {
+            namespace: environment.variables.KUBE_NAMESPACE,
+          },
+        },
+      }),
+    },
+  ];
+};

package/scripts/magic/types/deploy/types.ts

@@ -0,0 +1,29 @@
+export type DeployConfigBase = {};
+export type DeployConfigKubernetes = {
+  type: "kubernetes";
+  additionalHelmArgs?: string[];
+  values?: {
+    application?: {
+      resources?: {
+        limits?: {
+          cpu?: any;
+        };
+        requests?: {
+          cpu?: any;
+        };
+      };
+    } & Record<string, any>;
+  } & Record<string, any>;
+} & DeployConfigBase;
+
+export type DeployConfigServerless = {
+  type: "serverless";
+};
+export type DeployConfig = DeployConfigKubernetes | DeployConfigServerless;
+
+export const isOfType = <T extends DeployConfig["type"]>(
+  t: DeployConfig,
+  type: T
+): t is Extract<DeployConfig, { type: T }> => {
+  return t.type === type;
+};

package/scripts/magic/types/gitlab-types.ts

@@ -0,0 +1,77 @@
+import { Environment } from "./context";
+
+export type GitlabStage =
+  | "setup"
+  | "test"
+  | "build"
+  | "deploy"
+  | "verify"
+  | "actions";
+
+export type Artifacts = {
+  paths: string[];
+};
+export type GitlabJobCache = {
+  key: string;
+  policy?: string;
+  paths: string[];
+};
+export type GitlabRule = {
+  if: string;
+  when?: "always" | "on_success" | "manual" | "never"; // todo
+};
+export type Retry = {
+  max: number;
+  when: string[];
+};
+
+export type Service = {
+  name: string;
+  command: string[];
+};
+
+export type GitlabEnvironment = {
+  url: string;
+  name: string;
+  kubernetes?: {
+    namespace?: string;
+  };
+};
+export type GitlabJobDef = {
+  stage: GitlabStage;
+  before_script?: string[];
+  script: string[];
+  interruptible?: boolean;
+  needs?: string[];
+  rules?: GitlabRule[];
+  cache?: GitlabJobCache | GitlabJobCache[];
+  artifacts?: Artifacts;
+  retry?: Retry;
+  services?: Service[];
+  image?: string;
+  variables?: GitlabVariables;
+  dependencies?: string[];
+  environment?: GitlabEnvironment;
+};
+
+export type GitlabVariables = Record<string, string | undefined>;
+
+export type JobEnvDependent = {
+  name: string;
+  env: true;
+  job: (environment: Environment) => GitlabJobDef;
+};
+
+export type JobNonEnv = {
+  name: string;
+  env: false;
+  job: () => GitlabJobDef;
+};
+export type GitlabJob = JobEnvDependent | JobNonEnv;
+export type GitlabJobs = GitlabJob[];
+
+export type GitlabPipeline = {
+  variables: GitlabVariables;
+  stages: GitlabStage[];
+  jobs: GitlabJobs;
+};

package/scripts/magic/types/index.ts

@@ -0,0 +1,3 @@
+export * from "./config";
+export * from "./gitlab-types";
+export * from "./context";