@catladder/pipeline 0.0.5 → 1.0.0

package/scripts/magic/createChildPipeline/index.ts

@@ -1,176 +0,0 @@
-import { merge } from "lodash";
-import slugify from "slugify";
-import { GitlabJobDef, GitlabJobs } from "../types/gitlab-types";
-
-import { Config, DefaultEnvConfig } from "../types/config";
-import { CommitInfo, Context, Environment } from "../types/context";
-import { deployJobCreators, getDeployDefaults } from "../deploy";
-import { buildJobCreators, getBuildDefaults } from "../build";
-import { BuildConfig } from "../build/types";
-import { DeployConfig } from "../deploy/types";
-
-const createRawJobs = (context: Context): GitlabJobs => {
-  const buildJobs =
-    buildJobCreators[context.componentConfig.build.type](context);
-  const deployJobs =
-    deployJobCreators[context.componentConfig.deploy.type](context);
-
-  return [...buildJobs, ...deployJobs];
-};
-
-const getEnvironment = (
-  config: Config,
-  componentName: string,
-  env: string,
-  commitInfo: CommitInfo
-): Environment => {
-  const componentConfig = config.components[componentName];
-  if (!componentConfig) {
-    throw new Error("unknown component " + componentName);
-  }
-
-  const defaultConfig = componentConfig;
-  const envConfig = componentConfig.env?.[env] ?? {};
-
-  const mergedConfig = merge({}, defaultConfig, envConfig);
-
-  const publicEnvVars = mergedConfig.vars?.public ?? {};
-  const secretEnvVarsRaw = mergedConfig.vars?.secret ?? {}; // TODO, map to gl secrets?
-  const secretEnvVars = {}; // TODO
-  const referencedRaw = mergedConfig.vars?.fromComponents ?? {};
-
-  const referenced = Object.entries(referencedRaw).reduce(
-    (acc, [otherApp, mapping]) => {
-      // TODO: prevent infinit looop
-      const { variables } = getEnvironment(config, otherApp, env, commitInfo);
-
-      return Object.fromEntries(
-        Object.entries(mapping).map(([ourKey, otherKey]) => [
-          ourKey,
-          variables[otherKey],
-        ])
-      );
-    },
-    {}
-  );
-
-  const envType = componentConfig.env?.[env]?.type ?? env;
-  const environmentName =
-    envType === "review"
-      ? `${env}-${componentName}/${commitInfo.refName}`
-      : `${env}-${componentName}`;
-
-  const KUBE_APP_NAME =
-    envType === "review"
-      ? `${componentName}-${commitInfo.refSlug}`
-      : componentName;
-
-  const KUBE_NAMESPACE = `${config.customerName}-${config.appName}-${env}`;
-
-  const APP_SLUG = slugify(KUBE_APP_NAME);
-
-  const HOST_CANONICAL = `${config.appName}-${APP_SLUG}.${config.customerName}.panter.cloud`;
-
-  const hostname = mergedConfig.hostname ?? HOST_CANONICAL;
-  const url = `https://${hostname}`;
-
-  const predefinedVariables = {
-    HOST_CANONICAL,
-    ROOT_URL: url,
-    KUBE_NAMESPACE,
-    KUBE_APP_NAME,
-    ENV_SHORT: env,
-  };
-  const variables = {
-    ...predefinedVariables,
-    ...publicEnvVars,
-    ...secretEnvVars,
-    ...referenced,
-  };
-
-  return {
-    hostname,
-    fullName: environmentName,
-    shortName: env,
-    url: url,
-    variables,
-  };
-};
-
-export const createContext = (
-  componentName: string,
-  config: Config,
-  env: string
-): Context => {
-  const rawConfig = config.components[componentName];
-  if (!rawConfig) {
-    throw new Error("unknown component " + componentName);
-  }
-  // envs can override the config
-  const envConfig = rawConfig.env?.[env] ?? {};
-  const componentConfigWithoutDefaults = merge({}, rawConfig, envConfig);
-  // fill in defaults of build and deploy
-  const defaults: {
-    build: Partial<BuildConfig>;
-    deploy: Partial<DeployConfig>;
-  } = {
-    build: getBuildDefaults[componentConfigWithoutDefaults.build.type](),
-    deploy: getDeployDefaults[componentConfigWithoutDefaults.deploy.type](),
-  };
-  const componentConfig = merge({}, componentConfigWithoutDefaults, defaults);
-  const commit = {
-    refName: process.env.CI_COMMIT_REF_NAME ?? "unknown",
-    refSlug: process.env.CI_COMMIT_REF_SLUG ?? "unknown",
-  };
-  return {
-    fullConfig: config,
-    componentConfig,
-    componentName,
-    environment: getEnvironment(config, componentName, env, commit),
-    commit: commit,
-  };
-};
-
-const getFullJobName = (name: string, componentName: string, env?: string) => {
-  if (env) {
-    return `${env} ${componentName} ${name}`;
-  }
-  return `${componentName} ${name}`;
-};
-
-const replaceReferences = (
-  def: GitlabJobDef,
-  componentName: string,
-  env?: string
-): GitlabJobDef => {
-  return {
-    ...def,
-    needs: def.needs?.map((n) => getFullJobName(n, componentName, env)),
-    dependencies: def.dependencies?.map((n) =>
-      getFullJobName(n, componentName, env)
-    ),
-  };
-};
-
-export const createJobs = (
-  envs: string[],
-  config: Config,
-  componentName: string
-): Record<string, GitlabJobDef> => {
-  return envs.reduce((acc, env) => {
-    const context = createContext(componentName, config, env);
-    const jobs = createRawJobs(context);
-    return {
-      ...acc,
-      ...jobs.reduce((acc, { name, job, perEnv = true }) => {
-        const def = job;
-
-        return {
-          ...acc,
-          [getFullJobName(name, componentName, perEnv ? env : undefined)]:
-            replaceReferences(def, componentName, perEnv ? env : undefined),
-        };
-      }, {}),
-    };
-  }, {});
-};

package/scripts/magic/createPipeline.ts

@@ -1,52 +0,0 @@
-import { createJobs } from "./createChildPipeline";
-import { Config, ENV_TYPES, PipelineTrigger } from "./types/config";
-import { GitlabJobDef } from "./types/gitlab-types";
-
-export const createPipeline = async (
-  trigger: PipelineTrigger,
-  config: Config
-) => {
-  const envs = Object.keys(ENV_TYPES).filter(
-    (e) => ENV_TYPES[e as keyof typeof ENV_TYPES].trigger === trigger
-  );
-  const components = Object.keys(config.components);
-
-  // 2. write the triggering pipeline
-
-  const jobs = components.reduce<Record<string, GitlabJobDef>>(
-    (acc, componentName) => {
-      return {
-        ...acc,
-        ...createJobs(envs, config, componentName),
-      };
-    },
-    {}
-  );
-
-  const rules = [
-    // same as rules "always", but with `changes` to only trigger changed branches
-    { if: "$CI_COMMIT_TAG" },
-    {
-      if: "$CI_COMMIT_MESSAGE =~ /^chore(release).*/",
-      when: "never",
-    },
-    { if: "$CI_COMMIT_BRANCH =~ /^[0-9]+.([0-9]+|x).x$/" },
-    {
-      if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH",
-    },
-
-    { if: "$CI_MERGE_REQUEST_ID" },
-  ];
-
-  const mainPipeline = {
-    image:
-      "git.panter.ch:5001/catladder/gitlab-ci/pipeline:$PIPELINE_IMAGE_TAG",
-    workflow: {
-      rules,
-    },
-    stages: ["setup", "test", "build", "deploy", "verify", "actions"],
-    ...jobs,
-  };
-
-  return mainPipeline;
-};

package/scripts/magic/deploy/index.ts

@@ -1,19 +0,0 @@
-import { GitlabJobs } from "../types/gitlab-types";
-import { ComponentConfig } from "../types/config";
-import { Context } from "../types/context";
-import { createKubernetesDeployJobs, getKubernetesDefault } from "./kubernetes";
-import { DeployConfig } from "./types";
-
-export const deployJobCreators: {
-  [type in DeployConfig["type"]]: (context: Context) => GitlabJobs;
-} = {
-  kubernetes: createKubernetesDeployJobs,
-  serverless: () => [],
-};
-
-export const getDeployDefaults: {
-  [type in DeployConfig["type"]]: () => Partial<DeployConfig>;
-} = {
-  kubernetes: getKubernetesDefault,
-  serverless: () => ({}),
-};

package/scripts/magic/deploy/kubernetes.ts

@@ -1,65 +0,0 @@
-import { GitlabJobs } from "../types/gitlab-types";
-import { Context } from "../types/context";
-import { DeployConfigKubernetes, isOfType } from "./types";
-
-export const createKubernetesDeployJobs = (context: Context): GitlabJobs => {
-  const deployConfig = context.componentConfig.deploy;
-  if (!isOfType(deployConfig, "kubernetes")) {
-    // should not happen
-    throw new Error("deploy config is not kubernetes");
-  }
-
-  const defaltKubeValues = {
-    application: {
-      hostname: context.environment.hostname,
-      command: context.componentConfig.build.startCommand,
-    },
-  };
-  const kubeValues = {
-    ...defaltKubeValues,
-    ...deployConfig.values,
-  }; // TODO: merge with some defaults
-  return [
-    {
-      name: "deploy-to-kubernetes",
-
-      job: {
-        variables: {
-          ...context.environment.variables,
-
-          // TODO: refactor and unify with other stages
-          HELM_EXPERIMENTAL_OCI: "1",
-          IMAGE_PULL_SECRET: `gitlab-registry-${context.componentName}`,
-          KUBE_VALUES: JSON.stringify(kubeValues),
-          HELM_GITLAB_CHART_PATH: "catladder/helm-charts",
-          HELM_GITLAB_CHART_NAME: "the-panter-chart",
-          COMPONENT_NAME: context.componentName,
-          // TODO: unify with docker build stage
-          IMAGE_TAG: "$CI_COMMIT_SHA",
-          HELM_GITLAB_CHART_VERSION: "3.2.0", // TODO, we could actually just ship the chart directly here
-          RELEASE_NAME: `${context.fullConfig.customerName}-${context.fullConfig.appName}-$CI_ENVIRONMENT_SLUG`,
-        },
-        stage: "deploy",
-        dependencies: [],
-        script: [
-          "kubernetesEnsureNamespace",
-          "kubernetesCreateSecret",
-          "kubernetesDeploy",
-        ],
-        environment: {
-          name: context.environment.fullName,
-          url: context.environment.url,
-          kubernetes: {
-            namespace: context.environment.variables.KUBE_NAMESPACE,
-          },
-        },
-      },
-    },
-  ];
-};
-
-export const getKubernetesDefault = (): Partial<DeployConfigKubernetes> => {
-  return {
-    values: {},
-  };
-};

package/scripts/magic/deploy/types.ts

@@ -1,29 +0,0 @@
-export type DeployConfigBase = {};
-export type DeployConfigKubernetes = {
-  type: "kubernetes";
-  additionalHelmArgs?: string[];
-  values?: {
-    application?: {
-      resources?: {
-        limits?: {
-          cpu?: any;
-        };
-        requests?: {
-          cpu?: any;
-        };
-      };
-    } & Record<string, any>;
-  } & Record<string, any>;
-} & DeployConfigBase;
-
-export type DeployConfigServerless = {
-  type: "serverless";
-};
-export type DeployConfig = DeployConfigKubernetes | DeployConfigServerless;
-
-export const isOfType = <T extends DeployConfig["type"]>(
-  t: DeployConfig,
-  type: T
-): t is Extract<DeployConfig, { type: T }> => {
-  return t.type === type;
-};

package/scripts/magic/index.ts

@@ -1,2 +0,0 @@
-export { createPipeline } from "./createPipeline";
-export * from "./types";

package/scripts/magic/sample-config.ts

@@ -1,78 +0,0 @@
-import { Config } from "./types/config";
-
-const sampleConfig: Config = {
-  customerName: "pan",
-  appName: "fonsi",
-  components: {
-    web: {
-      dir: "web",
-      build: {
-        type: "node",
-      },
-      deploy: {
-        type: "kubernetes",
-        values: {
-          application: {
-            resources: {
-              limits: {
-                cpu: "100m",
-              },
-            },
-          },
-        },
-      },
-      vars: {
-        public: {
-          GOOGLE_TAG_MANAGER_ID: "1234",
-        },
-        fromComponents: {
-          api: {
-            API_URL: "ROOT_URL",
-          },
-        },
-      },
-      env: {
-        prod: {
-          hostname: "fudilo.ch",
-          vars: {},
-          deploy: {
-            type: "kubernetes",
-            values: {
-              application: {
-                resources: {
-                  limits: {
-                    cpu: "1000m",
-                  },
-                },
-              },
-            },
-          },
-        },
-        myCustomProd: {
-          type: "prod",
-          hostname: "fudilo2.ch",
-        },
-      },
-    },
-    api: {
-      env: {
-        prod: {
-          hostname: "api.fudilo.ch",
-        },
-        myCustomProd: {
-          type: "prod",
-          hostname: "api.fudilo2.ch",
-        },
-      },
-      build: {
-        type: "node",
-      },
-      deploy: {
-        type: "kubernetes",
-      },
-      dir: "api",
-    },
-  },
-};
-
-export default sampleConfig;

package/scripts/magic/types/config.ts

@@ -1,54 +0,0 @@
-import { BuildConfig } from "../build/types";
-import { DeployConfig } from "../deploy/types";
-
-export type PipelineTrigger = "mainBranch" | "mr" | "taggedRelease";
-
-export const ENV_TYPES = {
-  dev: {
-    trigger: "mainBranch",
-  },
-  review: {
-    trigger: "mr",
-  },
-  prod: {
-    trigger: "taggedRelease",
-  },
-  stage: {
-    trigger: "taggedRelease",
-  },
-} as const;
-type EnvType = keyof typeof ENV_TYPES;
-
-export type DefaultEnvConfig = {
-  deploy: DeployConfig;
-  build: BuildConfig;
-  vars?: {
-    public?: Record<string, any>;
-    secret?: Record<string, string>;
-    fromComponents?: {
-      [otherApp: string]: Record<string, string>;
-    };
-  };
-};
-type EnvConfig<E extends EnvType = EnvType> = {
-  type?: E;
-  hostname?: string;
-} & Partial<DefaultEnvConfig>;
-
-export type Env = {
-  dev?: EnvConfig<"dev">;
-  stage?: EnvConfig<"stage">;
-  review?: EnvConfig<"review">;
-  prod?: EnvConfig<"prod">;
-} & Record<string, EnvConfig>;
-
-export type ComponentConfig = {
-  env?: Env;
-  dir: string;
-} & DefaultEnvConfig;
-
-export type Config = {
-  customerName: string;
-  appName: string;
-  components: Record<string, ComponentConfig>;
-};

package/scripts/magic/types/context.ts

@@ -1,21 +0,0 @@
-import { ComponentConfig, Config } from "./config";
-
-export type Environment = {
-  hostname: string;
-  fullName: string;
-  shortName: string;
-  url: string;
-  variables: Record<string, string>;
-};
-
-export type CommitInfo = {
-  refName: string;
-  refSlug: string;
-};
-export type Context = {
-  componentConfig: ComponentConfig;
-  componentName: string;
-  fullConfig: Config;
-  environment: Environment;
-  commit: CommitInfo;
-};

package/scripts/magic/types/gitlab-types.ts

@@ -1,69 +0,0 @@
-export type GitlabStage =
-  | "setup"
-  | "test"
-  | "build"
-  | "deploy"
-  | "verify"
-  | "actions";
-
-export type Artifacts = {
-  paths: string[];
-};
-export type GitlabJobCache = {
-  key: string;
-  policy?: string;
-  paths: string[];
-};
-export type GitlabRule = {
-  if: string;
-  when?: "always" | "on_success" | "manual" | "never"; // todo
-};
-export type Retry = {
-  max: number;
-  when: string[];
-};
-
-export type Service = {
-  name: string;
-  command: string[];
-};
-
-export type GitlabEnvironment = {
-  url: string;
-  name: string;
-  kubernetes?: {
-    namespace?: string;
-  };
-};
-export type GitlabJobDef = {
-  stage: GitlabStage;
-  before_script?: string[];
-  script: string[];
-  interruptible?: boolean;
-  needs?: string[];
-  rules?: GitlabRule[];
-  cache?: GitlabJobCache | GitlabJobCache[];
-  artifacts?: Artifacts;
-  retry?: Retry;
-  services?: Service[];
-  image?: string;
-  variables?: GitlabVariables;
-  dependencies?: string[];
-  environment?: GitlabEnvironment;
-  allow_failure?: boolean;
-};
-
-export type GitlabVariables = Record<string, string | undefined>;
-
-export type GitlabJob = {
-  name: string;
-  perEnv?: boolean;
-  job: GitlabJobDef;
-};
-export type GitlabJobs = GitlabJob[];
-
-export type GitlabPipeline = {
-  variables: GitlabVariables;
-  stages: GitlabStage[];
-  jobs: GitlabJobs;
-};

package/scripts/magic.ts

@@ -1,51 +0,0 @@
-#!/usr/bin/env ts-node
-
-import { existsSync, writeFileSync } from "fs";
-import { parse } from "yaml";
-import { PipelineTrigger } from "./magic/types/config";
-import { createPipeline } from "./magic/createPipeline";
-
-const {
-  CI_MERGE_REQUEST_ID,
-  CI_COMMIT_TAG,
-  CI_COMMIT_BRANCH,
-  CI_DEFAULT_BRANCH,
-} = process.env;
-
-const isDefaultBranch = CI_COMMIT_BRANCH == CI_DEFAULT_BRANCH;
-const isHotfixBranch = false; // TODO:  $CI_COMMIT_BRANCH =~ /^[0-9]+\.([0-9]+|x)\.x$/
-const isMergeRequest = Boolean(CI_MERGE_REQUEST_ID);
-const isTaggedRelease = Boolean(CI_COMMIT_TAG);
-
-const fullPath = (ext: string) => process.cwd() + "/catladder." + ext;
-const readConfig = () => {
-  const found = ["ts", "js", "yml", "yaml"].find((extension) =>
-    existsSync(fullPath(extension))
-  );
-  if (found) {
-    if (found === "ts" || found === "js") {
-      return require(fullPath(found)).default;
-    } else {
-      return parse(fullPath(found));
-    }
-  }
-};
-
-const trigger: PipelineTrigger | null =
-  isMergeRequest || isHotfixBranch
-    ? "mr"
-    : isDefaultBranch
-    ? "mainBranch"
-    : isTaggedRelease
-    ? "taggedRelease"
-    : null;
-if (trigger) {
-  const config = readConfig();
-  createPipeline(trigger, config).then((mainPipeline) => {
-    writeFileSync(`__pipeline.yml`, JSON.stringify(mainPipeline, null, 2), {
-      encoding: "utf-8",
-    });
-  });
-} else {
-  throw new Error("no matching trigger");
-}