@catladder/pipeline 0.0.7 → 1.0.2

package/src/deploy/kubernetes/deployJob.ts

@@ -0,0 +1,116 @@
+import { GitlabJobs } from "../../types/gitlab-types";
+import { Context } from "../../types/context";
+import { isOfDeployType } from "../types";
+import { getRunnerImage } from "../../runner";
+import { getBaseDeploymentJob, getBaseDeploymentStopJob } from "../base";
+import { merge } from "lodash";
+import { dump } from "js-yaml";
+import { createMongodbBaseConfig } from "./mongodb";
+import { createCloudsqlBaseConfig } from "./cloudsql";
+
+export const createKubernetesDeployJobs = (context: Context): GitlabJobs => {
+  const deployConfig = context.componentConfig.deploy;
+  if (deployConfig === false) {
+    return [];
+  }
+  if (!isOfDeployType(deployConfig, "kubernetes")) {
+    // should not happen
+    throw new Error("deploy config is not kubernetes");
+  }
+
+  const allEnvVars = context.environment.envVars;
+  /**
+   * separate by secrets and public.
+   * we evalulate the actual values later, but want to store the secrets in kubernetes secrets
+   */
+  const env = Object.entries(allEnvVars).reduce<{
+    secret: Record<string, string>;
+    public: Record<string, string>;
+  }>(
+    (acc, [key, value]) => {
+      if (value?.startsWith("$CL_")) {
+        acc.secret = {
+          ...acc.secret,
+          [key]: value,
+        };
+        return acc;
+      }
+      acc.public = {
+        ...acc.public,
+        [key]: value,
+      };
+      return acc;
+    },
+    {
+      secret: {},
+      public: {},
+    }
+  );
+
+  const defaultKubeValues = {
+    application: {
+      hostname: context.environment.hostname,
+      command: context.componentConfig.build.startCommand,
+    },
+
+    env: env,
+    ...(deployConfig.values?.cloudsql?.enabled
+      ? createCloudsqlBaseConfig(context)
+      : {}),
+    ...(deployConfig.values?.mongodb?.enabled
+      ? createMongodbBaseConfig(context)
+      : {}),
+  };
+
+  const kubeValues = merge({}, defaultKubeValues, deployConfig.values);
+
+  const kubernetesEnvironment = {
+    namespace: context.environment.envVars.KUBE_NAMESPACE,
+  };
+  const shared = {
+    job: {
+      image: getRunnerImage("kubernetes"),
+      variables: {
+        ...context.environment.envVars,
+        HELM_EXPERIMENTAL_OCI: "1",
+        IMAGE_PULL_SECRET: `gitlab-registry-${context.componentName}`,
+        KUBE_VALUES: dump(kubeValues, {
+          lineWidth: -1,
+          quotingType: "'",
+          forceQuotes: true,
+        }),
+        HELM_GITLAB_CHART_NAME: "the-panter-chart",
+        HELM_ARGS: deployConfig.additionalHelmArgs,
+        COMPONENT_NAME: context.componentName,
+        // TODO: unify with docker build stage
+        IMAGE_TAG: "$CI_COMMIT_SHA",
+      },
+    },
+  };
+
+  const baseDeploymentJob = getBaseDeploymentJob(context);
+  const baseStopJob = getBaseDeploymentStopJob(context);
+
+  return [
+    merge({}, baseDeploymentJob, shared, {
+      job: {
+        script: [
+          "kubernetesEnsureNamespace",
+          "kubernetesCreateSecret",
+          "kubernetesDeploy",
+        ],
+        environment: {
+          kubernetes: kubernetesEnvironment,
+        },
+      },
+    }),
+    merge({}, baseStopJob, shared, {
+      job: {
+        script: ["kubernetesDelete"],
+        environment: {
+          kubernetes: kubernetesEnvironment,
+        },
+      },
+    }),
+  ];
+};

package/src/deploy/kubernetes/index.ts

@@ -0,0 +1 @@
+export * from "./deployJob";

package/src/deploy/kubernetes/mongodb.ts

@@ -0,0 +1,20 @@
+import { Context } from "../..";
+
+export const createMongodbBaseConfig = (context: Context) => {
+  return {
+    mongodb: {
+      enabled: true,
+      backup: {
+        enabled: ["prod", "stage"].includes(context.environment.envType),
+      },
+    },
+    "mongodb-replicaset": {
+      replicas: 1,
+      persistentVolume: {
+        storageClass: ["prod", "stage"].includes(context.environment.envType)
+          ? "fast"
+          : "standard",
+      },
+    },
+  };
+};

package/src/deploy/types.ts

@@ -0,0 +1,60 @@
+// eslint-disable-next-line @typescript-eslint/ban-types
+export type DeployConfigBase = {};
+type AllowUnknownProps<T extends Record<string, unknown>> = T &
+  Record<string, unknown>;
+export type DeployConfigKubernetes = {
+  type: "kubernetes";
+  cluster?: string;
+  additionalHelmArgs?: string[];
+  values?: AllowUnknownProps<{
+    cloudsql?: {
+      enabled: boolean;
+      instanceId?: string;
+      projectId?: string;
+      region?: string;
+    };
+    mongodb?: AllowUnknownProps<{
+      enabled: boolean;
+    }>;
+    mailhog?: {
+      enabled: boolean;
+    };
+    jobs?: Record<string, AllowUnknownProps<{ command: string }>>;
+    cronjobs?: Record<
+      string,
+      AllowUnknownProps<{
+        schedule: string;
+        command: string;
+        concurrencyPolicy?: "Forbid" | "Allow" | "Replace";
+      }>
+    >;
+    application?: AllowUnknownProps<{
+      redirects?: AllowUnknownProps<{ host: string }>[];
+      replicas?: number;
+      resources?: {
+        limits?: {
+          cpu?: string;
+          memory?: string;
+        };
+        requests?: {
+          cpu?: string;
+          memory?: string;
+        };
+      };
+    }>;
+  }>;
+} & DeployConfigBase;
+
+type CustomDeployConfig = {
+  type: "custom";
+  script: string[];
+};
+
+export type DeployConfig = DeployConfigKubernetes;
+
+export const isOfDeployType = <T extends Array<DeployConfig["type"]>>(
+  t: DeployConfig | false,
+  ...types: T
+): t is Extract<DeployConfig, { type: T[number] }> => {
+  return t && types.includes(t.type);
+};

package/src/index.ts

@@ -0,0 +1,8 @@
+export * from "./pipeline/";
+export * from "./types";
+export * from "./rules";
+export * from "./runner";
+export * from "./config";
+export * from "./context";
+export * from "./build";
+export * from "./deploy";

package/{scripts/magic/tests/createPipeline.test.ts → src/pipeline/createChildPipeline.test.ts}

@@ -1,7 +1,7 @@
 import { Config } from "../types";
-import { createPipeline } from "../createPipeline";
+import { createChildPipeline } from "./createChildPipeline";
 
-describe("createPipeline", () => {
+describe("createChildPipeline", () => {
   describe("node-app to kuberntes", () => {
     const config: Config = {
       appName: "test",
@@ -20,12 +20,12 @@ describe("createPipeline", () => {
     };
 
     it("creates a pipeline for a single app on the main branch", async () => {
-      const { image, stages, workflow, ...jobs } = await createPipeline(
+      const { image, stages, workflow, ...jobs } = await createChildPipeline(
         "mainBranch",
         config
       );
       expect(image).toEqual(
-        "git.panter.ch:5001/catladder/gitlab-ci/pipeline:$PIPELINE_IMAGE_TAG"
+        "git.panter.ch:5001/catladder/catladder/base-pipeline:"
       );
 
       expect(Object.keys(jobs)).toEqual([
@@ -35,6 +35,7 @@ describe("createPipeline", () => {
         "dev my-app app-build",
         "dev my-app docker-build",
         "dev my-app deploy-to-kubernetes",
+        "dev my-app kubernetes-stop",
       ]);
     });
   });

package/src/pipeline/createChildPipeline.ts

@@ -0,0 +1,51 @@
+import { getAllEnvsByTrigger, getAllEnvsInAllComponents } from "../config";
+import { createJobs } from "./createJobs";
+import { RULES_ALWAYS } from "../rules";
+import { getRunnerImage } from "../runner";
+import { Config, PipelineTrigger } from "../types/config";
+import { GitlabJobDef } from "../types/gitlab-types";
+const baseStages = ["setup", "test", "build", "deploy", "verify", "stop"];
+export const createChildPipeline = async (
+  trigger: PipelineTrigger,
+  config: Config
+) => {
+  const components = Object.keys(config.components);
+
+  // 2. write the triggering pipeline
+
+  const jobs = await components.reduce<Promise<Record<string, GitlabJobDef>>>(
+    async (acc, componentName) => {
+      const envs = getAllEnvsByTrigger(config, componentName, trigger);
+      return {
+        ...(await acc),
+        ...(await createJobs(envs, config, componentName, trigger)),
+      };
+    },
+    Promise.resolve({})
+  );
+
+  // while technically not required, we group different envs in its own stage
+  // each job from `createJobs` that is defined as `envMode: "stagePerEnv"` will have `deploy dev`, etc. instead of just `deploy`
+  // this is just so that it looks nicer in gitlab and makes running mutliple manual tasks more easy to use
+
+  const allEnvs = getAllEnvsInAllComponents(config);
+  const stages = baseStages.reduce<string[]>(
+    (acc, baseStage) => [
+      ...acc,
+      baseStage,
+      ...allEnvs.map((e) => `${baseStage} ${e}`),
+    ],
+    []
+  );
+
+  const childPipeline = {
+    image: getRunnerImage("jobs"), // default image
+    workflow: {
+      rules: RULES_ALWAYS,
+    },
+    stages,
+    ...jobs,
+  };
+
+  return childPipeline as typeof childPipeline & Record<string, GitlabJobDef>;
+};

package/src/pipeline/createJobs.ts

@@ -0,0 +1,158 @@
+import { isObject } from "lodash";
+import { BUILD_TYPES } from "../build";
+import { createContext } from "../context";
+import { DEPLOY_TYPES } from "../deploy";
+import { Config, PipelineTrigger } from "../types/config";
+import { Context, CommitInfo } from "../types/context";
+import { GitlabJob, GitlabJobDef, GitlabJobs } from "../types/gitlab-types";
+import { notNil } from "../utils";
+import { getYarnInfo } from "./yarnInfo";
+
+const createRawJobs = (context: Context): GitlabJobs => {
+  if (context.componentConfig.deploy === false) {
+    return [];
+  }
+  const buildJobs =
+    BUILD_TYPES[context.componentConfig.build.type].jobs(context);
+  const deployJobs =
+    DEPLOY_TYPES[context.componentConfig.deploy.type].jobs(context);
+
+  return [...buildJobs, ...deployJobs];
+};
+const getFullJobName = (
+  name: string,
+  componentName: string,
+  env?: string | null
+) => {
+  if (env) {
+    return `${componentName} ${name} | ${env} `;
+  }
+  return `${componentName} ${name}`;
+};
+
+const getFullReferencedJobName = (
+  referencedJobName: string,
+  componentName: string,
+  env: string,
+  allRawJobs: GitlabJobs
+) => {
+  const referencedJob = allRawJobs.find((j) => j.name === referencedJobName);
+  if (!referencedJob) {
+    throw new Error("unknown job referenced: " + referencedJobName);
+  }
+  const envToSet = referencedJob.envMode !== "none" ? env : null;
+  return getFullJobName(referencedJobName, componentName, envToSet);
+};
+// replaces references to other jobs with the full name
+// the full name contains the componentname and the env name (if any)
+const replaceReferences = (
+  job: GitlabJob,
+  componentName: string,
+  env: string,
+  allRawJobs: GitlabJobs
+): GitlabJobDef => {
+  const def = job.job;
+  const stage =
+    job.envMode === "stagePerEnv" ? `${def.stage} ${env}` : def.stage;
+  return {
+    ...def,
+    stage,
+    needs: def.needs?.map((n) =>
+      isObject(n)
+        ? {
+            job: getFullReferencedJobName(
+              n.job,
+              componentName,
+              env,
+              allRawJobs
+            ),
+            artifacts: n.artifacts,
+          }
+        : getFullReferencedJobName(n, componentName, env, allRawJobs)
+    ),
+    environment: def.environment?.on_stop
+      ? {
+          ...def.environment,
+          on_stop: getFullReferencedJobName(
+            def.environment.on_stop,
+            componentName,
+            env,
+            allRawJobs
+          ),
+        }
+      : def.environment,
+    dependencies: def.dependencies?.map((n) =>
+      getFullReferencedJobName(n, componentName, env, allRawJobs)
+    ),
+  };
+};
+
+// this can be removed once https://gitlab.com/gitlab-org/gitlab/-/issues/220758 is resolved
+const addStageNeeds = (jobs: GitlabJobs): GitlabJobs => {
+  // when a job defines needsStages, we add these as needs
+  return jobs.map((job) => {
+    if (!job.needsStages || job.needsStages.length === 0) {
+      return job;
+    }
+
+    const neededJobs = jobs
+      .map((j) => {
+        const neededStage = job.needsStages?.find(
+          (s) => j.job.stage === s.stage
+        );
+        if (neededStage) {
+          return {
+            job: j.name,
+            artifacts: neededStage.artifacts ?? false,
+          };
+        }
+      })
+      .filter(notNil);
+    return {
+      ...job,
+      job: {
+        ...job.job,
+        needs: [...(job.job.needs ?? []), ...neededJobs],
+      },
+    };
+  });
+};
+export const createJobs = async (
+  envs: string[],
+  config: Config,
+  componentName: string,
+  trigger: PipelineTrigger
+): Promise<Record<string, GitlabJobDef>> => {
+  const commitInfo: CommitInfo = {
+    refName: process.env.CI_COMMIT_REF_NAME ?? "unknown",
+    refSlug: process.env.CI_COMMIT_REF_SLUG ?? "unknown",
+    trigger,
+  };
+
+  const yarnInfo = await getYarnInfo(config, componentName);
+
+  return envs.reduce((acc, env) => {
+    const context = createContext(
+      config,
+      componentName,
+      env,
+      commitInfo,
+      yarnInfo
+    );
+    const jobs = addStageNeeds(createRawJobs(context));
+
+    return {
+      ...acc,
+      ...jobs.reduce((acc, job) => {
+        return {
+          ...acc,
+          [getFullJobName(
+            job.name,
+            componentName,
+            job.envMode !== "none" ? env : undefined
+          )]: replaceReferences(job, componentName, env, jobs),
+        };
+      }, {}),
+    };
+  }, {});
+};

package/src/pipeline/index.ts

@@ -0,0 +1,2 @@
+export * from "./createChildPipeline";
+export * from "./createJobs";

package/src/pipeline/yarnInfo.ts

@@ -0,0 +1,54 @@
+import { exec } from "child-process-promise";
+import { Config, YarnInfo } from "../types";
+import { pathEqual } from "path-equal";
+import memoizee from "memoizee";
+
+const execOrFail = async (cmd: string, onFail: string): Promise<string> => {
+  try {
+    return await exec(cmd).then((r) => r.stdout);
+  } catch (e) {
+    return onFail ?? null;
+  }
+};
+
+const getYarnVersion = memoizee(
+  async () => {
+    return await execOrFail("yarn --version", "");
+  },
+  { promise: true }
+);
+
+const getWorkspaces = memoizee(
+  async (isClassic: boolean): Promise<YarnInfo["workspaces"]> => {
+    return isClassic
+      ? Object.values(
+          JSON.parse(
+            JSON.parse(await execOrFail("yarn workspaces --json info", "{}"))
+              ?.data ?? "{}"
+          )
+        )
+      : JSON.parse(`[${await execOrFail("yarn workspaces list --json", "")}]`);
+  },
+  { promise: true }
+);
+export const getYarnInfo = async (
+  config: Config,
+  componentName: string
+): Promise<YarnInfo> => {
+  const version = await getYarnVersion();
+  if (!version) throw new Error("could not get yarn version");
+  const isClassic = version.startsWith("1");
+
+  const component = config.components[componentName];
+  const workspaces = await getWorkspaces(isClassic);
+  const componentIsInWorkspace = workspaces.some((w) =>
+    pathEqual(component.dir, w.location)
+  );
+
+  return {
+    workspaces,
+    version,
+    isClassic,
+    componentIsInWorkspace,
+  };
+};

package/src/rules/index.ts

@@ -0,0 +1,37 @@
+import { GitlabRule } from "../types";
+
+const NEVER_ON_RELEASE_COMMIT: GitlabRule = {
+  if: "$CI_COMMIT_MESSAGE =~ /^chore\\(release\\).*/",
+  when: "never",
+};
+export const RULES_ALWAYS: GitlabRule[] = [
+  {
+    if: "$CI_COMMIT_TAG",
+  },
+  NEVER_ON_RELEASE_COMMIT,
+  {
+    if: "$CI_COMMIT_BRANCH =~ /^[0-9]+.([0-9]+|x).x$/",
+  },
+  {
+    if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH",
+  },
+  {
+    if: "$CI_MERGE_REQUEST_ID",
+  },
+];
+
+export const RULES_RELEASE: GitlabRule[] = [
+  NEVER_ON_RELEASE_COMMIT,
+  {
+    if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $AUTO_RELEASE == "true"',
+    when: "on_success",
+  },
+  {
+    if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH",
+    when: "manual",
+  },
+  {
+    if: "$CI_COMMIT_BRANCH =~ /^[0-9]+.([0-9]+|x).x$/", // hotfix branches
+    when: "manual",
+  },
+];

package/src/runner/index.ts

@@ -0,0 +1,10 @@
+import { PIPELINE_IMAGE_TAG, DOCKER_REGISTRY } from "../constants";
+
+type RunnerImageName =
+  | "jobs"
+  | "kubernetes"
+  | "base-pipeline"
+  | "docker-build"
+  | "semantic-release";
+export const getRunnerImage = (imageName: RunnerImageName) =>
+  DOCKER_REGISTRY + "/" + imageName + ":" + PIPELINE_IMAGE_TAG;

package/src/types/config.ts

@@ -0,0 +1,88 @@
+import { BuildConfig } from "../build/types";
+import { DeployConfig } from "../deploy/types";
+import { PartialDeep } from "type-fest";
+export type PipelineTrigger = "mainBranch" | "mr" | "taggedRelease";
+
+/**
+ * all env types with their trigger.
+ * Each env type has a default env with the same name which is always included
+ */
+export const ENV_TYPES = {
+  dev: {
+    triggers: ["mainBranch", "taggedRelease"], // we also trigger dev on tagged release, so that the versions match
+  },
+  review: {
+    triggers: ["mr"],
+  },
+  stage: {
+    triggers: ["taggedRelease"],
+  },
+  prod: {
+    triggers: ["taggedRelease"],
+  },
+  local: {
+    triggers: [],
+  },
+} as const;
+
+/**
+ *
+ * @param trigger a trigger
+ * @returns array of env types for that trigger. this is also the list of default envs
+ */
+export const getEnvTypesByTrigger = (trigger: PipelineTrigger) =>
+  Object.entries(ENV_TYPES)
+    .filter(([, e]) =>
+      (e.triggers as readonly PipelineTrigger[]).includes(trigger)
+    )
+    .map(([e]) => e as EnvType);
+
+export const DEFAULT_ENVS = Object.keys(ENV_TYPES);
+export const DEFAULT_ENV_TYPES = DEFAULT_ENVS as EnvType[];
+export type EnvType = keyof typeof ENV_TYPES;
+
+export const isKnowEnvType = (env: string): env is EnvType => {
+  return env in ENV_TYPES;
+};
+
+export type EnvVars = {
+  public?: Record<string, any>;
+  secret?: string[];
+  fromComponents?: {
+    [otherApp: string]: Record<string, string>;
+  };
+};
+
+export type DefaultEnvConfig = {
+  deploy: DeployConfig | false;
+  build: BuildConfig;
+  vars?: EnvVars;
+};
+
+export type DevLocalEnvConfig = {
+  vars?: EnvVars;
+  port?: number;
+};
+export type EnvConfig<E extends EnvType = EnvType> = {
+  type?: E;
+  hostname?: string;
+} & PartialDeep<DefaultEnvConfig>;
+
+export type Env = {
+  local?: DevLocalEnvConfig;
+  dev?: EnvConfig<"dev"> | false;
+  stage?: EnvConfig<"stage"> | false;
+  review?: EnvConfig<"review"> | false;
+  prod?: EnvConfig<"prod"> | false;
+} & Record<string, EnvConfig | false>;
+
+export type ComponentConfig = {
+  env?: Env;
+  dir: string;
+} & DefaultEnvConfig;
+
+export type Config = {
+  customerName: string;
+  appName: string;
+  components: Record<string, ComponentConfig>;
+};

package/src/types/context.ts

@@ -0,0 +1,36 @@
+import { PipelineTrigger, ComponentConfig, Config, EnvType } from "./config";
+
+export type Environment = {
+  hostname: string;
+  fullName: string;
+  shortName: string;
+  slug: string;
+  url: string;
+  /**
+   * env vars contain all build-time env vars. secrets have to be resolved (they are stored in gitlab)
+   */
+  envVars: Record<string, string>;
+  envType: EnvType;
+  secretEnvVarKeys: string[];
+};
+
+export type CommitInfo = {
+  refName: string;
+  refSlug: string;
+  trigger: PipelineTrigger;
+};
+
+export type YarnInfo = {
+  version: string;
+  workspaces: { location: string }[];
+  isClassic: boolean;
+  componentIsInWorkspace: boolean;
+};
+export type Context = {
+  componentName: string;
+  componentConfig: ComponentConfig;
+  fullConfig: Config;
+  environment: Environment;
+  commitInfo?: CommitInfo;
+  yarnInfo?: YarnInfo;
+};