@catladder/pipeline 0.0.1

package/scripts/magic.ts

@@ -0,0 +1,51 @@
+#!/usr/bin/env ts-node
+
+import { existsSync, writeFileSync } from "fs";
+import { parse } from "yaml";
+import { PipelineTrigger } from "./magic/types/config";
+import { createPipeline } from "./magic/createPipeline";
+
+const {
+  CI_MERGE_REQUEST_ID,
+  CI_COMMIT_TAG,
+  CI_COMMIT_BRANCH,
+  CI_DEFAULT_BRANCH,
+} = process.env;
+
+const isDefaultBranch = CI_COMMIT_BRANCH == CI_DEFAULT_BRANCH;
+const isHotfixBranch = false; // TODO:  $CI_COMMIT_BRANCH =~ /^[0-9]+\.([0-9]+|x)\.x$/
+const isMergeRequest = Boolean(CI_MERGE_REQUEST_ID);
+const isTaggedRelease = Boolean(CI_COMMIT_TAG);
+
+const fullName = (ext: string) => "catladder." + ext;
+const readConfig = () => {
+  const found = ["ts", "js", "yml", "yaml"].find((extension) =>
+    existsSync(fullName(extension))
+  );
+  if (found) {
+    if (found === "ts" || found === "js") {
+      return require(fullName(found));
+    } else {
+      return parse(fullName(found));
+    }
+  }
+};
+
+const trigger: PipelineTrigger | null =
+  isMergeRequest || isHotfixBranch
+    ? "mr"
+    : isDefaultBranch
+    ? "mainBranch"
+    : isTaggedRelease
+    ? "taggedRelease"
+    : null;
+if (trigger) {
+  const config = readConfig();
+  createPipeline(trigger, config).then((mainPipeline) => {
+    writeFileSync(`__pipeline.yml`, JSON.stringify(mainPipeline, null, 2), {
+      encoding: "utf-8",
+    });
+  });
+} else {
+  throw new Error("no matching trigger");
+}

package/scripts/monorepoGenerate.ts

@@ -0,0 +1,104 @@
+#!/usr/bin/env ts-node
+
+import { readdirSync, readFileSync, lstatSync, existsSync } from "fs";
+import { join } from "path";
+import { parse } from "yaml";
+import yargs from "yargs";
+
+import { hideBin } from "yargs/helpers";
+import { extractBuildEnv } from "./utils/extractBuildEnv";
+type Type = "test" | "deploy";
+const args = yargs(hideBin(process.argv)).array("env").argv as Record<
+  string,
+  unknown
+>;
+
+const thisGitlabCi = parse(
+  readFileSync(".gitlab-ci.yml", { encoding: "utf-8" })
+);
+
+const subcis = readdirSync(".")
+  .filter((file) => lstatSync(file).isDirectory())
+  .map((dir) => ({
+    dirname: dir,
+    ci: join(dir, ".gitlab-ci.yml"),
+  }))
+  .filter(({ ci }) => existsSync(ci));
+
+const makeConfig = async ({
+  subapp,
+  type,
+  ci,
+  env,
+}: {
+  subapp: string;
+  type: Type;
+  env: string;
+  ci: string;
+}) => {
+  const changes = [subapp + "/**/*"];
+  const rules = [
+    // same as rules "always", but with `changes` to only trigger changed branches
+    { if: "$CI_COMMIT_TAG" },
+    {
+      if: "$CI_COMMIT_MESSAGE =~ /^chore(release).*/",
+      when: "never",
+    },
+    { if: "$CI_COMMIT_BRANCH =~ /^[0-9]+.([0-9]+|x).x$/" },
+    {
+      if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH",
+      changes,
+    },
+    // add it manually when no changes
+    {
+      if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH",
+      when: "manual",
+    },
+    { if: "$CI_MERGE_REQUEST_ID", changes },
+    // add it manually when no changes
+    { if: "$CI_MERGE_REQUEST_ID", when: "manual" },
+  ];
+
+  return {
+    variables: {
+      // merge
+      ...thisGitlabCi.variables,
+      APP_DIR: subapp,
+      VALUES_DIR: subapp,
+      COMPONENT_NAME: subapp,
+      PARENT_PIPELINE_ID: "$CI_PIPELINE_ID",
+      [type.toUpperCase() + "_ONLY"]: "1",
+      ...(await extractBuildEnv(subapp, env)),
+    },
+    stage: type,
+    rules: rules,
+    trigger: {
+      include: ci,
+      strategy: "depend",
+    },
+  };
+};
+const generateSubPipeline = async (type: Type, envs: string[]) => {
+  return await envs.reduce(async (acc, env) => {
+    return {
+      ...(await acc),
+      ...(await subcis.reduce(
+        async (inneracc, { dirname, ci }) => ({
+          ...(await inneracc),
+
+          [`${env}-${dirname} ${type}`]: await makeConfig({
+            subapp: dirname,
+            ci,
+            type,
+            env,
+          }),
+        }),
+        {} as Promise<Record<string, unknown>>
+      )),
+    };
+  }, {} as Promise<Record<string, unknown>>);
+};
+
+generateSubPipeline(args.t as Type, args.env as string[]).then((p) =>
+  console.log(JSON.stringify(p, null, 2))
+);

package/scripts/printAllValues.ts

@@ -0,0 +1,17 @@
+#!/usr/bin/env ts-node
+
+import yargs from "yargs";
+import { hideBin } from "yargs/helpers";
+import { extractAllValues } from "./utils/extractAllValues";
+
+const args = yargs(hideBin(process.argv)).argv as Record<string, unknown>;
+
+extractAllValues(args.d as string, args.e as string)
+  .then(async (values) => {
+    console.log(JSON.stringify(values, null, 2) ?? "");
+  })
+  .catch((e) => {
+    console.error("could not extract values", {
+      error: e,
+    });
+  });

package/scripts/printBuildEnv.ts

@@ -0,0 +1,22 @@
+#!/usr/bin/env ts-node
+
+import { isObject } from "lodash";
+import yargs from "yargs";
+import { hideBin } from "yargs/helpers";
+import { extractBuildEnv } from "./utils/extractBuildEnv";
+
+const args = yargs(hideBin(process.argv)).argv as Record<string, unknown>;
+
+extractBuildEnv(args.d as string, args.e as string)
+  .then(async (values) => {
+    Object.entries(values).forEach(([key, value]) =>
+      console.log(
+        `${key}: "${isObject(value) ? JSON.stringify(value) : value}"`
+      )
+    );
+  })
+  .catch((e) => {
+    console.error("could not extract values", {
+      error: e,
+    });
+  });

package/scripts/utils/extractAllValues.ts

@@ -0,0 +1,32 @@
+import { readFileSync } from "fs";
+import { merge } from "lodash";
+import { parse } from "yaml";
+import { extractValuesFromEnvVars } from "./extractValuesFromEnvVars";
+import { extractValuesFromMr } from "./extractValuesFromMr";
+
+const { CI_PROJECT_ID, CI_MERGE_REQUEST_IID } = process.env;
+
+const readYaml = (path: string) => {
+  try {
+    return parse(readFileSync(path, { encoding: "utf-8" }));
+  } catch (e) {
+    return {};
+  }
+};
+
+export const extractAllValues = async (
+  appDirectory: string,
+  env: string | null
+) => {
+  const valuesFromEnvVars = extractValuesFromEnvVars();
+  const valuesDefault = readYaml(appDirectory + "/values.yml");
+  const valuesEnv = env
+    ? readYaml(appDirectory + "/values-" + env + ".yml")
+    : {};
+  const valuesFromMR =
+    CI_PROJECT_ID && CI_MERGE_REQUEST_IID
+      ? await extractValuesFromMr(CI_PROJECT_ID, Number(CI_MERGE_REQUEST_IID))
+      : {};
+
+  return merge({}, valuesFromEnvVars, valuesDefault, valuesEnv, valuesFromMR);
+};

package/scripts/utils/extractBuildEnv.ts

@@ -0,0 +1,92 @@
+import { extractAllValues } from "./extractAllValues";
+import slugify from "slugify";
+
+type EnvVars = Record<string, string>;
+export const extractBuildEnv = async (
+  componentName: string,
+  env: string | null,
+  visitedReferencedApps: { [app: string]: EnvVars } = {} // this is just to avoid infinit recursion
+): Promise<EnvVars> => {
+  const values = await extractAllValues(componentName, env);
+  const {
+    APP_NAME,
+    CUSTOMER_NAME,
+    CI_COMMIT_REF_SLUG,
+    CI_MERGE_REQUEST_IID,
+    CI_ENVIRONMENT_SLUG,
+  } = process.env;
+
+  const isNotMonorepo = componentName === ".";
+
+  const KUBE_APP_NAME = CI_MERGE_REQUEST_IID
+    ? `${componentName}-${CI_COMMIT_REF_SLUG}`
+    : componentName;
+  const KUBE_APP_SLUG = isNotMonorepo
+    ? CI_ENVIRONMENT_SLUG
+    : slugify(KUBE_APP_NAME);
+  const CANONICAL_URL = `https://${APP_NAME}-${KUBE_APP_SLUG}.${CUSTOMER_NAME}.panter.cloud`;
+  const defaultBuildEnv = {
+    KUBE_APP_NAME,
+    CANONICAL_URL,
+    IMAGE_PULL_SECRET: `gitlab-registry-${componentName}`,
+    ROOT_URL: values?.application?.host || CANONICAL_URL,
+  };
+  const publicBuildEnv = values?.buildEnv?.public ?? {};
+  const merged = {
+    ...defaultBuildEnv,
+    ...publicBuildEnv,
+  };
+  if (process.env.DEBUG) {
+    console.log({ values, defaultBuildEnv, publicBuildEnv });
+  }
+  // extract referenced build env (only works in monorepo
+  // like this
+  /*
+  fromComponents:
+    vendure:
+      API_URL: ROOT_URL
+*/
+  const fromComponents: {
+    [otherApp: string]: {
+      [ourKey: string]: string;
+    };
+  } = values?.buildEnv?.fromComponents ?? {};
+  const referencedBuildEnv = await Object.entries(fromComponents).reduce(
+    async (acc, [otherApp, mapping]) => {
+      if (process.env.DEBUG) {
+        console.log("extract reference build env", otherApp, mapping);
+      }
+      if (!mapping) {
+        return await acc;
+      }
+
+      // prevent infinit loop
+      const otherBuildEnv =
+        visitedReferencedApps[otherApp] ??
+        (await extractBuildEnv(otherApp, env, {
+          ...visitedReferencedApps,
+          [componentName]: merged,
+        }));
+      if (process.env.DEBUG) {
+        console.log({ otherBuildEnv });
+      }
+      // resolve mapping
+      return {
+        ...(await acc),
+        ...Object.entries(mapping).reduce(
+          (inneracc, [ourKey, otherKey]) => ({
+            ...inneracc,
+            [ourKey]: otherBuildEnv[otherKey],
+          }),
+          {} as EnvVars
+        ),
+      };
+    },
+    {} as Promise<EnvVars>
+  );
+
+  return {
+    ...merged,
+    ...referencedBuildEnv,
+  };
+};

package/scripts/utils/extractValuesFromEnvVars.ts

@@ -0,0 +1,21 @@
+#!/usr/bin/env ts-node
+import { set } from "lodash";
+
+export const extractValuesFromEnvVars = () => {
+  const values = {};
+
+  const PREFIX = "DEFAULT_VALUE_";
+  for (const [key, value] of Object.entries(process.env).filter(([key]) =>
+    key.startsWith(PREFIX)
+  )) {
+    const path = key.replace(PREFIX, "").split("_");
+    let theValue;
+    try {
+      theValue = JSON.parse(value ?? "");
+    } catch (e) {
+      theValue = value;
+    }
+    set(values, path, theValue);
+  }
+  return values;
+};

package/scripts/utils/extractValuesFromMr.ts

@@ -0,0 +1,39 @@
+import { Gitlab } from "@gitbeaker/node"; // Just the Project Resource
+import { parse } from "yaml";
+import { merge } from "lodash";
+
+const { GL_TOKEN, CI_SERVER_URL } = process.env;
+
+const gitlabApi = new Gitlab({
+  token: GL_TOKEN,
+  host: CI_SERVER_URL,
+});
+const getMRInfo = async (projectId: string, mrId: number) => {
+  return await gitlabApi.MergeRequests.show(projectId, mrId);
+};
+
+export const extractValuesFromMRMessage = (mrMessage: string) => {
+  const codeBlocks = Array.from(mrMessage.matchAll(/```.*([^`]+)```/gm)).map(
+    (match) => match[1]
+  );
+
+  return codeBlocks.reduce((values = {}, block) => {
+    if (block.includes("$env") || block.includes("$values"))
+      try {
+        const { $values, $env } = parse(block);
+        if ($values) merge(values, $values);
+        if ($env) merge(values, { env: { public: $env } });
+        return values;
+      } catch (e) {
+        console.warn(e);
+      }
+  }, {});
+};
+
+export const extractValuesFromMr = async (
+  projectId: string,
+  mergeRequestId: number
+) => {
+  const mrInfo = await getMRInfo(projectId, mergeRequestId);
+  return extractValuesFromMRMessage(mrInfo.description);
+};

package/tests/extractValuesFromMr.test.ts

@@ -0,0 +1,156 @@
+import { extractValuesFromMRMessage } from "../scripts/utils/extractValuesFromMr";
+
+describe("extractValuesFromMr", () => {
+  describe("extractValuesFromMRMessage", () => {
+    it("can extract $values yaml", () => {
+      const result = extractValuesFromMRMessage(`
+      
+      blabla
+
+      test
+
+      \`\`\`
+      $values:
+        application:
+          host: www.foo.com
+        env:
+          public:
+            API_KEY: fooooxxx
+            BLA: "blubb"
+       \`\`\`
+      `);
+      expect(result).toEqual({
+        application: {
+          host: "www.foo.com",
+        },
+        env: {
+          public: {
+            API_KEY: "fooooxxx",
+            BLA: "blubb",
+          },
+        },
+      });
+    });
+    it("can handle $env as shortcut", () => {
+      const result = extractValuesFromMRMessage(`
+      
+      blabla
+
+      test
+
+      \`\`\`
+      $env:
+        API_KEY: fooooxxx
+        BLA: "blubb"
+     
+       \`\`\`
+      `);
+      expect(result).toEqual({
+        env: {
+          public: {
+            API_KEY: "fooooxxx",
+            BLA: "blubb",
+          },
+        },
+      });
+    });
+
+    it("ignores language in backticks", () => {
+      const result = extractValuesFromMRMessage(`
+      
+      blabla
+
+      test
+
+      \`\`\`yaml
+      $env:
+        API_KEY: fooooxxx
+        BLA: "blubb"
+     
+       \`\`\`
+      `);
+      expect(result).toEqual({
+        env: {
+          public: {
+            API_KEY: "fooooxxx",
+            BLA: "blubb",
+          },
+        },
+      });
+    });
+    it("can handle mixed blocks", () => {
+      const result = extractValuesFromMRMessage(`
+      
+      blabla
+
+      test
+
+      \`\`\`
+      $values:
+        env:
+          public:
+            FOO: "bar"
+
+      $env:
+        API_KEY: fooooxxx
+        BLA: "blubb"
+     
+      \`\`\`
+      `);
+      expect(result).toEqual({
+        env: {
+          public: {
+            FOO: "bar",
+            API_KEY: "fooooxxx",
+            BLA: "blubb",
+          },
+        },
+      });
+    });
+
+    it("can handle multiple blocks", () => {
+      const result = extractValuesFromMRMessage(`
+      
+      blabla
+
+      test
+
+      \`\`\`
+      $values:
+        env:
+          public:
+            FOO: "bar"
+
+      $env:
+        API_KEY: fooooxxx
+        BLA: "blubb"
+     
+      \`\`\`
+
+
+      fooo
+
+      \`\`\`
+      $values:
+        env:
+          public:
+            FOO: "blubb"
+
+      $env:
+        TEST: test
+     
+      \`\`\`
+      `);
+      expect(result).toEqual({
+        env: {
+          public: {
+            FOO: "blubb",
+            API_KEY: "fooooxxx",
+            BLA: "blubb",
+            TEST: "test",
+          },
+        },
+      });
+    });
+  });
+});

package/tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "compilerOptions": {
+    "lib": ["es2020", "DOM"],
+    "strict": true,
+    "downlevelIteration": true,
+    "declaration": true,
+    "outDir": "dist",
+    "esModuleInterop": true
+  }
+}