@catladder/pipeline 0.0.7 → 1.0.2

package/scripts/magic/createChildPipeline/index.ts

@@ -1,60 +0,0 @@
-import { merge } from "lodash";
-import slugify from "slugify";
-import { buildJobCreators } from "../build";
-import { createContext } from "../context";
-import { deployJobCreators } from "../deploy";
-import { Config } from "../types/config";
-import { CommitInfo, Context, Environment } from "../types/context";
-import { GitlabJobDef, GitlabJobs } from "../types/gitlab-types";
-
-const createRawJobs = (context: Context): GitlabJobs => {
-  const buildJobs =
-    buildJobCreators[context.componentConfig.build.type](context);
-  const deployJobs =
-    deployJobCreators[context.componentConfig.deploy.type](context);
-
-  return [...buildJobs, ...deployJobs];
-};
-const getFullJobName = (name: string, componentName: string, env?: string) => {
-  if (env) {
-    return `${env} ${componentName} ${name}`;
-  }
-  return `${componentName} ${name}`;
-};
-
-const replaceReferences = (
-  def: GitlabJobDef,
-  componentName: string,
-  env?: string
-): GitlabJobDef => {
-  return {
-    ...def,
-    needs: def.needs?.map((n) => getFullJobName(n, componentName, env)),
-    dependencies: def.dependencies?.map((n) =>
-      getFullJobName(n, componentName, env)
-    ),
-  };
-};
-
-export const createJobs = (
-  envs: string[],
-  config: Config,
-  componentName: string
-): Record<string, GitlabJobDef> => {
-  return envs.reduce((acc, env) => {
-    const context = createContext(componentName, config, env);
-    const jobs = createRawJobs(context);
-    return {
-      ...acc,
-      ...jobs.reduce((acc, { name, job, perEnv = true }) => {
-        const def = job;
-
-        return {
-          ...acc,
-          [getFullJobName(name, componentName, perEnv ? env : undefined)]:
-            replaceReferences(def, componentName, perEnv ? env : undefined),
-        };
-      }, {}),
-    };
-  }, {});
-};

package/scripts/magic/createPipeline.ts

@@ -1,52 +0,0 @@
-import { createJobs } from "./createChildPipeline";
-import { Config, ENV_TYPES, PipelineTrigger } from "./types/config";
-import { GitlabJobDef } from "./types/gitlab-types";
-
-export const createPipeline = async (
-  trigger: PipelineTrigger,
-  config: Config
-) => {
-  const envs = Object.keys(ENV_TYPES).filter(
-    (e) => ENV_TYPES[e as keyof typeof ENV_TYPES].trigger === trigger
-  );
-  const components = Object.keys(config.components);
-
-  // 2. write the triggering pipeline
-
-  const jobs = components.reduce<Record<string, GitlabJobDef>>(
-    (acc, componentName) => {
-      return {
-        ...acc,
-        ...createJobs(envs, config, componentName),
-      };
-    },
-    {}
-  );
-
-  const rules = [
-    // same as rules "always", but with `changes` to only trigger changed branches
-    { if: "$CI_COMMIT_TAG" },
-    {
-      if: "$CI_COMMIT_MESSAGE =~ /^chore(release).*/",
-      when: "never",
-    },
-    { if: "$CI_COMMIT_BRANCH =~ /^[0-9]+.([0-9]+|x).x$/" },
-    {
-      if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH",
-    },
-
-    { if: "$CI_MERGE_REQUEST_ID" },
-  ];
-
-  const mainPipeline = {
-    image:
-      "git.panter.ch:5001/catladder/gitlab-ci/pipeline:$PIPELINE_IMAGE_TAG",
-    workflow: {
-      rules,
-    },
-    stages: ["setup", "test", "build", "deploy", "verify", "actions"],
-    ...jobs,
-  };
-
-  return mainPipeline;
-};

package/scripts/magic/deploy/index.ts

@@ -1,19 +0,0 @@
-import { GitlabJobs } from "../types/gitlab-types";
-import { ComponentConfig } from "../types/config";
-import { Context } from "../types/context";
-import { createKubernetesDeployJobs, getKubernetesDefault } from "./kubernetes";
-import { DeployConfig } from "./types";
-
-export const deployJobCreators: {
-  [type in DeployConfig["type"]]: (context: Context) => GitlabJobs;
-} = {
-  kubernetes: createKubernetesDeployJobs,
-  serverless: () => [],
-};
-
-export const getDeployDefaults: {
-  [type in DeployConfig["type"]]: () => Partial<DeployConfig>;
-} = {
-  kubernetes: getKubernetesDefault,
-  serverless: () => ({}),
-};

package/scripts/magic/deploy/kubernetes.ts

@@ -1,65 +0,0 @@
-import { GitlabJobs } from "../types/gitlab-types";
-import { Context } from "../types/context";
-import { DeployConfigKubernetes, isOfType } from "./types";
-
-export const createKubernetesDeployJobs = (context: Context): GitlabJobs => {
-  const deployConfig = context.componentConfig.deploy;
-  if (!isOfType(deployConfig, "kubernetes")) {
-    // should not happen
-    throw new Error("deploy config is not kubernetes");
-  }
-
-  const defaltKubeValues = {
-    application: {
-      hostname: context.environment.hostname,
-      command: context.componentConfig.build.startCommand,
-    },
-  };
-  const kubeValues = {
-    ...defaltKubeValues,
-    ...deployConfig.values,
-  }; // TODO: merge with some defaults
-  return [
-    {
-      name: "deploy-to-kubernetes",
-
-      job: {
-        variables: {
-          ...context.environment.variables,
-
-          // TODO: refactor and unify with other stages
-          HELM_EXPERIMENTAL_OCI: "1",
-          IMAGE_PULL_SECRET: `gitlab-registry-${context.componentName}`,
-          KUBE_VALUES: JSON.stringify(kubeValues),
-          HELM_GITLAB_CHART_PATH: "catladder/helm-charts",
-          HELM_GITLAB_CHART_NAME: "the-panter-chart",
-          COMPONENT_NAME: context.componentName,
-          // TODO: unify with docker build stage
-          IMAGE_TAG: "$CI_COMMIT_SHA",
-          HELM_GITLAB_CHART_VERSION: "3.2.0", // TODO, we could actually just ship the chart directly here
-          RELEASE_NAME: `${context.fullConfig.customerName}-${context.fullConfig.appName}-$CI_ENVIRONMENT_SLUG`,
-        },
-        stage: "deploy",
-        dependencies: [],
-        script: [
-          "kubernetesEnsureNamespace",
-          "kubernetesCreateSecret",
-          "kubernetesDeploy",
-        ],
-        environment: {
-          name: context.environment.fullName,
-          url: context.environment.url,
-          kubernetes: {
-            namespace: context.environment.variables.KUBE_NAMESPACE,
-          },
-        },
-      },
-    },
-  ];
-};
-
-export const getKubernetesDefault = (): Partial<DeployConfigKubernetes> => {
-  return {
-    values: {},
-  };
-};

package/scripts/magic/deploy/types.ts

@@ -1,29 +0,0 @@
-export type DeployConfigBase = {};
-export type DeployConfigKubernetes = {
-  type: "kubernetes";
-  additionalHelmArgs?: string[];
-  values?: {
-    application?: {
-      resources?: {
-        limits?: {
-          cpu?: any;
-        };
-        requests?: {
-          cpu?: any;
-        };
-      };
-    } & Record<string, any>;
-  } & Record<string, any>;
-} & DeployConfigBase;
-
-export type DeployConfigServerless = {
-  type: "serverless";
-};
-export type DeployConfig = DeployConfigKubernetes | DeployConfigServerless;
-
-export const isOfType = <T extends DeployConfig["type"]>(
-  t: DeployConfig,
-  type: T
-): t is Extract<DeployConfig, { type: T }> => {
-  return t.type === type;
-};

package/scripts/magic/index.ts

@@ -1,2 +0,0 @@
-export { createPipeline } from "./createPipeline";
-export * from "./types";

package/scripts/magic/sample-config.ts

@@ -1,78 +0,0 @@
-import { Config } from "./types/config";
-
-const sampleConfig: Config = {
-  customerName: "pan",
-  appName: "fonsi",
-  components: {
-    web: {
-      dir: "web",
-      build: {
-        type: "node",
-      },
-      deploy: {
-        type: "kubernetes",
-        values: {
-          application: {
-            resources: {
-              limits: {
-                cpu: "100m",
-              },
-            },
-          },
-        },
-      },
-      vars: {
-        public: {
-          GOOGLE_TAG_MANAGER_ID: "1234",
-        },
-        fromComponents: {
-          api: {
-            API_URL: "ROOT_URL",
-          },
-        },
-      },
-      env: {
-        prod: {
-          hostname: "fudilo.ch",
-          vars: {},
-          deploy: {
-            type: "kubernetes",
-            values: {
-              application: {
-                resources: {
-                  limits: {
-                    cpu: "1000m",
-                  },
-                },
-              },
-            },
-          },
-        },
-        myCustomProd: {
-          type: "prod",
-          hostname: "fudilo2.ch",
-        },
-      },
-    },
-    api: {
-      env: {
-        prod: {
-          hostname: "api.fudilo.ch",
-        },
-        myCustomProd: {
-          type: "prod",
-          hostname: "api.fudilo2.ch",
-        },
-      },
-      build: {
-        type: "node",
-      },
-      deploy: {
-        type: "kubernetes",
-      },
-      dir: "api",
-    },
-  },
-};
-
-export default sampleConfig;

package/scripts/magic/types/config.ts

@@ -1,54 +0,0 @@
-import { BuildConfig } from "../build/types";
-import { DeployConfig } from "../deploy/types";
-
-export type PipelineTrigger = "mainBranch" | "mr" | "taggedRelease";
-
-export const ENV_TYPES = {
-  dev: {
-    trigger: "mainBranch",
-  },
-  review: {
-    trigger: "mr",
-  },
-  prod: {
-    trigger: "taggedRelease",
-  },
-  stage: {
-    trigger: "taggedRelease",
-  },
-} as const;
-type EnvType = keyof typeof ENV_TYPES;
-
-export type DefaultEnvConfig = {
-  deploy: DeployConfig;
-  build: BuildConfig;
-  vars?: {
-    public?: Record<string, any>;
-    secret?: Record<string, string>;
-    fromComponents?: {
-      [otherApp: string]: Record<string, string>;
-    };
-  };
-};
-type EnvConfig<E extends EnvType = EnvType> = {
-  type?: E;
-  hostname?: string;
-} & Partial<DefaultEnvConfig>;
-
-export type Env = {
-  dev?: EnvConfig<"dev">;
-  stage?: EnvConfig<"stage">;
-  review?: EnvConfig<"review">;
-  prod?: EnvConfig<"prod">;
-} & Record<string, EnvConfig>;
-
-export type ComponentConfig = {
-  env?: Env;
-  dir: string;
-} & DefaultEnvConfig;
-
-export type Config = {
-  customerName: string;
-  appName: string;
-  components: Record<string, ComponentConfig>;
-};

package/scripts/magic/types/context.ts

@@ -1,21 +0,0 @@
-import { ComponentConfig, Config } from "./config";
-
-export type Environment = {
-  hostname: string;
-  fullName: string;
-  shortName: string;
-  url: string;
-  variables: Record<string, string>;
-};
-
-export type CommitInfo = {
-  refName: string;
-  refSlug: string;
-};
-export type Context = {
-  componentConfig: ComponentConfig;
-  componentName: string;
-  fullConfig: Config;
-  environment: Environment;
-  commit: CommitInfo;
-};

package/scripts/magic/types/gitlab-types.ts

@@ -1,69 +0,0 @@
-export type GitlabStage =
-  | "setup"
-  | "test"
-  | "build"
-  | "deploy"
-  | "verify"
-  | "actions";
-
-export type Artifacts = {
-  paths: string[];
-};
-export type GitlabJobCache = {
-  key: string;
-  policy?: string;
-  paths: string[];
-};
-export type GitlabRule = {
-  if: string;
-  when?: "always" | "on_success" | "manual" | "never"; // todo
-};
-export type Retry = {
-  max: number;
-  when: string[];
-};
-
-export type Service = {
-  name: string;
-  command: string[];
-};
-
-export type GitlabEnvironment = {
-  url: string;
-  name: string;
-  kubernetes?: {
-    namespace?: string;
-  };
-};
-export type GitlabJobDef = {
-  stage: GitlabStage;
-  before_script?: string[];
-  script: string[];
-  interruptible?: boolean;
-  needs?: string[];
-  rules?: GitlabRule[];
-  cache?: GitlabJobCache | GitlabJobCache[];
-  artifacts?: Artifacts;
-  retry?: Retry;
-  services?: Service[];
-  image?: string;
-  variables?: GitlabVariables;
-  dependencies?: string[];
-  environment?: GitlabEnvironment;
-  allow_failure?: boolean;
-};
-
-export type GitlabVariables = Record<string, string | undefined>;
-
-export type GitlabJob = {
-  name: string;
-  perEnv?: boolean;
-  job: GitlabJobDef;
-};
-export type GitlabJobs = GitlabJob[];
-
-export type GitlabPipeline = {
-  variables: GitlabVariables;
-  stages: GitlabStage[];
-  jobs: GitlabJobs;
-};

package/scripts/magic.ts

@@ -1,51 +0,0 @@
-#!/usr/bin/env ts-node
-
-import { existsSync, writeFileSync } from "fs";
-import { parse } from "yaml";
-import { PipelineTrigger } from "./magic/types/config";
-import { createPipeline } from "./magic/createPipeline";
-
-const {
-  CI_MERGE_REQUEST_ID,
-  CI_COMMIT_TAG,
-  CI_COMMIT_BRANCH,
-  CI_DEFAULT_BRANCH,
-} = process.env;
-
-const isDefaultBranch = CI_COMMIT_BRANCH == CI_DEFAULT_BRANCH;
-const isHotfixBranch = false; // TODO:  $CI_COMMIT_BRANCH =~ /^[0-9]+\.([0-9]+|x)\.x$/
-const isMergeRequest = Boolean(CI_MERGE_REQUEST_ID);
-const isTaggedRelease = Boolean(CI_COMMIT_TAG);
-
-const fullPath = (ext: string) => process.cwd() + "/catladder." + ext;
-const readConfig = () => {
-  const found = ["ts", "js", "yml", "yaml"].find((extension) =>
-    existsSync(fullPath(extension))
-  );
-  if (found) {
-    if (found === "ts" || found === "js") {
-      return require(fullPath(found)).default;
-    } else {
-      return parse(fullPath(found));
-    }
-  }
-};
-
-const trigger: PipelineTrigger | null =
-  isMergeRequest || isHotfixBranch
-    ? "mr"
-    : isDefaultBranch
-    ? "mainBranch"
-    : isTaggedRelease
-    ? "taggedRelease"
-    : null;
-if (trigger) {
-  const config = readConfig();
-  createPipeline(trigger, config).then((mainPipeline) => {
-    writeFileSync(`__pipeline.yml`, JSON.stringify(mainPipeline, null, 2), {
-      encoding: "utf-8",
-    });
-  });
-} else {
-  throw new Error("no matching trigger");
-}

package/scripts/monorepoGenerate.ts

@@ -1,104 +0,0 @@
-#!/usr/bin/env ts-node
-
-import { readdirSync, readFileSync, lstatSync, existsSync } from "fs";
-import { join } from "path";
-import { parse } from "yaml";
-import yargs from "yargs";
-
-import { hideBin } from "yargs/helpers";
-import { extractBuildEnv } from "./utils/extractBuildEnv";
-type Type = "test" | "deploy";
-const args = yargs(hideBin(process.argv)).array("env").argv as Record<
-  string,
-  unknown
->;
-
-const thisGitlabCi = parse(
-  readFileSync(".gitlab-ci.yml", { encoding: "utf-8" })
-);
-
-const subcis = readdirSync(".")
-  .filter((file) => lstatSync(file).isDirectory())
-  .map((dir) => ({
-    dirname: dir,
-    ci: join(dir, ".gitlab-ci.yml"),
-  }))
-  .filter(({ ci }) => existsSync(ci));
-
-const makeConfig = async ({
-  subapp,
-  type,
-  ci,
-  env,
-}: {
-  subapp: string;
-  type: Type;
-  env: string;
-  ci: string;
-}) => {
-  const changes = [subapp + "/**/*"];
-  const rules = [
-    // same as rules "always", but with `changes` to only trigger changed branches
-    { if: "$CI_COMMIT_TAG" },
-    {
-      if: "$CI_COMMIT_MESSAGE =~ /^chore(release).*/",
-      when: "never",
-    },
-    { if: "$CI_COMMIT_BRANCH =~ /^[0-9]+.([0-9]+|x).x$/" },
-    {
-      if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH",
-      changes,
-    },
-    // add it manually when no changes
-    {
-      if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH",
-      when: "manual",
-    },
-    { if: "$CI_MERGE_REQUEST_ID", changes },
-    // add it manually when no changes
-    { if: "$CI_MERGE_REQUEST_ID", when: "manual" },
-  ];
-
-  return {
-    variables: {
-      // merge
-      ...thisGitlabCi.variables,
-      APP_DIR: subapp,
-      VALUES_DIR: subapp,
-      COMPONENT_NAME: subapp,
-      PARENT_PIPELINE_ID: "$CI_PIPELINE_ID",
-      [type.toUpperCase() + "_ONLY"]: "1",
-      ...(await extractBuildEnv(subapp, env)),
-    },
-    stage: type,
-    rules: rules,
-    trigger: {
-      include: ci,
-      strategy: "depend",
-    },
-  };
-};
-const generateSubPipeline = async (type: Type, envs: string[]) => {
-  return await envs.reduce(async (acc, env) => {
-    return {
-      ...(await acc),
-      ...(await subcis.reduce(
-        async (inneracc, { dirname, ci }) => ({
-          ...(await inneracc),
-
-          [`${env}-${dirname} ${type}`]: await makeConfig({
-            subapp: dirname,
-            ci,
-            type,
-            env,
-          }),
-        }),
-        {} as Promise<Record<string, unknown>>
-      )),
-    };
-  }, {} as Promise<Record<string, unknown>>);
-};
-
-generateSubPipeline(args.t as Type, args.env as string[]).then((p) =>
-  console.log(JSON.stringify(p, null, 2))
-);

package/scripts/printAllValues.ts

@@ -1,17 +0,0 @@
-#!/usr/bin/env ts-node
-
-import yargs from "yargs";
-import { hideBin } from "yargs/helpers";
-import { extractAllValues } from "./utils/extractAllValues";
-
-const args = yargs(hideBin(process.argv)).argv as Record<string, unknown>;
-
-extractAllValues(args.d as string, args.e as string)
-  .then(async (values) => {
-    console.log(JSON.stringify(values, null, 2) ?? "");
-  })
-  .catch((e) => {
-    console.error("could not extract values", {
-      error: e,
-    });
-  });

package/scripts/printBuildEnv.ts

@@ -1,22 +0,0 @@
-#!/usr/bin/env ts-node
-
-import { isObject } from "lodash";
-import yargs from "yargs";
-import { hideBin } from "yargs/helpers";
-import { extractBuildEnv } from "./utils/extractBuildEnv";
-
-const args = yargs(hideBin(process.argv)).argv as Record<string, unknown>;
-
-extractBuildEnv(args.d as string, args.e as string)
-  .then(async (values) => {
-    Object.entries(values).forEach(([key, value]) =>
-      console.log(
-        `${key}: "${isObject(value) ? JSON.stringify(value) : value}"`
-      )
-    );
-  })
-  .catch((e) => {
-    console.error("could not extract values", {
-      error: e,
-    });
-  });

package/scripts/utils/extractAllValues.ts

@@ -1,32 +0,0 @@
-import { readFileSync } from "fs";
-import { merge } from "lodash";
-import { parse } from "yaml";
-import { extractValuesFromEnvVars } from "./extractValuesFromEnvVars";
-import { extractValuesFromMr } from "./extractValuesFromMr";
-
-const { CI_PROJECT_ID, CI_MERGE_REQUEST_IID } = process.env;
-
-const readYaml = (path: string) => {
-  try {
-    return parse(readFileSync(path, { encoding: "utf-8" }));
-  } catch (e) {
-    return {};
-  }
-};
-
-export const extractAllValues = async (
-  appDirectory: string,
-  env: string | null
-) => {
-  const valuesFromEnvVars = extractValuesFromEnvVars();
-  const valuesDefault = readYaml(appDirectory + "/values.yml");
-  const valuesEnv = env
-    ? readYaml(appDirectory + "/values-" + env + ".yml")
-    : {};
-  const valuesFromMR =
-    CI_PROJECT_ID && CI_MERGE_REQUEST_IID
-      ? await extractValuesFromMr(CI_PROJECT_ID, Number(CI_MERGE_REQUEST_IID))
-      : {};
-
-  return merge({}, valuesFromEnvVars, valuesDefault, valuesEnv, valuesFromMR);
-};