@catladder/pipeline 0.0.7 → 1.0.2

package/dist/types/gitlab-types.d.ts

@@ -0,0 +1,86 @@
+export declare type Artifacts = {
+  paths: string[];
+};
+export declare type GitlabJobCache = {
+  key: string;
+  policy?: string;
+  paths: string[];
+};
+export declare type GitlabRule = {
+  if?: string;
+  when?: "always" | "on_success" | "manual" | "never";
+  allow_failure?: boolean;
+};
+export declare type Retry = {
+  max: number;
+  when: string[];
+};
+export declare type Service = {
+  name: string;
+  command: string[];
+};
+export declare type GitlabEnvironment = {
+  url: string;
+  name: string;
+  kubernetes?: {
+    namespace?: string;
+  };
+  action?: "stop" | "start" | "prepare";
+  on_stop?: string;
+  auto_stop_in?: string;
+};
+export declare type GitlabJobDef = {
+  stage: string;
+  before_script?: string[];
+  script: (string | undefined)[];
+  interruptible?: boolean;
+  needs?: Array<string | {
+    job: string;
+    artifacts: boolean;
+  }>;
+  rules?: GitlabRule[];
+  cache?: GitlabJobCache | GitlabJobCache[];
+  artifacts?: Artifacts;
+  retry?: Retry;
+  services?: Service[];
+  image?: string;
+  variables?: GitlabVariables;
+  dependencies?: string[];
+  environment?: GitlabEnvironment;
+  allow_failure?: boolean;
+  trigger?: {
+    strategy: "depend";
+    include: Array<{
+      artifact: string;
+      job: string;
+    }>;
+  };
+};
+export declare const GITLAB_BASE_STAGES: readonly ["setup", "test", "build", "deploy", "verify", "stop"];
+export declare type GitlabBaseStage = typeof GITLAB_BASE_STAGES[number];
+export declare type GitlabVariables = Record<string, string | undefined>;
+export declare type GitlabJob = {
+  /**
+   * the name of the job (without any env or app prefix and suffix)
+   */
+  name: string;
+  /**
+   * envMode sets the behavior of the job regarding multiple envs:
+   * - none: the job does not run per env, but once for all envs
+   * - jobPerEnv: the job runs once per env
+   * - stagePerEnv: the job runs once per env and is organized in its own stage. This mproves usability in gitlab, but works the same as `jobPerEnv`
+   */
+
+  envMode: "jobPerEnv" | "stagePerEnv" | "none";
+  job: GitlabJobDef;
+  needsStages?: {
+    stage: GitlabBaseStage;
+    artifacts?: boolean;
+  }[];
+};
+export declare type GitlabJobs = GitlabJob[];
+export declare type GitlabPipeline = {
+  variables: GitlabVariables;
+  stages: string[];
+  jobs: GitlabJobs;
+};

package/dist/types/gitlab-types.js

@@ -0,0 +1,5 @@
+"use strict";
+
+exports.__esModule = true;
+exports.GITLAB_BASE_STAGES = void 0;
+exports.GITLAB_BASE_STAGES = ["setup", "test", "build", "deploy", "verify", "stop"];

package/dist/{scripts/magic/types → types}/index.d.ts

@@ -1,3 +1,3 @@
 export * from "./config";
 export * from "./gitlab-types";
-export * from "./context";
+export * from "./context";

package/dist/types/index.js

@@ -0,0 +1,26 @@
+"use strict";
+
+var __createBinding = this && this.__createBinding || (Object.create ? function (o, m, k, k2) {
+  if (k2 === undefined) k2 = k;
+  Object.defineProperty(o, k2, {
+    enumerable: true,
+    get: function () {
+      return m[k];
+    }
+  });
+} : function (o, m, k, k2) {
+  if (k2 === undefined) k2 = k;
+  o[k2] = m[k];
+});
+
+var __exportStar = this && this.__exportStar || function (m, exports) {
+  for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+
+exports.__esModule = true;
+
+__exportStar(require("./config"), exports);
+
+__exportStar(require("./gitlab-types"), exports);
+
+__exportStar(require("./context"), exports);

package/dist/utils/index.d.ts

@@ -0,0 +1,2 @@
+export declare function notNil<TValue>(value: TValue | null | undefined): value is TValue;
+export declare const ensureArray: <T>(s: T | T[]) => T[] | null;

package/dist/utils/index.js

@@ -0,0 +1,18 @@
+"use strict";
+
+exports.__esModule = true;
+exports.ensureArray = exports.notNil = void 0;
+
+var lodash_1 = require("lodash");
+
+function notNil(value) {
+  return value !== null && value !== undefined;
+}
+
+exports.notNil = notNil;
+
+var ensureArray = function (s) {
+  return (0, lodash_1.isNil)(s) ? null : Array.isArray(s) ? s : [s];
+};
+
+exports.ensureArray = ensureArray;

package/package.json

@@ -1,22 +1,43 @@
 {
   "name": "@catladder/pipeline",
-  "version": "0.0.7",
+  "version": "1.0.2",
   "scripts": {
-    "prepack": "yarn tsc"
+    "build:tsc": "yarn tsc",
+    "build": "yarn tsc && yarn build:inline-variables",
+    "build:watch": "yarn tsc -w",
+    "build:inline-variables": "yarn babel --keep-file-extension --extensions \".ts,.js\" --config-file ./babel-inline.json dist --out-dir dist",
+    "test": "jest",
+    "lint": "eslint \"src/**/*.ts\"",
+    "lint:fix": "eslint \"src/**/*.ts\" --fix"
+  },
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "bin": {
+    "catladder-gitlab": "./bin/catladder-gitlab.js"
   },
-  "main": "dist/scripts/magic/index.js",
-  "types": "dist/scripts/magic/index.d.ts",
   "devDependencies": {
+    "@babel/cli": "^7.16.0",
+    "@babel/core": "^7.0.0-0",
+    "@babel/plugin-syntax-typescript": "^7.16.7",
+    "@babel/preset-typescript": "^7.16.5",
+    "@types/child-process-promise": "^2.2.2",
+    "@types/jest": "^27.4.0",
     "@types/lodash": "^4.14.172",
-    "@types/node": "^16.6.1",
-    "@types/yargs": "^17.0.5",
-    "typescript": "^4.5.4"
+    "babel-plugin-transform-inline-environment-variables": "^0.4.3",
+    "eslint": "^8.7.0"
   },
   "dependencies": {
     "@gitbeaker/node": "^32.1.2",
+    "@types/node": "^17.0.10",
+    "child-process-promise": "^2.2.1",
+    "js-yaml": "^4.1.0",
     "lodash": "^4.17.21",
+    "memoizee": "^0.4.15",
+    "path-equal": "^1.1.3",
+    "prettier": "^2.5.1",
     "slugify": "^1.6.3",
-    "yaml": "^1.10.2",
-    "yargs": "^17.2.1"
+    "ts-node": "^10.4.0",
+    "type-fest": "^2.10.0",
+    "typescript": "^4.5.4"
   }
 }

package/src/build/base/constants.ts

@@ -0,0 +1,7 @@
+export const APP_BUILD_JOB_NAME = "🔨 app";
+export const RUNNER_BUILD_RESOURCE_VARIABLES = {
+  KUBERNETES_CPU_REQUEST: "0.5",
+  KUBERNETES_CPU_LIMIT: "2",
+  KUBERNETES_MEMORY_REQUEST: "512Mi",
+  KUBERNETES_MEMORY_LIMIT: "2Gi",
+};

package/src/build/base/createBuildJob.ts

@@ -0,0 +1,46 @@
+import { merge } from "lodash";
+import { Context, GitlabJob, GitlabJobDef } from "../..";
+import { BASE_RETRY } from "../../defaults";
+import { ensureArray } from "../../utils";
+import {
+  APP_BUILD_JOB_NAME,
+  RUNNER_BUILD_RESOURCE_VARIABLES,
+} from "./constants";
+
+export const createBuildJob = (
+  context: Context,
+  { script, ...def }: Partial<GitlabJobDef>
+): GitlabJob => {
+  const buildInfo = [
+    ". getCommitInfo", // TODO: inline
+    `echo '{"id":"'$BUILD_ID'","commit":"'$BUILD_COMMIT'","tag":"'$BUILD_TAG'","time":"'$BUILD_TIME'"}' > ${context.componentConfig.dir}/__build_info.json`,
+  ];
+
+  return {
+    name: APP_BUILD_JOB_NAME,
+    envMode: "jobPerEnv",
+    job: merge(
+      {
+        needs: [],
+        cache: [],
+        variables: {
+          ...RUNNER_BUILD_RESOURCE_VARIABLES,
+          ...context.environment.envVars,
+          ...(context.componentConfig.build.extraVars ?? {}),
+        },
+        retry: BASE_RETRY,
+        interruptible: true,
+        stage: "build",
+        script: [
+          ...buildInfo,
+          `cd ${context.componentConfig.dir}`,
+          ...(ensureArray(script) ?? []),
+        ],
+        artifacts: {
+          paths: [context.componentConfig.dir + "/__build_info.json"],
+        },
+      },
+      def
+    ),
+  };
+};

package/{scripts/magic → src}/build/docker.ts

@@ -1,13 +1,21 @@
+import { getRunnerImage } from "../runner";
 import { GitlabJobDef, Context } from "../types";
 
+const DOCKER_RUNNER_BUILD_VARIABLES = {
+  KUBERNETES_CPU_REQUEST: "0.5",
+  KUBERNETES_CPU_LIMIT: "1",
+  KUBERNETES_MEMORY_REQUEST: "1Gi",
+  KUBERNETES_MEMORY_LIMIT: "2Gi",
+};
+
+export const DOCKER_BUILD_JOB_NAME = "🔨 docker";
 export const createDockerBuildJob = (
   context: Context,
   { script }: Pick<GitlabJobDef, "script">
 ): GitlabJobDef => {
   const base: Omit<GitlabJobDef, "script"> = {
     stage: "build",
-    image:
-      "git.panter.ch:5001/catladder/gitlab-ci/docker-build:$PIPELINE_IMAGE_TAG",
+    image: getRunnerImage("docker-build"),
     interruptible: true,
     services: [
       {
@@ -16,6 +24,11 @@ export const createDockerBuildJob = (
       },
     ],
     variables: {
+      ...DOCKER_RUNNER_BUILD_VARIABLES,
+      DOCKERFILE_ADDITIONS:
+        context.componentConfig.build.docker?.additionsBegin?.join("\n"),
+      DOCKERFILE_ADDITIONS_END:
+        context.componentConfig.build.docker?.additionsEnd?.join("\n"),
       APP_DIR: context.componentConfig.dir,
       DOCKER_HOST: "tcp://0.0.0.0:2375",
       DOCKER_TLS_CERTDIR: "",

package/src/build/index.ts

@@ -0,0 +1,42 @@
+import { Context } from "../types/context";
+import { GitlabJobs } from "../types/gitlab-types";
+import { createNodeJobs, createStorybookJobs, createMeteorJobs } from "./node";
+
+import { BuildConfig } from "./types";
+export * from "./types";
+export * from "./node";
+
+export type BuildTypes = {
+  [type in BuildConfig["type"]]: {
+    jobs: (context: Context) => GitlabJobs;
+    defaults: () => Partial<Extract<BuildConfig, { type: type }>>;
+  };
+};
+
+export const BUILD_TYPES: BuildTypes = {
+  node: {
+    jobs: createNodeJobs,
+    defaults: () => ({
+      buildCommand: "yarn build",
+      startCommand: "yarn start",
+    }),
+  },
+  "node-static": {
+    jobs: createNodeJobs,
+    defaults: () => ({
+      buildCommand: "yarn build",
+    }),
+  },
+  storybook: {
+    jobs: createStorybookJobs,
+    defaults: () => ({
+      buildCommand: ["yarn storybook", "mv ./storybook-out /dist"],
+    }),
+  },
+  meteor: {
+    jobs: createMeteorJobs,
+    defaults: () => ({
+      startCommand: "node main.js",
+    }),
+  },
+};

package/src/build/node/buildJob.ts

@@ -0,0 +1,54 @@
+import { Context } from "../../types/context";
+import { GitlabJob, GitlabJobs } from "../../types/gitlab-types";
+import { ensureArray } from "../../utils";
+import { APP_BUILD_JOB_NAME } from "../base/constants";
+import { createBuildJob } from "../base/createBuildJob";
+import { createDockerBuildJob, DOCKER_BUILD_JOB_NAME } from "../docker";
+import { isOfBuildType } from "../types";
+import { getNextCache, getNodeCache } from "./cache";
+import { getYarnInstall } from "./yarn";
+
+export const createNodeBuildJobs = (context: Context): GitlabJobs => {
+  const buildConfig = context.componentConfig.build;
+
+  if (!isOfBuildType(buildConfig, "node", "node-static", "storybook")) {
+    throw new Error("deploy config is not node, node-static or storybook");
+  }
+
+  const yarnInstall = getYarnInstall(context);
+  const appBuildJob: GitlabJob | null =
+    buildConfig.buildCommand !== null
+      ? createBuildJob(context, {
+          cache: [...getNodeCache(context), ...getNextCache(context)],
+          script: [
+            ...yarnInstall,
+            ...(ensureArray(buildConfig.buildCommand) ?? []),
+          ],
+          artifacts: {
+            paths: [
+              context.componentConfig.dir + "/__build_info.json",
+              context.componentConfig.dir + "/dist",
+              context.componentConfig.dir + "/.next",
+            ],
+          },
+        })
+      : null;
+  return [
+    ...(appBuildJob ? [appBuildJob] : []),
+    {
+      name: DOCKER_BUILD_JOB_NAME,
+      envMode: "jobPerEnv",
+      job: {
+        ...createDockerBuildJob(context, {
+          script: [
+            buildConfig.type === "node-static" ||
+            buildConfig.type === "storybook"
+              ? "ensureNginxDockerfile"
+              : "ensureNodeDockerfile",
+          ],
+        }),
+        needs: appBuildJob ? [APP_BUILD_JOB_NAME] : [],
+      },
+    },
+  ];
+};

package/src/build/node/cache.ts

@@ -0,0 +1,36 @@
+import { join } from "path";
+import { Context } from "../../types/context";
+import { GitlabJobCache } from "../../types/gitlab-types";
+
+export const getNodeCache = (context: Context): GitlabJobCache[] => {
+  const componentIsInWorkspace = context.yarnInfo?.componentIsInWorkspace;
+
+  return [
+    {
+      // if component is in a shared workspace, use workspace cache. use individual cache else
+      key: componentIsInWorkspace
+        ? "node-modules-workspace"
+        : context.componentName + "-node-modules",
+      policy: "pull-push",
+      paths: [
+        ".yarn",
+        ...(componentIsInWorkspace
+          ? [
+              "node_modules",
+              ...(context.yarnInfo?.workspaces.map((w) =>
+                join(w.location, "node_modules")
+              ) ?? []),
+            ]
+          : [join(context.componentConfig.dir, "node_modules")]),
+      ],
+    },
+  ];
+};
+
+export const getNextCache = (context: Context): GitlabJobCache[] => [
+  {
+    key: context.componentName + "-next-cache",
+    policy: "pull-push",
+    paths: [context.componentConfig.dir + "/.next/cache/"],
+  },
+];

package/src/build/node/constants.ts

@@ -0,0 +1,6 @@
+export const NODE_RUNNER_BUILD_VARIABLES = {
+  KUBERNETES_CPU_REQUEST: "0.5",
+  KUBERNETES_CPU_LIMIT: "2",
+  KUBERNETES_MEMORY_REQUEST: "512Mi",
+  KUBERNETES_MEMORY_LIMIT: "2Gi",
+};

package/src/build/node/index.ts

@@ -0,0 +1,17 @@
+import type { Context } from "../../types/context";
+import type { GitlabJobs } from "../../types/gitlab-types";
+import { createNodeBuildJobs } from "./buildJob";
+import { createMeteorBuildJobs } from "./meteor";
+import { createNodeTestJobs } from "./testJob";
+
+export const createNodeJobs = (context: Context): GitlabJobs => {
+  return [...createNodeTestJobs(context), ...createNodeBuildJobs(context)];
+};
+
+export const createStorybookJobs = (context: Context): GitlabJobs => {
+  return [...createNodeBuildJobs(context)];
+};
+
+export const createMeteorJobs = (context: Context): GitlabJobs => {
+  return [...createNodeTestJobs(context), ...createMeteorBuildJobs(context)];
+};

package/src/build/node/meteor.ts

@@ -0,0 +1,75 @@
+import { join } from "path";
+import type { Context } from "../../types/context";
+import type {
+  GitlabJob,
+  GitlabJobCache,
+  GitlabJobs,
+} from "../../types/gitlab-types";
+
+import { APP_BUILD_JOB_NAME } from "../base/constants";
+import { createBuildJob } from "../base/createBuildJob";
+import { createDockerBuildJob, DOCKER_BUILD_JOB_NAME } from "../docker";
+import { isOfBuildType } from "../types";
+import { getNodeCache } from "./cache";
+import { getYarnInstall } from "./yarn";
+
+const getMeteorCache = (context: Context): GitlabJobCache[] => [
+  {
+    key: context.componentName + "meteor-build-cache",
+    policy: "pull-push",
+    paths: [
+      join(
+        context.componentConfig.dir,
+        ".meteor/local/resolver-result-cache.json"
+      ),
+      join(context.componentConfig.dir, ".meteor/local/plugin-cache"),
+      join(context.componentConfig.dir, ".meteor/local/isopacks"),
+      join(context.componentConfig.dir, ".meteor/local/bundler-cache/scanner"),
+    ],
+  },
+];
+export const createMeteorBuildJobs = (context: Context): GitlabJobs => {
+  const buildConfig = context.componentConfig.build;
+
+  if (!isOfBuildType(buildConfig, "meteor")) {
+    throw new Error("deploy config is not meteor");
+  }
+
+  const yarnInstall = getYarnInstall(context);
+  const appBuildJob: GitlabJob | null =
+    buildConfig.buildCommand !== null
+      ? createBuildJob(context, {
+          cache: [...getNodeCache(context), ...getMeteorCache(context)],
+          script: [
+            ...yarnInstall,
+
+            'echo "add healthcheck package"',
+            "meteor add panter:healthroute --allow-superuser",
+            "meteor add qualia:prod-shell --allow-superuser",
+
+            'TOOL_NODE_FLAGS="--max_old_space_size=3584 --min_semi_space_size=8 --max_semi_space_size=256 --optimize_for_size" meteor build ./dist --architecture os.linux.x86_64 --allow-superuser --server-only --directory',
+
+            "cp ./__build_info.json ./dist/bundle/programs/server",
+          ],
+          artifacts: {
+            paths: [
+              context.componentConfig.dir + "/__build_info.json",
+              context.componentConfig.dir + "/dist",
+            ],
+          },
+        })
+      : null;
+  return [
+    ...(appBuildJob ? [appBuildJob] : []),
+    {
+      name: DOCKER_BUILD_JOB_NAME,
+      envMode: "jobPerEnv",
+      job: {
+        ...createDockerBuildJob(context, {
+          script: ["ensureMeteorDockerfile"],
+        }),
+        needs: appBuildJob ? [APP_BUILD_JOB_NAME] : [],
+      },
+    },
+  ];
+};

package/src/build/node/testJob.ts

@@ -0,0 +1,80 @@
+import { BASE_RETRY } from "../../defaults";
+import { Context } from "../../types/context";
+import { GitlabJob, GitlabJobDef, GitlabJobs } from "../../types/gitlab-types";
+import { ensureArray, notNil } from "../../utils";
+import { getNodeCache } from "./cache";
+import { NODE_RUNNER_BUILD_VARIABLES } from "./constants";
+import { getYarnInstall } from "./yarn";
+
+export const createNodeTestJobs = (context: Context): GitlabJobs => {
+  // don't run tests after release
+  if (context.commitInfo?.trigger === "taggedRelease") {
+    return [];
+  }
+
+  const buildConfig = context.componentConfig.build;
+
+  const base: Omit<GitlabJobDef, "script"> = {
+    variables: {
+      APP_PATH: context.componentConfig.dir,
+      ...NODE_RUNNER_BUILD_VARIABLES,
+      ...(buildConfig.extraVars ?? {}),
+    },
+
+    stage: "test",
+    interruptible: true,
+    needs: [],
+    retry: BASE_RETRY,
+  };
+  const yarnInstall = getYarnInstall(context);
+  const auditJob: GitlabJob | null =
+    buildConfig.audit !== false
+      ? {
+          name: "🛡 audit",
+          envMode: "none",
+          job: {
+            ...base,
+            cache: undefined, // audit does not need yarn install and no cache
+            script: [
+              `cd ${context.componentConfig.dir}`,
+              ...(ensureArray(buildConfig.audit?.command) ?? ["yarn audit"]),
+            ],
+            allow_failure: true,
+          },
+        }
+      : null;
+
+  const lintJob: GitlabJob | null =
+    buildConfig.lint !== false
+      ? {
+          name: "👮 lint",
+          envMode: "none",
+          job: {
+            ...base,
+            cache: getNodeCache(context),
+            script: [
+              `cd ${context.componentConfig.dir}`,
+              ...yarnInstall,
+              ...(ensureArray(buildConfig.lint?.command) ?? ["yarn lint"]),
+            ],
+          },
+        }
+      : null;
+  const testJob: GitlabJob | null =
+    buildConfig.test !== false
+      ? {
+          name: "🧪 test",
+          envMode: "none",
+          job: {
+            ...base,
+            cache: getNodeCache(context),
+            script: [
+              `cd ${context.componentConfig.dir}`,
+              ...yarnInstall,
+              ...(ensureArray(buildConfig.test?.command) ?? ["yarn test"]),
+            ],
+          },
+        }
+      : null;
+  return [auditJob, lintJob, testJob].filter(notNil);
+};

package/src/build/node/yarn.ts

@@ -0,0 +1,8 @@
+import { Context } from "../../types";
+
+export const getYarnInstall = (context: Context) => [
+  "if [ -f ./.nvmrc ]; then source /root/.nvm/nvm.sh && nvm install <<< .nvmrc; fi",
+  context.yarnInfo?.isClassic
+    ? "yarn install --frozen-lockfile"
+    : "yarn install --immutable",
+];

package/src/build/tests/storybook.test.ts

@@ -0,0 +1,46 @@
+import { Config } from "../../types";
+import { createChildPipeline } from "../../";
+
+describe("storybook build", () => {
+  describe("creates two build jobs", () => {
+    const config: Config = {
+      appName: "test",
+      customerName: "pan",
+      components: {
+        "my-app": {
+          dir: "myapp",
+          build: {
+            type: "storybook",
+          },
+          deploy: {
+            type: "kubernetes",
+          },
+        },
+      },
+    };
+
+    it("creates a pipeline for storybook that does not contain test stages", async () => {
+      const { image, stages, workflow, ...jobs } = await createChildPipeline(
+        "mainBranch",
+        config
+      );
+
+      expect(Object.keys(jobs)).toEqual([
+        "dev my-app app-build",
+        "dev my-app docker-build",
+        "dev my-app deploy-to-kubernetes",
+        "dev my-app kubernetes-stop",
+      ]);
+    });
+
+    it("runs build and renames folder", async () => {
+      const { image, stages, workflow, ...jobs } = await createChildPipeline(
+        "mainBranch",
+        config
+      );
+      const buildJob = jobs["dev my-app app-build"];
+      expect(buildJob.script).toContain("yarn storybook");
+      expect(buildJob.script).toContain("mv ./storybook-out /dist");
+    });
+  });
+});