@catladder/pipeline 0.0.5 → 1.0.0

package/src/types/context.ts

@@ -0,0 +1,36 @@
+import { PipelineTrigger, ComponentConfig, Config, EnvType } from "./config";
+
+export type Environment = {
+  hostname: string;
+  fullName: string;
+  shortName: string;
+  slug: string;
+  url: string;
+  /**
+   * env vars contain all build-time env vars. secrets have to be resolved (they are stored in gitlab)
+   */
+  envVars: Record<string, string>;
+  envType: EnvType;
+  secretEnvVarKeys: string[];
+};
+
+export type CommitInfo = {
+  refName: string;
+  refSlug: string;
+  trigger: PipelineTrigger;
+};
+
+export type YarnInfo = {
+  version: string;
+  workspaces: { location: string }[];
+  isClassic: boolean;
+  componentIsInWorkspace: boolean;
+};
+export type Context = {
+  componentName: string;
+  componentConfig: ComponentConfig;
+  fullConfig: Config;
+  environment: Environment;
+  commitInfo?: CommitInfo;
+  yarnInfo?: YarnInfo;
+};

package/src/types/gitlab-types.ts

@@ -0,0 +1,94 @@
+export type Artifacts = {
+  paths: string[];
+};
+export type GitlabJobCache = {
+  key: string;
+  policy?: string;
+  paths: string[];
+};
+export type GitlabRule = {
+  if?: string;
+  when?: "always" | "on_success" | "manual" | "never"; // todo
+  allow_failure?: boolean;
+};
+export type Retry = {
+  max: number;
+  when: string[];
+};
+
+export type Service = {
+  name: string;
+  command: string[];
+};
+
+export type GitlabEnvironment = {
+  url: string;
+  name: string;
+  kubernetes?: {
+    namespace?: string;
+  };
+  action?: "stop" | "start" | "prepare"; // defaults to start
+  on_stop?: string; // other job to run on stop
+  auto_stop_in?: string;
+};
+export type GitlabJobDef = {
+  stage: string;
+  before_script?: string[];
+  script: (string | undefined)[];
+  interruptible?: boolean;
+  needs?: Array<string | { job: string; artifacts: boolean }>;
+  rules?: GitlabRule[];
+  cache?: GitlabJobCache | GitlabJobCache[];
+  artifacts?: Artifacts;
+  retry?: Retry;
+  services?: Service[];
+  image?: string;
+  variables?: GitlabVariables;
+  dependencies?: string[];
+  environment?: GitlabEnvironment;
+  allow_failure?: boolean;
+  trigger?: {
+    strategy: "depend";
+    include: Array<{
+      artifact: string;
+      job: string;
+    }>;
+  };
+};
+
+export const GITLAB_BASE_STAGES = [
+  "setup",
+  "test",
+  "build",
+  "deploy",
+  "verify",
+  "stop",
+] as const;
+export type GitlabBaseStage = typeof GITLAB_BASE_STAGES[number];
+export type GitlabVariables = Record<string, string | undefined>;
+
+export type GitlabJob = {
+  /**
+   * the name of the job (without any env or app prefix and suffix)
+   */
+  name: string;
+  /**
+   * envMode sets the behavior of the job regarding multiple envs:
+   * - none: the job does not run per env, but once for all envs
+   * - jobPerEnv: the job runs once per env
+   * - stagePerEnv: the job runs once per env and is organized in its own stage. This mproves usability in gitlab, but works the same as `jobPerEnv`
+   */
+  envMode: "jobPerEnv" | "stagePerEnv" | "none";
+  job: GitlabJobDef;
+  needsStages?: {
+    stage: GitlabBaseStage;
+    artifacts?: boolean;
+  }[];
+};
+export type GitlabJobs = GitlabJob[];
+
+export type GitlabPipeline = {
+  variables: GitlabVariables;
+  stages: string[];
+  jobs: GitlabJobs;
+};

package/src/utils/index.ts

@@ -0,0 +1,10 @@
+import { isNil } from "lodash";
+
+export function notNil<TValue>(
+  value: TValue | null | undefined
+): value is TValue {
+  return value !== null && value !== undefined;
+}
+
+export const ensureArray = <T>(s: T | T[]): T[] | null =>
+  isNil(s) ? null : Array.isArray(s) ? s : [s];

package/tsconfig.json

@@ -5,6 +5,8 @@
     "downlevelIteration": true,
     "declaration": true,
     "outDir": "dist",
-    "esModuleInterop": true
-  }
+    "esModuleInterop": true,
+    "incremental": true
+  },
+  "include": ["./src"]
 }

package/Dockerfile

@@ -1,84 +0,0 @@
-
-FROM node:14
-
-RUN rm /bin/sh && \
-    ln -s /bin/bash /bin/sh && \
-    mkdir -p /root/.nvm
-
-# add sources
-# Google Chrome (for testing purposes)
-RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \
-    sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list'
-
-# install some sources
-RUN apt-get update && apt-get install -y \
-    git ssh-client \
-    openssl \
-    curl \
-    bash \
-    apt-transport-https \
-    ca-certificates \
-    git \
-    gettext-base \
-    libxss1 \ 
-    google-chrome-stable
-
-
-
-# set high timeout for yarn, see https://github.com/dhis2/notes/issues/29
-RUN yarn config set network-timeout 600000 -g 
-
-# add semantic release
-RUN yarn global add \
-    semantic-release \
-    @semantic-release/commit-analyzer \
-    @semantic-release/release-notes-generator \
-    @semantic-release/git \
-    @semantic-release/changelog \
-    @semantic-release/gitlab;
-
-# kubernetes & helm
-ENV KUBERNETES_VERSION 1.18.20
-RUN curl -L -o /usr/bin/kubectl "https://storage.googleapis.com/kubernetes-release/release/v${KUBERNETES_VERSION}/bin/linux/amd64/kubectl" ;\
-    chmod +x /usr/bin/kubectl ;\
-    kubectl version --client
-
-# helm 2 for legacy reasons
-ENV HELM_2_VERSION 2.14.3
-RUN curl "https://get.helm.sh/helm-v${HELM_2_VERSION}-linux-amd64.tar.gz" | tar zx ;\
-    mv linux-amd64/helm /usr/bin/ ;\
-    helm version --client
-RUN cp /usr/bin/helm /usr/bin/helm2
-#  helm3
-ENV HELM_VERSION 3.6.3
-RUN curl "https://get.helm.sh/helm-v${HELM_VERSION}-linux-amd64.tar.gz" | tar zx ;\
-    mv linux-amd64/helm /usr/bin/helm3 ;\
-    helm3 version --client
-
-# install nvm && some common node versions
-RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash && \
-    source /root/.nvm/nvm.sh && \
-    chmod +x /root/.nvm/nvm.sh && \
-    /root/.nvm/nvm.sh install 10 && \
-    /root/.nvm/nvm.sh install 12 && \
-    /root/.nvm/nvm.sh install 14 && \
-    /root/.nvm/nvm.sh install 16
-
-# add meteor (used by some apps)
-# install also additional used released (should theoretically speed up ci-builds)
-RUN curl https://install.meteor.com/ | sh &&  \
-    meteor update --release 2.2 --allow-superuser
-
-# scripts
-
-RUN yarn global add ts-node typescript
-
-COPY ./scripts /scripts
-RUN chmod +x /scripts/*
-
-COPY package.json /scripts/
-COPY yarn.lock /scripts/
-
-WORKDIR /scripts
-RUN yarn
-ENV PATH="/scripts:${PATH}"

package/dist/scripts/magic/index.js

@@ -1,16 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __exportStar = (this && this.__exportStar) || function(m, exports) {
-    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
-};
-exports.__esModule = true;
-exports.createPipeline = void 0;
-var createPipeline_1 = require("./createPipeline");
-__createBinding(exports, createPipeline_1, "createPipeline");
-__exportStar(require("./types"), exports);

package/scripts/createSamplePipeline.ts

@@ -1,10 +0,0 @@
-#!/usr/bin/env ts-node
-import { writeFileSync } from "fs";
-import { createPipeline } from "./magic/createPipeline";
-import sampleConfig from "./magic/sample-config";
-
-createPipeline("taggedRelease", sampleConfig).then((mainPipeline) => {
-  writeFileSync(`__pipeline.yml`, JSON.stringify(mainPipeline, null, 2), {
-    encoding: "utf-8",
-  });
-});

package/scripts/getCommitInfo

@@ -1,11 +0,0 @@
-#!/bin/bash
-
-export BUILD_TAG="$(git describe --tags || git rev-parse HEAD)"
-export BUILD_ID="$BUILD_TAG"
-export BUILD_COMMIT="$(git rev-parse HEAD)"
-export BUILD_TIME="$(date)"
-echo "BUILD_TAG=$BUILD_TAG" >>build.env
-echo "BUILD_ID=$BUILD_ID" >>build.env
-echo "BUILD_COMMIT=$BUILD_COMMIT" >>build.env
-echo "BUILD_TIME=$BUILD_TIME" >>build.env
-echo '{"id":"'$BUILD_ID'","commit":"'$BUILD_COMMIT'","tag":"'$BUILD_TAG'","time":"'$BUILD_TIME'"}' >__build_info.json

package/scripts/kubernetesCreateSecret

@@ -1,9 +0,0 @@
-#!/bin/bash
-echo "KUBE_NAMESPACE: '$KUBE_NAMESPACE', IMAGE_PULL_SECRET: '$IMAGE_PULL_SECRET'"
-kubectl create secret -n "$KUBE_NAMESPACE" \
-    docker-registry $IMAGE_PULL_SECRET \
-    --docker-server="$CI_REGISTRY" \
-    --docker-username="${CI_DEPLOY_USER:-$CI_REGISTRY_USER}" \
-    --docker-password="${CI_DEPLOY_PASSWORD:-$CI_REGISTRY_PASSWORD}" \
-    --docker-email="$GITLAB_USER_EMAIL" \
-    -o yaml --dry-run | kubectl replace -n "$KUBE_NAMESPACE" --force -f -

package/scripts/kubernetesDelete

@@ -1,4 +0,0 @@
-#!/bin/bash
-RELEASE_NAME=${CUSTOMER_NAME}-${APP_NAME}-${CI_ENVIRONMENT_SLUG}
-echo "Delete $RELEASE_NAME"
-helm3 uninstall "$RELEASE_NAME" --namespace "$KUBE_NAMESPACE"

package/scripts/kubernetesDeploy

@@ -1,55 +0,0 @@
-#!/bin/bash
-
-echo "Deploy to kubernetes"
-echo "Release: $RELEASE_NAME"
-echo "URL (canonical): $HOST_CANONICAL"
-#helm3 init --client-only --stable-repo-url=https://charts.helm.sh/stable
-helm3 chart pull $CI_REGISTRY/$HELM_GITLAB_CHART_PATH/$HELM_GITLAB_CHART_NAME:v$HELM_GITLAB_CHART_VERSION
-helm3 chart export $CI_REGISTRY/$HELM_GITLAB_CHART_PATH/$HELM_GITLAB_CHART_NAME:v$HELM_GITLAB_CHART_VERSION -d /tmp/
-# use helmArgs defined in variables
-helmArgsEvaluated=$(echo $helmArgs | envsubst)
-
-# this bash-brainfuck simply splits $helmArgs by newline to an array...
-readarray -t helmArgsArray <<<"$helmArgsEvaluated"
-
-#Log the time until selfdestruct in case of review environment
-if [[ ${ENV_SHORT} == "review" ]]; then echo "NOTE - This deployment will stop itself after 2 weeks"; fi
-
-echo "all values"
-echo "$KUBE_VALUES"
-echo $KUBE_VALUES >__all_values.yml
-
-echo "doing helm upgrade"
-helm3 upgrade --install "$RELEASE_NAME" /tmp/$HELM_GITLAB_CHART_NAME \
-    --wait \
-    --timeout 15m0s \
-    --cleanup-on-fail \
-    --set global.componentName="$KUBE_APP_NAME" \
-    --set global.ENV_SHORT="$ENV_SHORT" \
-    --set global.CI_COMMIT_REF_SLUG="$CI_COMMIT_REF_SLUG" \
-    --set global.BUILD_ID="$BUILD_ID" \
-    --set image.repository="$CI_REGISTRY_IMAGE" \
-    --set image.pullSecret="$IMAGE_PULL_SECRET" \
-    --set image.name="$COMPONENT_NAME" \
-    --set image.tag="$IMAGE_TAG" \
-    --set image.pullPolicy=IfNotPresent \
-    --set storybook.enabled=$STORYBOOK \
-    --set application.hostCanonical="$HOST_CANONICAL" \
-    --set mongodb.enabled=$MONGODB_ENABLED \
-    --set mongodb.backup.enabled=$MONGODB_BACKUP_ENABLED \
-    --set mongodb-replicaset.replicas=$MONGODB_REPLICAS \
-    --set mongodb-replicaset.fullnameOverride=$KUBE_APP_NAME-mongodb-replicaset \
-    --set mongodb-replicaset.persistentVolume.storageClass="$MONGODB_STORAGE_CLASS" \
-    --set mailhog.fullnameOverride=$KUBE_APP_NAME-mailhog \
-    --set prisma.enabled=$PRISMA_ENABLED \
-    --set gitlab.visualReviewEnabled="$GITLAB_VISUAL_REVIEW_ENABLED" \
-    --set gitlab.app="$CI_PROJECT_PATH_SLUG" \
-    --set gitlab.projectPath="$CI_PROJECT_PATH" \
-    --set gitlab.mergeRequestId="$CI_MERGE_REQUEST_IID" \
-    --set gitlab.projectId="$CI_PROJECT_ID" \
-    --set gitlab.env="$CI_ENVIRONMENT_SLUG" \
-    --set namespace="$KUBE_NAMESPACE" \
-    --namespace="$KUBE_NAMESPACE" \
-    ${helmArgsArray[@]} \
-    --values __all_values.yml
-echo "Deployment successful 😻"

package/scripts/kubernetesEnsureNamespace

@@ -1,3 +0,0 @@
-#!/bin/bash
-echo "Ensure Namespace $KUBE_NAMESPACE"
-kubectl describe namespace "$KUBE_NAMESPACE" || kubectl create namespace "$KUBE_NAMESPACE"

package/scripts/magic/build/index.ts

@@ -1,16 +0,0 @@
-import { GitlabJobs } from "../types/gitlab-types";
-import { Context } from "../types/context";
-import { createNodeJobs, getNodeDefaults } from "./node";
-import { BuildConfig } from "./types";
-
-export const buildJobCreators: {
-  [type in BuildConfig["type"]]: (context: Context) => GitlabJobs;
-} = {
-  node: createNodeJobs,
-};
-
-export const getBuildDefaults: {
-  [type in BuildConfig["type"]]: () => Partial<BuildConfig>;
-} = {
-  node: getNodeDefaults,
-};

package/scripts/magic/build/node.ts

@@ -1,130 +0,0 @@
-import {
-  GitlabJobDef,
-  GitlabJobCache,
-  GitlabJobs,
-  Retry,
-} from "../types/gitlab-types";
-import { Context } from "../types/context";
-import { createDockerBuildJob } from "./docker";
-import { BuildConfigNode, isOfType } from "./types";
-
-const baseRetry: Retry = {
-  max: 2,
-  when: ["runner_system_failure", "stuck_or_timeout_failure"],
-};
-
-const getYarnInstall = (context: Context) => [
-  `cd ${context.componentConfig.dir}`,
-  "if [ -f ./.nvmrc ]; then source /root/.nvm/nvm.sh && nvm install <<< .nvmrc; fi",
-  "yarn install --frozen-lockfile",
-];
-const createNodeTestJobs = (context: Context): GitlabJobs => {
-  const cache: GitlabJobCache[] = [
-    {
-      key: "node-modules",
-      policy: "pull-push",
-      paths: ["node_modules", "**/node_modules/"],
-    },
-  ];
-
-  const base: Omit<GitlabJobDef, "script"> = {
-    variables: {
-      APP_PATH: context.componentConfig.dir,
-    },
-    cache,
-    stage: "test",
-    interruptible: true,
-    needs: [],
-    retry: baseRetry,
-  };
-  const yarnInstall = getYarnInstall(context);
-  return [
-    {
-      name: "audit",
-      perEnv: false,
-      job: {
-        ...base,
-        script: [...yarnInstall, "yarn audit"],
-        allow_failure: true,
-      },
-    },
-    {
-      name: "lint",
-      perEnv: false,
-      job: {
-        ...base,
-        script: [...yarnInstall, "yarn lint"],
-      },
-    },
-    {
-      name: "test",
-      perEnv: false,
-      job: {
-        ...base,
-        script: [...yarnInstall, "yarn test"],
-      },
-    },
-  ];
-};
-
-const createNodeBuildJobs = (context: Context): GitlabJobs => {
-  const buildConfig = context.componentConfig.build;
-  if (!isOfType(buildConfig, "node")) {
-    // should not happen
-    throw new Error("deploy config is not kubernetes");
-  }
-
-  const buildInfo = [
-    ". getCommitInfo", // TODO: inline
-    `echo '{"id":"'$BUILD_ID'","commit":"'$BUILD_COMMIT'","tag":"'$BUILD_TAG'","time":"'$BUILD_TIME'"}' > ${context.componentConfig.dir}/__build_info.json`,
-  ];
-
-  const yarnInstall = getYarnInstall(context);
-
-  return [
-    {
-      name: "app-build",
-      job: {
-        needs: [],
-        variables: context.environment.variables,
-        retry: baseRetry,
-        interruptible: true,
-        stage: "build",
-        script: [...buildInfo, ...yarnInstall, buildConfig.buildCommand!],
-        artifacts: {
-          paths: [
-            context.componentConfig.dir + "/__build_info.json",
-            context.componentConfig.dir + "/dist",
-            context.componentConfig.dir + "/.next",
-          ],
-        },
-      },
-    },
-    {
-      name: "docker-build",
-
-      job: {
-        ...createDockerBuildJob(context, {
-          script: [
-            buildConfig.runtime === "static"
-              ? "ensureNginxDockerfile"
-              : "ensureNodeDockerfile",
-          ], // TOOD: inline
-        }),
-        needs: ["app-build"],
-      },
-    },
-  ];
-};
-
-export const createNodeJobs = (context: Context): GitlabJobs => {
-  return [...createNodeTestJobs(context), ...createNodeBuildJobs(context)];
-};
-
-export const getNodeDefaults = (): Partial<BuildConfigNode> => {
-  return {
-    startCommand: "yarn start",
-    runtime: "dynamic",
-    buildCommand: "yarn build",
-  };
-};

package/scripts/magic/build/types.ts

@@ -1,17 +0,0 @@
-export type BuildConfigBase = {
-  startCommand?: string;
-};
-export type BuildConfigNode = {
-  type: "node";
-  runtime?: "dynamic" | "static"; // defaults to dynamic
-  buildCommand?: string;
-} & BuildConfigBase;
-
-export type BuildConfig = BuildConfigNode;
-
-export const isOfType = <T extends BuildConfig["type"]>(
-  t: BuildConfig,
-  type: T
-): t is Extract<BuildConfig, { type: T }> => {
-  return t.type === type;
-};