@catladder/pipeline 0.0.0-node16-ddfae617 → 0.0.0-prettier-cbceeed9

package/src/deploy/types/googleCloudRun.ts

@@ -0,0 +1,179 @@
+import type { DeployConfigBase } from "./base";
+
+export type Gcloudregion =
+  | "asia-east1"
+  | "asia-northeast1"
+  | "asia-northeast2"
+  | "europe-north1"
+  | "europe-southwest1"
+  | "europe-west1"
+  | "europe-west4"
+  | "europe-west8"
+  | "europe-west9"
+  | "me-west1"
+  | "us-central1"
+  | "us-east1"
+  | "us-east4"
+  | "us-east5"
+  | "us-south1"
+  | "us-west1"
+  | "asia-east2"
+  | "asia-northeast3"
+  | "asia-southeast1"
+  | "asia-southeast2"
+  | "asia-south1"
+  | "asia-south2"
+  | "australia-southeast1"
+  | "australia-southeast2"
+  | "europe-central2"
+  | "europe-west2"
+  | "europe-west3"
+  | "europe-west6"
+  | "northamerica-northeast1"
+  | "northamerica-northeast2"
+  | "southamerica-east1"
+  | "southamerica-west1"
+  | "us-west2"
+  | "us-west3"
+  | "us-west4";
+
+export type DeployConfigCloudRunCloudSql = {
+  type: "unmanaged";
+  instanceConnectionName: string;
+  /**
+   * the database username, defaults to "postgres"
+   */
+  dbUser?: string;
+
+  /**
+   * the prefix of the database, the full db name is this plus the environment slug prefix plus the componentName
+   *
+   * defaults to customerName-appName
+   */
+  dbNamePrefix?: string | false;
+
+  /**
+   * the base name of the db, defaults to the componentName
+   */
+  dbBaseName?: string;
+  /**
+   * whether to delete the database if the environment is stopped
+   * defaults to true for review envs, to false for every other environment
+   */
+  deleteDatabaseOnStop?: boolean;
+
+  /**
+   * add additional query params to the database connection string
+   */
+  dbAdditionalQueryParams?: Record<string, string | number> & {
+    /**
+     * set to 0 for prisma in review or dev apps where you reset the db occasionally without restarting any server
+     */
+    statement_cache_size?: number;
+  };
+};
+
+type Memory = `${number}${"M" | "G" | "Mi" | "Gi"}`;
+export type DeployConfigCloudRunService = {
+  /**
+   * command / entrypoint, fallsback to buildConfig.startcommand
+   */
+  command?: string;
+
+  /**
+   * how many instances to keep around when there are no requests. defaults to 0.
+   * Set to 1 for workers
+   */
+  minInstances?: number;
+
+  /**
+   * maximum amount of instances. Defaults to 100
+   */
+  maxInstances?: number;
+
+  /**
+   * set to true to allways allocate cpu, e.g. for workers. Careful that this is expensive!
+   */
+  noCpuThrottling?: true;
+
+  /**
+   * whether to allow public access (defaults to true)
+   */
+  allowUnauthenticated?: boolean;
+
+  /**
+   * memory limit. Defaults to 512MB
+   */
+  memory?: Memory;
+};
+
+export type DeployConfigCloudRunJobBase = {
+  /**
+   * command
+   */
+  command: string;
+
+  /**
+   * memory limit of the job, defaults to 51Mi
+   */
+  memory?: `${number}${"M" | "G" | "Mi" | "Gi"}`;
+};
+
+type Minute = string;
+type Hour = string;
+type DayOfMonth = string;
+type DayOfWeek = string;
+type Month = string;
+export type DeployConfigCloudRunJobWithSchedule =
+  DeployConfigCloudRunJobBase & {
+    when: "schedule";
+    schedule: `${Minute} ${Hour} ${DayOfMonth} ${Month} ${DayOfWeek}`;
+  };
+
+export type DeployConfigCloudRunJobNormal = DeployConfigCloudRunJobBase & {
+  when: "manual" | "preDeploy" | "postDeploy";
+};
+
+export type DeployConfigCloudRunJob =
+  | DeployConfigCloudRunJobNormal
+  | DeployConfigCloudRunJobWithSchedule;
+
+export type DeployConfigCloudRun = {
+  /**
+   * EXPERIMENTAL cloud run deployment.
+   *
+   * This will deploy a cloud run service.
+   *
+   * Requires that cloud run is enabled on the project, as well as cloud run api and artifacts registry.
+   */
+  type: "google-cloudrun";
+  /**
+   * the google cloud porject id
+   */
+  projectId: string;
+  /**
+   * the region. Set to europe-west6 for switzerland
+   */
+  region: Gcloudregion;
+
+  /**
+   * whether to enable the service, defaults to true
+   */
+  service?: boolean | DeployConfigCloudRunService;
+
+  /**
+   * deploy additional services with different entry pointss
+   */
+  additionalServices?: {
+    [name: string]: DeployConfigCloudRunService;
+  };
+
+  jobs?: {
+    [name: string]: DeployConfigCloudRunJob | false | null;
+  };
+
+  /**
+   * add cloudSql
+   */
+  cloudSql?: DeployConfigCloudRunCloudSql | false;
+} & DeployConfigBase;

package/src/deploy/types/index.ts

@@ -0,0 +1,28 @@
+import type { DeployConfigCustom } from "./custom";
+import type { DeployConfigDockerTag } from "./dockerTag";
+import type { DeployConfigCloudRun } from "./googleCloudRun";
+import type { DeployConfigKubernetes } from "./kubernetes";
+
+export * from "./custom";
+export * from "./googleCloudRun";
+export * from "./kubernetes";
+export * from "./dockerTag";
+
+export type DeployConfig =
+  | DeployConfigKubernetes
+  | DeployConfigCustom
+  | DeployConfigCloudRun
+  | DeployConfigDockerTag;
+
+export type DeployConfigType = DeployConfig["type"];
+export type DeployConfigGeneric<T extends DeployConfigType> = Extract<
+  DeployConfig,
+  { type: T }
+>;
+
+export const isOfDeployType = <T extends Array<DeployConfigType>>(
+  t: DeployConfig | false,
+  ...types: T
+): t is Extract<DeployConfig, { type: T[number] }> => {
+  return t && types.includes(t.type);
+};

package/src/deploy/types/kubernetes.ts

@@ -0,0 +1,301 @@
+import type { AllowUnknownProps, DeployConfigBase } from "./base";
+
+export type DeployConfigKubernetesClusterGCloud = {
+  type: "gcloud";
+  /**
+   * gcoud name of the cluster
+   */
+  name: string;
+  /**
+   * google cloud project id
+   */
+  projectId: string;
+  /**
+   * region
+   */
+  region: string;
+  /**
+   * domain of the cluster, for canonical domains without custom host (e.g. review and dev apps)
+   */
+  domainCanonical?: string;
+};
+export type DeployConfigKubernetesCluster = DeployConfigKubernetesClusterGCloud; // currently only this
+
+export type KubernetesAutoscaleMetricResourceTarget =
+  | {
+      type: "Utilization";
+      averageUtilization: number;
+    }
+  | {
+      type: "AverageValue";
+      averageValue: number | string;
+    };
+export type KubernetesAutoscaleMetric = {
+  type: "Resource";
+  resource: {
+    name: "cpu" | "memory";
+    target: KubernetesAutoscaleMetricResourceTarget;
+  };
+};
+export type KubernetesAutoscale = {
+  minReplicas: number;
+  maxReplicas: number;
+  /**
+   * declare a number of metrics (at least one). Usually you would use cpu as a metric
+   */
+  metrics: KubernetesAutoscaleMetric[];
+};
+export type KubernetesResourcesDef = {
+  limits?: {
+    cpu?: string;
+    memory?: string;
+  };
+  requests?: {
+    cpu?: string;
+    memory?: string;
+  };
+};
+type KubernetesHealthDef = {
+  httpGet?: {
+    path?: string;
+    port?: number;
+    httpHeaders?: {
+      name: string;
+      value: string;
+    }[];
+    failureThreshold?: number;
+    timeoutSeconds?: number;
+    periodSeconds?: number;
+    successThreshold?: number;
+  };
+};
+export type KubernetesUpdateStrategy = {
+  rollingUpdate: {
+    /**
+     * how many pods can be created additionally to the target number
+     * can be absolute number or percentage.
+     * defaults to 1 (kube default is 25%)
+     */
+    maxSurge: number | string;
+    /**
+     * how many pods are allowed to be unavailable
+     * can be absolute number or percentage
+     * defaults to 0 (kube default is 25%)
+     */
+    maxUnavailable: number | string;
+  };
+  type: "RollingUpdate";
+};
+export type KubernetesWorkerDef = {
+  enabled: boolean;
+  /**
+   * which command to run. defaults to `yarn start`
+   */
+  command?: string;
+  resources?: KubernetesResourcesDef;
+  service?: {
+    enabled?: boolean;
+  };
+};
+
+export type DeployConfigMongodbBase = {
+  enabled?: boolean;
+  dbName?: string;
+  persistence?: {
+    /**
+     * set premium-rwo for ssd
+     */
+    storageClass?: "standard-rwo" | "premium-rwo";
+
+    /**
+     * size of the volume, defaults to 8Gi
+     */
+    size?: string;
+  };
+  resources?: KubernetesResourcesDef;
+};
+
+export type DeployConfigMongodbStandalone = {
+  architecture: "standalone";
+};
+export type DeployConfigMongodbReplicaset = {
+  architecture: "replicaset";
+  /**
+   * defaults to 2
+   */
+  replicaCount?: number;
+};
+export type DeployConfigMongodb = DeployConfigMongodbBase &
+  (DeployConfigMongodbStandalone | DeployConfigMongodbReplicaset);
+
+export type DeployConfigKubernetesValues = AllowUnknownProps<{
+  /**
+   * enable cloudsql db. Currently you have to manually set it up
+   */
+  cloudsql?: {
+    enabled: boolean;
+    instanceId?: string;
+    projectId?: string;
+    region?: string;
+  };
+  /**
+   * enable mongodb. The mongodb is deployed using a helm chart.
+   * See https://github.com/bitnami/charts/tree/master/bitnami/mongodb
+   */
+  mongodb?: DeployConfigMongodb;
+  /**
+   * enable mailhog. Mailhog is a virtual mail server that catches all outgoing mailsl and show them in a mailbox.
+   * This is great for development as it prevents to accidentially send out real emails and helps with debugging outgoing mails.
+   *
+   * It will automaticaly inject a MAIL_URL env var.
+   *
+   * Turn it of for production.
+   */
+  mailhog?: {
+    enabled: boolean;
+  };
+  /**
+   * post-install/upgrade jobs. These use the app image and have all env vars available.
+   * Typically used for migrations and seeds
+   */
+  jobs?: Record<
+    string,
+    | false
+    | AllowUnknownProps<{
+        /**
+         * the command to execute
+         */
+        command: string;
+        /**
+         * comma-separated list of helm hooks, see https://helm.sh/docs/topics/charts_hooks/
+         *
+         * defaults to post-install,post-upgrade
+         */
+        hook?: string;
+      }>
+  >;
+  /**
+   * cronjobs that run periodically. These use the app image have all env vars available.
+   */
+  cronjobs?: Record<
+    string,
+    | false
+    | AllowUnknownProps<{
+        schedule: string;
+        command: string;
+        concurrencyPolicy?: "Forbid" | "Allow" | "Replace";
+      }>
+  >;
+  /**
+   * configuration for the application ("Deployment" in kubernetes)
+   */
+  application?:
+    | false
+    | AllowUnknownProps<{
+        /**
+         * command to start, defaults to build's startCommand
+         */
+        command?: string;
+
+        /**
+         * enable, disable app deployment, defaults to true
+         */
+        enabled?: boolean;
+        /**
+         * redirects
+         */
+        redirects?: AllowUnknownProps<{ host: string }>[];
+        /**
+         * Host aliases
+         */
+        hostAliases?: string[];
+        /**
+         * how many pods will be started
+         */
+        replicas?: number;
+
+        /**
+         * autoscale the pods horizontally
+         */
+        autoscale?: KubernetesAutoscale;
+
+        /**
+         * the update strategy, see https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#rolling-update-deployment
+         */
+        updateStrategy?: KubernetesUpdateStrategy;
+
+        /**
+         * kubernetes resources
+         */
+
+        resources?: KubernetesResourcesDef;
+        healthRoute?: string;
+        startupProbe?: KubernetesHealthDef;
+        readinessProbe?: KubernetesHealthDef;
+        livenessProbe?: KubernetesHealthDef;
+        /**
+         * specify a worker. A worker is a separate deployment that runs continously in the background.
+         * The worker pod can use the same entry point as the app or a different one (specify command).
+         * It has the same environment variables as the application and an additional IS_WORKER="true" variable
+         */
+        worker?: KubernetesWorkerDef;
+
+        /**
+         * kubernetes resources for jobs and cronjobs
+         */
+        jobDefaults?: {
+          resources: KubernetesResourcesDef;
+        };
+      }>;
+
+  /**
+   * load secrets from other components.
+   * Use this only in cases where the secret is not known on pipeline-time
+   * and only if the component is also in the same cluster.
+   *
+   * a typical case is reusing mongo_url from other component
+   *
+   * also make sure that the other component deploys first
+   *
+   * in all other cases use normal env var handling
+   */
+  secretsFromOtherComponent?: {
+    [envVar: string]: string;
+  };
+
+  /**
+   * map another services internal endpoint into a env var.
+   */
+  mapServiceUrlToEnv?: {
+    [envVar: string]: string;
+  };
+  /**
+   * Mount secrets as files in the filesystem.
+   * These secrets are still available as env vars, but will contain the path of the file instead
+   *
+   * @deprecated this will be removed in the future, because its very niche and not used
+   */
+  secretsAsFile?: string[];
+}>;
+
+export type DeployConfigKubernetes = {
+  type: "kubernetes";
+  /**
+   * cluster config to use.
+   */
+  cluster: DeployConfigKubernetesCluster;
+  /**
+   * prints out debug info (helm --debug)
+   */
+  debug?: boolean;
+  additionalHelmArgs?: string[];
+  /**
+   * values to configure the app
+   */
+  values?: DeployConfigKubernetesValues;
+  /**
+   * Custom Helm chart location.
+   * Not recommended to use.
+   */
+  chartName?: string;
+} & DeployConfigBase;

package/src/deploy/utils.ts

@@ -1,5 +1,6 @@
-import { Config, Context } from "../types";
-import { DeployConfigKubernetesCluster, isOfDeployType } from "./types";
+import type { Config, Context } from "../types";
+import type { DeployConfigKubernetesCluster } from "./types";
+import { isOfDeployType } from "./types";
 
 export const getFullKubernetesClusterName = (
   cluster: DeployConfigKubernetesCluster
@@ -22,6 +23,9 @@ export const contextIsStoppable = (context: Context) => {
     return true;
   }
 
+  if (isOfDeployType(deployConfig, "google-cloudrun")) {
+    return true;
+  }
   if (isOfDeployType(deployConfig, "custom") && deployConfig.stopScript) {
     return true;
   }

package/src/pipeline/commitInfo/getBuildId.ts

@@ -0,0 +1,6 @@
+import { exec } from "child-process-promise";
+
+export const getBuildId = async () =>
+  await exec("git describe --tags || git rev-parse HEAD").then((s) =>
+    s.stdout.trim()
+  );

package/src/pipeline/commitInfo/getCommitInfo.ts

@@ -0,0 +1,12 @@
+import type { CommitInfo } from "../../types";
+import { getBuildId } from "./getBuildId";
+export const getBaseCommitInfo = async (): Promise<
+  Omit<CommitInfo, "trigger">
+> => ({
+  refName: process.env.CI_COMMIT_REF_NAME ?? "unknown",
+  reviewSlug: process.env.CI_MERGE_REQUEST_IID
+    ? `mr${process.env.CI_MERGE_REQUEST_IID}`
+    : "unknown",
+  buildTime: new Date().toISOString(),
+  buildId: await getBuildId(),
+});

package/src/pipeline/createAllJobs.ts

@@ -0,0 +1,38 @@
+import { getAllEnvsByTrigger } from "../config";
+import type { Config, PipelineTrigger } from "../types";
+import type { CatladderJob } from "../types/jobs";
+import { createJobsForComponent } from "./createJobsForComponent";
+
+export type AllCatladderJobs = {
+  [componentName: string]: {
+    [env: string]: Array<CatladderJob>;
+  };
+};
+export const createAllJobs = async (
+  config: Config,
+  trigger: PipelineTrigger
+): Promise<AllCatladderJobs> => {
+  return Object.fromEntries(
+    await Promise.all(
+      Object.keys(config.components).map(async (componentName) => {
+        const envs = getAllEnvsByTrigger(config, componentName, trigger);
+        return [
+          componentName,
+          Object.fromEntries(
+            await Promise.all(
+              envs.map(async (env) => [
+                env,
+                await createJobsForComponent(
+                  config,
+                  componentName,
+                  env,
+                  trigger
+                ),
+              ])
+            )
+          ),
+        ];
+      })
+    )
+  );
+};

package/src/pipeline/createChildPipeline.ts

@@ -1,36 +1,25 @@
-import { getAllEnvsByTrigger, getAllEnvsInAllComponents } from "../config";
+import { getAllEnvsInAllComponents } from "../config";
 import { RULES_ALWAYS } from "../rules";
 import { getRunnerImage } from "../runner";
-import { GitlabPipeline, Pipeline, PipelineJob, PipelineType } from "../types";
-import { Config, PipelineTrigger } from "../types/config";
-import { createJobs } from "./createJobs";
+import type { GitlabPipeline, Pipeline, PipelineType } from "../types";
+import type { Config, PipelineTrigger } from "../types/config";
+import { BASE_STAGES } from "../types/jobs";
+import { createAllJobs } from "./createAllJobs";
+import { createGitlabJobs } from "./gitlab/createGitlabJobs";
 
-const baseStages = ["setup", "test", "build", "deploy", "verify", "stop"];
 export const createChildPipeline = async <T extends PipelineType>(
   type: T,
   trigger: PipelineTrigger,
   config: Config
 ): Promise<Pipeline<T>> => {
-  const components = Object.keys(config.components);
-
-  // 2. write the triggering pipeline
-  const jobs = await components.reduce<Promise<Record<string, PipelineJob<T>>>>(
-    async (acc, componentName) => {
-      const envs = getAllEnvsByTrigger(config, componentName, trigger);
-      return {
-        ...(await acc),
-        ...(await createJobs(type, envs, config, componentName, trigger)),
-      };
-    },
-    Promise.resolve({})
-  );
+  const jobs = await createAllJobs(config, trigger);
 
   // while technically not required, we group different envs in its own stage
   // each job from `createJobs` that is defined as `envMode: "stagePerEnv"` will have `deploy dev`, etc. instead of just `deploy`
   // this is just so that it looks nicer in gitlab and makes running mutliple manual tasks more easy to use
 
   const allEnvs = getAllEnvsInAllComponents(config);
-  const stages = baseStages.reduce<string[]>(
+  const stages = BASE_STAGES.reduce<string[]>(
     (acc, baseStage) => [
       ...acc,
       baseStage,
@@ -48,7 +37,7 @@ export const createChildPipeline = async <T extends PipelineType>(
         rules: RULES_ALWAYS,
       },
       stages,
-      jobs,
+      jobs: await createGitlabJobs(jobs),
     };
     return pipeline as Pipeline<T>;
   }

package/src/pipeline/createJobsForComponent.ts

@@ -0,0 +1,64 @@
+import { isFunction } from "lodash";
+import { BUILD_TYPES } from "../build";
+import { createContext } from "../context";
+import { DEPLOY_TYPES } from "../deploy";
+import type { Config, PipelineTrigger } from "../types/config";
+import type { CommitInfo, Context } from "../types/context";
+import type { CatladderJob } from "../types/jobs";
+import { getBaseCommitInfo } from "./commitInfo/getCommitInfo";
+import { getPackageManagerInfo } from "./packageManager";
+
+const injectDefaultVarsInCustomJobs = (
+  context: Context,
+  jobs: CatladderJob[]
+) =>
+  jobs.map(({ variables, ...job }) => ({
+    variables: {
+      ...(context.environment.envVars ?? {}),
+      ...(variables ?? {}),
+    },
+    ...job,
+  }));
+const getCustomJobs = (context: Context) => {
+  if (!context.componentConfig.customJobs) {
+    return [];
+  }
+  const rawJobs = isFunction(context.componentConfig.customJobs)
+    ? context.componentConfig.customJobs(context)
+    : context.componentConfig.customJobs;
+  return injectDefaultVarsInCustomJobs(context, rawJobs);
+};
+const createRawJobs = (context: Context): CatladderJob[] => {
+  if (context.componentConfig.deploy === false) {
+    return [];
+  }
+  const buildJobs =
+    BUILD_TYPES[context.componentConfig.build.type].jobs(context);
+  const deployJobs =
+    DEPLOY_TYPES[context.componentConfig.deploy.type].jobs(context);
+
+  const customJobs = getCustomJobs(context);
+  return [...buildJobs, ...deployJobs, ...customJobs];
+};
+export const createJobsForComponent = async (
+  config: Config,
+  componentName: string,
+  env: string,
+  trigger: PipelineTrigger
+): Promise<Array<CatladderJob>> => {
+  const commitInfo: CommitInfo = {
+    ...(await getBaseCommitInfo()),
+    trigger,
+  };
+
+  const packageManagerInfo = await getPackageManagerInfo(config, componentName);
+
+  const context = await createContext(
+    config,
+    componentName,
+    env,
+    commitInfo,
+    packageManagerInfo
+  );
+  return createRawJobs(context);
+};