@catladder/pipeline 0.0.0-more-collapse-deeb54d7 → 0.0.0-multiple-gitlab-includes-3df5566d

package/src/deploy/types/googleCloudRun.ts

@@ -62,6 +62,21 @@ export type DeployConfigCloudRunCloudSql = {
    */
   deleteDatabaseOnStop?: boolean;
 
+  /**
+   * format of the `DATABASE_URL` environment variable
+   *
+   * the options are:
+   * - prisma: adds [required but ignored hostname](https://www.prisma.io/docs/orm/overview/databases/postgresql#connecting-via-sockets):
+   *   `postgresql://$DB_USER:$DB_PASSWORD@localhost/$DB_NAME?host=/cloudsql/$CLOUD_SQL_INSTANCE_CONNECTION_NAME`
+   * - rails: percent-encodes the socket in the host part
+   *   `postgresql://$DB_USER:$DB_PASSWORD@%2Fcloudsql%2FprojectId%3Aregion%3Ainstancename/$DB_NAME?`
+   * - jdbc: for use with Google Cloud SQL Connector for Java
+   *   `jdbc:postgresql:///$DB_NAME?cloudSqlInstance=$CLOUD_SQL_INSTANCE_CONNECTION_NAME&socketFactory=com.google.cloud.sql.postgres.SocketFactory&user=$DB_USER&password=$DB_PASSWORD`
+   *
+   * defaults to prisma
+   */
+  dbConnectionStringFormat?: "prisma" | "rails" | "jdbc";
+
   /**
    * add additional query params to the database connection string
    */
@@ -74,11 +89,12 @@ export type DeployConfigCloudRunCloudSql = {
 };
 
 type Memory = `${number}${"M" | "G" | "Mi" | "Gi"}`;
+type Cpu = 1 | 2 | 4 | 6 | 8;
 export type DeployConfigCloudRunService = {
   /**
    * command / entrypoint, fallsback to buildConfig.startcommand
    */
-  command?: string;
+  command?: string | string[];
 
   /**
    * how many instances to keep around when there are no requests. defaults to 0.
@@ -97,24 +113,98 @@ export type DeployConfigCloudRunService = {
   noCpuThrottling?: true;
 
   /**
-   * whether to allow public access (defaults to true)
+   * whether to allow public access without authentification (defaults to true)
    */
   allowUnauthenticated?: boolean;
 
+  /**
+   * whether to expose the service to the internet.
+   *
+   * The options are:
+   * - all: expose to the internet and internal traffic
+   * - internal: only expose to internal traffic
+   * - internal-and-cloud-load-balancing: expose to internal traffic and cloud load balancing
+   *
+   * Defaults to "all"
+   */
+  ingress?: "all" | "internal" | "internal-and-cloud-load-balancing";
+
+  /**
+   * CPU limit.
+   */
+  cpu?: Cpu;
+
   /**
    * memory limit. Defaults to 512MB
    */
   memory?: Memory;
-};
+
+  /**
+   * timeout of the service, defaults to 5 minutes.
+   *
+   * you can specify something like 10m30s. if its just a number, it will be interpreted as seconds
+   *
+   * max supported is 60min
+   */
+  timeout?: string;
+
+  /**
+   *
+   * set the execution environment.
+   *
+   * see https://cloud.google.com/run/docs/about-execution-environments
+   *
+   * defaults to gen1 according to gcloud
+   */
+  executionEnvironment?: "gen2" | "gen1";
+
+  /* the vpc network, see https://cloud.google.com/sdk/gcloud/reference/run/deploy#--network  */
+  network?: string;
+
+  /**
+   * vpc subnet to use, see https://cloud.google.com/sdk/gcloud/reference/run/deploy#--subnet
+   */
+  subnet?: string;
+  /**
+   * vpc egress, see https://cloud.google.com/sdk/gcloud/reference/run/deploy#--vpc-egress
+   */
+  vpcEgress?: "all-traffic" | "private-ranges-only";
+
+  /**
+   * vpc connector
+   */
+  vpcConnector?: string;
+
+  /**
+   * Use http2 end-to-end. See https://cloud.google.com/run/docs/configuring/http2
+   *
+   * Defaults to false.
+   *
+   * Your service needs to be able to handle http2 requests.
+   * Its recommended to use http2 without tls, since cloud run handles the encryption for you. (so called "h2c" (http2 cleartext))
+   *
+   */
+  http2?: boolean;
+} & DeployConfigCloudRunWithVolumes;
 
 export type DeployConfigCloudRunJobBase = {
   /**
    * command
    */
-  command: string;
+  command: string | string[];
+
+  /**
+   * custom image to use. Defaults to the image from the build
+   */
+  image?: string;
+
+  /**
+   * CPU limit of the job.
+   */
+  cpu?: Cpu;
 
   /**
-   * memory limit of the job, defaults to 51Mi
+   * memory limit of the job, defaults to 512Mi
    */
   memory?: `${number}${"M" | "G" | "Mi" | "Gi"}`;
 
@@ -124,7 +214,12 @@ export type DeployConfigCloudRunJobBase = {
    * you can specify something like 10m30s
    */
   timeout?: string;
-};
+
+  /**
+   * number of tasks that may run concurrently, defaults to 1
+   */
+  parallelism?: number;
+} & DeployConfigCloudRunWithVolumes;
 
 type Minute = string;
 type Hour = string;
@@ -135,11 +230,25 @@ export type DeployConfigCloudRunJobWithSchedule =
   DeployConfigCloudRunJobBase & {
     when: "schedule";
     schedule: `${Minute} ${Hour} ${DayOfMonth} ${Month} ${DayOfWeek}`;
+    /**
+     * Max number of retries of the cloud scheduler.
+     * Note: the task itself is never retried.
+     *
+     * defaults to 0 (no retries)
+     */
     maxRetryAttempts?: 0 | 1 | 2 | 3 | 4 | 5;
   };
 
 export type DeployConfigCloudRunJobNormal = DeployConfigCloudRunJobBase & {
-  when: "manual" | "preDeploy" | "postDeploy";
+  when: "manual" | "preDeploy" | "postDeploy" | "preStop" | "postStop";
+};
+
+export type DeployConfigCloudRunWithVolumes = {
+  /**
+   * adds volumes to the service. This is an experimental feature and requires beta gcloud.
+   * sets executionEnvironment to gen2 automatically, since that is required
+   */
+  volumes?: DeployConfigCloudRunVolumes;
 };
 
 export type DeployConfigCloudRunJob =
@@ -148,9 +257,9 @@ export type DeployConfigCloudRunJob =
 
 export type DeployConfigCloudRun = {
   /**
-   * EXPERIMENTAL cloud run deployment.
+   * cloud run deployment.
    *
-   * This will deploy a cloud run service.
+   * This will deploy a cloud run service. You can optionally define one-time and scheduled jobs running on cloud run
    *
    * Requires that cloud run is enabled on the project, as well as cloud run api and artifacts registry.
    */
@@ -170,7 +279,7 @@ export type DeployConfigCloudRun = {
   service?: boolean | DeployConfigCloudRunService;
 
   /**
-   * deploy additional services with different entry pointss
+   * deploy additional services with different entry points.
    */
   additionalServices?: {
     [name: string]: DeployConfigCloudRunService;
@@ -184,4 +293,18 @@ export type DeployConfigCloudRun = {
    * add cloudSql
    */
   cloudSql?: DeployConfigCloudRunCloudSql | false;
+
+  debug?: boolean;
 } & DeployConfigBase;
+
+export type DeployConfigCloudRunVolumes = Record<
+  string,
+  DeployConfigCloudRunVolume
+>;
+
+export type DeployConfigCloudRunVolume = {
+  type: "cloud-storage";
+  bucket: string;
+  mountPath: string;
+  readonly?: boolean;
+};

package/src/deploy/types/index.ts

@@ -21,8 +21,8 @@ export type DeployConfigGeneric<T extends DeployConfigType> = Extract<
 >;
 
 export const isOfDeployType = <T extends Array<DeployConfigType>>(
-  t: DeployConfig | false,
+  t: DeployConfig | false | null | undefined,
   ...types: T
 ): t is Extract<DeployConfig, { type: T[number] }> => {
-  return t && types.includes(t.type);
+  return t && types.includes(t.type) ? true : false;
 };

package/src/deploy/types/kubernetes.ts

@@ -96,6 +96,10 @@ export type KubernetesWorkerDef = {
   service?: {
     enabled?: boolean;
   };
+  /**
+   * if the same livenessProbe as the application should be used, defaults to `true`
+   */
+  livenessProbe?: boolean;
 };
 
 type KubernetesTolerationExists = {
@@ -160,33 +164,6 @@ export type DeployConfigMongodbReplicaset = {
 export type DeployConfigMongodb = DeployConfigMongodbBase &
   (DeployConfigMongodbStandalone | DeployConfigMongodbReplicaset);
 
-/**
- * @deprecated use config with instanceConnectionName
- */
-type DeployConfigKubernetesValuesCloudSQLLegacy = {
-  enabled: boolean;
-  /**
-   * @deprecated use config with instanceConnectionName
-   */
-  instanceId?: string;
-  /**
-   * @deprecated use config with instanceConnectionName
-   */
-  projectId?: string;
-  /**
-   * @deprecated use config with instanceConnectionName
-   */
-  region?: string;
-  /**
-   * @deprecated use type "unmanaged".
-   *
-   * Be aware that "unmanaged" is not fully backward compatible with "legacy".
-   *
-   *
-   */
-  type?: "legacy";
-};
-
 export type DeployConfigKubernetesValuesCloudSQLUnmanaged = {
   enabled: boolean;
   instanceConnectionName: string;
@@ -216,8 +193,7 @@ export type DeployConfigKubernetesValuesCloudSQLUnmanaged = {
   dbBaseName?: string;
 };
 export type DeployConfigKubernetesValuesCloudSQL =
-  | DeployConfigKubernetesValuesCloudSQLLegacy
-  | DeployConfigKubernetesValuesCloudSQLUnmanaged;
+  DeployConfigKubernetesValuesCloudSQLUnmanaged;
 export type DeployConfigKubernetesValues = AllowUnknownProps<{
   /**
    * enable cloudsql db. Currently you have to manually set it up
@@ -354,13 +330,6 @@ export type DeployConfigKubernetesValues = AllowUnknownProps<{
   mapServiceUrlToEnv?: {
     [envVar: string]: string;
   };
-  /**
-   * Mount secrets as files in the filesystem.
-   * These secrets are still available as env vars, but will contain the path of the file instead
-   *
-   * @deprecated this will be removed in the future, because its very niche and not used
-   */
-  secretsAsFile?: string[];
 }>;
 
 export type DeployConfigKubernetes = {

package/src/deploy/utils.ts

@@ -1,9 +1,9 @@
-import type { Config, Context } from "../types";
+import type { Config, ComponentContext } from "../types";
 import type { DeployConfigKubernetesCluster } from "./types";
 import { isOfDeployType } from "./types";
 
 export const getFullKubernetesClusterName = (
-  cluster: DeployConfigKubernetesCluster
+  cluster: DeployConfigKubernetesCluster,
 ) => {
   if (cluster.type === "gcloud") {
     return `gke_${cluster.projectId}_${cluster.region}_${cluster.name}`;
@@ -12,13 +12,13 @@ export const getFullKubernetesClusterName = (
 
 export const getKubernetesNamespace = (
   config: Pick<Config, "customerName" | "appName">,
-  env: string
+  env: string,
 ) => {
   return `${config.customerName}-${config.appName}-${env}`;
 };
 
-export const contextIsStoppable = (context: Context) => {
-  const deployConfig = context.componentConfig.deploy;
+export const contextIsStoppable = (context: ComponentContext) => {
+  const deployConfig = context.deploy?.config;
   if (isOfDeployType(deployConfig, "kubernetes")) {
     return true;
   }

package/src/globalScriptFunctions/index.ts

@@ -0,0 +1,30 @@
+class GlobalScriptFunction {
+  constructor(
+    public name: string,
+    public script: string,
+  ) {}
+
+  toBashFunction() {
+    return `function ${this.name} () {
+${this.script.trim()}
+}`.trim();
+  }
+  invoke(...args: string[]) {
+    return `${this.name} ${args.join(" ")}`;
+  }
+}
+
+export const globalScriptFunctions = new Map<string, GlobalScriptFunction>();
+
+/**
+ * registers a global script function
+ * only use that in top-level scopes as it will attach the function to the global scope
+ */
+export const registerGlobalScriptFunction = (name: string, script: string) => {
+  if (globalScriptFunctions.has(name)) {
+    throw new Error(`Global script function ${name} already exists`);
+  }
+  const scriptFunction = new GlobalScriptFunction(name, script);
+  globalScriptFunctions.set(name, scriptFunction);
+  return scriptFunction;
+};

package/src/index.ts

@@ -1,4 +1,4 @@
-export * from "./pipeline/";
+export * from "./pipeline";
 export * from "./types";
 export * from "./rules";
 export * from "./runner";
@@ -6,3 +6,6 @@ export * from "./config";
 export * from "./context";
 export * from "./build";
 export * from "./deploy";
+export * from "./utils/writeFiles";
+export * from "./variables/VariableValue";
+export * from "./bash";

package/src/packageInfos.ts

@@ -0,0 +1,14 @@
+// we can't import the package.json directly, because the bundling process will inline it
+// but the version is only set during deployement, when its already bundled
+// instead we need to use require.main.path to get the path of the main module, which is actually the file in the bin
+// this will point to the root package.json
+let ownPkg = { version: "0.0.0" };
+try {
+  const path = require.main?.path + "/../package.json";
+
+  ownPkg = require(path);
+} catch (e) {
+  // ignore, currently in the gitlab-includes, this doesn't work
+}
+
+export default ownPkg;

package/src/pipeline/__tests__/__snapshots__/getPipelineStages.test.ts.snap

@@ -0,0 +1,69 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`getPipelineStages() should return all envs for SIMPLE_CONFIG1 1`] = `
+[
+  "setup",
+  "setup dev",
+  "setup review",
+  "setup stage",
+  "setup prod",
+  "setup dev2",
+  "setup review2",
+  "setup stage2",
+  "setup prod2",
+  "test",
+  "test dev",
+  "test review",
+  "test stage",
+  "test prod",
+  "test dev2",
+  "test review2",
+  "test stage2",
+  "test prod2",
+  "build",
+  "build dev",
+  "build review",
+  "build stage",
+  "build prod",
+  "build dev2",
+  "build review2",
+  "build stage2",
+  "build prod2",
+  "deploy",
+  "deploy dev",
+  "deploy review",
+  "deploy stage",
+  "deploy prod",
+  "deploy dev2",
+  "deploy review2",
+  "deploy stage2",
+  "deploy prod2",
+  "verify",
+  "verify dev",
+  "verify review",
+  "verify stage",
+  "verify prod",
+  "verify dev2",
+  "verify review2",
+  "verify stage2",
+  "verify prod2",
+  "rollback",
+  "rollback dev",
+  "rollback review",
+  "rollback stage",
+  "rollback prod",
+  "rollback dev2",
+  "rollback review2",
+  "rollback stage2",
+  "rollback prod2",
+  "stop",
+  "stop dev",
+  "stop review",
+  "stop stage",
+  "stop prod",
+  "stop dev2",
+  "stop review2",
+  "stop stage2",
+  "stop prod2",
+]
+`;

package/src/pipeline/__tests__/getPipelineStages.test.ts

@@ -0,0 +1,63 @@
+import { type DeployConfigKubernetesCluster } from "../..";
+import type { Config } from "../../types";
+import { getPipelineStages } from "../getPipelineStages";
+
+describe("getPipelineStages()", () => {
+  const cluster: DeployConfigKubernetesCluster = {
+    type: "gcloud",
+    name: "mega-cluster",
+    projectId: "super-google-project",
+    region: "ch-blabla",
+  };
+  const SIMPLE_CONFIG1: Config = {
+    appName: "my-app",
+    customerName: "pan",
+    components: {
+      app1: {
+        dir: "dir1",
+        build: {
+          type: "node",
+        },
+        deploy: { type: "kubernetes", cluster },
+      },
+      app2: {
+        dir: "dir2",
+        build: {
+          type: "node",
+        },
+        deploy: { type: "kubernetes", cluster },
+        env: {
+          dev2: {
+            type: "dev",
+          },
+          review2: {
+            type: "review",
+          },
+          stage2: {
+            type: "stage",
+          },
+          prod2: {
+            type: "prod",
+          },
+        },
+      },
+      app3: {
+        dir: "dir2",
+        build: {
+          type: "node",
+        },
+        deploy: { type: "kubernetes", cluster },
+        env: {
+          dev: false,
+          review: false,
+          stage: false,
+          prod: false,
+        },
+      },
+    },
+  };
+
+  it("should return all envs for SIMPLE_CONFIG1", () => {
+    expect(getPipelineStages(SIMPLE_CONFIG1)).toMatchSnapshot();
+  });
+});

package/src/pipeline/createAllJobs.ts

@@ -1,38 +1,72 @@
-import { getAllEnvsByTrigger } from "../config";
-import type { Config, PipelineTrigger } from "../types";
+import type { CreateAllComponentsContextProps } from "../context/createAllComponentsContext";
+import { createAllComponentsContext } from "../context/createAllComponentsContext";
+import { createWorkspaceContext } from "../context/createWorkspaceContext";
+import type { ComponentContext, WorkspaceContext } from "../types";
+import { componentContextHasWorkspaceBuild } from "../types";
 import type { CatladderJob } from "../types/jobs";
-import { createJobsForComponent } from "./createJobsForComponent";
+import { createJobsForComponentContext } from "./createJobsForComponent";
+import { createJobsForWorkspace } from "./createJobsForWorkspace";
 
 export type AllCatladderJobs = {
-  [componentName: string]: {
-    [env: string]: Array<CatladderJob>;
-  };
+  workspaces: Array<{
+    context: WorkspaceContext;
+    jobs: Array<CatladderJob>;
+  }>;
+  components: Array<{
+    context: ComponentContext;
+    jobs: Array<CatladderJob>;
+  }>;
 };
-export const createAllJobs = async (
-  config: Config,
-  trigger: PipelineTrigger
-): Promise<AllCatladderJobs> => {
-  return Object.fromEntries(
-    await Promise.all(
-      Object.keys(config.components).map(async (componentName) => {
-        const envs = getAllEnvsByTrigger(config, componentName, trigger);
-        return [
-          componentName,
-          Object.fromEntries(
-            await Promise.all(
-              envs.map(async (env) => [
-                env,
-                await createJobsForComponent(
-                  config,
-                  componentName,
-                  env,
-                  trigger
-                ),
-              ])
-            )
-          ),
-        ];
-      })
-    )
-  );
+
+export const createAllJobs = async ({
+  config,
+  trigger,
+  pipelineType,
+}: CreateAllComponentsContextProps): Promise<AllCatladderJobs> => {
+  const allComponentsContext = await createAllComponentsContext({
+    config,
+    trigger,
+    pipelineType,
+  });
+
+  return {
+    workspaces: await Promise.all(
+      Object.keys(config.builds ?? {}).map(async (workspaceName) => {
+        const componentsInAllEnvs = allComponentsContext.filter(
+          (context) =>
+            componentContextHasWorkspaceBuild(context) &&
+            context.build.workspaceName === workspaceName,
+        );
+
+        const allEnvs = [...new Set(componentsInAllEnvs.map(({ env }) => env))];
+
+        return await Promise.all(
+          allEnvs.map(async (env) => {
+            const workspaceContext = await createWorkspaceContext({
+              components: componentsInAllEnvs.filter(
+                ({ env: componentEnv }) => componentEnv === env,
+              ),
+              workspaceName,
+              config,
+              trigger,
+              pipelineType,
+              env,
+            });
+
+            return {
+              context: workspaceContext,
+              jobs: createJobsForWorkspace(workspaceContext),
+            };
+          }),
+        );
+      }),
+    ).then((f) => f.flat()),
+
+    components: allComponentsContext.map((context) => {
+      return {
+        context,
+        jobs: createJobsForComponentContext(context),
+      };
+    }),
+  };
 };