@catladder/pipeline 1.75.4 → 1.76.0

package/examples/__utils__/helpers.ts

@@ -0,0 +1,14 @@
+import type { Config } from "../../src";
+import { createChildPipeline } from "../../src";
+
+const ALL_TRIGGERS = ["mainBranch", "taggedRelease", "mr"] as const;
+
+export const createAllPipelines = async (config: Config) =>
+  Object.fromEntries(
+    await Promise.all(
+      ALL_TRIGGERS.map(async (trigger) => [
+        trigger,
+        await createChildPipeline("gitlab", trigger, config),
+      ])
+    )
+  );

package/examples/{cloud-sql.ts → cloud-run-with-sql.ts}

@@ -1,6 +1,6 @@
 import type { Config } from "../src";
-
-const config: Config<{ CustomEnvs: "asdf" | "bla" }> = {
+import { createAllPipelines } from "./__utils__/helpers";
+const config: Config = {
   appName: "test-app",
   customerName: "pan",
   components: {
@@ -11,32 +11,29 @@ const config: Config<{ CustomEnvs: "asdf" | "bla" }> = {
       },
       deploy: {
         type: "google-cloudrun",
-        projectId: "asdf",
+        projectId: "google-project-id",
         region: "europe-west6",
         cloudSql: {
           type: "unmanaged",
           instanceConnectionName: "projectId:region:instancename",
+          dbUser: "my-user",
         },
         jobs: {
           migration: {
             when: "postDeploy",
             command: "yarn migrate",
           },
-        },
-      },
-      env: {
-        local: {
-          port: 4000,
-        },
-        asdf: {
-          type: "dev",
-        },
-        bla: {
-          type: "dev",
+          ["send-reminders"]: {
+            when: "schedule",
+            command: "yarn job:send-reminders",
+            schedule: "0 * * * *",
+          },
         },
       },
     },
   },
 };
 
-export default config;
+it("matches snapshot", async () => {
+  expect(await createAllPipelines(config)).toMatchSnapshot();
+});

package/examples/custom-build-job.ts

@@ -1,4 +1,5 @@
 import type { Config } from "../src";
+import { createAllPipelines } from "./__utils__/helpers";
 
 const config: Config = {
   appName: "test-app",
@@ -22,4 +23,6 @@ const config: Config = {
   },
 };
 
-export default config;
+it("matches snapshot", async () => {
+  expect(await createAllPipelines(config)).toMatchSnapshot();
+});

package/examples/custom-envs.ts

@@ -0,0 +1,31 @@
+import type { Config } from "../src";
+import { createAllPipelines } from "./__utils__/helpers";
+
+const config: Config<{ CustomEnvs: "asdf" | "bla" }> = {
+  appName: "test-app",
+  customerName: "pan",
+  components: {
+    api: {
+      dir: "api",
+      build: {
+        type: "node",
+      },
+      deploy: false,
+      env: {
+        local: {
+          port: 4000,
+        },
+        asdf: {
+          type: "dev",
+        },
+        bla: {
+          type: "dev",
+        },
+      },
+    },
+  },
+};
+
+it("matches snapshot", async () => {
+  expect(await createAllPipelines(config)).toMatchSnapshot();
+});

package/examples/wait-for-other-deploy.ts

@@ -0,0 +1,36 @@
+import type { Config } from "../src";
+import { createAllPipelines } from "./__utils__/helpers";
+
+const config: Config<{ CustomEnvs: "asdf" | "bla" }> = {
+  appName: "test-app",
+  customerName: "pan",
+  components: {
+    api: {
+      dir: "api",
+      build: {
+        type: "node",
+      },
+      deploy: {
+        type: "custom",
+        requiresDocker: false,
+        script: ["yarn deploy"],
+      },
+    },
+    www: {
+      dir: "www",
+      build: {
+        type: "node",
+      },
+      deploy: {
+        waitFor: ["api"],
+        type: "custom",
+        requiresDocker: false,
+        script: ["yarn deploy"],
+      },
+    },
+  },
+};
+
+it("matches snapshot", async () => {
+  expect(await createAllPipelines(config)).toMatchSnapshot();
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@catladder/pipeline",
-  "version": "1.75.4",
+  "version": "1.76.0",
   "scripts": {
     "build:tsc": "yarn tsc",
     "build": "yarn build:compile && yarn build:inline-variables && yarn build:bundle",
@@ -8,7 +8,6 @@
     "build:compile": "yarn tsc",
     "build:inline-variables": "yarn babel --keep-file-extension --extensions \".ts,.js\" --config-file ./babel-inline.json dist --out-dir dist",
     "build:watch": "yarn tsc -w",
-    "test": "jest",
     "lint": "eslint \"src/**/*.ts\"",
     "lint:fix": "eslint \"src/**/*.ts\" --fix"
   },
@@ -23,6 +22,7 @@
     "@babel/core": "^7.0.0-0",
     "@babel/plugin-syntax-typescript": "^7.16.7",
     "@babel/preset-typescript": "^7.16.5",
+    "@jest/globals": "^29.3.1",
     "@types/child-process-promise": "^2.2.2",
     "@types/jest": "^27.4.0",
     "@types/lodash": "^4.14.185",

package/src/config/{configruedEnvs.test.ts → __tests__/configruedEnvs.test.ts}

@@ -1,6 +1,6 @@
-import { getAllEnvsByTrigger } from ".";
-import type { DeployConfigKubernetesCluster } from "..";
-import type { Config } from "../types";
+import { getAllEnvsByTrigger } from "..";
+import type { DeployConfigKubernetesCluster } from "../..";
+import type { Config } from "../../types";
 
 describe("getAllEnvsByTrigger", () => {
   const cluster: DeployConfigKubernetesCluster = {

package/src/context/{resolveReferences.test.ts → __tests__/resolveReferences.test.ts}

@@ -1,7 +1,7 @@
 import {
   resolveReferences,
   translateLegacyFromComponents,
-} from "./resolveReferences";
+} from "../resolveReferences";
 
 describe("resolveReferences", () => {
   it("replaces occurences of ${componentName:VARIABLE_NAME}", async () => {
@@ -30,12 +30,12 @@ describe("resolveReferences", () => {
     });
   });
 
-  it("replaces self references with structure ${VARIABLE_NAME}", () => {
+  it("replaces self references with structure ${VARIABLE_NAME}", async () => {
     const variables = {
       FOO: "hello world",
       BAR: "this: ${FOO}!",
     };
-    const result = resolveReferences(variables);
+    const result = await resolveReferences(variables);
 
     expect(result).toEqual({
       FOO: "hello world",

package/src/deploy/cloudRun/index.ts

@@ -31,11 +31,12 @@ const getCloudSqlVariables = ({
     )
       .map(([key, value]) => `&${key}=${value}`)
       .join("");
+
     return {
       CLOUD_SQL_INSTANCE_CONNECTION_NAME:
         deployConfigRaw.cloudSql.instanceConnectionName,
       DB_NAME: DB_NAME,
-      DB_USER: "postgres",
+      DB_USER: deployConfigRaw.cloudSql.dbUser ?? "postgres",
       DB_PASSWORD: "$" + getSecretVarName(env, componentName, "DB_PASSWORD"),
       DATABASE_URL: `postgresql://$DB_USER:$DB_PASSWORD@localhost/$DB_NAME?host=/cloudsql/$CLOUD_SQL_INSTANCE_CONNECTION_NAME${additionalQueryParamsString}`,
       DATABASE_JDBC_URL: `jdbc:postgresql:///$DB_NAME?cloudSqlInstance=$CLOUD_SQL_INSTANCE_CONNECTION_NAME&socketFactory=com.google.cloud.sql.postgres.SocketFactory&user=$DB_USER&password=$DB_PASSWORD${additionalQueryParamsString}`,

package/src/deploy/types/googleCloudRun.ts

@@ -40,6 +40,11 @@ export type Gcloudregion =
 export type DeployConfigCloudRunCloudSql = {
   type: "unmanaged";
   instanceConnectionName: string;
+  /**
+   * the database username, defaults to "postgres"
+   */
+  dbUser?: string;
+
   /**
    * the prefix of the database, the full db name is this plus the environment slug
    *

package/src/pipeline/commitInfo/getBuildId.ts

@@ -0,0 +1,6 @@
+import { exec } from "child-process-promise";
+
+export const getBuildId = async () =>
+  await exec("git describe --tags || git rev-parse HEAD").then((s) =>
+    s.stdout.trim()
+  );

package/src/pipeline/commitInfo/getCommitInfo.ts

@@ -0,0 +1,12 @@
+import type { CommitInfo } from "../../types";
+import { getBuildId } from "./getBuildId";
+export const getBaseCommitInfo = async (): Promise<
+  Omit<CommitInfo, "trigger">
+> => ({
+  refName: process.env.CI_COMMIT_REF_NAME ?? "unknown",
+  reviewSlug: process.env.CI_MERGE_REQUEST_IID
+    ? `mr${process.env.CI_MERGE_REQUEST_IID}`
+    : "unknown",
+  buildTime: new Date().toISOString(),
+  buildId: await getBuildId(),
+});

package/src/pipeline/createChildPipeline.ts

@@ -1,7 +1,12 @@
 import { getAllEnvsByTrigger, getAllEnvsInAllComponents } from "../config";
 import { RULES_ALWAYS } from "../rules";
 import { getRunnerImage } from "../runner";
-import type { GitlabPipeline, Pipeline, PipelineJob, PipelineType } from "../types";
+import type {
+  GitlabPipeline,
+  Pipeline,
+  PipelineJob,
+  PipelineType,
+} from "../types";
 import type { Config, PipelineTrigger } from "../types/config";
 import { BASE_STAGES } from "../types/jobs";
 import { createJobs } from "./createJobs";

package/src/pipeline/createJobs.ts

@@ -1,4 +1,3 @@
-import { exec } from "child-process-promise";
 import { isFunction, isObject } from "lodash";
 import { BUILD_TYPES } from "../build";
 import { createContext } from "../context";
@@ -6,8 +5,9 @@ import { DEPLOY_TYPES } from "../deploy";
 import type { PipelineJob, PipelineType } from "../types";
 import type { Config, PipelineTrigger } from "../types/config";
 import type { CommitInfo, Context } from "../types/context";
-import type { CatladderJob } from "../types/jobs";
+import type { CatladderJob, CatladderJobNeed } from "../types/jobs";
 import { notNil } from "../utils";
+import { getBaseCommitInfo } from "./commitInfo/getCommitInfo";
 import { makeGitlabJob } from "./gitlab/makeGitlabJob";
 import { getPackageManagerInfo } from "./packageManager";
 
@@ -153,14 +153,7 @@ export const createJobs = async <T extends PipelineType>(
   trigger: PipelineTrigger
 ): Promise<Record<string, PipelineJob<T>>> => {
   const commitInfo: CommitInfo = {
-    refName: process.env.CI_COMMIT_REF_NAME ?? "unknown",
-    reviewSlug: process.env.CI_MERGE_REQUEST_IID
-      ? `mr${process.env.CI_MERGE_REQUEST_IID}`
-      : "unknown",
-    buildTime: new Date().toISOString(),
-    buildId: await exec("git describe --tags || git rev-parse HEAD").then((s) =>
-      s.stdout.trim()
-    ),
+    ...(await getBaseCommitInfo()),
     trigger,
   };
 

package/src/pipeline/gitlab/makeGitlabJob.ts

@@ -1,16 +1,22 @@
+import { isObject } from "lodash";
 import { BASE_RETRY } from "../../defaults";
 import type { GitlabJobDef } from "../../types";
-import type { CatladderJob } from "../../types/jobs";
+import type { CatladderJob, CatladderJobNeed } from "../../types/jobs";
+const getJobName = (need: CatladderJobNeed) =>
+  isObject(need) ? need.job : need;
 
 export const makeGitlabJob = ({
   envMode,
   needsStages,
   needsOtherComponent,
   name,
+  needs,
   ...rest
 }: CatladderJob<string>): GitlabJobDef => {
   return {
     ...rest,
+    // sort in a predictable manner for snapshot tests
+    needs: needs?.sort((a, b) => getJobName(a).localeCompare(getJobName(b))),
     retry: BASE_RETRY,
     interruptible: true,
   };

package/src/pipeline/packageManager.ts

@@ -1,43 +1,9 @@
-import { exec } from "child-process-promise";
-import type { Config, PackageManagerInfo } from "../types";
-import { pathEqual } from "path-equal";
-import memoizee from "memoizee";
-import { join } from "path";
 import { existsSync } from "fs";
+import { join } from "path";
+import { pathEqual } from "path-equal";
+import type { Config, PackageManagerInfo } from "../types";
+import { getWorkspaces, getYarnVersion } from "./yarn/yarnUtils";
 
-const execOrFail = async (cmd: string, onFail: string): Promise<string> => {
-  try {
-    return await exec(cmd).then((r) => r.stdout);
-  } catch (e) {
-    return onFail ?? null;
-  }
-};
-
-const getYarnVersion = memoizee(
-  async () => {
-    return await execOrFail("yarn --version", "");
-  },
-  { promise: true }
-);
-
-const getWorkspaces = memoizee(
-  async (isClassic: boolean): Promise<PackageManagerInfo["workspaces"]> => {
-    return isClassic
-      ? Object.values(
-          JSON.parse(
-            JSON.parse(await execOrFail("yarn workspaces --json info", "{}"))
-              ?.data ?? "{}"
-          )
-        )
-      : JSON.parse(
-          `[${(await execOrFail("yarn workspaces list --json --verbose", ""))
-            .trim()
-            .split("\n")
-            .join(",")}]`
-        );
-  },
-  { promise: true }
-);
 export const getPackageManagerInfo = async (
   config: Config,
   componentName: string

package/src/pipeline/yarn/yarnUtils.ts

@@ -0,0 +1,37 @@
+import { exec } from "child-process-promise";
+import memoizee from "memoizee";
+import type { PackageManagerInfo } from "../../types";
+
+const execOrFail = async (cmd: string, onFail: string): Promise<string> => {
+  try {
+    return await exec(cmd).then((r) => r.stdout);
+  } catch (e) {
+    return onFail ?? null;
+  }
+};
+// export for mocking
+export const getYarnVersion = memoizee(
+  async () => {
+    return await execOrFail("yarn --version", "");
+  },
+  { promise: true }
+);
+// export for mocking
+export const getWorkspaces = memoizee(
+  async (isClassic: boolean): Promise<PackageManagerInfo["workspaces"]> => {
+    return isClassic
+      ? Object.values(
+          JSON.parse(
+            JSON.parse(await execOrFail("yarn workspaces --json info", "{}"))
+              ?.data ?? "{}"
+          )
+        )
+      : JSON.parse(
+          `[${(await execOrFail("yarn workspaces list --json --verbose", ""))
+            .trim()
+            .split("\n")
+            .join(",")}]`
+        );
+  },
+  { promise: true }
+);

package/src/types/jobs.ts

@@ -18,6 +18,7 @@ export const BASE_STAGES = [
 ] as const;
 export type BaseStage = typeof BASE_STAGES[number];
 
+export type CatladderJobNeed = string | { job: string; artifacts: boolean };
 export type CatladderJob<S = BaseStage> = {
   /**
    * the name of the job (without any env or app prefix and suffix)
@@ -52,7 +53,7 @@ export type CatladderJob<S = BaseStage> = {
   /**
    * does this require another job (from the same component)?
    */
-  needs?: Array<string | { job: string; artifacts: boolean }>;
+  needs?: Array<CatladderJobNeed>;
 
   needsOtherComponent?: Array<{
     componentName: string;

package/tsconfig.json

@@ -10,5 +10,5 @@
     "esModuleInterop": true,
     "incremental": true
   },
-  "include": ["./src"]
+  "include": ["./src", "tests"]
 }

package/dist/build/tests/storybook.test.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/pipeline/createChildPipeline.test.d.ts

@@ -1 +0,0 @@
-export {};

package/src/build/tests/storybook.test.ts

@@ -1,54 +0,0 @@
-import type { Config } from "../../types";
-import { createChildPipeline } from "../../";
-
-describe("storybook build", () => {
-  describe("creates two build jobs", () => {
-    const config: Config = {
-      appName: "test",
-      customerName: "pan",
-      components: {
-        "my-app": {
-          dir: "myapp",
-          build: {
-            type: "storybook",
-          },
-          deploy: {
-            type: "kubernetes",
-            cluster: {
-              name: "production",
-              projectId: "bla",
-              region: "europe",
-              type: "gcloud",
-            },
-          },
-        },
-      },
-    };
-
-    it("creates a pipeline for storybook that does not contain test stages", async () => {
-      const { jobs } = await createChildPipeline(
-        "gitlab",
-        "mainBranch",
-        config
-      );
-
-      expect(Object.keys(jobs)).toEqual([
-        "dev my-app app-build",
-        "dev my-app docker-build",
-        "dev my-app deploy-to-kubernetes",
-        "dev my-app kubernetes-stop",
-      ]);
-    });
-
-    it("runs build and renames folder", async () => {
-      const { jobs } = await createChildPipeline(
-        "gitlab",
-        "mainBranch",
-        config
-      );
-      const buildJob = jobs["dev my-app app-build"];
-      expect(buildJob.script).toContain("yarn storybook");
-      expect(buildJob.script).toContain("mv ./storybook-out /dist");
-    });
-  });
-});

package/src/pipeline/createChildPipeline.test.ts

@@ -1,49 +0,0 @@
-import type { Config } from "../types";
-import { createChildPipeline } from "./createChildPipeline";
-
-describe("createChildPipeline", () => {
-  describe("node-app to kuberntes", () => {
-    const config: Config = {
-      appName: "test",
-      customerName: "pan",
-      components: {
-        "my-app": {
-          dir: "myapp",
-          build: {
-            type: "node",
-          },
-          deploy: {
-            type: "kubernetes",
-            cluster: {
-              type: "gcloud",
-              name: "mega-cluster",
-              projectId: "super-google-project",
-              region: "ch-blabla",
-            },
-          },
-        },
-      },
-    };
-
-    it("creates a pipeline for a single app on the main branch", async () => {
-      const { image, jobs } = await createChildPipeline(
-        "gitlab",
-        "mainBranch",
-        config
-      );
-      expect(image).toEqual(
-        "git.panter.ch:5001/catladder/catladder/base-pipeline:"
-      );
-
-      expect(Object.keys(jobs)).toEqual([
-        "my-app audit",
-        "my-app lint",
-        "my-app test",
-        "dev my-app app-build",
-        "dev my-app docker-build",
-        "dev my-app deploy-to-kubernetes",
-        "dev my-app kubernetes-stop",
-      ]);
-    });
-  });
-});