@catladder/pipeline 0.0.0-refactor-gcloud-api-5bd4bbed → 0.0.0-refactor-needs-aae6e67d

package/examples/cloud-run-memory-limit.ts

@@ -0,0 +1,28 @@
+import type { Config } from "../src";
+import { createAllPipelines } from "./__utils__/helpers";
+const config: Config = {
+  appName: "test-app",
+  customerName: "pan",
+  components: {
+    api: {
+      dir: "api",
+      build: {
+        type: "node",
+      },
+      deploy: {
+        type: "google-cloudrun",
+        projectId: "google-project-id",
+        region: "europe-west6",
+
+        service: {
+          // defaults to 512Mi
+          memory: "1024Mi",
+        },
+      },
+    },
+  },
+};
+
+it("matches snapshot", async () => {
+  expect(await createAllPipelines(config)).toMatchSnapshot();
+});

package/examples/cloud-run-no-cpu-throttling.ts

@@ -0,0 +1,33 @@
+import type { Config } from "../src";
+import { createAllPipelines } from "./__utils__/helpers";
+const config: Config = {
+  appName: "test-app",
+  customerName: "pan",
+  components: {
+    api: {
+      dir: "api",
+      build: {
+        type: "node",
+      },
+      deploy: {
+        type: "google-cloudrun",
+        projectId: "google-project-id",
+        region: "europe-west6",
+
+        service: {
+          minInstances: 1,
+          // careful, this is expensive!
+          // this will always allocate the cpu
+          // use this if you have background tasks.
+
+          // consider using cloud task instead of this
+          noCpuThrottling: true,
+        },
+      },
+    },
+  },
+};
+
+it("matches snapshot", async () => {
+  expect(await createAllPipelines(config)).toMatchSnapshot();
+});

package/examples/cloud-run-with-sql-reuse-db.ts

@@ -0,0 +1,54 @@
+import type { Config, DeployConfigCloudRunCloudSql } from "../src";
+import { createAllPipelines } from "./__utils__/helpers";
+
+const CLOUD_SQL: DeployConfigCloudRunCloudSql = {
+  type: "unmanaged",
+  instanceConnectionName: "projectId:region:instancename",
+  dbUser: "my-user",
+};
+const config: Config = {
+  appName: "test-app",
+  customerName: "pan",
+  components: {
+    api: {
+      dir: "api",
+      build: {
+        type: "node",
+      },
+      deploy: {
+        type: "google-cloudrun",
+        projectId: "google-project-id",
+        region: "europe-west6",
+        cloudSql: CLOUD_SQL,
+      },
+    },
+    worker: {
+      dir: "api",
+      build: {
+        type: "node",
+        buildCommand: "yarn build:worker",
+      },
+      deploy: {
+        type: "google-cloudrun",
+        projectId: "google-project-id",
+        region: "europe-west6",
+        cloudSql: {
+          ...CLOUD_SQL,
+          // override the db base name to the other component's name
+          // if ommited, this defaults to the component's name
+          dbBaseName: "api",
+        },
+      },
+      vars: {
+        public: {
+          // need to reference the password
+          DB_PASSWORD: "${api:DB_PASSWORD}",
+        },
+      },
+    },
+  },
+};
+
+it("matches snapshot", async () => {
+  expect(await createAllPipelines(config)).toMatchSnapshot();
+});

package/examples/cloud-run-with-sql.ts

@@ -49,7 +49,7 @@ const config: Config = {
       },
       vars: {
         public: {
-          API_URL: "${api:ROOT_URL}",
+          API_URL: "${api:ROOT_URL}/graphql",
         },
       },
     },

package/examples/rails-k8s-with-worker.ts

@@ -0,0 +1,95 @@
+import type { Config } from "../src";
+import { createAllPipelines } from "./__utils__/helpers";
+
+// the image version should match your `.ruby-version`
+const RAILS_TEST_STAGE_IMAGE = "ruby:3.2.1"
+
+const config: Config = {
+  appName: "test-app",
+  customerName: "pan",
+  components: {
+    app: {
+      dir: ".",
+      vars: {
+        public: {
+          RAILS_ENV: "production",
+        },
+        secret: [
+          "SECRET_KEY_BASE",
+        ]
+      },
+      build: {
+        type: "rails",
+        test: {
+          // if operating system dependencies are required you can either
+          // - build a dedicated image
+          // - use `command` to apt-get install them before running the test command
+          jobImage: RAILS_TEST_STAGE_IMAGE,
+        },
+        lint: {
+          jobImage: RAILS_TEST_STAGE_IMAGE,
+        },
+        audit: {
+          jobImage: RAILS_TEST_STAGE_IMAGE,
+        },
+        cnbBuilder: {
+          buildVars: { SECRET_KEY_BASE: "dummy-value" },
+        },
+      },
+      deploy: {
+        type: "kubernetes",
+        cluster: {
+          name: "some-cluster-name",
+          region: "europe-west6",
+          projectId: "some-project-id",
+          type: "gcloud",
+          domainCanonical: "panter.cloud",
+        },
+        values: {
+          cloudsql: { enabled: true, projectId: "some-project-id" },
+          application: {
+            // if deployed to Google Cloud Run use a gem like cloudtasker instead of a permanently running expensive worker
+            worker: {
+              enabled: true,
+              command: "launcher bundle exec rake jobs:work",
+            },
+          },
+          jobs: {
+            "db-migrate": {
+              hook: "post-install,post-upgrade",
+              command:
+                "launcher bundle exec rake db:migrate",
+            },
+          },
+        },
+      },
+      env: {
+        review: {
+          deploy: {
+            values: {
+              jobs: {
+                "db-prepare-seed": {
+                  hook: "post-install",
+                  command:
+                    "launcher bundle exec rake db:prepare db:seed",
+                },
+                "db-migrate": {
+                  hook: "post-upgrade",
+                  command:
+                    "launcher bundle exec rake db:migrate",
+                },
+              },
+            },
+          },
+        },
+        prod: {
+          host: "my-fancy-website.com"
+        }
+      },
+    },
+  },
+};
+
+it("matches snapshot", async () => {
+  expect(await createAllPipelines(config)).toMatchSnapshot();
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@catladder/pipeline",
-  "version": "0.0.0-refactor-gcloud-api-5bd4bbed",
+  "version": "0.0.0-refactor-needs-aae6e67d",
   "scripts": {
     "build:tsc": "yarn tsc",
     "build": "yarn build:compile && yarn build:inline-variables && yarn build:bundle",

package/src/build/custom/buildJob.ts

@@ -29,6 +29,7 @@ export const createCustomBuildJobs = (context: Context): CatladderJob[] => {
           variables: {
             ...RUNNER_BUILD_VARIABLES,
           },
+          cache: buildConfig.jobCache,
           services: buildConfig.jobServices,
           script: [...(ensureArray(buildConfig.buildCommand) ?? [])],
           artifacts: {

package/src/build/index.ts

@@ -51,6 +51,12 @@ export const BUILD_TYPES: BuildTypes = {
   },
   rails: {
     jobs: createRailsJobs,
-    defaults: () => ({}),
+    defaults: () => ({
+      startCommand: "/cnb/process/web",
+      cnbBuilder: {
+        image: "heroku/buildpacks:20",
+        packVersion: "0.28.0",
+      }
+    }),
   },
 };

package/src/build/rails/build.ts

@@ -1,27 +1,34 @@
 import type { Context } from "../..";
+import { isOfBuildType } from "../types";
 import type { CatladderJob } from "../../types/jobs";
 import { createDockerBuildJobBase, gitlabDockerLogin } from "../docker";
 
 export const createRailsBuildJobs = (context: Context): CatladderJob[] => {
+  const buildConfig = context.componentConfig.build
+  if (!isOfBuildType(buildConfig, "rails")) {
+    // should not happen
+    throw new Error("build type is not rails");
+  }
+
+  const cnbConf = buildConfig.cnbBuilder;
+
+  // backwards compatabilty with CNB_ENV_VARS
+  // TODO: remove when all projects are migrated
+  const packEnvArgs = buildConfig.extraVars?.CNB_ENV_VARS?.split(" ").map(v => `--env '${v}'`)
+    ?? Object.entries(cnbConf?.buildVars ?? {}).map(([k, v]) => `--env '${k}${v ? `=${v}` : ""}'`)
+
   return [
     createDockerBuildJobBase(context, {
-      variables: {
-        CNB_BUILDER: "heroku/buildpacks:18",
-        CNB_ENV_VARS: "BUNDLE_GIT__PANTER__CH",
-        CNB_EXTRA_ARGS: "",
-        CNB_PACK_VERSION: "0.20.0",
-        ...context.componentConfig.build.extraVars,
-      },
+      variables: context.componentConfig.build.extraVars,
       // custom script
       script: [
         gitlabDockerLogin,
         `docker pull $DOCKER_CACHE_IMAGE || true`,
-        `wget --output-document=- https://github.com/buildpacks/pack/releases/download/v$CNB_PACK_VERSION/pack-v$CNB_PACK_VERSION-linux.tgz | tar -zx --directory /usr/local/bin pack`,
+        `wget --output-document=- https://github.com/buildpacks/pack/releases/download/v${cnbConf?.packVersion}/pack-v${cnbConf?.packVersion}-linux.tgz | tar -zx --directory /usr/local/bin pack`,
         `chmod +x /usr/local/bin/pack`,
         //  replace private git ssh gem sources with https to make bundler with credentials via env var work
         `sed --in-place 's|git@\\([^:]*\\):|https://\\1/|g' Gemfile Gemfile.lock`,
-        `for v in $CNB_ENV_VARS; do env_args="$env_args --env $v"; done`,
-        `pack build $DOCKER_IMAGE:$DOCKER_IMAGE_TAG --builder $CNB_BUILDER --publish --cache-image $DOCKER_CACHE_IMAGE $env_args $CNB_EXTRA_ARGS`,
+        `pack build "$DOCKER_IMAGE:$DOCKER_IMAGE_TAG" --builder '${cnbConf?.image}' --publish --cache-image "$DOCKER_CACHE_IMAGE" ${packEnvArgs.join(" ")} ${cnbConf?.packExtraArgs?.join(" ") ?? ""}`
       ],
     }),
   ];

package/src/build/rails/index.ts

@@ -1,5 +1,5 @@
-import { Context } from "../../types";
-import { CatladderJob } from "../../types/jobs";
+import type { Context } from "../../types";
+import type { CatladderJob } from "../../types/jobs";
 import { createRailsBuildJobs } from "./build";
 import { createRailsTestJobs } from "./test";
 

package/src/build/types.ts

@@ -1,4 +1,6 @@
-import type { GitlabJobService } from "../types";
+import type { GitlabJobImage, GitlabJobService } from "../types";
+
+import type { CatladderJob } from "../types/jobs";
 
 export type BuildConfigBase = {
   /**
@@ -29,7 +31,7 @@ export type BuildConfigBase = {
     | false
     | {
         command?: string | string[];
-        jobImage?: string;
+        jobImage?: GitlabJobImage;
       };
 
   /**
@@ -39,7 +41,7 @@ export type BuildConfigBase = {
     | false
     | {
         command?: string | string[];
-        jobImage?: string;
+        jobImage?: GitlabJobImage;
       };
 
   /**
@@ -49,7 +51,7 @@ export type BuildConfigBase = {
     | false
     | {
         command?: string | string[];
-        jobImage?: string;
+        jobImage?: GitlabJobImage;
       };
 
   /**
@@ -70,7 +72,7 @@ export type BuildConfigBase = {
      * eg. `["dist/test-results/TEST-*.xml", "dist/rspec.xml", ...]`
      */
     junit?: string[];
-  }
+  };
 };
 
 export type BuildConfigNodeBase = BuildConfigBase;
@@ -113,11 +115,44 @@ export type BuildConfigCustom = BuildConfigBase & {
   type: "custom";
   jobImage: string;
   jobServices?: GitlabJobService[];
+
   docker: BuildConfigCustomDocker;
+
+  /**
+   * customize cache for the job
+   */
+  jobCache?: CatladderJob["cache"];
 };
 
 export type BuildConfigRails = BuildConfigBase & {
   type: "rails";
+  cnbBuilder?: {
+    /**
+     * The Cloud Native Buildpacks builder image to use.
+     * See e.g. https://github.com/heroku/builder or others.
+     * Default: heroku/buildpacks:20
+     */
+    image?: string;
+    /**
+     * The version of the Cloud Native Buildpacks pack command to use.
+     * See https://buildpacks.io/docs/tools/pack/
+     * Default: 0.28.0
+     */
+    packVersion?: string;
+    /**
+     * Additional command arguments passed to the pack command.
+     * See https://buildpacks.io/docs/tools/pack/
+     */
+    packExtraArgs?: string[];
+    /**
+     * Variables needed for Bundler and Rails asset precompilation.
+     *
+     * Specify a key with the value `undefined` to inherit it from the EnvVars.
+     *
+     * See https://github.com/rails/rails/issues/32947
+     */
+    buildVars?: Record<string, string | undefined>;
+  };
 };
 
 export type BuildConfigStorybook = BuildConfigNodeBase & {

package/src/context/__tests__/resolveReferences.test.ts

@@ -17,12 +17,9 @@ describe("resolveReferences", () => {
         FOO: "foo from frontend",
       },
     };
-    const result = await resolveReferences(
-      variables,
-      async (componentName, variableName) => {
-        return otherVariables[componentName]?.[variableName];
-      }
-    );
+    const result = await resolveReferences(variables, async (componentName) => {
+      return otherVariables[componentName];
+    });
 
     expect(result).toEqual({
       a: "hello world",
@@ -53,12 +50,9 @@ describe("resolveReferences", () => {
         FOO: "foo from api",
       },
     };
-    const result = await resolveReferences(
-      variables,
-      async (componentName, variableName) => {
-        return otherVariables[componentName]?.[variableName];
-      }
-    );
+    const result = await resolveReferences(variables, async (componentName) => {
+      return otherVariables[componentName];
+    });
 
     expect(result).toEqual({
       FOO: "hello from foo from api",
@@ -72,12 +66,9 @@ describe("resolveReferences", () => {
       b: "a not found value looks like this: '${api:FOO}'",
     };
     const otherVariables: Record<string, Record<string, string>> = {};
-    const result = await resolveReferences(
-      variables,
-      async (componentName, variableName) => {
-        return otherVariables[componentName]?.[variableName];
-      }
-    );
+    const result = await resolveReferences(variables, async (componentName) => {
+      return otherVariables[componentName];
+    });
 
     expect(result).toEqual({
       a: "hello world",
@@ -97,12 +88,9 @@ describe("resolveReferences", () => {
         FOO: "hi, ${api:FOO}",
       },
     };
-    const result = await resolveReferences(
-      variables,
-      async (componentName, variableName) => {
-        return otherVariables[componentName]?.[variableName] ?? "";
-      }
-    );
+    const result = await resolveReferences(variables, async (componentName) => {
+      return otherVariables[componentName];
+    });
 
     expect(result).toEqual({
       a: "value is hi, foo from api!",
@@ -121,12 +109,9 @@ describe("resolveReferences", () => {
         FOO: "frontend ${api:FOO}",
       },
     };
-    const result = await resolveReferences(
-      variables,
-      async (componentName, variableName) => {
-        return otherVariables[componentName]?.[variableName] ?? "";
-      }
-    );
+    const result = await resolveReferences(variables, async (componentName) => {
+      return otherVariables[componentName];
+    });
 
     expect(result).toEqual({
       a: "value is frontend api ${frontend:FOO}!",

package/src/context/getEnvironment.ts

@@ -31,6 +31,7 @@ export const getEnvironment = async (
       url,
     },
     fullName: envContext.fullName,
+    slugPrefix: envContext.environmentSlugPrefix,
     slug: envContext.environmentSlug,
     shortName: env,
     envVars,

package/src/context/getEnvironmentContext.ts

@@ -5,17 +5,16 @@ import type { EnvironmentContext } from "../types/environmentContext";
 import { getEnvConfig } from "./getEnvConfig";
 import { getEnvType } from "./getEnvType";
 
-const getEnvironmentSlug = (
+const getEnvironmentSlugPrefix = (
   envConfig: EnvConfigWithComponent,
   env: string,
-  componentName: string,
   commitInfo?: CommitInfo
 ) => {
   const envType = getEnvType(env, envConfig);
 
   return envType === "review" && commitInfo
-    ? `${env}-${commitInfo.reviewSlug}-${componentName}`
-    : `${env}-${componentName}`;
+    ? `${env}-${commitInfo.reviewSlug}`
+    : `${env}`;
 };
 
 export const getEnvironmentContext = (
@@ -27,13 +26,13 @@ export const getEnvironmentContext = (
   const envConfigRaw = getEnvConfig(config, componentName, env);
   const envType = getEnvType(env, envConfigRaw);
 
-  const environmentSlug = getEnvironmentSlug(
+  const environmentSlugPrefix = getEnvironmentSlugPrefix(
     envConfigRaw,
     env,
-    componentName,
     commitInfo
   );
 
+  const environmentSlug = `${environmentSlugPrefix}-${componentName}`;
   const gitlabEnvironmentName =
     envType === "review" && commitInfo
       ? `${env}/${commitInfo.refName}/${componentName}`
@@ -45,6 +44,7 @@ export const getEnvironmentContext = (
     envConfigRaw,
     deployConfigRaw: envConfigRaw.deploy,
     buildConfigRaw: envConfigRaw.build,
+    environmentSlugPrefix,
     environmentSlug,
     gitlabEnvironmentName,
     fullName,

package/src/context/getEnvironmentVariables.ts

@@ -53,7 +53,7 @@ export const getEnvironmentVariables = async (
   } else {
     const additionalEnvVars = deployConfigRaw
       ? DEPLOY_TYPES[deployConfigRaw.type].getAdditionalEnvVars(
-          environmentContext
+          environmentContext as never
         )
       : {};
 
@@ -66,7 +66,9 @@ export const getEnvironmentVariables = async (
     predefinedVariables = {
       ...basePredefinedVariables,
       // Rails before 6.1 (mis)uses the `HOST` environment variable to specify the IP to bind to
-      ...(config.components[componentName].build.type === "rails" ? {} : {HOST: host}),
+      ...(config.components[componentName].build.type === "rails"
+        ? {}
+        : { HOST: host }),
       ROOT_URL: url,
       ...additionalEnvVars,
     };
@@ -75,7 +77,7 @@ export const getEnvironmentVariables = async (
 
   const additionalSecretKeys = deployConfigRaw
     ? DEPLOY_TYPES[deployConfigRaw.type].additionalSecretKeys(
-        environmentContext
+        environmentContext as never
       )
     : [];
 
@@ -115,7 +117,7 @@ export const getEnvironmentVariables = async (
 
   const envVars = await resolveReferences(
     envVarsRaw,
-    async (otherComponentName, variableName, alreadyVisited) => {
+    async (otherComponentName, alreadyVisited) => {
       const { envVars: otherEnvVars } = await getEnvironmentVariables(
         config,
         otherComponentName,
@@ -123,7 +125,7 @@ export const getEnvironmentVariables = async (
         commitInfo,
         alreadyVisited
       );
-      return otherEnvVars[variableName];
+      return otherEnvVars;
     },
     alreadyVisited
   );

package/src/context/resolveReferences.ts

@@ -7,9 +7,8 @@ export const resolveReferences = async (
   vars: Record<string, string>,
   getOtherVariables?: (
     componentName: string,
-    variableName: string,
     alreadyVisited: Record<string, Record<string, boolean>>
-  ) => Promise<string | null>,
+  ) => Promise<Record<string, string | null>>,
   alreadyVisitedBase: Record<string, Record<string, boolean>> = {}
 ) => {
   const replaceSingleValue = async (
@@ -30,10 +29,8 @@ export const resolveReferences = async (
             },
           });
           const result = componentName
-            ? (await getOtherVariables?.(
-                componentName,
-                variableName,
-                newAlreadyVisited
+            ? (await getOtherVariables?.(componentName, newAlreadyVisited).then(
+                (r) => r?.[variableName]
               )) ?? null
             : vars[variableName]; // is self reference
 

package/src/deploy/base.ts

@@ -44,7 +44,7 @@ export const getBaseDeploymentJob = (context: Context): JobWithoutScript => {
     name: DEPLOY_JOB_NAME,
     envMode: "stagePerEnv", // makes it easier to run manual tasks er env
 
-    needsOtherComponent: context.componentConfig.deploy
+    needs: context.componentConfig.deploy
       ? context.componentConfig.deploy.waitFor?.map((c) => ({
           componentName: c,
           job: DEPLOY_JOB_NAME,

package/src/deploy/cloudRun/deployJob.ts

@@ -116,20 +116,18 @@ export const createGoogleCloudRunDeployJobs = (
         : undefined;
 
     const argsString = createArgsString({
-      command: command ? '"' + command.split(" ").join(",") + '"' : undefined, // not sure if quotes are needed
+      // command as empty string resets it to default (uses the image's entrypoint)
+      command: command ? '"' + command.split(" ").join(",") + '"' : '""',
       ...commonDeployArgs,
       "env-vars-file": "____envvars.yaml",
-      "min-instances": customConfig?.minInstances,
-      "max-instances": customConfig?.maxInstances,
-      "no-cpu-throttling": customConfig?.noCpuThrottling,
+      "min-instances": customConfig?.minInstances ?? 0,
+      "max-instances": customConfig?.maxInstances ?? 100,
+      "cpu-throttling": customConfig?.noCpuThrottling !== true,
+      memory: customConfig?.memory,
+      "allow-unauthenticated": customConfig?.allowUnauthenticated ?? true,
     });
-    const authenticationString =
-      customConfig?.allowUnauthenticated ?? true
-        ? "--allow-unauthenticated"
-        : "--no-allow-unauthenticated";
-    return `gcloud run deploy ${serviceName}${
-      nameSuffix ?? ""
-    } ${argsString} ${authenticationString}`;
+
+    return `gcloud run deploy ${serviceName}${nameSuffix ?? ""} ${argsString}`;
   };
 
   const getJobCreateScriptsForJob = (

package/src/deploy/cloudRun/index.ts

@@ -12,7 +12,7 @@ export const GCLOUD_RUN_CANONICAL_HOST_SUFFIX =
 
 const getCloudSqlVariables = ({
   deployConfigRaw,
-  environmentSlug,
+  environmentSlugPrefix,
   env,
   componentName,
   fullConfig,
@@ -21,7 +21,8 @@ const getCloudSqlVariables = ({
     const DB_NAME = [
       deployConfigRaw.cloudSql.dbNamePrefix ??
         `${fullConfig.customerName}-${fullConfig.appName}`,
-      environmentSlug,
+      environmentSlugPrefix,
+      deployConfigRaw.cloudSql.dbBaseName ?? componentName,
     ]
       .filter(Boolean)
       .join("-");
@@ -67,9 +68,9 @@ export const GCLOUD_RUN_DEPLOY_TYPE: DeployTypeDefinition<"google-cloudrun"> = {
     const HOST_CANONICAL =
       fullName.toLowerCase() +
       "-" +
-      process.env[
+      (process.env[
         getSecretVarName(env, componentName, GCLOUD_RUN_CANONICAL_HOST_SUFFIX)
-      ];
+      ] ?? "unknown-host.example.com");
 
     const jobTriggers =
       deployConfigRaw && deployConfigRaw.jobs