@catladder/pipeline 0.0.0-fix-command-710f75f1 → 0.0.0-fix-error-sql-db-delete-b3878c2e

package/examples/cloud-run-with-sql-reuse-db.ts

@@ -0,0 +1,54 @@
+import type { Config, DeployConfigCloudRunCloudSql } from "../src";
+import { createAllPipelines } from "./__utils__/helpers";
+
+const CLOUD_SQL: DeployConfigCloudRunCloudSql = {
+  type: "unmanaged",
+  instanceConnectionName: "projectId:region:instancename",
+  dbUser: "my-user",
+};
+const config: Config = {
+  appName: "test-app",
+  customerName: "pan",
+  components: {
+    api: {
+      dir: "api",
+      build: {
+        type: "node",
+      },
+      deploy: {
+        type: "google-cloudrun",
+        projectId: "google-project-id",
+        region: "europe-west6",
+        cloudSql: CLOUD_SQL,
+      },
+    },
+    worker: {
+      dir: "api",
+      build: {
+        type: "node",
+        buildCommand: "yarn build:worker",
+      },
+      deploy: {
+        type: "google-cloudrun",
+        projectId: "google-project-id",
+        region: "europe-west6",
+        cloudSql: {
+          ...CLOUD_SQL,
+          // override the db base name to the other component's name
+          // if ommited, this defaults to the component's name
+          dbBaseName: "api",
+        },
+      },
+      vars: {
+        public: {
+          // need to reference the password
+          DB_PASSWORD: "${api:DB_PASSWORD}",
+        },
+      },
+    },
+  },
+};
+
+it("matches snapshot", async () => {
+  expect(await createAllPipelines(config)).toMatchSnapshot();
+});

package/examples/custom-deploy.ts

@@ -0,0 +1,25 @@
+import type { Config } from "../src";
+import { createAllPipelines } from "./__utils__/helpers";
+
+const config: Config = {
+  appName: "test-app",
+  customerName: "pan",
+  components: {
+    www: {
+      dir: "www",
+      build: {
+        type: "node",
+      },
+      deploy: {
+        type: "custom",
+        requiresDocker: true,
+        script: ["echo 'would deploy'"],
+        stopScript: ["echo 'would stop'"],
+      },
+    },
+  },
+};
+
+it("matches snapshot", async () => {
+  expect(await createAllPipelines(config)).toMatchSnapshot();
+});

package/examples/kubernetes-application-customization.ts

@@ -0,0 +1,57 @@
+import type { Config } from "../src";
+import { createAllPipelines } from "./__utils__/helpers";
+const config: Config = {
+  appName: "test-app",
+  customerName: "pan",
+  components: {
+    api: {
+      dir: "api",
+      build: {
+        type: "node",
+      },
+      deploy: {
+        type: "kubernetes",
+        cluster: {
+          name: "some-cluster-name",
+          region: "europe-west6",
+          projectId: "some-project-id",
+          type: "gcloud",
+          domainCanonical: "panter.cloud",
+        },
+
+        values: {
+          application: {
+            command: "node main.js",
+            autoscale: {
+              minReplicas: 2,
+              maxReplicas: 5,
+              metrics: [
+                {
+                  type: "Resource",
+                  resource: {
+                    name: "cpu",
+
+                    target: {
+                      type: "Utilization",
+                      averageUtilization: 0.5,
+                    },
+                  },
+                },
+              ],
+            },
+            resources: {
+              limits: {
+                cpu: "1",
+                memory: "2048Mi",
+              },
+            },
+          },
+        },
+      },
+    },
+  },
+};
+
+it("matches snapshot", async () => {
+  expect(await createAllPipelines(config)).toMatchSnapshot();
+});

package/examples/kubernetes-with-cloud-sql-legacy.ts

@@ -0,0 +1,33 @@
+import type { Config } from "../src";
+import { createAllPipelines } from "./__utils__/helpers";
+const config: Config = {
+  appName: "test-app",
+  customerName: "pan",
+  components: {
+    api: {
+      dir: "api",
+      build: {
+        type: "node",
+      },
+      deploy: {
+        type: "kubernetes",
+        cluster: {
+          name: "some-cluster-name",
+          region: "europe-west6",
+          projectId: "some-project-id",
+          type: "gcloud",
+          domainCanonical: "panter.cloud",
+        },
+        values: {
+          cloudsql: {
+            enabled: true,
+          },
+        },
+      },
+    },
+  },
+};
+
+it("matches snapshot", async () => {
+  expect(await createAllPipelines(config)).toMatchSnapshot();
+});

package/examples/rails-k8s-with-worker.ts

@@ -0,0 +1,95 @@
+import type { Config } from "../src";
+import { createAllPipelines } from "./__utils__/helpers";
+
+// the image version should match your `.ruby-version`
+const RAILS_TEST_STAGE_IMAGE = "ruby:3.2.1"
+
+const config: Config = {
+  appName: "test-app",
+  customerName: "pan",
+  components: {
+    app: {
+      dir: ".",
+      vars: {
+        public: {
+          RAILS_ENV: "production",
+        },
+        secret: [
+          "SECRET_KEY_BASE",
+        ]
+      },
+      build: {
+        type: "rails",
+        test: {
+          // if operating system dependencies are required you can either
+          // - build a dedicated image
+          // - use `command` to apt-get install them before running the test command
+          jobImage: RAILS_TEST_STAGE_IMAGE,
+        },
+        lint: {
+          jobImage: RAILS_TEST_STAGE_IMAGE,
+        },
+        audit: {
+          jobImage: RAILS_TEST_STAGE_IMAGE,
+        },
+        cnbBuilder: {
+          buildVars: { SECRET_KEY_BASE: "dummy-value" },
+        },
+      },
+      deploy: {
+        type: "kubernetes",
+        cluster: {
+          name: "some-cluster-name",
+          region: "europe-west6",
+          projectId: "some-project-id",
+          type: "gcloud",
+          domainCanonical: "panter.cloud",
+        },
+        values: {
+          cloudsql: { enabled: true, projectId: "some-project-id" },
+          application: {
+            // if deployed to Google Cloud Run use a gem like cloudtasker instead of a permanently running expensive worker
+            worker: {
+              enabled: true,
+              command: "launcher bundle exec rake jobs:work",
+            },
+          },
+          jobs: {
+            "db-migrate": {
+              hook: "post-install,post-upgrade",
+              command:
+                "launcher bundle exec rake db:migrate",
+            },
+          },
+        },
+      },
+      env: {
+        review: {
+          deploy: {
+            values: {
+              jobs: {
+                "db-prepare-seed": {
+                  hook: "post-install",
+                  command:
+                    "launcher bundle exec rake db:prepare db:seed",
+                },
+                "db-migrate": {
+                  hook: "post-upgrade",
+                  command:
+                    "launcher bundle exec rake db:migrate",
+                },
+              },
+            },
+          },
+        },
+        prod: {
+          host: "my-fancy-website.com"
+        }
+      },
+    },
+  },
+};
+
+it("matches snapshot", async () => {
+  expect(await createAllPipelines(config)).toMatchSnapshot();
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@catladder/pipeline",
-  "version": "0.0.0-fix-command-710f75f1",
+  "version": "0.0.0-fix-error-sql-db-delete-b3878c2e",
   "scripts": {
     "build:tsc": "yarn tsc",
     "build": "yarn build:compile && yarn build:inline-variables && yarn build:bundle",

package/src/build/custom/buildJob.ts

@@ -29,6 +29,7 @@ export const createCustomBuildJobs = (context: Context): CatladderJob[] => {
           variables: {
             ...RUNNER_BUILD_VARIABLES,
           },
+          cache: buildConfig.jobCache,
           services: buildConfig.jobServices,
           script: [...(ensureArray(buildConfig.buildCommand) ?? [])],
           artifacts: {

package/src/build/docker.ts

@@ -45,20 +45,24 @@ export const requiresDockerBuild = (context: Context) => {
   return false;
 };
 
+const getDockerBaseVariables = () => ({
+  DOCKER_HOST: "tcp://0.0.0.0:2375",
+  DOCKER_TLS_CERTDIR: "",
+  DOCKER_DRIVER: "overlay2",
+  DOCKER_BUILDKIT: "1", // see https://docs.docker.com/develop/develop-images/build_enhancements/
+});
+
 export const getDockerBuildVariables = (context: Context) => {
   return {
     ...DOCKER_RUNNER_BUILD_VARIABLES,
-    DOCKER_BUILDKIT: "1", // see https://docs.docker.com/develop/develop-images/build_enhancements/
     DOCKERFILE_ADDITIONS:
       context.componentConfig.build.docker?.additionsBegin?.join("\n"),
     DOCKERFILE_ADDITIONS_END:
       context.componentConfig.build.docker?.additionsEnd?.join("\n"),
     APP_DIR: context.componentConfig.dir,
-    DOCKER_HOST: "tcp://0.0.0.0:2375",
-    DOCKER_TLS_CERTDIR: "",
-    DOCKER_DIR: ".", // relative to componentdir
 
-    DOCKER_DRIVER: "overlay2",
+    DOCKER_DIR: ".", // relative to componentdir
+    ...getDockerBaseVariables(),
 
     ...getDockerImageVariables(context),
   };
@@ -74,7 +78,7 @@ export const getDockerJobBaseProps = (context: Context) => {
         command: ["--tls=false"],
       },
     ],
-    variables: getDockerBuildVariables(context),
+    variables: getDockerBaseVariables(),
   };
 };
 export const createDockerBuildJobBase = (
@@ -89,6 +93,9 @@ export const createDockerBuildJobBase = (
       ...getDockerJobBaseProps(context),
       script: script || [],
     },
+    {
+      variables: getDockerBuildVariables(context),
+    },
     def
   );
 };

package/src/build/index.ts

@@ -51,6 +51,12 @@ export const BUILD_TYPES: BuildTypes = {
   },
   rails: {
     jobs: createRailsJobs,
-    defaults: () => ({}),
+    defaults: () => ({
+      startCommand: "/cnb/process/web",
+      cnbBuilder: {
+        image: "heroku/buildpacks:20",
+        packVersion: "0.28.0",
+      },
+    }),
   },
 };

package/src/build/node/cache.ts

@@ -31,8 +31,8 @@ export const getNodeModulesCache = (
   const componentIsInWorkspace =
     context.packageManagerInfo?.componentIsInWorkspace;
 
-   // We intentionally do not use the contents of yarn.lock as a cache key, as yarn install should always guarantee that the files are updated, but it can still use part of the cache if not all packages are up-to-date.
-   // It would slow down all pipelines whenever one adds a new dependency as it will need to download all node_modules again.
+  // We intentionally do not use the contents of yarn.lock as a cache key, as yarn install should always guarantee that the files are updated, but it can still use part of the cache if not all packages are up-to-date.
+  // It would slow down all pipelines whenever one adds a new dependency as it will need to download all node_modules again.
   return [
     {
       // if component is in a shared workspace, use workspace cache. use individual cache else

package/src/build/rails/build.ts

@@ -1,27 +1,37 @@
 import type { Context } from "../..";
+import { isOfBuildType } from "../types";
 import type { CatladderJob } from "../../types/jobs";
 import { createDockerBuildJobBase, gitlabDockerLogin } from "../docker";
 
 export const createRailsBuildJobs = (context: Context): CatladderJob[] => {
+  const buildConfig = context.componentConfig.build;
+  if (!isOfBuildType(buildConfig, "rails")) {
+    // should not happen
+    throw new Error("build type is not rails");
+  }
+
+  const cnbConf = buildConfig.cnbBuilder;
+
+  const packEnvArgs = Object.entries(cnbConf?.buildVars ?? {})
+    .map(([k, v]) => `--env '${k}${v ? `=${v}` : ""}'`)
+    .join(" ");
+
   return [
     createDockerBuildJobBase(context, {
-      variables: {
-        CNB_BUILDER: "heroku/buildpacks:18",
-        CNB_ENV_VARS: "BUNDLE_GIT__PANTER__CH",
-        CNB_EXTRA_ARGS: "",
-        CNB_PACK_VERSION: "0.20.0",
-        ...context.componentConfig.build.extraVars,
-      },
+      variables: context.componentConfig.build.extraVars,
       // custom script
       script: [
         gitlabDockerLogin,
         `docker pull $DOCKER_CACHE_IMAGE || true`,
-        `wget --output-document=- https://github.com/buildpacks/pack/releases/download/v$CNB_PACK_VERSION/pack-v$CNB_PACK_VERSION-linux.tgz | tar -zx --directory /usr/local/bin pack`,
+        `wget --output-document=- https://github.com/buildpacks/pack/releases/download/v${cnbConf?.packVersion}/pack-v${cnbConf?.packVersion}-linux.tgz | tar -zx --directory /usr/local/bin pack`,
         `chmod +x /usr/local/bin/pack`,
         //  replace private git ssh gem sources with https to make bundler with credentials via env var work
         `sed --in-place 's|git@\\([^:]*\\):|https://\\1/|g' Gemfile Gemfile.lock`,
-        `for v in $CNB_ENV_VARS; do env_args="$env_args --env $v"; done`,
-        `pack build $DOCKER_IMAGE:$DOCKER_IMAGE_TAG --builder $CNB_BUILDER --publish --cache-image $DOCKER_CACHE_IMAGE $env_args $CNB_EXTRA_ARGS`,
+        `pack build "$DOCKER_IMAGE:$DOCKER_IMAGE_TAG" --builder '${
+          cnbConf?.image
+        }' --publish --cache-image "$DOCKER_CACHE_IMAGE" ${packEnvArgs} ${
+          cnbConf?.packExtraArgs?.join(" ") ?? ""
+        }`,
       ],
     }),
   ];

package/src/build/rails/index.ts

@@ -1,5 +1,5 @@
-import { Context } from "../../types";
-import { CatladderJob } from "../../types/jobs";
+import type { Context } from "../../types";
+import type { CatladderJob } from "../../types/jobs";
 import { createRailsBuildJobs } from "./build";
 import { createRailsTestJobs } from "./test";
 

package/src/build/rails/test.ts

@@ -17,66 +17,70 @@ export const createRailsTestJobs = (context: Context): CatladderJob[] => {
     envMode: "none",
   };
   const defaultImage = "docker.io/ruby";
-  const bundlerCacheDir = "tmp/cache"
+  const bundlerCacheDir = "tmp/cache";
   const bundlerInstall = [
     `bundle config set path '${bundlerCacheDir}'`,
-    "bundle install -j $(nproc)"
-  ]
+    "bundle install -j $(nproc)",
+  ];
   const bundlerCache = {
     key: {
       files: ["Gemfile.lock"],
-      prefix: "$CI_JOB_IMAGE" // a changed image might have different OS libraries which no longer work with the cached gems
+      prefix: "$CI_JOB_IMAGE", // a changed image might have different OS libraries which no longer work with the cached gems
     },
-    paths: [bundlerCacheDir]
-  }
+    paths: [bundlerCacheDir],
+  };
   const auditJob: CatladderJob | null =
     buildConfig.audit !== false
       ? {
-        name: "🛡 audit",
+          name: "🛡 audit",
 
-        ...base,
-        cache: undefined, // audit does not need bundle install and no cache
-        image: buildConfig.audit?.jobImage ?? defaultImage,
-        script: [
-          `cd ${context.componentConfig.dir}`,
-          ...(ensureArray(buildConfig.audit?.command) ?? [
-            "gem install bundler-audit",
-            "bundle audit check"
-          ]),
-        ],
-        allow_failure: true,
-      }
+          ...base,
+          cache: undefined, // audit does not need bundle install and no cache
+          image: buildConfig.audit?.jobImage ?? defaultImage,
+          script: [
+            `cd ${context.componentConfig.dir}`,
+            ...(ensureArray(buildConfig.audit?.command) ?? [
+              "gem install bundler-audit",
+              "bundle audit check",
+            ]),
+          ],
+          allow_failure: true,
+        }
       : null;
 
   const lintJob: CatladderJob | null =
     buildConfig.lint !== false
       ? {
-        name: "👮 lint",
+          name: "👮 lint",
 
-        ...base,
-        cache: bundlerCache,
-        image: buildConfig.lint?.jobImage ?? defaultImage,
-        script: [
-          `cd ${context.componentConfig.dir}`,
-          ...bundlerInstall,
-          ...(ensureArray(buildConfig.lint?.command) ?? ["bundle exec rubocop"]),
-        ],
-      }
+          ...base,
+          cache: bundlerCache,
+          image: buildConfig.lint?.jobImage ?? defaultImage,
+          script: [
+            `cd ${context.componentConfig.dir}`,
+            ...bundlerInstall,
+            ...(ensureArray(buildConfig.lint?.command) ?? [
+              "bundle exec rubocop",
+            ]),
+          ],
+        }
       : null;
   const testJob: CatladderJob | null =
     buildConfig.test !== false
       ? {
-        name: "🧪 test",
+          name: "🧪 test",
 
-        ...base,
-        cache: bundlerCache,
-        image: buildConfig.test?.jobImage ?? defaultImage,
-        script: [
-          `cd ${context.componentConfig.dir}`,
-          ...bundlerInstall,
-          ...(ensureArray(buildConfig.test?.command) ?? ["bundle exec rspec"]),
-        ],
-      }
+          ...base,
+          cache: bundlerCache,
+          image: buildConfig.test?.jobImage ?? defaultImage,
+          script: [
+            `cd ${context.componentConfig.dir}`,
+            ...bundlerInstall,
+            ...(ensureArray(buildConfig.test?.command) ?? [
+              "bundle exec rspec",
+            ]),
+          ],
+        }
       : null;
   return [auditJob, lintJob, testJob].filter(notNil);
 };

package/src/build/types.ts

@@ -1,4 +1,6 @@
-import type { GitlabJobService } from "../types";
+import type { GitlabJobImage, GitlabJobService } from "../types";
+
+import type { CatladderJob } from "../types/jobs";
 
 export type BuildConfigBase = {
   /**
@@ -29,7 +31,7 @@ export type BuildConfigBase = {
     | false
     | {
         command?: string | string[];
-        jobImage?: string;
+        jobImage?: GitlabJobImage;
       };
 
   /**
@@ -39,7 +41,7 @@ export type BuildConfigBase = {
     | false
     | {
         command?: string | string[];
-        jobImage?: string;
+        jobImage?: GitlabJobImage;
       };
 
   /**
@@ -49,7 +51,7 @@ export type BuildConfigBase = {
     | false
     | {
         command?: string | string[];
-        jobImage?: string;
+        jobImage?: GitlabJobImage;
       };
 
   /**
@@ -70,7 +72,7 @@ export type BuildConfigBase = {
      * eg. `["dist/test-results/TEST-*.xml", "dist/rspec.xml", ...]`
      */
     junit?: string[];
-  }
+  };
 };
 
 export type BuildConfigNodeBase = BuildConfigBase;
@@ -113,11 +115,44 @@ export type BuildConfigCustom = BuildConfigBase & {
   type: "custom";
   jobImage: string;
   jobServices?: GitlabJobService[];
+
   docker: BuildConfigCustomDocker;
+
+  /**
+   * customize cache for the job
+   */
+  jobCache?: CatladderJob["cache"];
 };
 
 export type BuildConfigRails = BuildConfigBase & {
   type: "rails";
+  cnbBuilder?: {
+    /**
+     * The Cloud Native Buildpacks builder image to use.
+     * See e.g. https://github.com/heroku/builder or others.
+     * Default: heroku/buildpacks:20
+     */
+    image?: string;
+    /**
+     * The version of the Cloud Native Buildpacks pack command to use.
+     * See https://buildpacks.io/docs/tools/pack/
+     * Default: 0.28.0
+     */
+    packVersion?: string;
+    /**
+     * Additional command arguments passed to the pack command.
+     * See https://buildpacks.io/docs/tools/pack/
+     */
+    packExtraArgs?: string[];
+    /**
+     * Variables needed for Bundler and Rails asset precompilation.
+     *
+     * Specify a key with the value `undefined` to inherit it from the EnvVars.
+     *
+     * See https://github.com/rails/rails/issues/32947
+     */
+    buildVars?: Record<string, string | undefined>;
+  };
 };
 
 export type BuildConfigStorybook = BuildConfigNodeBase & {

package/src/config/configruedEnvs.ts

@@ -1,11 +1,5 @@
-import type {
-  Config,
-  EnvType,
-  PipelineTrigger} from "../types";
-import {
-  DEFAULT_ENV_TYPES,
-  getEnvTypesByTrigger
-} from "../types";
+import type { Config, EnvType, PipelineTrigger } from "../types";
+import { DEFAULT_ENV_TYPES, getEnvTypesByTrigger } from "../types";
 
 const getConfiguredAndDefaultEnvs = (
   config: Config,

package/src/context/getEnvironment.ts

@@ -31,6 +31,7 @@ export const getEnvironment = async (
       url,
     },
     fullName: envContext.fullName,
+    slugPrefix: envContext.environmentSlugPrefix,
     slug: envContext.environmentSlug,
     shortName: env,
     envVars,