@catladder/pipeline 0.0.0-refactore-catenv-command-validates-argument-78d05852 → 0.0.0-reference-database-url-1befbbee

package/src/build/node/meteor.ts

@@ -1,33 +1,34 @@
-import { join } from "path";
 import { getRunnerImage } from "../../runner";
-import type { GitlabJobCache } from "../../types";
-import type { Context } from "../../types/context";
+import type {
+  ComponentContext,
+  ComponentContextWithBuild,
+} from "../../types/context";
 
 import type { CatladderJob } from "../../types/jobs";
 
-import { createBuildJobs } from "../base";
-import { getDockerBuildDefaultScript } from "../docker";
+import { createComponentBuildJobs } from "../base";
+import { getDockerBuildScriptWithBuiltInDockerFile } from "../docker";
+import type { CacheConfig } from "../types";
 import { isOfBuildType } from "../types";
 import { getNodeCache } from "./cache";
 import { getYarnInstall } from "./yarn";
 
-const getMeteorCache = (context: Context): GitlabJobCache[] => [
+const getMeteorCache = (context: ComponentContext): CacheConfig[] => [
   {
-    key: context.componentName + "meteor-build-cache",
+    key: "meteor-build-cache",
     policy: "pull-push",
     paths: [
-      join(
-        context.componentConfig.dir,
-        ".meteor/local/resolver-result-cache.json"
-      ),
-      join(context.componentConfig.dir, ".meteor/local/plugin-cache"),
-      join(context.componentConfig.dir, ".meteor/local/isopacks"),
-      join(context.componentConfig.dir, ".meteor/local/bundler-cache/scanner"),
+      ".meteor/local/resolver-result-cache.json",
+      ".meteor/local/plugin-cache",
+      ".meteor/local/isopacks",
+      ".meteor/local/bundler-cache/scanner",
     ],
   },
 ];
-export const createMeteorBuildJobs = (context: Context): CatladderJob[] => {
-  const buildConfig = context.componentConfig.build;
+export const createMeteorBuildJobs = (
+  context: ComponentContextWithBuild,
+): CatladderJob[] => {
+  const buildConfig = context.build.config;
 
   if (!isOfBuildType(buildConfig, "meteor")) {
     throw new Error("deploy config is not meteor");
@@ -35,7 +36,7 @@ export const createMeteorBuildJobs = (context: Context): CatladderJob[] => {
 
   const yarnInstall = getYarnInstall(context);
 
-  return createBuildJobs(context, {
+  return createComponentBuildJobs(context, {
     appBuild:
       buildConfig.buildCommand !== null
         ? {
@@ -55,16 +56,10 @@ export const createMeteorBuildJobs = (context: Context): CatladderJob[] => {
 
               "cp ./__build_info.json ./dist/bundle/programs/server",
             ],
-            artifacts: {
-              paths: [
-                context.componentConfig.dir + "/__build_info.json",
-                context.componentConfig.dir + "/dist",
-              ],
-            },
           }
         : undefined,
     dockerBuild: {
-      script: getDockerBuildDefaultScript(context, "ensureMeteorDockerfile"),
+      script: getDockerBuildScriptWithBuiltInDockerFile(context, "meteor"),
       variables: {
         METEOR_INSTALL_SCRIPTS: buildConfig.installScripts ? "true" : "",
       },

package/src/build/node/testJob.ts

@@ -1,31 +1,50 @@
 import { getRunnerImage } from "../../runner";
-import type { Context } from "../../types/context";
+import type { WorkspaceContext } from "../../types/context";
+import {
+  componentContextIsStandaloneBuild,
+  type ComponentContext,
+} from "../../types/context";
 import type { CatladderJob } from "../../types/jobs";
-import { ensureArray, notNil } from "../../utils";
+import { ensureArrayOrNull, notNil } from "../../utils";
+import { createArtifactsConfig } from "../base/createArtifactsConfig";
 import { getNodeCache } from "./cache";
 import { NODE_RUNNER_BUILD_VARIABLES } from "./constants";
 import { ensureNodeVersion, getYarnInstall } from "./yarn";
-import { createArtifactsConfig } from "../base/createArtifactsConfig";
+import { createJobCacheFromCacheConfigs } from "../cache/createJobCache";
 
-export const createNodeTestJobs = (context: Context): CatladderJob[] => {
+export const createNodeTestJobs = (
+  context: ComponentContext | WorkspaceContext,
+): CatladderJob[] => {
   // don't run tests after release
-  if (context.commitInfo?.trigger === "taggedRelease") {
+  // TODO: this will be replaced by using rules
+  if (context.trigger === "taggedRelease") {
+    return [];
+  }
+  // if its not a standalone build, we don't need to run tests
+  if (
+    context.type === "component" &&
+    !componentContextIsStandaloneBuild(context)
+  ) {
     return [];
   }
 
-  const buildConfig = context.componentConfig.build;
   const defaultImage = getRunnerImage("jobs-default");
   const base: Omit<CatladderJob, "script" | "name"> = {
     variables: {
-      APP_PATH: context.componentConfig.dir,
-      ...NODE_RUNNER_BUILD_VARIABLES,
-      ...context.environment.jobOnlyVars.build.envVars,
-      ...(buildConfig.extraVars ?? {}),
+      APP_PATH: context.build.dir,
+      ...(context.type === "component"
+        ? {
+            ...context.environment.jobOnlyVars.build.envVars,
+          }
+        : {}),
     },
+
+    runnerVariables: NODE_RUNNER_BUILD_VARIABLES,
     stage: "test",
     needs: [],
     envMode: "none",
   };
+  const buildConfig = context.build.config;
   const yarnInstall = getYarnInstall(context);
   const auditJob: CatladderJob | null =
     buildConfig.audit !== false
@@ -35,18 +54,18 @@ export const createNodeTestJobs = (context: Context): CatladderJob[] => {
           image: buildConfig.audit?.jobImage ?? defaultImage,
           cache: undefined, // audit does not need yarn install and no cache
           script: [
-            `cd ${context.componentConfig.dir}`,
-            ...(ensureArray(buildConfig.audit?.command) ?? [
-              context.packageManagerInfo?.isClassic
+            `cd ${context.build.dir}`,
+            ...(ensureArrayOrNull(buildConfig.audit?.command) ?? [
+              context.packageManagerInfo.isClassic
                 ? "yarn audit"
                 : "yarn npm audit --environment production", // yarn 2
             ]),
           ],
           allow_failure: true,
           ...createArtifactsConfig(
-            context.componentConfig.dir,
+            context.build.dir,
             buildConfig.audit?.artifactsReports,
-            buildConfig.audit?.artifacts
+            buildConfig.audit?.artifacts,
           ),
         }
       : null;
@@ -57,17 +76,17 @@ export const createNodeTestJobs = (context: Context): CatladderJob[] => {
           name: "👮 lint",
           ...base,
           image: buildConfig.lint?.jobImage ?? defaultImage,
-          cache: getNodeCache(context),
+          cache: createJobCacheFromCacheConfigs(context, getNodeCache(context)),
           script: [
             ...ensureNodeVersion(context),
-            `cd ${context.componentConfig.dir}`,
+            `cd ${context.build.dir}`,
             ...yarnInstall,
-            ...(ensureArray(buildConfig.lint?.command) ?? ["yarn lint"]),
+            ...(ensureArrayOrNull(buildConfig.lint?.command) ?? ["yarn lint"]),
           ],
           ...createArtifactsConfig(
-            context.componentConfig.dir,
+            context.build.dir,
             buildConfig.lint?.artifactsReports,
-            buildConfig.lint?.artifacts
+            buildConfig.lint?.artifacts,
           ),
         }
       : null;
@@ -79,17 +98,17 @@ export const createNodeTestJobs = (context: Context): CatladderJob[] => {
           ...base,
           image:
             buildConfig.test?.jobImage ?? getRunnerImage("jobs-testing-chrome"),
-          cache: getNodeCache(context),
+          cache: createJobCacheFromCacheConfigs(context, getNodeCache(context)),
           script: [
             ...ensureNodeVersion(context),
-            `cd ${context.componentConfig.dir}`,
+            `cd ${context.build.dir}`,
             ...yarnInstall,
-            ...(ensureArray(buildConfig.test?.command) ?? ["yarn test"]),
+            ...(ensureArrayOrNull(buildConfig.test?.command) ?? ["yarn test"]),
           ],
           ...createArtifactsConfig(
-            context.componentConfig.dir,
+            context.build.dir,
             buildConfig.test?.artifactsReports,
-            buildConfig.test?.artifacts
+            buildConfig.test?.artifacts,
           ),
         }
       : null;

package/src/build/node/yarn.ts

@@ -1,4 +1,5 @@
-import type { Context } from "../../types";
+import { BashExpression } from "../../bash/BashExpression";
+import { type Context } from "../../types";
 import { ensureArray } from "../../utils";
 import { collapseableSection } from "../../utils/gitlab";
 
@@ -8,7 +9,7 @@ const YARN_INSTALL_CLASSIC = `yarn install --frozen-lockfile`;
 const YARN_BERRY_PROD_REBUILD = `yarn workspaces focus --production && yarn rebuild`;
 
 const getYarnInstallCommand = (context: Context) => {
-  if (context.packageManagerInfo?.isClassic) {
+  if (context.packageManagerInfo.isClassic) {
     return YARN_INSTALL_CLASSIC;
   }
 
@@ -18,7 +19,7 @@ const getYarnInstallCommand = (context: Context) => {
 export const ensureNodeVersion = (context: Context) =>
   collapseableSection(
     "nodeinstall",
-    "Ensure node version"
+    "Ensure node version",
   )([
     "if [ -f ~/.nvm/nvm.sh ];  then source ~/.nvm/nvm.sh; fi",
     "if command -v nvm &> /dev/null && [ -f ./.nvmrc ]; then nvm install; fi",
@@ -28,23 +29,25 @@ export const getYarnInstall = (
   context: Context,
   options?: {
     noCustomPostInstall: boolean;
-  }
+  },
 ) => {
   const postInstall =
-    "postInstall" in context.componentConfig.build
-      ? context.componentConfig.build.postInstall
+    context.type !== "workspace" &&
+    context.build.type !== "disabled" &&
+    "postInstall" in context.build.config
+      ? context.build.config.postInstall
       : null;
   return [
     ...ensureNodeVersion(context),
     ...collapseableSection(
       "yarninstall",
-      "Yarn install"
+      "Yarn install",
     )([getYarnInstallCommand(context)]),
     ...(postInstall && !options?.noCustomPostInstall
       ? collapseableSection(
           "postinstall",
-          "Custom post install"
-        )(ensureArray(postInstall) ?? [])
+          "Custom post install",
+        )(ensureArray(postInstall))
       : []),
   ];
 };
@@ -52,18 +55,20 @@ export const getYarnInstall = (
 const DOCKER_COPY_FILES = `COPY --chown=node:node $APP_DIR .`;
 
 export const getDockerAppCopyAndBuildScript = (context: Context) => {
-  if (context.packageManagerInfo?.isClassic) {
-    return `
+  if (context.packageManagerInfo.isClassic) {
+    return new BashExpression(
+      `
 RUN ${YARN_INSTALL_CLASSIC} --production --ignore-scripts
 ${DOCKER_COPY_FILES}
 RUN ${YARN_INSTALL_CLASSIC} --production 
-    `.trim();
+    `.trim(),
+    );
   }
 
   // yarn >= 4 ships with build in plugins, see https://github.com/yarnpkg/berry/pull/4253
   // trying to import those fail on this version
   const doesNotShipWithBuiltInPlugins = ["2", "3"].some((v) =>
-    context.packageManagerInfo?.version.startsWith(v)
+    context.packageManagerInfo.version.startsWith(v),
   );
   const maybeAddWorkspaceToolsCommand = doesNotShipWithBuiltInPlugins
     ? "RUN yarn plugin import workspace-tools"
@@ -71,10 +76,12 @@ RUN ${YARN_INSTALL_CLASSIC} --production
 
   // copy first everything and then install
   // rebuild first does not work as it will run postinstall and that might require files in the app
-  return `
+  return new BashExpression(
+    `
 ${DOCKER_COPY_FILES}
 ${maybeAddWorkspaceToolsCommand}
 RUN ${YARN_BERRY_PROD_REBUILD}
 
-    `.trim();
+    `.trim(),
+  );
 };

package/src/build/rails/build.ts

@@ -1,6 +1,7 @@
-import type { Context } from "../..";
+import type { ComponentContextWithBuild } from "../..";
+import { componentContextIsStandaloneBuild } from "../..";
 import type { CatladderJob } from "../../types/jobs";
-import { createBuildJobs } from "../base";
+import { createComponentBuildJobs } from "../base";
 import {
   getDockerBuildDefaultScript,
   gitlabDockerLogin,
@@ -8,15 +9,22 @@ import {
 } from "../docker";
 import { isOfBuildType } from "../types";
 
-export const createRailsBuildJobs = (context: Context): CatladderJob[] => {
-  const buildConfig = context.componentConfig.build;
+export const createRailsBuildJobs = (
+  context: ComponentContextWithBuild,
+): CatladderJob[] => {
+  const buildConfig = context.build.config;
   if (!isOfBuildType(buildConfig, "rails")) {
     // should not happen
     throw new Error("build type is not rails");
   }
 
+  // if its not a standalone build, we don't need to run tests
+  if (!componentContextIsStandaloneBuild(context)) {
+    throw new Error("workspace builds are not supported for rails apps");
+  }
+
   if (hasDockerfile(context)) {
-    return createBuildJobs(context, {
+    return createComponentBuildJobs(context, {
       appBuild: undefined,
       dockerBuild: {
         script: getDockerBuildDefaultScript(context),
@@ -30,18 +38,16 @@ export const createRailsBuildJobs = (context: Context): CatladderJob[] => {
   const packEnvArgs = Object.entries(cnbConf?.buildVars ?? {})
     .map(([k, v]) => `--env '${k}${v ? `=${v}` : ""}'`)
     .join(" ");
-  return createBuildJobs(context, {
+  return createComponentBuildJobs(context, {
     appBuild: undefined,
     dockerBuild: {
       variables: {
         ...context.environment.jobOnlyVars.build.envVars,
-        ...context.componentConfig.build.extraVars,
       },
       // custom script
       script: [
         ...gitlabDockerLogin(context),
-        `cd ${context.componentConfig.dir}`,
-        `docker pull $DOCKER_CACHE_IMAGE || true`,
+        `cd ${context.build.dir}`,
         `wget --output-document=- https://github.com/buildpacks/pack/releases/download/v${cnbConf?.packVersion}/pack-v${cnbConf?.packVersion}-linux.tgz | tar -zx --directory /usr/local/bin pack`,
         `chmod +x /usr/local/bin/pack`,
         //  replace private git ssh gem sources with https to make bundler with credentials via env var work

package/src/build/rails/index.ts

@@ -1,8 +1,10 @@
-import type { Context } from "../../types";
+import type { ComponentContextWithBuild } from "../../types";
 import type { CatladderJob } from "../../types/jobs";
 import { createRailsBuildJobs } from "./build";
 import { createRailsTestJobs } from "./test";
 
-export const createRailsJobs = (context: Context): CatladderJob[] => {
+export const createRailsJobs = (
+  context: ComponentContextWithBuild,
+): CatladderJob[] => {
   return [...createRailsTestJobs(context), ...createRailsBuildJobs(context)];
 };

package/src/build/rails/test.ts

@@ -1,19 +1,29 @@
-import type { Context } from "../..";
+import {
+  componentContextIsStandaloneBuild,
+  type ComponentContext,
+} from "../..";
 import type { CatladderJob } from "../../types/jobs";
-import { ensureArray, notNil } from "../../utils";
+import { ensureArrayOrNull, notNil } from "../../utils";
 
-export const createRailsTestJobs = (context: Context): CatladderJob[] => {
+export const createRailsTestJobs = (
+  context: ComponentContext,
+): CatladderJob[] => {
   // don't run tests after release
-  if (context.commitInfo?.trigger === "taggedRelease") {
+  // TODO: this will be replaced by using rules
+  if (context.trigger === "taggedRelease") {
     return [];
   }
 
-  const buildConfig = context.componentConfig.build;
+  // if its not a standalone build, we don't need to run tests
+  if (!componentContextIsStandaloneBuild(context)) {
+    return [];
+  }
+
+  const buildConfig = context.build.config;
 
   const base: Omit<CatladderJob, "script" | "name"> = {
     variables: {
       ...context.environment.jobOnlyVars.build.envVars,
-      ...(buildConfig.extraVars ?? {}),
     },
     stage: "test",
     needs: [],
@@ -42,8 +52,8 @@ export const createRailsTestJobs = (context: Context): CatladderJob[] => {
           image:
             buildConfig.audit?.jobImage ?? buildConfig.jobImage ?? defaultImage,
           script: [
-            `cd ${context.componentConfig.dir}`,
-            ...(ensureArray(buildConfig.audit?.command) ?? [
+            `cd ${context.build.dir}`,
+            ...(ensureArrayOrNull(buildConfig.audit?.command) ?? [
               "gem install bundler-audit",
               "bundle audit check",
             ]),
@@ -62,9 +72,9 @@ export const createRailsTestJobs = (context: Context): CatladderJob[] => {
           image:
             buildConfig.lint?.jobImage ?? buildConfig.jobImage ?? defaultImage,
           script: [
-            `cd ${context.componentConfig.dir}`,
+            `cd ${context.build.dir}`,
             ...bundlerInstall,
-            ...(ensureArray(buildConfig.lint?.command) ?? [
+            ...(ensureArrayOrNull(buildConfig.lint?.command) ?? [
               "bundle exec rubocop",
             ]),
           ],
@@ -80,12 +90,31 @@ export const createRailsTestJobs = (context: Context): CatladderJob[] => {
           image:
             buildConfig.test?.jobImage ?? buildConfig.jobImage ?? defaultImage,
           script: [
-            `cd ${context.componentConfig.dir}`,
+            `cd ${context.build.dir}`,
             ...bundlerInstall,
-            ...(ensureArray(buildConfig.test?.command) ?? [
+            ...(ensureArrayOrNull(buildConfig.test?.command) ?? [
+              "bundle exec rake db:test:prepare",
+              "bundle exec rake assets:precompile assets:clean",
               "bundle exec rspec",
             ]),
           ],
+          runnerVariables: {
+            RAILS_ENV: "test",
+            DATABASE_URL: "postgresql://postgres@database",
+          },
+          services: [
+            {
+              name:
+                buildConfig.test && "databaseImage" in buildConfig.test
+                  ? (buildConfig?.test?.databaseImage ??
+                    "docker.io/postgres:latest")
+                  : "docker.io/postgres:latest",
+              alias: "database",
+              variables: {
+                POSTGRES_HOST_AUTH_METHOD: "trust",
+              },
+            },
+          ],
         }
       : null;
   return [auditJob, lintJob, testJob].filter(notNil);

package/src/build/sbom.ts

@@ -1,30 +1,40 @@
-import type { Context } from "../types/context";
+import { isStandaloneBuildConfig } from ".";
+import type { ComponentContextWithBuild } from "../types/context";
 import type { CatladderJob } from "../types/jobs";
-import { ensureArray } from "../utils";
+import { ensureArrayOrNull } from "../utils";
 
 export const SBOM_BUILD_JOB_NAME = "🧾 sbom";
 export const SBOM_FILE = "__sbom.json";
 
-export const createSbomBuildJob = (context: Context): CatladderJob => {
-  const buildConfig = context.componentConfig.build;
+export const createSbomBuildJob = (
+  context: ComponentContextWithBuild,
+): CatladderJob => {
+  const buildConfig = context.build.config;
 
-  const defaultImage = "aquasec/trivy:0.38.3";
+  const defaultImage = {
+    name: "aquasec/trivy:0.58.2",
+    entrypoint: [""],
+  };
   const defaultScript = [
     `trivy fs --quiet --format cyclonedx --output "${SBOM_FILE}" ${
-      context.packageManagerInfo?.componentIsInWorkspace
+      context.packageManagerInfo.componentIsInWorkspace
         ? "."
-        : context.componentConfig.dir
+        : context.build.dir
     }`,
   ];
 
   const image =
-    buildConfig.type === "custom" && buildConfig.sbom !== false
-      ? buildConfig.sbom?.jobImage ?? defaultImage
+    isStandaloneBuildConfig(buildConfig) &&
+    buildConfig.type === "custom" &&
+    buildConfig.sbom !== false
+      ? (buildConfig.sbom?.jobImage ?? defaultImage)
       : defaultImage;
 
   const script =
-    buildConfig.type === "custom" && buildConfig.sbom !== false
-      ? ensureArray(buildConfig.sbom?.command) ?? defaultScript
+    isStandaloneBuildConfig(buildConfig) &&
+    buildConfig.type === "custom" &&
+    buildConfig.sbom !== false
+      ? (ensureArrayOrNull(buildConfig.sbom?.command) ?? defaultScript)
       : defaultScript;
 
   return {