@catladder/pipeline 1.101.4 → 1.103.0

package/examples/custom-deploy.ts

@@ -13,6 +13,9 @@ const config: Config = {
       deploy: {
         type: "custom",
         requiresDocker: true,
+        jobVars: {
+          secret: ["DEPLOY_API_KEY"],
+        },
         script: ["echo 'would deploy'"],
         stopScript: ["echo 'would stop'"],
       },

package/examples/native-app.ts

@@ -0,0 +1,81 @@
+import type { Config } from "../src";
+import type { CatladderJob } from "../src/types/jobs";
+import { createAllPipelines } from "./__utils__/helpers";
+const APP_GEM_CACHE: CatladderJob["cache"] = [
+  {
+    key: {
+      files: ["app/Gemfile.lock"],
+    },
+    paths: ["app/vendor"],
+  },
+];
+
+const config: Config = {
+  appName: "test-app",
+  customerName: "pan",
+  components: {
+    app: {
+      dir: "app",
+      dotEnv: true,
+      vars: {
+        public: {
+          GRAPHQL_URL: "${api:ROOT_URL}/graphql",
+        },
+        secret: [
+          "APP_STORE_CONNECT_API_KEY_CONTENT",
+          "APP_STORE_CONNECT_ISSUER_ID",
+          "APP_STORE_CONNECT_API_KEY_ID",
+        ],
+      },
+      build: {
+        extraVars: {
+          LC_A: "L=en_US.UTF-8",
+          LANG: "en_US.UTF-8",
+        },
+        type: "node",
+        buildCommand: [
+          "bundle config set --local path 'vendor/ruby'",
+          "gem install bundler",
+          "bundle install",
+          "bundle exec pod install --project-directory=ios",
+          "bundle exec fastlane build",
+        ],
+        jobTags: ["mac-runner"],
+        jobCache: APP_GEM_CACHE,
+      },
+
+      deploy: {
+        type: "custom",
+        extraVars: {
+          LC_A: "L=en_US.UTF-8",
+          LANG: "en_US.UTF-8",
+        },
+        requiresDocker: false,
+        script: [
+          "bundle config set --local path 'vendor/ruby'",
+          "gem install bundler",
+          "bundle install",
+          "bundle exec fastlane deploy_test",
+        ],
+        jobTags: ["mac-runner"],
+        jobCache: APP_GEM_CACHE,
+      },
+    },
+
+    api: {
+      dir: "api",
+      build: {
+        type: "node",
+      },
+      deploy: {
+        type: "google-cloudrun",
+        projectId: "asdf",
+        region: "asia-east1",
+      },
+    },
+  },
+};
+
+it("matches snapshot", async () => {
+  expect(await createAllPipelines(config)).toMatchSnapshot();
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@catladder/pipeline",
-  "version": "1.101.4",
+  "version": "1.103.0",
   "scripts": {
     "build:tsc": "yarn tsc",
     "build": "yarn build:compile && yarn build:inline-variables && yarn build:bundle",

package/src/build/base/createBuildJob.ts

@@ -10,6 +10,7 @@ import {
   RUNNER_BUILD_RESOURCE_VARIABLES,
 } from "./constants";
 import { writeBuildInfo } from "./writeBuildInfo";
+import { writeDotEnv } from "./writeDotEnv";
 
 export const createBuildJob = (
   context: Context,
@@ -27,10 +28,12 @@ export const createBuildJob = (
         ...RUNNER_BUILD_RESOURCE_VARIABLES,
         ...(variables ?? {}),
         ...context.environment.envVars,
+        ...context.environment.jobOnlyVars.build.envVars,
         ...(context.componentConfig.build.extraVars ?? {}),
       },
 
       script: [
+        ...(context.componentConfig.dotEnv ? writeDotEnv(context) : []),
         ...writeBuildInfo(context),
         ...ensureNodeVersion(context), // in pure node repos, we might want to have the nvmrc file in top-level
         `cd ${context.componentConfig.dir}`,

package/src/build/base/writeDotEnv.ts

@@ -0,0 +1,21 @@
+import type { Context } from "../../types";
+
+/**
+ * writes a .env file in the components folder
+ * @param context
+ * @returns
+ */
+export const writeDotEnv = (context: Context) => {
+  const envVars = context.environment.envVars;
+
+  // make key=value and sanitize multiline
+  const keyValueString = Object.entries(envVars)
+    .map(([key, value]) => `${key}=${value.replaceAll("\n", "\\n")}`)
+    .join("\n");
+
+  return [
+    `cat <<EOF > ${context.componentConfig.dir}/.env
+${keyValueString}
+EOF`,
+  ];
+};

package/src/build/custom/testJob.ts

@@ -29,6 +29,7 @@ export const createCustomTestJobs = (context: Context): CatladderJob[] => {
     variables: {
       APP_PATH: context.componentConfig.dir,
       ...RUNNER_CUSTOM_TEST_VARIABLES,
+      ...context.environment.jobOnlyVars.build.envVars,
       ...(buildConfig.extraVars ?? {}),
     },
     services: buildConfig.jobServices,

package/src/build/node/testJob.ts

@@ -18,6 +18,7 @@ export const createNodeTestJobs = (context: Context): CatladderJob[] => {
     variables: {
       APP_PATH: context.componentConfig.dir,
       ...NODE_RUNNER_BUILD_VARIABLES,
+      ...context.environment.jobOnlyVars.build.envVars,
       ...(buildConfig.extraVars ?? {}),
     },
     stage: "test",

package/src/build/rails/build.ts

@@ -18,7 +18,10 @@ export const createRailsBuildJobs = (context: Context): CatladderJob[] => {
 
   return [
     createDockerBuildJobBase(context, {
-      variables: context.componentConfig.build.extraVars,
+      variables: {
+        ...context.environment.jobOnlyVars.build.envVars,
+        ...context.componentConfig.build.extraVars,
+      },
       // custom script
       script: [
         gitlabDockerLogin,

package/src/build/rails/test.ts

@@ -11,7 +11,10 @@ export const createRailsTestJobs = (context: Context): CatladderJob[] => {
   const buildConfig = context.componentConfig.build;
 
   const base: Omit<CatladderJob, "script" | "name"> = {
-    variables: buildConfig.extraVars ?? {},
+    variables: {
+      ...context.environment.jobOnlyVars.build.envVars,
+      ...(buildConfig.extraVars ?? {}),
+    },
     stage: "test",
     needs: [],
     envMode: "none",

package/src/build/types.ts

@@ -1,4 +1,4 @@
-import type { GitlabJobImage, GitlabJobService } from "../types";
+import type { EnvVars, GitlabJobImage, GitlabJobService } from "../types";
 
 import type { CatladderJob } from "../types/jobs";
 
@@ -17,8 +17,16 @@ export type BuildConfigBase = {
    * command to run on the image to start the app (e.g. yarn start)
    */
   startCommand?: string;
+
+  /**
+   * vars that only exist in the build jobs.
+   * Can curently not reference other variables from other components
+   */
+  jobVars?: EnvVars;
   /**
    * additional env vars for the buid jobs
+   *
+   * @deprecated use jobVars
    */
   extraVars?: Record<string, string>;
   /**

package/src/context/getEnvironment.ts

@@ -10,8 +10,12 @@ export const getEnvironment = async (
   env: string,
   commitInfo?: CommitInfo
 ): Promise<Environment> => {
-  const { envVars, secretEnvVarKeys, host, url } =
-    await getEnvironmentVariables(config, componentName, env, commitInfo);
+  const variables = await getEnvironmentVariables(
+    config,
+    componentName,
+    env,
+    commitInfo
+  );
 
   const envContext = getEnvironmentContext(
     config,
@@ -24,17 +28,15 @@ export const getEnvironment = async (
 
   return {
     envType,
-    host,
-    url,
+
     gitlabEnvironment: {
       name: envContext.gitlabEnvironmentName,
-      url,
+      url: variables.url,
     },
     fullName: envContext.fullName,
     slugPrefix: envContext.environmentSlugPrefix,
     slug: envContext.environmentSlug,
     shortName: env,
-    envVars,
-    secretEnvVarKeys,
+    ...variables,
   };
 };

package/src/context/getEnvironmentVariables.ts

@@ -1,6 +1,10 @@
-import { isObject, merge } from "lodash";
+import { merge } from "lodash";
 import { DEPLOY_TYPES } from "../deploy";
-import type { CommitInfo, Context } from "../types";
+import type {
+  CommitInfo,
+  Context,
+  EnvironmentEnvVarPart as EnvironmentVariables,
+} from "../types";
 import type { Config, DevLocalEnvConfig } from "../types/config";
 
 import { getEnvironmentContext } from "./getEnvironmentContext";
@@ -8,6 +12,12 @@ import {
   resolveReferences,
   translateLegacyFromComponents,
 } from "./resolveReferences";
+import {
+  stringListToSecreteEnvVarList,
+  makeSecretEnvVarMapping,
+  stringifyValues,
+} from "./utils/envVars";
+import { transformJobOnlyVars } from "./transformJobOnlyVars";
 
 export type SecretEnvVar = {
   key: string;
@@ -20,12 +30,7 @@ export const getEnvironmentVariables = async (
   env: string,
   commitInfo?: CommitInfo,
   alreadyVisited: Record<string, Record<string, boolean>> = {} // to prevent endless loop
-): Promise<{
-  envVars: Record<string, string>;
-  secretEnvVarKeys: SecretEnvVar[];
-  host: string;
-  url: string;
-}> => {
+): Promise<EnvironmentVariables> => {
   const environmentContext = getEnvironmentContext(
     config,
     env,
@@ -33,7 +38,8 @@ export const getEnvironmentVariables = async (
     commitInfo
   );
 
-  const { envConfigRaw, deployConfigRaw, envType } = environmentContext;
+  const { envConfigRaw, deployConfigRaw, buildConfigRaw, envType } =
+    environmentContext;
 
   const basePredefinedVariables = {
     ENV_SHORT: env,
@@ -90,31 +96,24 @@ export const getEnvironmentVariables = async (
     : [];
 
   const secretEnvVarKeys: SecretEnvVar[] = [
-    ...(envConfigRaw.vars?.secret ?? []).map((key) => ({ key })),
+    ...stringListToSecreteEnvVarList(envConfigRaw.vars?.secret ?? []),
     ...additionalSecretKeys,
   ];
-  const secretEnvVars = Object.fromEntries(
-    secretEnvVarKeys.map(({ key }) => [
-      key,
-      `$${getSecretVarName(env, componentName, key)}`,
-    ])
+  const secretEnvVars = makeSecretEnvVarMapping(
+    env,
+    componentName,
+    secretEnvVarKeys
   );
-
   // this is deprecated, we now support: $componentname:FOO
   const legacyFromComponents = envConfigRaw.vars?.fromComponents ?? {};
-  const publicEnvVarsRawWithLegasyFromComponents = merge(
+  const publicEnvVarsRawWithLegacyFromComponents = merge(
     {},
     translateLegacyFromComponents(legacyFromComponents),
     publicEnvVarsRaw
   );
 
-  const publicEnvVarsRawSanitized = Object.fromEntries(
-    Object.entries(publicEnvVarsRawWithLegasyFromComponents).map(
-      ([key, value]) => [
-        key,
-        isObject(value) ? JSON.stringify(value) : `${value}`,
-      ]
-    )
+  const publicEnvVarsRawSanitized = stringifyValues(
+    publicEnvVarsRawWithLegacyFromComponents
   );
 
   const envVarsRaw = addIndexVar({
@@ -141,6 +140,19 @@ export const getEnvironmentVariables = async (
   return {
     envVars,
     secretEnvVarKeys,
+    jobOnlyVars: {
+      build: await transformJobOnlyVars(
+        env,
+        componentName,
+        (buildConfigRaw && buildConfigRaw.jobVars) || null
+      ),
+      deploy: await transformJobOnlyVars(
+        env,
+        componentName,
+        (deployConfigRaw && deployConfigRaw.jobVars) || null
+      ),
+    },
+
     host,
     url,
   };

package/src/context/transformJobOnlyVars.ts

@@ -0,0 +1,42 @@
+import type { EnvironmentEnvVars } from "..";
+import type { EnvVars } from "../types/config";
+import {
+  makeSecretEnvVarMapping,
+  stringListToSecreteEnvVarList,
+  stringifyValues,
+} from "./utils/envVars";
+
+/**
+ * transform EnvVars for environment.
+ *
+ * currently does not resolve references and is only used for additional job-only-vars
+ * @param vars
+ * @returns
+ */
+export const transformJobOnlyVars = async (
+  env: string,
+  componentName: string,
+  vars: EnvVars | null
+): Promise<EnvironmentEnvVars> => {
+  if (!vars) {
+    return {
+      envVars: {},
+      secretEnvVarKeys: [],
+    };
+  }
+
+  const publicVars = stringifyValues(vars?.public ?? {});
+  const secretEnvVarKeys = stringListToSecreteEnvVarList(vars?.secret ?? []);
+  const deployJobOnlySecretEnvVars = vars?.secret
+    ? makeSecretEnvVarMapping(env, componentName, secretEnvVarKeys)
+    : {};
+
+  const envVars = {
+    ...deployJobOnlySecretEnvVars,
+    ...publicVars,
+  };
+  return {
+    envVars,
+    secretEnvVarKeys,
+  };
+};

package/src/context/utils/envVars.ts

@@ -0,0 +1,26 @@
+import { isObject } from "lodash";
+import type { SecretEnvVar } from "../getEnvironmentVariables";
+import { getSecretVarName } from "../getEnvironmentVariables";
+
+export const stringifyValues = (obj: Record<string, unknown>) =>
+  Object.fromEntries(
+    Object.entries(obj).map(([key, value]) => [
+      key,
+      isObject(value) ? JSON.stringify(value) : `${value}`,
+    ])
+  );
+
+export const stringListToSecreteEnvVarList = (keys: string[]): SecretEnvVar[] =>
+  keys.map((key) => ({ key }));
+export const makeSecretEnvVarMapping = (
+  env: string,
+  componentName: string,
+  secretEnvVars: SecretEnvVar[]
+) => {
+  return Object.fromEntries(
+    secretEnvVars.map(({ key }) => [
+      key,
+      `$${getSecretVarName(env, componentName, key)}`,
+    ])
+  );
+};

package/src/deploy/base.ts

@@ -76,6 +76,7 @@ export const getBaseDeploymentJob = (context: Context): JobWithoutScript => {
       ...DEPLOY_RUNNER_VARIABLES,
       ...context.environment.envVars,
       ...(hasDocker ? getDockerImageVariables(context) : {}),
+      ...context.environment.jobOnlyVars.deploy.envVars,
       ...(context.componentConfig.deploy
         ? context.componentConfig.deploy.extraVars ?? {}
         : {}),
@@ -115,6 +116,7 @@ export const getBaseDeploymentStopJob = (
     ],
     variables: {
       ...DEPLOY_RUNNER_VARIABLES,
+      ...context.environment.jobOnlyVars.deploy.envVars,
       GIT_STRATEGY: "none",
     },
     stage: "stop",
@@ -138,6 +140,7 @@ export const getBaseRollbackJob = (context: Context): JobWithoutScript => {
     ],
     variables: {
       ...DEPLOY_RUNNER_VARIABLES,
+      ...context.environment.jobOnlyVars.deploy.envVars,
       GIT_STRATEGY: "none",
     },
     stage: "rollback",

package/src/deploy/types/base.ts

@@ -1,3 +1,5 @@
+import type { EnvVars } from "../../types/config";
+
 export type DeployConfigBase = {
   /**
    * whether to deploy automatically or manual. If not defined, these rules apply:
@@ -17,8 +19,16 @@ export type DeployConfigBase = {
    */
   jobTags?: string[];
 
+  /**
+   * vars that only exist in the deploy job.
+   * Can curently not reference other variables from other components
+   */
+  jobVars?: EnvVars;
+
   /**
    * additional env vars for the deploy job
+   *
+   * @deprecated use deploy.jobVars for deploy job specific variables
    */
   extraVars?: Record<string, string>;
 };

package/src/types/config.ts

@@ -144,6 +144,13 @@ export type ComponentConfig<C extends ConfigProps = never> = {
    * This feedback will help us to generalize use cases
    */
   customJobs?: CatladderJob[] | ((context: Context) => CatladderJob[]);
+
+  /**
+   * experimental. whether to create a .env localy and during build jobs
+   *
+   * Be careful, this will overwrite any existing .env file!
+   */
+  dotEnv?: boolean;
 } & DefaultEnvConfig;
 
 export type ConfigProps = {

package/src/types/context.ts

@@ -6,6 +6,22 @@ import type {
   EnvType,
 } from "./config";
 
+export type EnvironmentEnvVars = {
+  envVars: Record<string, string>;
+  secretEnvVarKeys: SecretEnvVar[];
+};
+export type EnvironmentEnvVarPart = {
+  host: string;
+  url: string;
+
+  /**
+   * vars that only are injected in certain jobs, but not elsewhere
+   */
+  jobOnlyVars: {
+    deploy: EnvironmentEnvVars;
+    build: EnvironmentEnvVars;
+  };
+} & EnvironmentEnvVars;
 export type Environment = {
   host: string;
   url: string;
@@ -21,13 +37,9 @@ export type Environment = {
   shortName: string;
   slugPrefix: string;
   slug: string;
-  /**
-   * env vars contain all build-time env vars. secrets have to be resolved (they are stored in gitlab)
-   */
-  envVars: Record<string, string>;
+
   envType: EnvType;
-  secretEnvVarKeys: SecretEnvVar[];
-};
+} & EnvironmentEnvVarPart;
 
 export type CommitInfo = {
   refName: string;

package/tsconfig.json

@@ -1,6 +1,6 @@
 {
   "compilerOptions": {
-    "lib": ["es2020", "DOM"],
+    "lib": ["es2021", "DOM"],
     "strict": true,
     "module": "CommonJS",
     "downlevelIteration": true,