@catladder/pipeline 1.170.0 → 2.0.0

package/examples/{workspace-api-www-custom-cache.test.ts → cloud-run-nextjs.test.ts}

@@ -1,11 +1,11 @@
 import { createYamlLocalPipeline } from "./__utils__/helpers";
-import config from "./workspace-api-www-custom-cache";
+import config from "./cloud-run-nextjs";
 
 /**
  * This test is auto-generated.
  * Modifications will be overwritten on every `yarn test` run!
  */
 
-it("matches snapshot for workspace-api-www-custom-cache local pipeline YAML", async () => {
+it("matches snapshot for cloud-run-nextjs local pipeline YAML", async () => {
   expect(await createYamlLocalPipeline(config)).toMatchSnapshot();
 });

package/examples/cloud-run-nextjs.ts

@@ -0,0 +1,28 @@
+import type { Config } from "../src";
+
+const config: Config = {
+  appName: "test-app",
+  customerName: "pan",
+  components: {
+    www: {
+      dir: "www",
+      build: {
+        type: "node",
+        cache: {
+          paths: [".next/cache"],
+        },
+      },
+      deploy: {
+        type: "google-cloudrun",
+        projectId: "google-project-id",
+        region: "europe-west6",
+      },
+    },
+  },
+};
+
+export default config;
+
+export const information = {
+  title: "Cloud Run: With SQL",
+};

package/examples/cloud-run-with-sql.ts

@@ -5,7 +5,6 @@ const config: Config = {
   customerName: "pan",
   components: {
     api: {
-      dotEnv: true,
       dir: "api",
       build: {
         type: "node",

package/examples/kubernetes-application-customization.ts

@@ -6,6 +6,7 @@ const config: Config = {
   components: {
     api: {
       dir: "api",
+      dotEnv: false,
       build: {
         type: "node",
       },

package/examples/kubernetes-with-cloud-sql.ts

@@ -6,6 +6,7 @@ const config: Config = {
   components: {
     api: {
       dir: "api",
+      dotEnv: false,
       build: {
         type: "node",
       },

package/examples/kubernetes-with-jobs.ts

@@ -6,6 +6,7 @@ const config: Config = {
   components: {
     api: {
       dir: "api",
+      dotEnv: false,
       build: {
         type: "node",
       },

package/examples/kubernetes-with-mongodb.ts

@@ -6,6 +6,7 @@ const config: Config = {
   components: {
     api: {
       dir: "api",
+      dotEnv: false,
       build: {
         type: "node",
       },

package/examples/meteor-kubernetes.ts

@@ -6,7 +6,7 @@ const config: Config = {
   components: {
     web: {
       dir: "app",
-
+      dotEnv: false,
       build: { type: "meteor" },
       deploy: {
         type: "kubernetes",

package/examples/native-app.ts

@@ -16,7 +16,6 @@ const config: Config = {
   components: {
     app: {
       dir: "app",
-      dotEnv: true,
       vars: {
         public: {
           GRAPHQL_URL: "${api:ROOT_URL}/graphql",
@@ -28,9 +27,11 @@ const config: Config = {
         ],
       },
       build: {
-        extraVars: {
-          LC_A: "L=en_US.UTF-8",
-          LANG: "en_US.UTF-8",
+        jobVars: {
+          public: {
+            LC_A: "L=en_US.UTF-8",
+            LANG: "en_US.UTF-8",
+          },
         },
         type: "node",
         buildCommand: [
@@ -46,9 +47,11 @@ const config: Config = {
 
       deploy: {
         type: "custom",
-        extraVars: {
-          LC_A: "L=en_US.UTF-8",
-          LANG: "en_US.UTF-8",
+        jobVars: {
+          public: {
+            LC_A: "L=en_US.UTF-8",
+            LANG: "en_US.UTF-8",
+          },
         },
         requiresDocker: false,
         script: [

package/examples/rails-k8s-with-worker.ts

@@ -9,6 +9,8 @@ const config: Config = {
   components: {
     app: {
       dir: ".",
+      dotEnv: false,
+      envDTs: false,
       vars: {
         public: {
           RAILS_ENV: "production",
@@ -44,7 +46,11 @@ const config: Config = {
           domainCanonical: "panter.cloud",
         },
         values: {
-          cloudsql: { enabled: true, projectId: "some-project-id" },
+          cloudsql: {
+            enabled: true,
+            type: "unmanaged",
+            instanceConnectionName: "myproject:europe-west6:instance-name",
+          },
           application: {
             // if deployed to Google Cloud Run use a gem like cloudtasker instead of a permanently running expensive worker
             worker: {

package/examples/{kubernetes-with-cloud-sql-legacy.test.ts → referencing-other-vars.test.ts}

@@ -1,11 +1,11 @@
 import { createYamlLocalPipeline } from "./__utils__/helpers";
-import config from "./kubernetes-with-cloud-sql-legacy";
+import config from "./referencing-other-vars";
 
 /**
  * This test is auto-generated.
  * Modifications will be overwritten on every `yarn test` run!
  */
 
-it("matches snapshot for kubernetes-with-cloud-sql-legacy local pipeline YAML", async () => {
+it("matches snapshot for referencing-other-vars local pipeline YAML", async () => {
   expect(await createYamlLocalPipeline(config)).toMatchSnapshot();
 });

package/examples/referencing-other-vars.ts

@@ -0,0 +1,83 @@
+import type { Config } from "../src";
+
+const config: Config = {
+  appName: "test-app",
+  customerName: "pan",
+  components: {
+    app1: {
+      dir: "app1",
+      build: {
+        type: "node",
+      },
+      vars: {
+        secret: ["SECRET1"],
+        public: {
+          foo: "foo-value",
+          bar: "bar-value",
+          foo3: "from app3: ${app3:foo3}",
+          circle:
+            'this is from app3 that has reference to app1: "${app3:transitive}"', // not officially recommended, but may work in some cases
+        },
+      },
+      deploy: {
+        type: "google-cloudrun",
+        projectId: "asdf",
+        region: "asia-east1",
+      },
+    },
+    app2: {
+      dir: "app2",
+      build: {
+        type: "node",
+      },
+      vars: {
+        secret: ["SECRET2"],
+        public: {
+          foo2: "foo-value-2",
+          referencingSecret: "secret1: ${app1:SECRET1}, secret2: ${SECRET2}",
+          foo1: "this is from app1: ${app1:foo}",
+          selfReference: "this is from self: ${foo2}",
+          selfReference2: "this is from self: ${foo1}",
+          app1Api: "${app1:ROOT_URL}/graphql",
+        },
+      },
+      deploy: {
+        type: "google-cloudrun",
+        projectId: "asdf",
+        region: "asia-east1",
+      },
+    },
+    app3: {
+      dir: "kube",
+      build: {
+        type: "node",
+      },
+      vars: {
+        public: {
+          foo3: "foo-value-3",
+          foo2: "this is from app2: ${app2:foo2}",
+          transitive: "this is from app2: ${app2:foo1}",
+          transitiveWithSecret: "this is from app2: ${app2:referencingSecret}",
+          someJson:
+            '[{"name": "app1", "url": "${app1:ROOT_URL}"}, {"name": "app2", "url": "${app2:ROOT_URL}"}, {"name": "app3", "url": "${ROOT_URL}"}]',
+        },
+      },
+      deploy: {
+        type: "kubernetes",
+        cluster: {
+          name: "some-cluster-name",
+          region: "europe-west6",
+          projectId: "some-project-id",
+          type: "gcloud",
+          domainCanonical: "panter.cloud",
+        },
+      },
+    },
+  },
+};
+
+export default config;
+
+export const information = {
+  title: "Multiline Environment Variables",
+};

package/examples/workspace-api-www-turbo-cache.test.ts

@@ -0,0 +1,11 @@
+import { createYamlLocalPipeline } from "./__utils__/helpers";
+import config from "./workspace-api-www-turbo-cache";
+
+/**
+ * This test is auto-generated.
+ * Modifications will be overwritten on every `yarn test` run!
+ */
+
+it("matches snapshot for workspace-api-www-turbo-cache local pipeline YAML", async () => {
+  expect(await createYamlLocalPipeline(config)).toMatchSnapshot();
+});

package/examples/{workspace-api-www-custom-cache.ts → workspace-api-www-turbo-cache.ts}

@@ -12,13 +12,12 @@ const config: Config = {
     myWorkspace: {
       type: "node",
       cache: {
-        paths: [".my-cache"],
+        paths: [".turbo"],
       },
     },
   },
   components: {
     api: {
-      dotEnv: true,
       dir: "api",
       build: {
         from: "myWorkspace",
@@ -29,10 +28,12 @@ const config: Config = {
       deploy: DEPLOY_CONFIG,
     },
     www: {
-      dotEnv: true,
       dir: "www",
       build: {
         from: "myWorkspace",
+        cache: {
+          paths: [".next/cache"],
+        },
       },
       deploy: DEPLOY_CONFIG,
       vars: {

package/examples/workspace-api-www.ts

@@ -15,7 +15,6 @@ const config: Config = {
   },
   components: {
     api: {
-      dotEnv: true,
       dir: "api",
       build: {
         from: "myWorkspace",
@@ -23,10 +22,12 @@ const config: Config = {
       deploy: DEPLOY_CONFIG,
     },
     www: {
-      dotEnv: true,
       dir: "www",
       build: {
         from: "myWorkspace",
+        cache: {
+          paths: [".next/cache"],
+        },
       },
       deploy: DEPLOY_CONFIG,
       vars: {

package/package.json

@@ -53,11 +53,10 @@
     }
   ],
   "license": "MIT",
-  "version": "1.170.0",
+  "version": "2.0.0",
   "scripts": {
     "build:tsc": "yarn tsc",
-    "build": "yarn build:compile && yarn build:inline-variables && yarn build:bundle",
-    "build:bundle": "./scripts/bundle",
+    "build": "yarn build:compile && yarn build:inline-variables",
     "build:compile": "yarn tsc",
     "build:inline-variables": "yarn babel --keep-file-extension --extensions \".ts,.js\" --config-file ./babel-inline.json dist --out-dir dist",
     "build:watch": "yarn tsc -w",
@@ -69,9 +68,6 @@
   "types": "dist/index.d.ts",
   "main": "dist/index.js",
   "sideEffects": false,
-  "bin": {
-    "catladder-gitlab": "./bin/catladder-gitlab.js"
-  },
   "devDependencies": {
     "@babel/cli": "^7.16.0",
     "@babel/core": "^7.0.0-0",

package/src/bash/BashExpression.ts

@@ -12,6 +12,16 @@ export class BashExpression {
     return this.value.toString();
   }
 
+  public replace(
+    searchValue: any,
+
+    replacer: (substring: string, ...args: any[]) => string,
+  ) {
+    return new BashExpression(
+      this.value.toString().replace(searchValue, replacer),
+    );
+  }
+
   /**
    *
    * @returns a bash expression to lowercase the string
@@ -66,16 +76,3 @@ export const joinBashExpressions = (
     return parts.join(joiner);
   }
 };
-
-/**
- * escapes a string or bash expression for bash
- * it mainly escapes double quotes
- */
-export const bashEscape = (value: StringOrBashExpression | any) => {
-  if (value instanceof BashExpression) {
-    return value.toString(); // no need to escape bash expressions, as we want them to evaluate
-  }
-  // we wrap it in double quotes, so we need to escape them
-  if (typeof value === "string") return value.replace(/"/g, '\\"');
-  return value;
-};

package/src/bash/bashEscape.ts

@@ -0,0 +1,158 @@
+import { registerGlobalScriptFunction } from "../globalScriptFunctions";
+import type { VariableValue } from "../variables/VariableValue";
+import { VariableValueContainingReferences } from "../variables/VariableValueContainingReferences";
+import { BashExpression } from "./BashExpression";
+
+export type QuotesEscapeMode = "single" | "double";
+
+export type EscapeOptions = {
+  quotes?: QuotesEscapeMode | false;
+};
+/**
+ * escapes a string or bash expression for bash
+ * it either can escape single or double quotes (double is default)
+ */
+export const bashEscape = (
+  value: VariableValue | any,
+  options: EscapeOptions = {
+    quotes: "double",
+  },
+) => {
+  if (value instanceof BashExpression) {
+    // no need to escape
+    return escapeBashExpression(value, options);
+  }
+  if (value instanceof VariableValueContainingReferences) {
+    return value.toString(options);
+  }
+  return escapeString(value, options);
+};
+
+export const escapeString = (
+  value: string | null | undefined,
+  { quotes }: EscapeOptions = {
+    quotes: "double",
+  },
+) => {
+  const quoteEscaped = quotes
+    ? quotes === "single"
+      ? escapeSingleQuotes(value)
+      : escapeDoubleQuotes(value)
+    : value;
+
+  return quoteEscaped;
+};
+
+export const escapeBashExpression = (
+  value: BashExpression,
+  options: EscapeOptions,
+) => {
+  // no need to escape, we just return the string
+  return value;
+};
+
+export const escapeDoubleQuotes = (value: string | null | undefined) =>
+  value?.toString().replace(/"/g, '\\"');
+
+export const escapeSingleQuotes = (value: string | null | undefined) =>
+  value?.toString().replace(/'/g, "\\'");
+
+/**
+ *
+ * escape env vars for .env files.
+ * unfortunatly, the format has many limitations. In order to be very forgiving, we need to do some magic here:
+ *
+ * - when the value contains no newlines, we are fine
+ * - if the value contains newlines, we need to wrap it in quotes. And thats where the problem begins:
+ * - you can't escape quotes. this is a limitation of dotenv and node
+ * - you can have inner quotes, but they break in node.js (not in dotenv though), see https://github.com/nodejs/node/issues/54134
+ * - so we need to quote cleverly
+ * - to make things worse, we need to check whether we have a simple stirng or a bash expression, that needs to be evalulated first...
+ *
+ *  what an absolute nightmare.
+ *
+ * - other languages are currently only partially supported, since most .env implementations are slightly different
+ */
+export const escapeForDotEnv = (
+  value: VariableValue | undefined | null,
+): string => {
+  if (value === undefined || value === null) {
+    return "";
+  }
+  if (typeof value === "string") {
+    // if string contains newlines, we need to wrap it in quotes
+    // we additionaly escape newlines, that give best compatibility
+    if (value.includes("\n")) {
+      const newlinesReplaces = value.replace(/\n/g, "\\n");
+
+      // default to ", but if this is not possible, we try to use ' or `
+      const quote = value.includes(`"`)
+        ? value.includes(`'`)
+          ? value.includes("`")
+            ? // If all quote types are present, default to double quotes. This works in dotenv, but not in node.js because of the bug mentioned abouve
+              '"'
+            : "`"
+          : "'"
+        : '"';
+
+      // if we found a quote, we can wrap the string in it
+      return `${quote}${newlinesReplaces}${quote}`;
+    } else {
+      // otherwise we can return as is
+      return value;
+    }
+  } else if (value instanceof BashExpression) {
+    return escapeBashExpressionForDotEnv(value);
+  } else if (value instanceof VariableValueContainingReferences) {
+    // instead of doing it part-wise, we just do it all at once
+
+    const containsAnyBashExpression = value.parts.some(
+      (part) => part instanceof BashExpression,
+    );
+    if (!containsAnyBashExpression) {
+      return escapeForDotEnv(
+        value.toString({
+          quotes: "double",
+        }),
+      );
+    } else {
+      const result = escapeBashExpressionForDotEnv(
+        new BashExpression(
+          value.toString({
+            quotes: "double",
+          }),
+        ),
+      );
+
+      return result;
+    }
+  } else {
+    return value;
+  }
+};
+// basically the same thing as above for bash
+// thx chatgpt for this
+const escapeForDotEnvScript = registerGlobalScriptFunction(
+  "escapeForDotEnv",
+  `
+ input="\${1:-$(cat)}"
+ input="\${input//$'\\n'/\\\\n}"
+  if [[ "$input" == *\\\\n* ]]; then
+    if [[ "$input" == *\\"* && "$input" == *\\'* && "$input" == *\\\`* ]]; then
+      printf "\\"%s\\"\\n" "$input" ${/* fallback to double quotes */ ""}
+    elif [[ "$input" == *\\"* && "$input" == *\\'* ]]; then
+      printf "\`%s\`\\n" "$input"
+    elif [[ "$input" == *\\"* ]]; then
+      printf "'%s'\\n" "$input"
+    else
+      printf "\\"%s\\"\\n" "$input"
+    fi
+  else
+    printf "%s\\n" "$input"
+  fi
+  `,
+);
+
+const escapeBashExpressionForDotEnv = (value: BashExpression) => {
+  return value.transformWithCommand(escapeForDotEnvScript.name).toString();
+};

package/src/bash/bashYaml.ts

@@ -1,6 +1,7 @@
 import type { ScalarTag } from "yaml";
 import { stringify } from "yaml";
 import { BashExpression } from "./BashExpression";
+import { VariableValueContainingReferences } from "../variables/VariableValueContainingReferences";
 
 const bashExpressionType: ScalarTag = {
   tag: "",
@@ -12,12 +13,45 @@ const bashExpressionType: ScalarTag = {
     // we create a BLOCK_LITERAL
     // but because bash will interpret the value, it may resolve to a multiline string
     // so we need to indent every line using sed
-    return "|-\n" + node.value.transformWithCommand(`sed 's/^/${ctx.indent}/'`);
+    return (
+      "|-\n" +
+      ctx.indent +
+      indentNewlinesInBashExpression(node.value, ctx.indent)
+    );
   },
 
   identify: (v: any) => v instanceof BashExpression,
 };
 
+const indentNewlinesInBashExpression = (
+  expression: BashExpression,
+  indent: string,
+) => {
+  return expression.transformWithCommand(`sed '1!s/^/${indent}/'`);
+};
+
+const variableContainingReferences: ScalarTag = {
+  tag: "",
+  resolve: (str) => {
+    // not really needed,but let's make typescript happy
+    return new VariableValueContainingReferences(str);
+  },
+  stringify(node: any, ctx: any) {
+    const value: VariableValueContainingReferences = node.value;
+    const stringified = value.parts
+      .map((part) =>
+        part instanceof BashExpression
+          ? indentNewlinesInBashExpression(part, ctx.indent)
+          : // indent every new line
+            part.toString().replace(/\n/g, `\n${ctx.indent}`),
+      )
+      .join("");
+    return "|-\n" + ctx.indent + stringified;
+  },
+
+  identify: (v: any) => v instanceof VariableValueContainingReferences,
+};
+
 /***
  * creates a yaml string that can be used in bash scripts
  * it handles BashExpressions correctly so that they can be evaluated in bash
@@ -26,7 +60,7 @@ export const yamlBashString = (value: any) => {
   return stringify(value, {
     defaultStringType: "BLOCK_LITERAL",
     defaultKeyType: "PLAIN",
-    customTags: [bashExpressionType],
+    customTags: [bashExpressionType, variableContainingReferences],
     aliasDuplicateObjects: false,
     lineWidth: 0,
   });

package/src/bash/getInjectVarsScript.ts

@@ -1,6 +1,6 @@
 import type { UnspecifiedEnvVars } from "../types";
 import { isNil } from "lodash";
-import { bashEscape } from "./BashExpression";
+import { bashEscape } from "./bashEscape";
 
 export const getInjectVarsScript = (vars?: UnspecifiedEnvVars) => {
   if (!vars) return [];
@@ -8,5 +8,14 @@ export const getInjectVarsScript = (vars?: UnspecifiedEnvVars) => {
   return Object.entries(vars)
     .filter(([, value]) => !isNil(value))
 
-    .map(([key, value]) => `export ${key}="${value ? bashEscape(value) : ""}"`);
+    .map(
+      ([key, value]) =>
+        `export ${key}="${
+          value
+            ? bashEscape(value, {
+                quotes: "double",
+              })
+            : ""
+        }"`,
+    );
 };

package/src/bash/index.ts

@@ -0,0 +1,2 @@
+export * from "./bashEscape";
+export * from "./BashExpression";

package/src/build/base/createAppBuildJob.ts

@@ -39,7 +39,6 @@ export const createAppBuildJob = (
           ? {
               ...context.environment.envVars,
               ...context.environment.jobOnlyVars.build.envVars,
-              ...(context.build.config.extraVars ?? {}),
             }
           : {}),
       },

package/src/build/base/writeDotEnv.ts

@@ -1,7 +1,8 @@
 import { isNil } from "lodash";
+import { escapeForDotEnv } from "../../bash/bashEscape";
+import { ALL_BUILD_VARIABLES } from "../../context/getBuildInfoVariables";
 import type { ComponentContext } from "../../types";
 import { collapseableSection } from "../../utils/gitlab";
-import { ALL_BUILD_VARIABLES } from "../../context/getBuildInfoVariables";
 
 /**
  * writes a .env file in the components folder
@@ -18,14 +19,13 @@ export const writeDotEnv = (context: ComponentContext) => {
     // filter out build variables, since they may interfer with caching like turbo
     // build variables are rarely used anyway and we may treat them differently in the future
     .filter(([key]) => !ALL_BUILD_VARIABLES.includes(key))
-    .map(
-      ([key, value]) => `${key}=${value?.toString().replaceAll("\n", "\\n")}`,
-    )
+
+    .map(([key, value]) => `${key}=${escapeForDotEnv(value)}`)
     .join("\n");
 
   return collapseableSection(
     "write-dotenv-" + context.name,
-    "write dot env",
+    "write dot env for " + context.name,
   )([
     `cat <<EOF > ${context.build.dir}/.env
 ${keyValueString}
@@ -36,5 +36,5 @@ EOF`,
 export const componentContextNeedsBuildTimeDotEnv = (
   context: ComponentContext,
 ) => {
-  return context.componentConfig.dotEnv === true; // don't build when set to `local`
+  return (context.componentConfig.dotEnv ?? true) === true; // don't build when set to `local`
 };