@catladder/pipeline 0.0.0-refactore-catenv-command-validates-argument-78d05852 → 0.0.0-reference-database-url-1befbbee

package/examples/workspace-api-www.ts

@@ -0,0 +1,46 @@
+import type { Config, DeployConfig } from "../src";
+
+const DEPLOY_CONFIG: DeployConfig = {
+  type: "google-cloudrun",
+  projectId: "google-project-id",
+  region: "europe-west6",
+};
+const config = {
+  appName: "test-app",
+  customerName: "pan",
+  builds: {
+    myWorkspace: {
+      type: "node",
+    },
+  },
+  components: {
+    api: {
+      dir: "api",
+      build: {
+        from: "myWorkspace",
+      },
+      deploy: DEPLOY_CONFIG,
+    },
+    www: {
+      dir: "www",
+      build: {
+        from: "myWorkspace",
+        cache: {
+          paths: [".next/cache"],
+        },
+      },
+      deploy: DEPLOY_CONFIG,
+      vars: {
+        public: {
+          API_URL: "${api:ROOT_URL}/graphql",
+        },
+      },
+    },
+  },
+} satisfies Config;
+
+export default config;
+
+export const information = {
+  title: "Workspace API and WWW",
+};

package/package.json

@@ -1,8 +1,9 @@
 {
   "name": "@catladder/pipeline",
-  "description": "CI/CD and Devops-Workflow",
-  "homepage": "https://git.panter.ch/catladder/catladder/#readme",
+  "description": "Panter workflow for cloud CI/CD and DevOps",
+  "homepage": "https://catladder.git.panter.biz/catladder/docs",
   "bugs": "https://git.panter.ch/catladder/catladder/issues",
+  "readme": "https://git.panter.ch/catladder/catladder/-/blob/main/pipeline/README.md",
   "repository": {
     "type": "git",
     "url": "git+https://git.panter.ch/catladder/catladder.git",
@@ -52,48 +53,44 @@
     }
   ],
   "license": "MIT",
-  "version": "0.0.0-refactore-catenv-command-validates-argument-78d05852",
+  "version": "0.0.0-reference-database-url-1befbbee",
   "stableVersion": "0.1.0",
   "scripts": {
     "build:tsc": "yarn tsc",
-    "build": "yarn build:compile && yarn build:inline-variables && yarn build:bundle",
-    "build:bundle": "./scripts/bundle",
+    "build": "yarn build:compile && yarn build:inline-variables",
     "build:compile": "yarn tsc",
     "build:inline-variables": "yarn babel --keep-file-extension --extensions \".ts,.js\" --config-file ./babel-inline.json dist --out-dir dist",
     "build:watch": "yarn tsc -w",
     "lint": "eslint \"src/**/*.ts\"",
     "lint:fix": "eslint \"src/**/*.ts\" --fix",
-    "generate:gitlab-ci-yml-types": "ts-node ./scripts/generate-gitlab-ci-types.ts"
+    "generate:gitlab-ci-yml-types": "ts-node ./scripts/generate-gitlab-ci-types.ts",
+    "generate:examples-test": "tsx scripts/generate-examples-test.ts"
   },
   "types": "dist/index.d.ts",
   "main": "dist/index.js",
   "sideEffects": false,
-  "bin": {
-    "catladder-gitlab": "./bin/catladder-gitlab.js"
-  },
   "devDependencies": {
-    "@babel/cli": "^7.16.0",
-    "@babel/core": "^7.0.0-0",
-    "@babel/plugin-syntax-typescript": "^7.16.7",
-    "@babel/preset-typescript": "^7.16.5",
-    "@jest/globals": "^29.3.1",
-    "@types/child-process-promise": "^2.2.2",
-    "@types/jest": "^27.4.0",
-    "@types/lodash": "^4.14.185",
-    "@types/node": "^17.0.10",
+    "@babel/cli": "^7.24.8",
+    "@babel/core": "^7.25.2",
+    "@babel/plugin-syntax-typescript": "^7.25.4",
+    "@babel/preset-typescript": "^7.24.7",
+    "@jest/globals": "^29.7.0",
+    "@types/child-process-promise": "^2.2.6",
+    "@types/jest": "^29.5.12",
+    "@types/lodash": "^4.17.7",
+    "@types/node": "^22.5.1",
     "babel-plugin-transform-inline-environment-variables": "^0.4.3",
     "child-process-promise": "^2.2.1",
-    "eslint": "^8.7.0",
-    "js-yaml": "^4.1.0",
+    "glob": "^11.0.0",
     "json-schema-to-typescript": "^13.0.2",
     "lodash": "^4.17.21",
-    "memoizee": "^0.4.15",
-    "path-equal": "1.1.3",
-    "slugify": "^1.6.3"
+    "memoizee": "^0.4.17",
+    "path-equal": "1.2.5",
+    "slugify": "^1.6.6",
+    "typescript": "^5.4.5",
+    "yaml": "^2.5.0"
   },
   "dependencies": {
-    "string-replace-async": "^2.0.0",
-    "ts-node": "^10.4.0",
-    "typescript": "^4.5.4"
+    "tsx": "^4.19.0"
   }
 }

package/scripts/generate-examples-test.ts

@@ -0,0 +1,84 @@
+import { glob } from "glob";
+import { resolve, join, basename } from "path";
+import { existsSync } from "fs";
+import { readFile, writeFile, rm } from "fs/promises";
+import { format } from "prettier";
+
+const pipelineWorkspaceRoot = resolve(__dirname, "..");
+const examplesPath = join(pipelineWorkspaceRoot, "examples");
+const exampleIgnoresPath = join(examplesPath, ".test-gen-ignore");
+
+const defaultExampleIgnores = ["*.test.ts"];
+
+async function getExampleIgnores(): Promise<string[]> {
+  if (!existsSync(exampleIgnoresPath)) {
+    await writeFile(exampleIgnoresPath, defaultExampleIgnores.join("\n"));
+  }
+  try {
+    const ignoresFile = await readFile(exampleIgnoresPath, {
+      encoding: "utf-8",
+    });
+    return [
+      ...new Set([
+        ...defaultExampleIgnores,
+        ...ignoresFile
+          .split("\n")
+          .filter((line) => !line.match(/^\s*# [\s\w]*/))
+          .map((line) => line.trim())
+          .filter(Boolean),
+      ]),
+    ];
+  } catch (error) {
+    console.error(
+      `Error reading example ignores file: ${exampleIgnoresPath}`,
+      error,
+    );
+    return defaultExampleIgnores;
+  }
+}
+
+async function main() {
+  const exampleIgnores = await getExampleIgnores();
+  const [exampleFiles, existingTestFiles] = await Promise.all([
+    glob("*.ts", {
+      ignore: exampleIgnores,
+      cwd: examplesPath,
+    }),
+    glob("*.test.ts", {
+      ignore: ["rails-k8s-with-worker-dockerfile.test.ts"],
+      cwd: examplesPath,
+    }),
+  ]);
+  await Promise.all(
+    existingTestFiles.map((testFile) => rm(join(examplesPath, testFile))),
+  );
+  await Promise.all(
+    exampleFiles.map(async (exampleFile) => {
+      const kebabName = basename(exampleFile, ".ts");
+      const testFileContent = [
+        `import { createYamlLocalPipeline } from "./__utils__/helpers";`,
+        `import config from "./${kebabName}";`,
+        `\n/**`,
+        ` * This test is auto-generated.`,
+        ` * Modifications will be overwritten on every \`yarn test\` run!`,
+        ` */\n`,
+
+        `it("matches snapshot for ${kebabName} local pipeline YAML", async () => {`,
+        `  expect(await createYamlLocalPipeline(config)).toMatchSnapshot();`,
+        `});\n`,
+      ].join("\n");
+      const formattedTest = await format(testFileContent, {
+        parser: "typescript",
+      });
+      await writeFile(
+        join(examplesPath, `${kebabName}.test.ts`),
+        formattedTest,
+        {
+          encoding: "utf-8",
+        },
+      );
+    }),
+  );
+}
+
+main().catch(console.error);

package/scripts/generate-gitlab-ci-types.ts

@@ -8,7 +8,7 @@ const typeOutputPath = join(rootPath, "src/types/gitlab-ci-yml.ts");
 
 export const generateFromCurrentSchema = async () => {
   const response = await fetch(
-    "https://gitlab.com/gitlab-org/gitlab/-/raw/master/app/assets/javascripts/editor/schema/ci.json"
+    "https://gitlab.com/gitlab-org/gitlab/-/raw/master/app/assets/javascripts/editor/schema/ci.json",
   );
   const gitLabCiJsonSchema = await response.json();
   const schema = await compile(gitLabCiJsonSchema, "GitlabCiYml", {
@@ -40,7 +40,7 @@ export const generateFromCurrentSchema = async () => {
   const sanitizedSchenma = schema.replace(
     // eslint-disable-next-line no-useless-escape
     /export interface HttpsGitlabComGitlabCiYml \{\n[ a-zA-Z0-9?$:;\n_!"\{\}\/*@\[\],.|\(\)]*\[k\: string\]: Job;\n\}/m,
-    ""
+    "",
   );
   if (existsSync(typeOutputPath)) {
     await rm(typeOutputPath);

package/scripts/tsconfig.json

@@ -0,0 +1,7 @@
+{
+  "extends": "../tsconfig.json",
+  "compilerOptions": {
+    "target": "ES2020"
+  },
+  "include": ["**/*.ts"]
+}

package/src/bash/BashExpression.ts

@@ -0,0 +1,78 @@
+/**
+ * this class represents a string that can be evaluated in bash scripts.
+ *
+ * you can do basic transforms like lowercase, but that does not return you a lowercase string, but instead an experssion represeting a lowercase string
+ */
+export class BashExpression {
+  constructor(private value: string | BashExpression) {}
+  public toJSON(): string {
+    return this.toString();
+  }
+  public toString(): string {
+    return this.value.toString();
+  }
+
+  public replace(
+    searchValue: any,
+
+    replacer: (substring: string, ...args: any[]) => string,
+  ) {
+    return new BashExpression(
+      this.value.toString().replace(searchValue, replacer),
+    );
+  }
+
+  /**
+   *
+   * @returns a bash expression to lowercase the string
+   */
+  public toLowerCase(): BashExpression {
+    return this.transformWithCommand("awk '{print tolower($0)}'");
+  }
+  /**
+   * concats a value to this one and returns a new BashExpression
+   * @param value
+   * @returns
+   */
+  public concat(...values: Array<string | BashExpression>): BashExpression {
+    return new BashExpression(
+      this.toString().concat(...values.map((v) => v.toString())),
+    );
+  }
+
+  public transformWithCommand(command: string): BashExpression {
+    return new BashExpression(
+      // see https://stackoverflow.com/a/2264537
+      `$(printf %s "${this.toString()}" | ${command})`,
+    );
+  }
+}
+
+export type StringOrBashExpression = string | BashExpression;
+
+export const getBashVariable = (name: string) => new BashExpression(`$${name}`);
+
+/**
+ * joins bash expressions together with a joiner
+ * returns a bash expression if any of the parts is a bash expression
+ * returns a string otherwise
+ *
+ * @param parts
+ * @param joiner
+ * @returns
+ */
+export const joinBashExpressions = (
+  parts: Array<StringOrBashExpression>,
+  joiner = "",
+): StringOrBashExpression => {
+  const anyIsBashExpression = parts.some((p) => p instanceof BashExpression);
+  if (anyIsBashExpression) {
+    return new BashExpression(
+      parts
+        .map((p) => (p instanceof BashExpression ? p.toString() : p))
+        .join(joiner),
+    );
+  } else {
+    return parts.join(joiner);
+  }
+};

package/src/bash/bashEscape.ts

@@ -0,0 +1,164 @@
+import { registerGlobalScriptFunction } from "../globalScriptFunctions";
+import type { VariableValue } from "../variables/VariableValue";
+import { VariableValueContainingReferences } from "../variables/VariableValueContainingReferences";
+import { BashExpression } from "./BashExpression";
+
+export type QuotesEscapeMode = "single" | "double";
+
+export type EscapeOptions = {
+  quotes?: QuotesEscapeMode | false;
+};
+/**
+ * escapes a string or bash expression for bash
+ * it either can escape single or double quotes (double is default)
+ */
+export const bashEscape = (
+  value: VariableValue | any,
+  options: EscapeOptions = {
+    quotes: "double",
+  },
+) => {
+  if (value instanceof BashExpression) {
+    // no need to escape
+    return escapeBashExpression(value, options);
+  }
+  if (value instanceof VariableValueContainingReferences) {
+    return value.toString(options);
+  }
+  return escapeString(value, options);
+};
+
+export const escapeString = (
+  value: string | null | undefined,
+  { quotes }: EscapeOptions = {
+    quotes: "double",
+  },
+) => {
+  const quoteEscaped = quotes
+    ? quotes === "single"
+      ? escapeSingleQuotes(value)
+      : escapeDoubleQuotes(value)
+    : value;
+
+  return quoteEscaped;
+};
+
+export const escapeBashExpression = (
+  value: BashExpression,
+  options: EscapeOptions,
+) => {
+  // no need to escape, we just return the string
+  return value;
+};
+
+export const escapeDoubleQuotes = (value: string | null | undefined) =>
+  value?.toString().replace(/"/g, '\\"');
+
+export const escapeSingleQuotes = (value: string | null | undefined) =>
+  value?.toString().replace(/'/g, "\\'");
+
+export type EscapeForDotEnvOptions = {
+  quoteMode: "auto" | "always";
+};
+/**
+ *
+ * escape env vars for .env files.
+ * unfortunatly, the format has many limitations. In order to be very forgiving, we need to do some magic here:
+ *
+ * - when the value contains no newlines, we are fine
+ * - if the value contains newlines, we need to wrap it in quotes. And thats where the problem begins:
+ * - you can't escape quotes. this is a limitation of dotenv and node
+ * - you can have inner quotes, but they break in node.js (not in dotenv though), see https://github.com/nodejs/node/issues/54134
+ * - so we need to quote cleverly
+ * - to make things worse, we need to check whether we have a simple stirng or a bash expression, that needs to be evalulated first...
+ *
+ *  what an absolute nightmare.
+ *
+ * - other languages are currently only partially supported, since most .env implementations are slightly different
+ */
+export const escapeForDotEnv = (
+  value: VariableValue | undefined | null,
+  options: EscapeForDotEnvOptions = {
+    quoteMode: "auto",
+  },
+): string => {
+  if (value === undefined || value === null) {
+    return "";
+  }
+  if (typeof value === "string") {
+    // if string contains newlines, we need to wrap it in quotes
+    // we additionaly escape newlines, that give best compatibility
+    if (options.quoteMode === "always" || value.includes("\n")) {
+      const newlinesReplaces = value.replace(/\n/g, "\\n");
+
+      // default to ", but if this is not possible, we try to use ' or `
+      const quote = value.includes(`"`)
+        ? value.includes(`'`)
+          ? value.includes("`")
+            ? // If all quote types are present, default to double quotes. This works in dotenv, but not in node.js because of the bug mentioned abouve
+              '"'
+            : "`"
+          : "'"
+        : '"';
+
+      // if we found a quote, we can wrap the string in it
+      return `${quote}${newlinesReplaces}${quote}`;
+    } else {
+      // otherwise we can return as is
+      return value;
+    }
+  } else if (value instanceof BashExpression) {
+    return escapeBashExpressionForDotEnv(value);
+  } else if (value instanceof VariableValueContainingReferences) {
+    // instead of doing it part-wise, we just do it all at once
+
+    const containsAnyBashExpression = value.parts.some(
+      (part) => part instanceof BashExpression,
+    );
+    if (!containsAnyBashExpression) {
+      return escapeForDotEnv(
+        value.toString({
+          quotes: "double",
+        }),
+      );
+    } else {
+      const result = escapeBashExpressionForDotEnv(
+        new BashExpression(
+          value.toString({
+            quotes: "double",
+          }),
+        ),
+      );
+
+      return result;
+    }
+  } else {
+    return value;
+  }
+};
+// basically the same thing as above for bash
+// thx chatgpt for this
+const escapeForDotEnvScript = registerGlobalScriptFunction(
+  "escapeForDotEnv",
+  `
+ input="\${1:-$(cat)}"
+ input="\${input//$'\\n'/\\\\n}"
+  if [[ "$input" == *\\\\n* ]]; then
+    if [[ "$input" == *\\"* && "$input" == *\\'* && "$input" == *\\\`* ]]; then
+      printf "\\"%s\\"\\n" "$input" ${/* fallback to double quotes */ ""}
+    elif [[ "$input" == *\\"* && "$input" == *\\'* ]]; then
+      printf "\`%s\`\\n" "$input"
+    elif [[ "$input" == *\\"* ]]; then
+      printf "'%s'\\n" "$input"
+    else
+      printf "\\"%s\\"\\n" "$input"
+    fi
+  else
+    printf "%s\\n" "$input"
+  fi
+  `,
+);
+
+const escapeBashExpressionForDotEnv = (value: BashExpression) => {
+  return value.transformWithCommand(escapeForDotEnvScript.name).toString();
+};

package/src/bash/bashExpressionPerPipelineType.ts

@@ -0,0 +1,16 @@
+import type { PipelineType } from "../types/pipeline";
+import type { BashExpression } from "./BashExpression";
+
+export type BashExpressionPerPipelineType = {
+  [type in PipelineType]?: BashExpression | string;
+} & {
+  default: BashExpression | string;
+};
+
+export const getBashExpressionPerPipelineType = (
+  definition: BashExpressionPerPipelineType,
+  pipelineType?: PipelineType,
+) => {
+  if (!pipelineType) return definition.default;
+  return definition[pipelineType] || definition.default;
+};

package/src/bash/bashYaml.ts

@@ -0,0 +1,80 @@
+import type { ScalarTag } from "yaml";
+import { stringify } from "yaml";
+import { BashExpression } from "./BashExpression";
+import { VariableValueContainingReferences } from "../variables/VariableValueContainingReferences";
+
+const bashExpressionType: ScalarTag = {
+  tag: "",
+  resolve: (str) => {
+    // not really needed,but let's make typescript happy
+    return new BashExpression(str);
+  },
+  stringify(node: any, ctx: any) {
+    // we create a BLOCK_LITERAL
+    // but because bash will interpret the value, it may resolve to a multiline string
+    // so we need to indent every line using sed
+    return (
+      "|-\n" +
+      ctx.indent +
+      indentNewlinesInBashExpression(node.value, ctx.indent)
+    );
+  },
+
+  identify: (v: any) => v instanceof BashExpression,
+};
+
+const indentNewlinesInBashExpression = (
+  expression: BashExpression,
+  indent: string,
+) => {
+  return expression.transformWithCommand(`sed '1!s/^/${indent}/'`);
+};
+
+const variableContainingReferences: ScalarTag = {
+  tag: "",
+  resolve: (str) => {
+    // not really needed,but let's make typescript happy
+    return new VariableValueContainingReferences(str);
+  },
+  stringify(node: any, ctx: any) {
+    const value: VariableValueContainingReferences = node.value;
+    const stringified = value.parts
+      .map((part) =>
+        part instanceof BashExpression
+          ? indentNewlinesInBashExpression(part, ctx.indent)
+          : // indent every new line
+            part.toString().replace(/\n/g, `\n${ctx.indent}`),
+      )
+      .join("");
+    return "|-\n" + ctx.indent + stringified;
+  },
+
+  identify: (v: any) => v instanceof VariableValueContainingReferences,
+};
+
+/***
+ * creates a yaml string that can be used in bash scripts
+ * it handles BashExpressions correctly so that they can be evaluated in bash
+ */
+export const yamlBashString = (value: any) => {
+  return stringify(value, {
+    defaultStringType: "BLOCK_LITERAL",
+    defaultKeyType: "PLAIN",
+    customTags: [bashExpressionType, variableContainingReferences],
+    aliasDuplicateObjects: false,
+    lineWidth: 0,
+  });
+};
+
+/**
+ * creates a bash script that writes a yaml string to a file
+ * @param value the yaml value to write, it supports BashExpressions
+ */
+export const writeBashYamlToFileScript = (value: any, filename: string) => {
+  return [
+    `cat > ${filename} <<EOF
+${yamlBashString(value)}
+EOF
+`,
+  ];
+};

package/src/bash/getInjectVarsScript.ts

@@ -0,0 +1,21 @@
+import type { UnspecifiedEnvVars } from "../types";
+import { isNil } from "lodash";
+import { bashEscape } from "./bashEscape";
+
+export const getInjectVarsScript = (vars?: UnspecifiedEnvVars) => {
+  if (!vars) return [];
+  // filter out null and undefined values
+  return Object.entries(vars)
+    .filter(([, value]) => !isNil(value))
+
+    .map(
+      ([key, value]) =>
+        `export ${key}="${
+          value
+            ? bashEscape(value, {
+                quotes: "double",
+              })
+            : ""
+        }"`,
+    );
+};

package/src/bash/index.ts

@@ -0,0 +1,2 @@
+export * from "./bashEscape";
+export * from "./BashExpression";

package/src/build/artifacts/createBuildJobArtifact.ts

@@ -0,0 +1,63 @@
+import { join } from "path";
+import {
+  componentContextIsStandaloneBuild,
+  type ComponentContext,
+  type Context,
+} from "../../types";
+import type { CatladderJob } from "../../types/jobs";
+import { uniq } from "lodash";
+import { componentContextNeedsBuildTimeDotEnv } from "../base/writeDotEnv";
+
+export const createBuildJobArtifacts = (
+  context: Context,
+): CatladderJob["artifacts"] => {
+  const paths =
+    context.type === "workspace"
+      ? context.components.flatMap((c) => getArtifactsPathForComponent(c))
+      : getArtifactsPathForComponent(context, ["__build_info.json"]);
+  // esxclude .env files if generated
+  const exclude =
+    context.type === "workspace"
+      ? context.components.flatMap((c) => getDotEnvPathsForComponent(c))
+      : getDotEnvPathsForComponent(context);
+  return {
+    paths: uniq(paths).sort((a, b) => a.localeCompare(b)),
+    ...(exclude.length > 0 ? { exclude } : {}),
+    expire_in: "1 day",
+    when: "always",
+    reports:
+      // TODO: support for junit reports in other builds
+      context.type === "component" && componentContextIsStandaloneBuild(context)
+        ? {
+            junit: context.build.config.artifactsReports?.junit?.map((p) =>
+              join(context.build.dir, p),
+            ),
+          }
+        : {},
+  };
+};
+const getArtifactsPathForComponent = (
+  c: ComponentContext,
+  additionalPaths?: string[],
+): string[] => {
+  return [
+    ...(c.build.type !== "disabled"
+      ? (c.build.config.artifactsPaths ?? [])
+      : []),
+    ...(additionalPaths ?? []),
+  ]?.flatMap((artifact) =>
+    c.build
+      // in theory, we only need "direct",
+      // but in some cases project may have packages in the workspace that create build artifacts, which aren't components
+      // this highly depends on the build tool. To be safe, we get all
+      .getComponentDirs("all")
+      .flatMap((cDir) => join(cDir, artifact)),
+  );
+};
+
+const getDotEnvPathsForComponent = (c: ComponentContext): string[] => {
+  if (componentContextNeedsBuildTimeDotEnv(c)) {
+    return [join(c.build.dir, ".env")];
+  }
+  return [];
+};