@catladder/pipeline 0.0.0-node16-ddfae617 → 0.0.0-perf-cli-reduce-bundle-size-with-tsconfig-optimization-3556aaa4

package/examples/rails-k8s-with-worker.ts

@@ -0,0 +1,95 @@
+import type { Config } from "../src";
+
+// the image version should match your `.ruby-version`
+const RAILS_TEST_STAGE_IMAGE = "ruby:3.2.1";
+
+const config: Config = {
+  appName: "test-app",
+  customerName: "pan",
+  components: {
+    app: {
+      dir: ".",
+      vars: {
+        public: {
+          RAILS_ENV: "production",
+        },
+        secret: ["SECRET_KEY_BASE"],
+      },
+      build: {
+        type: "rails",
+        test: {
+          // if operating system dependencies are required you can either
+          // - build a dedicated image
+          // - use `command` to apt-get install them before running the test command
+          jobImage: RAILS_TEST_STAGE_IMAGE,
+        },
+        lint: {
+          jobImage: RAILS_TEST_STAGE_IMAGE,
+        },
+        audit: {
+          jobImage: RAILS_TEST_STAGE_IMAGE,
+        },
+        // only needed if you do not have a Dockerfile
+        cnbBuilder: {
+          buildVars: { SECRET_KEY_BASE: "dummy-value" },
+        },
+      },
+      deploy: {
+        type: "kubernetes",
+        cluster: {
+          name: "some-cluster-name",
+          region: "europe-west6",
+          projectId: "some-project-id",
+          type: "gcloud",
+          domainCanonical: "panter.cloud",
+        },
+        values: {
+          cloudsql: { enabled: true, projectId: "some-project-id" },
+          application: {
+            // if deployed to Google Cloud Run use a gem like cloudtasker instead of a permanently running expensive worker
+            worker: {
+              enabled: true,
+              command: "launcher bundle exec rake jobs:work",
+              // the delayed_job gem does not have a http health endpoint
+              // disable probe to avoid worker restarts
+              livenessProbe: false,
+            },
+          },
+          jobs: {
+            "db-migrate": {
+              hook: "post-install,post-upgrade",
+              command: "launcher bundle exec rake db:migrate",
+            },
+          },
+        },
+      },
+      env: {
+        review: {
+          deploy: {
+            values: {
+              jobs: {
+                "db-prepare-seed": {
+                  hook: "post-install",
+                  command: "launcher bundle exec rake db:prepare db:seed",
+                },
+                "db-migrate": {
+                  hook: "post-upgrade",
+                  command: "launcher bundle exec rake db:migrate",
+                },
+              },
+            },
+          },
+        },
+        prod: {
+          host: "my-fancy-website.com",
+        },
+      },
+    },
+  },
+};
+
+export default config;
+
+export const information = {
+  title: "K8s: Rails with Worker",
+};

package/examples/wait-for-other-deploy.test.ts

@@ -0,0 +1,18 @@
+import {
+  createAllPipelines,
+  createYamlLocalPipeline,
+} from "./__utils__/helpers";
+import config from "./wait-for-other-deploy";
+
+/**
+ * This test is auto-generated.
+ * Modifications will be overwritten on every `yarn test` run!
+ */
+
+it("matches snapshot for wait-for-other-deploy", async () => {
+  expect(await createAllPipelines(config)).toMatchSnapshot();
+});
+
+it("matches snapshot for cloud-run-memory-limit local pipeline YAML", async () => {
+  expect(await createYamlLocalPipeline(config)).toMatchSnapshot();
+});

package/examples/wait-for-other-deploy.ts

@@ -0,0 +1,37 @@
+import type { Config } from "../src";
+
+const config: Config<{ CustomEnvs: "asdf" | "bla" }> = {
+  appName: "test-app",
+  customerName: "pan",
+  components: {
+    api: {
+      dir: "api",
+      build: {
+        type: "node",
+      },
+      deploy: {
+        type: "custom",
+        requiresDocker: false,
+        script: ["yarn deploy"],
+      },
+    },
+    www: {
+      dir: "www",
+      build: {
+        type: "node",
+      },
+      deploy: {
+        waitFor: ["api"],
+        type: "custom",
+        requiresDocker: false,
+        script: ["yarn deploy"],
+      },
+    },
+  },
+};
+
+export default config;
+
+export const information = {
+  title: "Wait for other deployment",
+};

package/package.json

@@ -1,17 +1,75 @@
 {
   "name": "@catladder/pipeline",
-  "version": "0.0.0-node16-ddfae617",
+  "description": "Panter workflow for cloud CI/CD and DevOps",
+  "homepage": "https://catladder.git.panter.biz/catladder/docs",
+  "bugs": "https://git.panter.ch/catladder/catladder/issues",
+  "readme": "https://git.panter.ch/catladder/catladder/-/blob/main/pipeline/README.md",
+  "repository": {
+    "type": "git",
+    "url": "git+https://git.panter.ch/catladder/catladder.git",
+    "directory": "pipeline"
+  },
+  "keywords": [
+    "GitLab",
+    "cloud",
+    "cli",
+    "gitlab-ci",
+    "ci/cd",
+    "pipeline",
+    "deployment",
+    "devops",
+    "automation",
+    "typescript"
+  ],
+  "author": "Marco Wettstein <maw@panter.ch>",
+  "contributors": [
+    {
+      "name": "Marco Wettstein",
+      "email": "maw@panter.ch"
+    },
+    {
+      "name": "Michael Leu",
+      "email": "leu@panter.ch"
+    },
+    {
+      "name": "Luboš Matejčík",
+      "email": "lma@panter.ch"
+    },
+    {
+      "name": "Kaspar Vollenweider",
+      "email": "vok@panter.ch"
+    },
+    {
+      "name": "Silvan Adrian",
+      "email": "sia@panter.ch"
+    },
+    {
+      "name": "Stefan Bakocs",
+      "email": "stb@panter.ch"
+    },
+    {
+      "name": "Lukas Brezina",
+      "email": "brl@panter.ch"
+    }
+  ],
+  "license": "MIT",
+  "version": "0.0.0-perf-cli-reduce-bundle-size-with-tsconfig-optimization-3556aaa4",
+  "stableVersion": "0.1.0",
   "scripts": {
     "build:tsc": "yarn tsc",
-    "build": "yarn tsc && yarn build:inline-variables",
-    "build:watch": "yarn tsc -w",
+    "build": "yarn build:compile && yarn build:inline-variables && yarn build:bundle",
+    "build:bundle": "./scripts/bundle",
+    "build:compile": "yarn tsc",
     "build:inline-variables": "yarn babel --keep-file-extension --extensions \".ts,.js\" --config-file ./babel-inline.json dist --out-dir dist",
-    "test": "jest",
+    "build:watch": "yarn tsc -w",
     "lint": "eslint \"src/**/*.ts\"",
-    "lint:fix": "eslint \"src/**/*.ts\" --fix"
+    "lint:fix": "eslint \"src/**/*.ts\" --fix",
+    "generate:gitlab-ci-yml-types": "ts-node ./scripts/generate-gitlab-ci-types.ts",
+    "generate:examples-test": "tsx scripts/generate-examples-test.ts"
   },
-  "main": "dist/index.js",
   "types": "dist/index.d.ts",
+  "main": "dist/index.js",
+  "sideEffects": false,
   "bin": {
     "catladder-gitlab": "./bin/catladder-gitlab.js"
   },
@@ -20,23 +78,24 @@
     "@babel/core": "^7.0.0-0",
     "@babel/plugin-syntax-typescript": "^7.16.7",
     "@babel/preset-typescript": "^7.16.5",
+    "@jest/globals": "^29.3.1",
     "@types/child-process-promise": "^2.2.2",
     "@types/jest": "^27.4.0",
-    "@types/lodash": "^4.14.172",
+    "@types/lodash": "^4.14.185",
+    "@types/node": "^20.14.2",
     "babel-plugin-transform-inline-environment-variables": "^0.4.3",
-    "eslint": "^8.7.0"
-  },
-  "dependencies": {
-    "@types/node": "^17.0.10",
     "child-process-promise": "^2.2.1",
-    "js-yaml": "^4.1.0",
+    "glob": "^10.4.2",
+    "json-schema-to-typescript": "^13.0.2",
     "lodash": "^4.17.21",
     "memoizee": "^0.4.15",
-    "path-equal": "^1.1.3",
+    "path-equal": "1.1.3",
     "slugify": "^1.6.3",
-    "ts-node": "^10.4.0",
-    "type-fest": "^2.10.0",
-    "typescript": "^4.5.4"
+    "tsx": "^4.16.2",
+    "typescript": "^5.4.5",
+    "yaml": "^2.4.5"
   },
-  "stableVersion": "0.1.0"
+  "dependencies": {
+    "tsx": "^4.16.2"
+  }
 }

package/scripts/bundle

@@ -0,0 +1,2 @@
+#!/bin/bash
+ncc build dist/catladder-gitlab.js -o dist/bundles/catladder-gitlab -e 'tsx/cjs/api' $( ((SHOULD_MINIFY == 1)) && printf %s '-m')

package/scripts/generate-examples-test.ts

@@ -0,0 +1,93 @@
+import { glob } from "glob";
+import { resolve, join, basename } from "path";
+import { existsSync } from "fs";
+import { readFile, writeFile, rm } from "fs/promises";
+import { format } from "prettier";
+
+const pipelineWorkspaceRoot = resolve(__dirname, "..");
+const examplesPath = join(pipelineWorkspaceRoot, "examples");
+const exampleIgnoresPath = join(examplesPath, ".test-gen-ignore");
+
+const defaultExampleIgnores = ["*.test.ts"];
+
+async function getExampleIgnores(): Promise<string[]> {
+  if (!existsSync(exampleIgnoresPath)) {
+    await writeFile(exampleIgnoresPath, defaultExampleIgnores.join("\n"));
+  }
+  try {
+    const ignoresFile = await readFile(exampleIgnoresPath, {
+      encoding: "utf-8",
+    });
+    return [
+      ...new Set([
+        ...defaultExampleIgnores,
+        ...ignoresFile
+          .split("\n")
+          .filter((line) => !line.match(/^\s*# [\s\w]*/))
+          .map((line) => line.trim())
+          .filter(Boolean),
+      ]),
+    ];
+  } catch (error) {
+    console.error(
+      `Error reading example ignores file: ${exampleIgnoresPath}`,
+      error,
+    );
+    return defaultExampleIgnores;
+  }
+}
+
+type ExampleTestLines = {
+  importLine: string;
+  testString: string;
+  kebabName: string;
+  camelName: string;
+};
+
+async function main() {
+  const exampleIgnores = await getExampleIgnores();
+  const [exampleFiles, existingTestFiles] = await Promise.all([
+    glob("*.ts", {
+      ignore: exampleIgnores,
+      cwd: examplesPath,
+    }),
+    glob("*.test.ts", {
+      ignore: ["rails-k8s-with-worker-dockerfile.test.ts"],
+      cwd: examplesPath,
+    }),
+  ]);
+  await Promise.all(
+    existingTestFiles.map((testFile) => rm(join(examplesPath, testFile))),
+  );
+  await Promise.all(
+    exampleFiles.map(async (exampleFile) => {
+      const kebabName = basename(exampleFile, ".ts");
+      const testFileContent = [
+        `import { createAllPipelines, createYamlLocalPipeline } from "./__utils__/helpers";`,
+        `import config from "./${kebabName}";`,
+        `\n/**`,
+        ` * This test is auto-generated.`,
+        ` * Modifications will be overwritten on every \`yarn test\` run!`,
+        ` */\n`,
+        `it("matches snapshot for ${kebabName}", async () => {`,
+        `  expect(await createAllPipelines(config)).toMatchSnapshot();`,
+        `});\n`,
+        `it("matches snapshot for cloud-run-memory-limit local pipeline YAML", async () => {`,
+        `  expect(await createYamlLocalPipeline(config)).toMatchSnapshot();`,
+        `});\n`,
+      ].join("\n");
+      const formattedTest = await format(testFileContent, {
+        parser: "typescript",
+      });
+      await writeFile(
+        join(examplesPath, `${kebabName}.test.ts`),
+        formattedTest,
+        {
+          encoding: "utf-8",
+        },
+      );
+    }),
+  );
+}
+
+main().catch(console.error);

package/scripts/generate-gitlab-ci-types.ts

@@ -0,0 +1,51 @@
+import { compile } from "json-schema-to-typescript";
+import { resolve, join } from "path";
+import { existsSync } from "fs";
+import { rm, writeFile } from "fs/promises";
+
+const rootPath = resolve(__dirname, "..");
+const typeOutputPath = join(rootPath, "src/types/gitlab-ci-yml.ts");
+
+export const generateFromCurrentSchema = async () => {
+  const response = await fetch(
+    "https://gitlab.com/gitlab-org/gitlab/-/raw/master/app/assets/javascripts/editor/schema/ci.json",
+  );
+  const gitLabCiJsonSchema = await response.json();
+  const schema = await compile(gitLabCiJsonSchema, "GitlabCiYml", {
+    additionalProperties: true,
+    enableConstEnums: true,
+    declareExternallyReferenced: true,
+    unreachableDefinitions: true,
+    strictIndexSignatures: true,
+    unknownAny: false,
+    ignoreMinAndMaxItems: true,
+    format: true,
+    style: {
+      bracketSpacing: true,
+      printWidth: 100,
+      semi: true,
+      tabWidth: 2,
+      useTabs: false,
+      trailingComma: "es5",
+      endOfLine: "lf",
+    },
+    cwd: rootPath,
+  });
+  /**
+   * Remove part that failes with tsc error
+   * ```
+   * error TS2411: Property '$schema' of type 'string | undefined' is not assignable to 'string' index type 'JobTemplate1'.
+   * ```
+   */
+  const sanitizedSchenma = schema.replace(
+    // eslint-disable-next-line no-useless-escape
+    /export interface HttpsGitlabComGitlabCiYml \{\n[ a-zA-Z0-9?$:;\n_!"\{\}\/*@\[\],.|\(\)]*\[k\: string\]: Job;\n\}/m,
+    "",
+  );
+  if (existsSync(typeOutputPath)) {
+    await rm(typeOutputPath);
+  }
+  await writeFile(typeOutputPath, sanitizedSchenma, { encoding: "utf-8" });
+};
+
+generateFromCurrentSchema();

package/scripts/tsconfig.json

@@ -0,0 +1,7 @@
+{
+  "extends": "../tsconfig.json",
+  "compilerOptions": {
+    "target": "ES2020"
+  },
+  "include": ["**/*.ts"]
+}

package/src/bash/BashExpression.ts

@@ -0,0 +1,81 @@
+/**
+ * this class represents a string that can be evaluated in bash scripts.
+ *
+ * you can do basic transforms like lowercase, but that does not return you a lowercase string, but instead an experssion represeting a lowercase string
+ */
+export class BashExpression {
+  constructor(private value: string | BashExpression) {}
+  public toJSON(): string {
+    return this.toString();
+  }
+  public toString(): string {
+    return this.value.toString();
+  }
+
+  /**
+   *
+   * @returns a bash expression to lowercase the string
+   */
+  public toLowerCase(): BashExpression {
+    return this.transformWithCommand("awk '{print tolower($0)}'");
+  }
+  /**
+   * concats a value to this one and returns a new BashExpression
+   * @param value
+   * @returns
+   */
+  public concat(...values: Array<string | BashExpression>): BashExpression {
+    return new BashExpression(
+      this.toString().concat(...values.map((v) => v.toString())),
+    );
+  }
+
+  public transformWithCommand(command: string): BashExpression {
+    return new BashExpression(
+      // see https://stackoverflow.com/a/2264537
+      `$(printf %s "${this.toString()}" | ${command})`,
+    );
+  }
+}
+
+export type StringOrBashExpression = string | BashExpression;
+
+export const getBashVariable = (name: string) => new BashExpression(`$${name}`);
+
+/**
+ * joins bash expressions together with a joiner
+ * returns a bash expression if any of the parts is a bash expression
+ * returns a string otherwise
+ *
+ * @param parts
+ * @param joiner
+ * @returns
+ */
+export const joinBashExpressions = (
+  parts: Array<StringOrBashExpression>,
+  joiner = "",
+): StringOrBashExpression => {
+  const anyIsBashExpression = parts.some((p) => p instanceof BashExpression);
+  if (anyIsBashExpression) {
+    return new BashExpression(
+      parts
+        .map((p) => (p instanceof BashExpression ? p.toString() : p))
+        .join(joiner),
+    );
+  } else {
+    return parts.join(joiner);
+  }
+};
+
+/**
+ * escapes a string or bash expression for bash
+ * it mainly escapes double quotes
+ */
+export const bashEscape = (value: StringOrBashExpression | any) => {
+  if (value instanceof BashExpression) {
+    return value.toString(); // no need to escape bash expressions, as we want them to evaluate
+  }
+  // we wrap it in double quotes, so we need to escape them
+  if (typeof value === "string") return value.replace(/"/g, '\\"');
+  return value;
+};

package/src/bash/bashExpressionPerPipelineType.ts

@@ -0,0 +1,16 @@
+import type { PipelineType } from "../types/pipeline";
+import type { BashExpression } from "./BashExpression";
+
+export type BashExpressionPerPipelineType = {
+  [type in PipelineType]?: BashExpression | string;
+} & {
+  default: BashExpression | string;
+};
+
+export const getBashExpressionPerPipelineType = (
+  definition: BashExpressionPerPipelineType,
+  pipelineType?: PipelineType,
+) => {
+  if (!pipelineType) return definition.default;
+  return definition[pipelineType] || definition.default;
+};

package/src/bash/bashYaml.ts

@@ -0,0 +1,46 @@
+import type { ScalarTag } from "yaml";
+import { stringify } from "yaml";
+import { BashExpression } from "./BashExpression";
+
+const bashExpressionType: ScalarTag = {
+  tag: "",
+  resolve: (str) => {
+    // not really needed,but let's make typescript happy
+    return new BashExpression(str);
+  },
+  stringify(node: any, ctx: any) {
+    // we create a BLOCK_LITERAL
+    // but because bash will interpret the value, it may resolve to a multiline string
+    // so we need to indent every line using sed
+    return "|-\n" + node.value.transformWithCommand(`sed 's/^/${ctx.indent}/'`);
+  },
+
+  identify: (v: any) => v instanceof BashExpression,
+};
+
+/***
+ * creates a yaml string that can be used in bash scripts
+ * it handles BashExpressions correctly so that they can be evaluated in bash
+ */
+export const yamlBashString = (value: any) => {
+  return stringify(value, {
+    defaultStringType: "BLOCK_LITERAL",
+    defaultKeyType: "PLAIN",
+    customTags: [bashExpressionType],
+    aliasDuplicateObjects: false,
+    lineWidth: 0,
+  });
+};
+
+/**
+ * creates a bash script that writes a yaml string to a file
+ * @param value the yaml value to write, it supports BashExpressions
+ */
+export const writeBashYamlToFileScript = (value: any, filename: string) => {
+  return [
+    `cat > ${filename} <<EOF
+${yamlBashString(value)}
+EOF
+`,
+  ];
+};

package/src/bash/getInjectVarsScript.ts

@@ -0,0 +1,12 @@
+import type { UnspecifiedEnvVars } from "../types";
+import { isNil } from "lodash";
+import { bashEscape } from "./BashExpression";
+
+export const getInjectVarsScript = (vars?: UnspecifiedEnvVars) => {
+  if (!vars) return [];
+  // filter out null and undefined values
+  return Object.entries(vars)
+    .filter(([, value]) => !isNil(value))
+
+    .map(([key, value]) => `export ${key}="${value ? bashEscape(value) : ""}"`);
+};

package/src/bash/replaceAsync.ts

@@ -0,0 +1,50 @@
+import { BashExpression } from "./BashExpression";
+
+// from https://github.com/dsblv/string-replace-async/blob/main/index.js
+// and adjusted a bit
+export default async function replaceAsync(
+  string: string | BashExpression,
+  searchValue: any,
+  replacer: (
+    substring: string,
+    ...args: any[]
+  ) => Promise<string | BashExpression>,
+) {
+  const wasBashExpression = string instanceof BashExpression;
+  try {
+    // 1. Run fake pass of `replace`, collect values from `replacer` calls
+    // 2. Resolve them with `Promise.all`
+    // 3. Run `replace` with resolved values
+    const values: Array<Promise<string | BashExpression>> = [];
+    String.prototype.replace.call(
+      string.toString(),
+      searchValue,
+      function (...args) {
+        // eslint-disable-next-line prefer-spread
+        const result = replacer.apply(undefined, args);
+        values.push(result);
+        return "";
+      },
+    );
+    const resolvedValues = await Promise.all(values);
+
+    const containsBashExpression = resolvedValues.some(
+      (value) => value instanceof BashExpression,
+    );
+
+    const result = String.prototype.replace.call(
+      string.toString(),
+      searchValue,
+      function () {
+        return resolvedValues.shift()?.toString() ?? "";
+      },
+    );
+    if (wasBashExpression || containsBashExpression) {
+      return new BashExpression(result);
+    } else {
+      return result;
+    }
+  } catch (error) {
+    return Promise.reject(error);
+  }
+}