@catladder/pipeline 1.170.0 → 2.0.0

package/src/runner/index.ts

@@ -5,7 +5,6 @@ export type RunnerImageName =
   | "jobs-meteor"
   | "jobs-testing-chrome"
   | "kubernetes"
-  | "base-pipeline"
   | "docker-build"
   | "gcloud"
   | "semantic-release";

package/src/types/config.ts

@@ -73,14 +73,6 @@ export type EnvVars = {
    * secret env vars. These vars can be managed with catladder/cli
    */
   secret?: string[];
-  /**
-   * @deprecated, use ${componentName:variableName} instead
-   *
-   * With fromComponents you can inject env vars from other components.
-   */
-  fromComponents?: {
-    [otherApp: string]: Record<string, string>;
-  };
 };
 
 export type DefaultEnvConfig = {
@@ -169,12 +161,14 @@ export type ComponentConfig<C extends ConfigProps = never> = {
    * Use this in combination with @dotenvx/dotenvx or node 20 to start apps locally with .env files
    * During build and runtime, it relies on the env vars set in the environment
    *
+   * @defaultValue true
+   *
    */
   dotEnv?: boolean | "local";
   /**
    * whether to create a env.d.ts localy and during build jobs
    *
-   * Be careful, this will overwrite any existing env.d.ts file!
+   * @defaultValue true
    */
   envDTs?: boolean;
 } & DefaultEnvConfig;
@@ -184,6 +178,9 @@ export type ConfigProps = {
 };
 
 export type Config<C extends ConfigProps = never> = {
+  /**
+   * the pipeline type to generate, defaults to gitlab
+   */
   pipelineType?: PipelineType;
 
   /**

package/src/types/context.ts

@@ -6,6 +6,7 @@ import type {
 } from "../build";
 import type { PredefinedVariables, SecretEnvVar } from "../context";
 import type { DeployConfig } from "../deploy";
+import type { VariableValue } from "../variables/VariableValue";
 import type {
   ComponentConfig,
   Config,
@@ -17,7 +18,7 @@ import type { PipelineType } from "./pipeline";
 
 export type UnspecifiedEnvVars = Record<
   string,
-  StringOrBashExpression | undefined | null
+  VariableValue | undefined | null
 >;
 
 export type EnvironmentEnvVars<
@@ -46,10 +47,7 @@ export type Environment = {
    * the full name of the app. We use this as RELEASE_NAME in kubernetes and the service name in google cloud run
    */
   fullName: StringOrBashExpression;
-  /**
-   * @deprecated this is the same as context.env, use that instead
-   */
-  shortName: string;
+
   /**
    * the environment slug without component name.
    */
@@ -158,10 +156,6 @@ export type ComponentContext<
    * the name of the component
    */
   name: string;
-  /**
-   * @deprecated use name instead
-   */
-  componentName: string;
 
   /**
    * the merged component config.

package/src/types/gitlab-types.ts

@@ -66,4 +66,5 @@ export interface GitlabPipeline extends Pick<JobTemplate, "variables"> {
   };
   stages: string[];
   jobs: Record<string, GitlabJobDef>;
+  before_script?: string[];
 }

package/src/types/jobs.ts

@@ -70,14 +70,6 @@ export type CatladderJob<S = BaseStage> = {
    */
   needs?: Array<CatladderJobNeed>;
 
-  /**
-   * @deprecated set `componentName` to `needs` if you want to wait for a job from another component
-   */
-  needsOtherComponent?: Array<{
-    componentName: string;
-    job: string;
-    artifacts: boolean;
-  }>;
   /**
    * cache config, we use here the same shape as gitlab itself
    */

package/src/utils/gitlab.ts

@@ -1,3 +1,5 @@
+import { registerGlobalScriptFunction } from "../globalScriptFunctions";
+
 export const allowFailureInScripts = (script: string[]): string[] => [
   "set +e", // disable fail job on error
   ...script,
@@ -22,12 +24,27 @@ export const repeatOnFailure = (
   `;
 };
 
+const start = registerGlobalScriptFunction(
+  "collapseable_section_start",
+  `local section_title="\${1}"
+  local section_description="\${2:-$section_title}"
+  echo -e "section_start:\`date +%s\`:\${section_title}[collapsed=true]\\r\\e[0K$\{section_description}"
+`,
+);
+
+const end = registerGlobalScriptFunction(
+  "collapseable_section_end",
+  `local section_title="\${1}"
+  echo -e "section_end:\`date +%s\`:\${section_title}\\r\\e[0K"
+`,
+);
+
 export const collapseableSection =
   (name: string, header: string) =>
   (commands: string[]): string[] => {
     return [
-      `echo -e "\\e[0Ksection_start:$(date +%s):${name}[collapsed=true]\\r\\e[0K${header}"`,
+      start.invoke(`"${name}"`, `"${header}"`),
       ...commands,
-      `echo -e "\\e[0Ksection_end:$(date +%s):${name}\\r\\e[0K"`,
+      end.invoke(`"${name}"`),
     ];
   };

package/src/utils/writeFiles.ts

@@ -1,11 +1,10 @@
 import { writeFile } from "fs/promises";
 import { stringify } from "yaml";
-import packageInfos from "../packageInfos";
 
 export const getAutoGeneratedHeader = (commentChar: string) => {
   return [
     "-------------------------------------------------",
-    `🐱 🔨 This file is generated by catladder v${packageInfos.version}`,
+    `🐱 🔨 This file is generated by catladder`,
     `🚨 Do not edit this file manually 🚨`,
     "-------------------------------------------------",
   ]

package/src/variables/VariableValue.ts

@@ -0,0 +1,6 @@
+import type { BashExpression } from "../bash/BashExpression";
+import type { VariableValueContainingReferences } from "./VariableValueContainingReferences";
+export type VariableValue =
+  | VariableValueContainingReferences
+  | BashExpression
+  | string;

package/src/variables/VariableValueContainingReferences.ts

@@ -0,0 +1,89 @@
+import { BashExpression } from "../bash/BashExpression";
+import type { EscapeOptions } from "../bash/bashEscape";
+import { escapeBashExpression, escapeString } from "../bash/bashEscape";
+
+export class UnresolvableReference {
+  constructor(public reference: VariableReference) {}
+  public toString() {
+    return `Unresolvable reference: ${this.reference.toString()}`;
+  }
+}
+export class VariableReference {
+  public componentName: string;
+  public variableName: string;
+
+  constructor(componentName: string, variableName: string) {
+    this.componentName = componentName;
+    this.variableName = variableName;
+  }
+
+  public toString() {
+    return `\${${this.componentName}:${this.variableName}}`;
+  }
+}
+
+type VariableValuePart =
+  | string
+  | BashExpression
+  | VariableReference
+  | UnresolvableReference;
+
+type MaybeArray<T> = T | T[];
+export class VariableValueContainingReferences {
+  public parts: VariableValuePart[];
+  constructor(
+    parts: MaybeArray<VariableValuePart | VariableValueContainingReferences>,
+  ) {
+    this.parts = (Array.isArray(parts) ? parts : [parts]).flatMap((part) =>
+      part instanceof VariableValueContainingReferences
+        ? part.parts
+        : part === ""
+          ? []
+          : [part],
+    );
+  }
+
+  public toString(
+    options: EscapeOptions = {
+      quotes: false,
+    },
+  ) {
+    return this.parts
+      .map((part) => {
+        if (typeof part === "string") {
+          return escapeString(part, options);
+        } else if (part instanceof BashExpression) {
+          return escapeBashExpression(part, options);
+        } else {
+          return part.toString();
+        }
+      })
+      .join("");
+  }
+}
+
+// regex to resolve references in catladder variables
+// those expressions have the pattern ${componentName:variableName}
+const REGEX = /\$\{(([^:}]+):)?([^}]+)}/gm;
+export const createVariableValueContainingReferencesFromString = (
+  value: string,
+  options: { componentName: string },
+) => {
+  const parts: Array<VariableValuePart> = [];
+  let match: any;
+  let lastIndex = 0;
+  while ((match = REGEX.exec(value)) !== null) {
+    const [fullMatch, _, otherComponentName, variableName] = match;
+
+    parts.push(value.slice(lastIndex, match.index));
+    parts.push(
+      new VariableReference(
+        otherComponentName || options.componentName,
+        variableName,
+      ),
+    );
+    lastIndex = REGEX.lastIndex;
+  }
+  parts.push(value.slice(lastIndex));
+  return new VariableValueContainingReferences(parts);
+};

package/src/variables/__tests__/resolveAllReferences.test.ts

@@ -0,0 +1,110 @@
+import {
+  VariableValueContainingReferences,
+  createVariableValueContainingReferencesFromString,
+} from "../VariableValueContainingReferences";
+import { resolveAllReferences } from "../resolveAllReferences";
+
+describe("replaceAllReferences", () => {
+  it("should replace all references recursivly", async () => {
+    const values = {
+      myVar: createVariableValueContainingReferencesFromString(
+        "the 2nd component has ${component2:variable1} and self reference ${variable2}",
+        {
+          componentName: "component1",
+        },
+      ),
+      myOtherVar: createVariableValueContainingReferencesFromString(
+        "the third component has ${component3:variable1}, isn't that cool?",
+        {
+          componentName: "component1",
+        },
+      ),
+      myThirdVar: createVariableValueContainingReferencesFromString(
+        "value from component3: ${component3:variable3}",
+        {
+          componentName: "component1",
+        },
+      ),
+    };
+
+    const getEnvVars = async (componentName: string) => {
+      return {
+        variable1: createVariableValueContainingReferencesFromString(
+          `i am variable1 from ${componentName}`,
+          { componentName },
+        ),
+        variable2: createVariableValueContainingReferencesFromString(
+          `foo from ${componentName}`,
+          {
+            componentName,
+          },
+        ),
+        variable3: createVariableValueContainingReferencesFromString(
+          `i am referencing \${component1:variable1}`,
+          { componentName },
+        ),
+      };
+    };
+
+    const result = await resolveAllReferences(values, getEnvVars);
+    expect(result).toEqual({
+      myVar: new VariableValueContainingReferences([
+        "the 2nd component has ",
+        "i am variable1 from component2",
+        " and self reference ",
+        "foo from component1",
+      ]),
+      myOtherVar: new VariableValueContainingReferences([
+        "the third component has ",
+        "i am variable1 from component3",
+        ", isn't that cool?",
+      ]),
+      myThirdVar: new VariableValueContainingReferences([
+        "value from component3: ",
+        "i am referencing ",
+        "i am variable1 from component1",
+      ]),
+    });
+  });
+
+  it("detects infinte loop", async () => {
+    const values = {
+      myVar: createVariableValueContainingReferencesFromString(
+        "i am referencing ${component2:variable1}",
+        {
+          componentName: "component1",
+        },
+      ),
+    };
+
+    const getEnvVars = async (componentName: string) => {
+      if (componentName === "component2") {
+        return {
+          variable1: createVariableValueContainingReferencesFromString(
+            `i am referencing \${component3:variable1}`,
+            { componentName },
+          ),
+        };
+      } else if (componentName === "component3") {
+        return {
+          variable1: createVariableValueContainingReferencesFromString(
+            `i am referencing \${component1:variable1}`,
+            { componentName },
+          ),
+        };
+      } else {
+        return {
+          variable1: createVariableValueContainingReferencesFromString(
+            `i am referencing \${component2:variable1}`,
+            { componentName },
+          ),
+        };
+      }
+    };
+
+    // expect to throw an error
+    await expect(resolveAllReferences(values, getEnvVars)).rejects.toThrow(
+      "Infinite loop detected in these variables: myVar (last reference: ${component1:variable1})",
+    );
+  });
+});

package/src/variables/__tests__/resolveAllReferencesOnce.test.ts

@@ -0,0 +1,64 @@
+import {
+  VariableReference,
+  VariableValueContainingReferences,
+  createVariableValueContainingReferencesFromString,
+} from "../VariableValueContainingReferences";
+import { resolveAllReferencesOnce } from "../resolveAllReferencesOnce";
+
+describe("resolveAllReferencesOnce", () => {
+  it("should replace all references", async () => {
+    const values = {
+      myVar: createVariableValueContainingReferencesFromString(
+        "the 2nd component has ${component2:variable1} and self reference ${variable2}",
+        { componentName: "component1" },
+      ),
+      myOtherVar: createVariableValueContainingReferencesFromString(
+        "the third component has ${component3:variable1}, isn't that cool?",
+        { componentName: "component1" },
+      ),
+      myThirdVar: createVariableValueContainingReferencesFromString(
+        "value from component3: ${component3:variable3}",
+        { componentName: "component1" },
+      ),
+    };
+
+    const getEnvVars = async (componentName: string) => {
+      return {
+        variable1: createVariableValueContainingReferencesFromString(
+          `i am variable1 from ${componentName}`,
+          { componentName },
+        ),
+        variable2: createVariableValueContainingReferencesFromString(
+          `foo from ${componentName}`,
+          {
+            componentName,
+          },
+        ),
+        variable3: createVariableValueContainingReferencesFromString(
+          `i am referencing \${component1:variable1}`,
+          { componentName },
+        ),
+      };
+    };
+
+    const result = await resolveAllReferencesOnce(values, getEnvVars);
+    expect(result).toEqual({
+      myVar: new VariableValueContainingReferences([
+        "the 2nd component has ",
+        "i am variable1 from component2",
+        " and self reference ",
+        "foo from component1",
+      ]),
+      myOtherVar: new VariableValueContainingReferences([
+        "the third component has ",
+        "i am variable1 from component3",
+        ", isn't that cool?",
+      ]),
+      myThirdVar: new VariableValueContainingReferences([
+        "value from component3: ",
+        "i am referencing ",
+        new VariableReference("component1", "variable1"),
+      ]),
+    });
+  });
+});

package/src/variables/__tests__/resolveReferencesOnce.test.ts

@@ -0,0 +1,117 @@
+import {
+  UnresolvableReference,
+  VariableReference,
+  createVariableValueContainingReferencesFromString,
+} from "../VariableValueContainingReferences";
+import { resolveReferencesOnce } from "../resolveReferencesOnce";
+
+describe("resolveReferencesOnce", () => {
+  it("should replace a reference to another component", async () => {
+    const value = createVariableValueContainingReferencesFromString(
+      "the other component has ${otherComponent:variableName}, isn't that cool?",
+      {
+        componentName: "myComponent",
+      },
+    );
+
+    const result = resolveReferencesOnce(
+      value,
+      ({ componentName, variableName }) => {
+        return createVariableValueContainingReferencesFromString(
+          `replaced ${componentName}:${variableName}`,
+          { componentName },
+        );
+      },
+    );
+
+    expect(result.parts).toEqual([
+      "the other component has ",
+      "replaced otherComponent:variableName",
+      ", isn't that cool?",
+    ]);
+  });
+  it("should keep references in replacement", async () => {
+    const value = createVariableValueContainingReferencesFromString(
+      "the other component has ${otherComponent:variableName}, isn't that cool?",
+      {
+        componentName: "myComponent",
+      },
+    );
+
+    const result = resolveReferencesOnce(
+      value,
+      ({ componentName, variableName }) => {
+        return createVariableValueContainingReferencesFromString(
+          `replaced ${componentName}:${variableName}, contains reference to \${thirdComponent:x}`,
+          { componentName },
+        );
+      },
+    );
+
+    expect(result.parts).toEqual([
+      "the other component has ",
+
+      "replaced otherComponent:variableName, contains reference to ",
+      new VariableReference("thirdComponent", "x"),
+      ", isn't that cool?",
+    ]);
+  });
+
+  it("can be run multiple times", async () => {
+    const value = createVariableValueContainingReferencesFromString(
+      "the other component has ${otherComponent:variableName}, isn't that cool?",
+      {
+        componentName: "myComponent",
+      },
+    );
+    const replacer = ({
+      componentName,
+      variableName,
+    }: {
+      componentName: string;
+      variableName: string;
+    }) => {
+      return createVariableValueContainingReferencesFromString(
+        componentName === "thirdComponent"
+          ? "value from third component"
+          : `replaced ${componentName}:${variableName}, contains reference to \${thirdComponent:x}`,
+        { componentName },
+      );
+    };
+    const result = resolveReferencesOnce(
+      resolveReferencesOnce(value, replacer),
+      replacer,
+    );
+
+    expect(result.parts).toEqual([
+      "the other component has ",
+      "replaced otherComponent:variableName, contains reference to ",
+      "value from third component",
+      ", isn't that cool?",
+    ]);
+  });
+
+  it("flags unresolveable", async () => {
+    const value = createVariableValueContainingReferencesFromString(
+      "the other component has ${otherComponent:variableName}, isn't that cool?",
+      {
+        componentName: "myComponent",
+      },
+    );
+
+    const result = resolveReferencesOnce(
+      value,
+      ({ componentName, variableName }) => {
+        return null;
+      },
+    );
+
+    expect(result.parts).toEqual([
+      "the other component has ",
+      new UnresolvableReference(
+        new VariableReference("otherComponent", "variableName"),
+      ),
+      ", isn't that cool?",
+    ]);
+  });
+});

package/src/variables/__tests__/variableValue.test.ts

@@ -0,0 +1,73 @@
+import {
+  VariableReference,
+  createVariableValueContainingReferencesFromString,
+} from "../VariableValueContainingReferences";
+
+describe("createVarialeValueFromString", () => {
+  it("should create a VariableValue from a simple string", () => {
+    const stringValue = "hello world";
+
+    const result = createVariableValueContainingReferencesFromString(
+      stringValue,
+      {
+        componentName: "myComponent",
+      },
+    );
+
+    expect(result.parts).toEqual([stringValue]);
+  });
+
+  it("extracts references to other components", () => {
+    const stringValue =
+      "the other component has ${otherComponent:variableName}, isn't that cool?";
+
+    const result = createVariableValueContainingReferencesFromString(
+      stringValue,
+      {
+        componentName: "myComponent",
+      },
+    );
+
+    expect(result.parts).toEqual([
+      "the other component has ",
+      new VariableReference("otherComponent", "variableName"),
+      ", isn't that cool?",
+    ]);
+  });
+
+  it("extracts self references", () => {
+    const stringValue = "my component has ${variableName}!";
+
+    const result = createVariableValueContainingReferencesFromString(
+      stringValue,
+      {
+        componentName: "myComponent",
+      },
+    );
+
+    expect(result.parts).toEqual([
+      "my component has ",
+      new VariableReference("myComponent", "variableName"),
+      "!",
+    ]);
+  });
+  it("extracts multiple references", () => {
+    const stringValue =
+      "my component has ${variableName} and the other component has ${otherComponent:variableName}!";
+
+    const result = createVariableValueContainingReferencesFromString(
+      stringValue,
+      {
+        componentName: "myComponent",
+      },
+    );
+
+    expect(result.parts).toEqual([
+      "my component has ",
+      new VariableReference("myComponent", "variableName"),
+      " and the other component has ",
+      new VariableReference("otherComponent", "variableName"),
+      "!",
+    ]);
+  });
+});

package/src/variables/resolveAllReferences.ts

@@ -0,0 +1,46 @@
+import type { VariableValue } from "./VariableValue";
+import type { VariableValueContainingReferences } from "./VariableValueContainingReferences";
+import { VariableReference } from "./VariableValueContainingReferences";
+import { resolveAllReferencesOnce as resolveAllReferencesOnce } from "./resolveAllReferencesOnce";
+
+export const resolveAllReferences = async (
+  values: Record<string, VariableValueContainingReferences>,
+  getEnvVars: (
+    componentName: string,
+  ) => Promise<Record<string, VariableValue | null | undefined>>,
+) => {
+  // replace until there aren't any references left
+  let result = values;
+
+  let i = 0;
+
+  while (
+    Object.values(result).some((value) =>
+      value.parts.some((part) => part instanceof VariableReference),
+    )
+  ) {
+    const replaced = await resolveAllReferencesOnce(result, getEnvVars);
+
+    result = replaced;
+    i++;
+    if (i > 1000) {
+      const unresolved = Object.entries(result).filter(([key, value]) =>
+        value.parts.some((part) => part instanceof VariableReference),
+      );
+
+      throw new Error(
+        "Infinite loop detected in these variables: " +
+          unresolved
+            .map(
+              ([key, value]) =>
+                `${key} (last reference: ${value.parts.find(
+                  (part) => part instanceof VariableReference,
+                )})`,
+            )
+            .join(", "),
+      );
+    }
+  }
+
+  return result;
+};