@catladder/pipeline 1.170.1 → 2.0.0

package/src/types/gitlab-types.ts

@@ -66,4 +66,5 @@ export interface GitlabPipeline extends Pick<JobTemplate, "variables"> {
   };
   stages: string[];
   jobs: Record<string, GitlabJobDef>;
+  before_script?: string[];
 }

package/src/types/jobs.ts

@@ -70,14 +70,6 @@ export type CatladderJob<S = BaseStage> = {
    */
   needs?: Array<CatladderJobNeed>;
 
-  /**
-   * @deprecated set `componentName` to `needs` if you want to wait for a job from another component
-   */
-  needsOtherComponent?: Array<{
-    componentName: string;
-    job: string;
-    artifacts: boolean;
-  }>;
   /**
    * cache config, we use here the same shape as gitlab itself
    */

package/src/utils/gitlab.ts

@@ -1,3 +1,5 @@
+import { registerGlobalScriptFunction } from "../globalScriptFunctions";
+
 export const allowFailureInScripts = (script: string[]): string[] => [
   "set +e", // disable fail job on error
   ...script,
@@ -22,12 +24,27 @@ export const repeatOnFailure = (
   `;
 };
 
+const start = registerGlobalScriptFunction(
+  "collapseable_section_start",
+  `local section_title="\${1}"
+  local section_description="\${2:-$section_title}"
+  echo -e "section_start:\`date +%s\`:\${section_title}[collapsed=true]\\r\\e[0K$\{section_description}"
+`,
+);
+
+const end = registerGlobalScriptFunction(
+  "collapseable_section_end",
+  `local section_title="\${1}"
+  echo -e "section_end:\`date +%s\`:\${section_title}\\r\\e[0K"
+`,
+);
+
 export const collapseableSection =
   (name: string, header: string) =>
   (commands: string[]): string[] => {
     return [
-      `echo -e "\\e[0Ksection_start:$(date +%s):${name}[collapsed=true]\\r\\e[0K${header}"`,
+      start.invoke(`"${name}"`, `"${header}"`),
       ...commands,
-      `echo -e "\\e[0Ksection_end:$(date +%s):${name}\\r\\e[0K"`,
+      end.invoke(`"${name}"`),
     ];
   };

package/src/utils/writeFiles.ts

@@ -1,11 +1,10 @@
 import { writeFile } from "fs/promises";
 import { stringify } from "yaml";
-import packageInfos from "../packageInfos";
 
 export const getAutoGeneratedHeader = (commentChar: string) => {
   return [
     "-------------------------------------------------",
-    `🐱 🔨 This file is generated by catladder v${packageInfos.version}`,
+    `🐱 🔨 This file is generated by catladder`,
     `🚨 Do not edit this file manually 🚨`,
     "-------------------------------------------------",
   ]

package/src/variables/VariableValue.ts

@@ -0,0 +1,6 @@
+import type { BashExpression } from "../bash/BashExpression";
+import type { VariableValueContainingReferences } from "./VariableValueContainingReferences";
+export type VariableValue =
+  | VariableValueContainingReferences
+  | BashExpression
+  | string;

package/src/variables/VariableValueContainingReferences.ts

@@ -0,0 +1,89 @@
+import { BashExpression } from "../bash/BashExpression";
+import type { EscapeOptions } from "../bash/bashEscape";
+import { escapeBashExpression, escapeString } from "../bash/bashEscape";
+
+export class UnresolvableReference {
+  constructor(public reference: VariableReference) {}
+  public toString() {
+    return `Unresolvable reference: ${this.reference.toString()}`;
+  }
+}
+export class VariableReference {
+  public componentName: string;
+  public variableName: string;
+
+  constructor(componentName: string, variableName: string) {
+    this.componentName = componentName;
+    this.variableName = variableName;
+  }
+
+  public toString() {
+    return `\${${this.componentName}:${this.variableName}}`;
+  }
+}
+
+type VariableValuePart =
+  | string
+  | BashExpression
+  | VariableReference
+  | UnresolvableReference;
+
+type MaybeArray<T> = T | T[];
+export class VariableValueContainingReferences {
+  public parts: VariableValuePart[];
+  constructor(
+    parts: MaybeArray<VariableValuePart | VariableValueContainingReferences>,
+  ) {
+    this.parts = (Array.isArray(parts) ? parts : [parts]).flatMap((part) =>
+      part instanceof VariableValueContainingReferences
+        ? part.parts
+        : part === ""
+          ? []
+          : [part],
+    );
+  }
+
+  public toString(
+    options: EscapeOptions = {
+      quotes: false,
+    },
+  ) {
+    return this.parts
+      .map((part) => {
+        if (typeof part === "string") {
+          return escapeString(part, options);
+        } else if (part instanceof BashExpression) {
+          return escapeBashExpression(part, options);
+        } else {
+          return part.toString();
+        }
+      })
+      .join("");
+  }
+}
+
+// regex to resolve references in catladder variables
+// those expressions have the pattern ${componentName:variableName}
+const REGEX = /\$\{(([^:}]+):)?([^}]+)}/gm;
+export const createVariableValueContainingReferencesFromString = (
+  value: string,
+  options: { componentName: string },
+) => {
+  const parts: Array<VariableValuePart> = [];
+  let match: any;
+  let lastIndex = 0;
+  while ((match = REGEX.exec(value)) !== null) {
+    const [fullMatch, _, otherComponentName, variableName] = match;
+
+    parts.push(value.slice(lastIndex, match.index));
+    parts.push(
+      new VariableReference(
+        otherComponentName || options.componentName,
+        variableName,
+      ),
+    );
+    lastIndex = REGEX.lastIndex;
+  }
+  parts.push(value.slice(lastIndex));
+  return new VariableValueContainingReferences(parts);
+};

package/src/variables/__tests__/resolveAllReferences.test.ts

@@ -0,0 +1,110 @@
+import {
+  VariableValueContainingReferences,
+  createVariableValueContainingReferencesFromString,
+} from "../VariableValueContainingReferences";
+import { resolveAllReferences } from "../resolveAllReferences";
+
+describe("replaceAllReferences", () => {
+  it("should replace all references recursivly", async () => {
+    const values = {
+      myVar: createVariableValueContainingReferencesFromString(
+        "the 2nd component has ${component2:variable1} and self reference ${variable2}",
+        {
+          componentName: "component1",
+        },
+      ),
+      myOtherVar: createVariableValueContainingReferencesFromString(
+        "the third component has ${component3:variable1}, isn't that cool?",
+        {
+          componentName: "component1",
+        },
+      ),
+      myThirdVar: createVariableValueContainingReferencesFromString(
+        "value from component3: ${component3:variable3}",
+        {
+          componentName: "component1",
+        },
+      ),
+    };
+
+    const getEnvVars = async (componentName: string) => {
+      return {
+        variable1: createVariableValueContainingReferencesFromString(
+          `i am variable1 from ${componentName}`,
+          { componentName },
+        ),
+        variable2: createVariableValueContainingReferencesFromString(
+          `foo from ${componentName}`,
+          {
+            componentName,
+          },
+        ),
+        variable3: createVariableValueContainingReferencesFromString(
+          `i am referencing \${component1:variable1}`,
+          { componentName },
+        ),
+      };
+    };
+
+    const result = await resolveAllReferences(values, getEnvVars);
+    expect(result).toEqual({
+      myVar: new VariableValueContainingReferences([
+        "the 2nd component has ",
+        "i am variable1 from component2",
+        " and self reference ",
+        "foo from component1",
+      ]),
+      myOtherVar: new VariableValueContainingReferences([
+        "the third component has ",
+        "i am variable1 from component3",
+        ", isn't that cool?",
+      ]),
+      myThirdVar: new VariableValueContainingReferences([
+        "value from component3: ",
+        "i am referencing ",
+        "i am variable1 from component1",
+      ]),
+    });
+  });
+
+  it("detects infinte loop", async () => {
+    const values = {
+      myVar: createVariableValueContainingReferencesFromString(
+        "i am referencing ${component2:variable1}",
+        {
+          componentName: "component1",
+        },
+      ),
+    };
+
+    const getEnvVars = async (componentName: string) => {
+      if (componentName === "component2") {
+        return {
+          variable1: createVariableValueContainingReferencesFromString(
+            `i am referencing \${component3:variable1}`,
+            { componentName },
+          ),
+        };
+      } else if (componentName === "component3") {
+        return {
+          variable1: createVariableValueContainingReferencesFromString(
+            `i am referencing \${component1:variable1}`,
+            { componentName },
+          ),
+        };
+      } else {
+        return {
+          variable1: createVariableValueContainingReferencesFromString(
+            `i am referencing \${component2:variable1}`,
+            { componentName },
+          ),
+        };
+      }
+    };
+
+    // expect to throw an error
+    await expect(resolveAllReferences(values, getEnvVars)).rejects.toThrow(
+      "Infinite loop detected in these variables: myVar (last reference: ${component1:variable1})",
+    );
+  });
+});

package/src/variables/__tests__/resolveAllReferencesOnce.test.ts

@@ -0,0 +1,64 @@
+import {
+  VariableReference,
+  VariableValueContainingReferences,
+  createVariableValueContainingReferencesFromString,
+} from "../VariableValueContainingReferences";
+import { resolveAllReferencesOnce } from "../resolveAllReferencesOnce";
+
+describe("resolveAllReferencesOnce", () => {
+  it("should replace all references", async () => {
+    const values = {
+      myVar: createVariableValueContainingReferencesFromString(
+        "the 2nd component has ${component2:variable1} and self reference ${variable2}",
+        { componentName: "component1" },
+      ),
+      myOtherVar: createVariableValueContainingReferencesFromString(
+        "the third component has ${component3:variable1}, isn't that cool?",
+        { componentName: "component1" },
+      ),
+      myThirdVar: createVariableValueContainingReferencesFromString(
+        "value from component3: ${component3:variable3}",
+        { componentName: "component1" },
+      ),
+    };
+
+    const getEnvVars = async (componentName: string) => {
+      return {
+        variable1: createVariableValueContainingReferencesFromString(
+          `i am variable1 from ${componentName}`,
+          { componentName },
+        ),
+        variable2: createVariableValueContainingReferencesFromString(
+          `foo from ${componentName}`,
+          {
+            componentName,
+          },
+        ),
+        variable3: createVariableValueContainingReferencesFromString(
+          `i am referencing \${component1:variable1}`,
+          { componentName },
+        ),
+      };
+    };
+
+    const result = await resolveAllReferencesOnce(values, getEnvVars);
+    expect(result).toEqual({
+      myVar: new VariableValueContainingReferences([
+        "the 2nd component has ",
+        "i am variable1 from component2",
+        " and self reference ",
+        "foo from component1",
+      ]),
+      myOtherVar: new VariableValueContainingReferences([
+        "the third component has ",
+        "i am variable1 from component3",
+        ", isn't that cool?",
+      ]),
+      myThirdVar: new VariableValueContainingReferences([
+        "value from component3: ",
+        "i am referencing ",
+        new VariableReference("component1", "variable1"),
+      ]),
+    });
+  });
+});

package/src/variables/__tests__/resolveReferencesOnce.test.ts

@@ -0,0 +1,117 @@
+import {
+  UnresolvableReference,
+  VariableReference,
+  createVariableValueContainingReferencesFromString,
+} from "../VariableValueContainingReferences";
+import { resolveReferencesOnce } from "../resolveReferencesOnce";
+
+describe("resolveReferencesOnce", () => {
+  it("should replace a reference to another component", async () => {
+    const value = createVariableValueContainingReferencesFromString(
+      "the other component has ${otherComponent:variableName}, isn't that cool?",
+      {
+        componentName: "myComponent",
+      },
+    );
+
+    const result = resolveReferencesOnce(
+      value,
+      ({ componentName, variableName }) => {
+        return createVariableValueContainingReferencesFromString(
+          `replaced ${componentName}:${variableName}`,
+          { componentName },
+        );
+      },
+    );
+
+    expect(result.parts).toEqual([
+      "the other component has ",
+      "replaced otherComponent:variableName",
+      ", isn't that cool?",
+    ]);
+  });
+  it("should keep references in replacement", async () => {
+    const value = createVariableValueContainingReferencesFromString(
+      "the other component has ${otherComponent:variableName}, isn't that cool?",
+      {
+        componentName: "myComponent",
+      },
+    );
+
+    const result = resolveReferencesOnce(
+      value,
+      ({ componentName, variableName }) => {
+        return createVariableValueContainingReferencesFromString(
+          `replaced ${componentName}:${variableName}, contains reference to \${thirdComponent:x}`,
+          { componentName },
+        );
+      },
+    );
+
+    expect(result.parts).toEqual([
+      "the other component has ",
+
+      "replaced otherComponent:variableName, contains reference to ",
+      new VariableReference("thirdComponent", "x"),
+      ", isn't that cool?",
+    ]);
+  });
+
+  it("can be run multiple times", async () => {
+    const value = createVariableValueContainingReferencesFromString(
+      "the other component has ${otherComponent:variableName}, isn't that cool?",
+      {
+        componentName: "myComponent",
+      },
+    );
+    const replacer = ({
+      componentName,
+      variableName,
+    }: {
+      componentName: string;
+      variableName: string;
+    }) => {
+      return createVariableValueContainingReferencesFromString(
+        componentName === "thirdComponent"
+          ? "value from third component"
+          : `replaced ${componentName}:${variableName}, contains reference to \${thirdComponent:x}`,
+        { componentName },
+      );
+    };
+    const result = resolveReferencesOnce(
+      resolveReferencesOnce(value, replacer),
+      replacer,
+    );
+
+    expect(result.parts).toEqual([
+      "the other component has ",
+      "replaced otherComponent:variableName, contains reference to ",
+      "value from third component",
+      ", isn't that cool?",
+    ]);
+  });
+
+  it("flags unresolveable", async () => {
+    const value = createVariableValueContainingReferencesFromString(
+      "the other component has ${otherComponent:variableName}, isn't that cool?",
+      {
+        componentName: "myComponent",
+      },
+    );
+
+    const result = resolveReferencesOnce(
+      value,
+      ({ componentName, variableName }) => {
+        return null;
+      },
+    );
+
+    expect(result.parts).toEqual([
+      "the other component has ",
+      new UnresolvableReference(
+        new VariableReference("otherComponent", "variableName"),
+      ),
+      ", isn't that cool?",
+    ]);
+  });
+});

package/src/variables/__tests__/variableValue.test.ts

@@ -0,0 +1,73 @@
+import {
+  VariableReference,
+  createVariableValueContainingReferencesFromString,
+} from "../VariableValueContainingReferences";
+
+describe("createVarialeValueFromString", () => {
+  it("should create a VariableValue from a simple string", () => {
+    const stringValue = "hello world";
+
+    const result = createVariableValueContainingReferencesFromString(
+      stringValue,
+      {
+        componentName: "myComponent",
+      },
+    );
+
+    expect(result.parts).toEqual([stringValue]);
+  });
+
+  it("extracts references to other components", () => {
+    const stringValue =
+      "the other component has ${otherComponent:variableName}, isn't that cool?";
+
+    const result = createVariableValueContainingReferencesFromString(
+      stringValue,
+      {
+        componentName: "myComponent",
+      },
+    );
+
+    expect(result.parts).toEqual([
+      "the other component has ",
+      new VariableReference("otherComponent", "variableName"),
+      ", isn't that cool?",
+    ]);
+  });
+
+  it("extracts self references", () => {
+    const stringValue = "my component has ${variableName}!";
+
+    const result = createVariableValueContainingReferencesFromString(
+      stringValue,
+      {
+        componentName: "myComponent",
+      },
+    );
+
+    expect(result.parts).toEqual([
+      "my component has ",
+      new VariableReference("myComponent", "variableName"),
+      "!",
+    ]);
+  });
+  it("extracts multiple references", () => {
+    const stringValue =
+      "my component has ${variableName} and the other component has ${otherComponent:variableName}!";
+
+    const result = createVariableValueContainingReferencesFromString(
+      stringValue,
+      {
+        componentName: "myComponent",
+      },
+    );
+
+    expect(result.parts).toEqual([
+      "my component has ",
+      new VariableReference("myComponent", "variableName"),
+      " and the other component has ",
+      new VariableReference("otherComponent", "variableName"),
+      "!",
+    ]);
+  });
+});

package/src/variables/resolveAllReferences.ts

@@ -0,0 +1,46 @@
+import type { VariableValue } from "./VariableValue";
+import type { VariableValueContainingReferences } from "./VariableValueContainingReferences";
+import { VariableReference } from "./VariableValueContainingReferences";
+import { resolveAllReferencesOnce as resolveAllReferencesOnce } from "./resolveAllReferencesOnce";
+
+export const resolveAllReferences = async (
+  values: Record<string, VariableValueContainingReferences>,
+  getEnvVars: (
+    componentName: string,
+  ) => Promise<Record<string, VariableValue | null | undefined>>,
+) => {
+  // replace until there aren't any references left
+  let result = values;
+
+  let i = 0;
+
+  while (
+    Object.values(result).some((value) =>
+      value.parts.some((part) => part instanceof VariableReference),
+    )
+  ) {
+    const replaced = await resolveAllReferencesOnce(result, getEnvVars);
+
+    result = replaced;
+    i++;
+    if (i > 1000) {
+      const unresolved = Object.entries(result).filter(([key, value]) =>
+        value.parts.some((part) => part instanceof VariableReference),
+      );
+
+      throw new Error(
+        "Infinite loop detected in these variables: " +
+          unresolved
+            .map(
+              ([key, value]) =>
+                `${key} (last reference: ${value.parts.find(
+                  (part) => part instanceof VariableReference,
+                )})`,
+            )
+            .join(", "),
+      );
+    }
+  }
+
+  return result;
+};

package/src/variables/resolveAllReferencesOnce.ts

@@ -0,0 +1,44 @@
+import type { VariableValue } from "./VariableValue";
+import type { VariableValueContainingReferences } from "./VariableValueContainingReferences";
+import { VariableReference } from "./VariableValueContainingReferences";
+import { resolveReferencesOnce } from "./resolveReferencesOnce";
+
+export const resolveAllReferencesOnce = async (
+  values: Record<string, VariableValueContainingReferences>,
+  getEnvVars: (
+    componentName: string,
+  ) => Promise<Record<string, VariableValue | undefined | null>>,
+) => {
+  const allReferences = Object.values(values).flatMap(
+    (value) =>
+      value?.parts.filter(
+        (part) => part instanceof VariableReference,
+      ) as VariableReference[],
+  );
+
+  const allComponentsUnique = Array.from(
+    new Set(allReferences.map((reference) => reference.componentName)),
+  );
+
+  const allEnvVarsInComponents = Object.fromEntries(
+    await Promise.all(
+      allComponentsUnique.map(async (componentName) => [
+        componentName,
+        await getEnvVars(componentName),
+      ]) as Array<
+        Promise<[string, Record<string, VariableValue | undefined | null>]>
+      >,
+    ),
+  );
+
+  return Object.fromEntries(
+    Object.entries(values).map(([key, value]) => [
+      key,
+      value !== null && value !== undefined
+        ? resolveReferencesOnce(value, ({ componentName, variableName }) => {
+            return allEnvVarsInComponents[componentName][variableName];
+          })
+        : value,
+    ]),
+  );
+};

package/src/variables/resolveReferencesOnce.ts

@@ -0,0 +1,29 @@
+import type { VariableValue } from "./VariableValue";
+import {
+  UnresolvableReference,
+  VariableReference,
+  VariableValueContainingReferences,
+} from "./VariableValueContainingReferences";
+
+type Resolver = (args: {
+  componentName: string;
+  variableName: string;
+}) => VariableValue | null | undefined;
+export const resolveReferencesOnce = (
+  value: VariableValueContainingReferences,
+  resolver: Resolver,
+) => {
+  const replacedParts = value.parts.map((part) => {
+    if (part instanceof VariableReference) {
+      const result = resolver({
+        componentName: part.componentName,
+        variableName: part.variableName,
+      });
+      return result ?? new UnresolvableReference(part);
+    } else {
+      return part;
+    }
+  });
+
+  return new VariableValueContainingReferences(replacedParts);
+};

package/bin/catladder-gitlab-dev.js

@@ -1,3 +0,0 @@
-#! /usr/bin/env node
-
-require("../dist/catladder-gitlab.js");

package/bin/catladder-gitlab.js

@@ -1,3 +0,0 @@
-#! /usr/bin/env node
-
-require("../dist/bundles/catladder-gitlab/index.js");

package/dist/bash/replaceAsync.d.ts

@@ -1,2 +0,0 @@
-import { BashExpression } from "./BashExpression";
-export default function replaceAsync(string: string | BashExpression, searchValue: any, replacer: (substring: string, ...args: any[]) => Promise<string | BashExpression>): Promise<any>;