@catladder/pipeline 0.0.0-more-collapse-deeb54d7 → 0.0.0-multiple-gitlab-includes-1a40efc0

package/src/types/gitlab-types.ts

@@ -1,18 +1,20 @@
+import type { GitlabJobWithContext } from "../pipeline/gitlab/createGitlabJobs";
 import type {
   Artifacts as GitlabCiArtifacts,
   JobTemplate,
-  Cache,
+  Cache as GitlabJobCache,
   Rules,
 } from "./gitlab-ci-yml";
 
 export { Retry, Image as GitlabJobImage } from "./gitlab-ci-yml";
 
 // Reports won't show up on MRs until https://gitlab.com/groups/gitlab-org/-/epics/8205
-export interface Artifacts extends Omit<GitlabCiArtifacts, "paths"> {
-  paths: string[];
-}
-export type GitlabJobCache = Cache;
-export type GitlabRule = Exclude<Rules, null>[number];
+export type Artifacts = GitlabCiArtifacts;
+export type { GitlabJobCache };
+export type GitlabRule = Exclude<
+  Exclude<Rules, null>[number],
+  string | string[] // we only use the object version
+>;
 export type GitlabEnvironment = Omit<
   Exclude<JobTemplate["environment"], undefined | string>,
   "deployment_tier"
@@ -64,5 +66,6 @@ export interface GitlabPipeline extends Pick<JobTemplate, "variables"> {
     rules: GitlabRule[];
   };
   stages: string[];
-  jobs: Record<string, GitlabJobDef>;
+  jobs: Record<string, GitlabJobWithContext>;
+  before_script?: string[];
 }

package/src/types/index.ts

@@ -1,4 +1,5 @@
 export * from "./config";
 export * from "./gitlab-types";
 export * from "./context";
+export * from "./jobDefinition";
 export * from "./pipeline";

package/src/types/jobDefinition.ts

@@ -0,0 +1,34 @@
+import type { CacheConfig } from "../build";
+import type { CatladderJob } from "./jobs";
+
+// new intermediate types. Currently not widely used, but we will built a bit  more abstraction on top of this
+
+export type AppBuildJobDefinition = Partial<
+  Omit<CatladderJob, "artifacts" | "cache">
+> & {
+  cache?: CacheConfig[];
+};
+
+export type DockerBuildJobDefinition = AppBuildJobDefinition; // currently the same
+
+export type DeployJobDefinition = Pick<
+  CatladderJob,
+  | "script"
+  | "variables"
+  | "image"
+  | "artifacts"
+  | "services"
+  | "runnerVariables"
+> & {
+  cache?: CacheConfig[];
+};
+
+export type StopJobDefinition = Pick<
+  CatladderJob,
+  "script" | "variables" | "image" | "runnerVariables"
+>;
+
+export type RollbackJobDefinition = Pick<
+  CatladderJob,
+  "script" | "variables" | "runnerVariables" | "image"
+>;

package/src/types/jobs.ts

@@ -1,12 +1,14 @@
+import type { UnspecifiedEnvVars } from "..";
 import type {
   Artifacts,
   GitlabEnvironment,
   GitlabJobCache,
-  GitlabRule,
-  GitlabJobImage,
   GitlabJobDef,
+  GitlabJobImage,
+  GitlabRule,
 } from "./gitlab-types";
 
+export type CatladderJobCache = GitlabJobCache;
 export const BASE_STAGES = [
   "setup",
   "test",
@@ -16,11 +18,18 @@ export const BASE_STAGES = [
   "rollback",
   "stop",
 ] as const;
-export type BaseStage = typeof BASE_STAGES[number];
+export type BaseStage = (typeof BASE_STAGES)[number];
 
 export type CatladderJobNeed =
   | string
-  | { job: string; artifacts: boolean; componentName?: string };
+  | { job: string; artifacts: boolean; componentName?: string }
+  | { job: string; artifacts: boolean; workspaceName: string };
+
+export type CatladderJobEnvironmentConfig = {
+  action?: GitlabEnvironment["action"];
+  on_stop?: GitlabEnvironment["on_stop"];
+  auto_stop_in?: string;
+};
 export type CatladderJob<S = BaseStage> = {
   /**
    * the name of the job (without any env or app prefix and suffix)
@@ -50,6 +59,7 @@ export type CatladderJob<S = BaseStage> = {
   needsStages?: {
     stage: S;
     artifacts?: boolean;
+    workspaceName?: string;
   }[];
 
   /**
@@ -60,18 +70,10 @@ export type CatladderJob<S = BaseStage> = {
    */
   needs?: Array<CatladderJobNeed>;
 
-  /**
-   * @deprecated set `componentName` to `needs` if you want to wait for a job from another component
-   */
-  needsOtherComponent?: Array<{
-    componentName: string;
-    job: string;
-    artifacts: boolean;
-  }>;
   /**
    * cache config, we use here the same shape as gitlab itself
    */
-  cache?: GitlabJobCache | GitlabJobCache[];
+  cache?: CatladderJobCache | CatladderJobCache[];
 
   /**
    * job artifacts, we also use gitlab shape here
@@ -91,14 +93,28 @@ export type CatladderJob<S = BaseStage> = {
   /**
    * variables to pass
    */
-  variables: GitlabJobDef["variables"];
+  variables: UnspecifiedEnvVars | undefined;
+
+  /**
+   * additional vars only for the runner.
+   * Also if you use services: that require env vars, you need to set them here.
+   *
+   */
+  runnerVariables?: Record<string, string>;
 
   /**
    * whether failures are allowed
    */
   allow_failure?: boolean;
 
-  environment?: GitlabEnvironment;
+  /**
+   * gitlab environment config, subject to change
+   */
+  environment?: CatladderJobEnvironmentConfig;
+
+  /** */
+
+  when?: GitlabRule["when"];
 
   rules?: GitlabRule[];
 

package/src/types/utils.ts

@@ -1,7 +1,7 @@
 export type PartialDeep<T> = T extends any[]
   ? T
   : T extends Record<string, any>
-  ? {
-      [P in keyof T]?: PartialDeep<T[P]>;
-    }
-  : T;
+    ? {
+        [P in keyof T]?: PartialDeep<T[P]>;
+      }
+    : T;

package/src/utils/gitlab.ts

@@ -1,3 +1,5 @@
+import { registerGlobalScriptFunction } from "../globalScriptFunctions";
+
 export const allowFailureInScripts = (script: string[]): string[] => [
   "set +e", // disable fail job on error
   ...script,
@@ -11,7 +13,7 @@ export const repeatOnFailure = (
   command: string,
   options: {
     pauseInSeconds: number;
-  }
+  },
 ): string => {
   return `
     until ${command}
@@ -22,12 +24,27 @@ export const repeatOnFailure = (
   `;
 };
 
+const start = registerGlobalScriptFunction(
+  "collapseable_section_start",
+  `local section_title="\${1}"
+  local section_description="\${2:-$section_title}"
+  echo -e "section_start:\`date +%s\`:\${section_title}[collapsed=true]\\r\\e[0K$\{section_description}"
+`,
+);
+
+const end = registerGlobalScriptFunction(
+  "collapseable_section_end",
+  `local section_title="\${1}"
+  echo -e "section_end:\`date +%s\`:\${section_title}\\r\\e[0K"
+`,
+);
+
 export const collapseableSection =
   (name: string, header: string) =>
   (commands: string[]): string[] => {
     return [
-      `echo -e "\\e[0Ksection_start:$(date +%s):${name}[collapsed=true]\\r\\e[0K${header}"`,
+      start.invoke(`"${name}"`, `"${header}"`),
       ...commands,
-      `echo -e "\\e[0Ksection_end:$(date +%s):${name}\\r\\e[0K"`,
+      end.invoke(`"${name}"`),
     ];
   };

package/src/utils/index.ts

@@ -1,19 +1,22 @@
 import { isNil, mergeWith } from "lodash";
 
 export function notNil<TValue>(
-  value: TValue | null | undefined
+  value: TValue | null | undefined,
 ): value is TValue {
   return value !== null && value !== undefined;
 }
 
-export const ensureArray = <T>(s: undefined | T | T[]): T[] | null =>
+export const ensureArrayOrNull = <T>(s: undefined | T | T[]): T[] | null =>
   isNil(s) ? null : Array.isArray(s) ? s : [s];
 
+export const ensureArray = <T>(s: undefined | T | T[]): T[] =>
+  ensureArrayOrNull(s) ?? [];
+
 // see https://github.com/lodash/lodash/issues/5384
 export const mergeWithMergingArrays = <A, B>(obj1: A, obj2: B): A & B => {
   return mergeWith({}, obj1, obj2, (prop1, prop2) =>
     Array.isArray(prop1) && Array.isArray(prop2)
       ? [...prop1, ...prop2]
-      : undefined
+      : undefined,
   );
 };

package/src/utils/jsonParse.ts

@@ -0,0 +1,7 @@
+export const jsonParseOrThrow = (str: string): any => {
+  try {
+    return JSON.parse(str);
+  } catch (e) {
+    throw new Error(`could not parse json: ${str}`);
+  }
+};

package/src/utils/removeUndefined.ts

@@ -0,0 +1,4 @@
+export const removeUndefined = <T extends object>(obj: T): T =>
+  Object.fromEntries(
+    Object.entries(obj).filter(([, value]) => value !== undefined),
+  ) as T;

package/src/utils/writeFiles.ts

@@ -0,0 +1,52 @@
+import { writeFile } from "fs/promises";
+import { stringify } from "yaml";
+
+export const getAutoGeneratedHeader = (commentChar: string) => {
+  return [
+    "-------------------------------------------------",
+    `🐱 🔨 This file is generated by catladder`,
+    `🚨 Do not edit this file manually 🚨`,
+    "-------------------------------------------------",
+  ]
+    .map((line) => `${commentChar} ${line}`)
+    .join("\n")
+    .concat("\n");
+};
+
+export const writeGeneratedFile = async (
+  path: string,
+  content: string,
+  {
+    commentChar,
+  }: {
+    commentChar: string;
+  },
+) => {
+  await writeFile(
+    path,
+    // need to spread out the jobs, forgot why
+    [getAutoGeneratedHeader(commentChar), content].join("\n"),
+    {
+      encoding: "utf-8",
+    },
+  );
+};
+
+type StringifyOptions = Exclude<
+  Parameters<typeof stringify>[2],
+  null | undefined | string | number
+>;
+export const yamlStringifyOptions: StringifyOptions = {
+  // prevents colapsing long command statements into multiple lines
+  lineWidth: 0,
+  // represent multi line commands as single line json strings
+  doubleQuotedAsJSON: true,
+  // Better readability when bash commands most often use double quotes
+  singleQuote: true,
+};
+
+export const writeYamlfile = async (path: string, data: any) => {
+  await writeGeneratedFile(path, stringify(data, yamlStringifyOptions), {
+    commentChar: "#",
+  });
+};

package/src/variables/VariableValue.ts

@@ -0,0 +1,6 @@
+import type { BashExpression } from "../bash/BashExpression";
+import type { VariableValueContainingReferences } from "./VariableValueContainingReferences";
+export type VariableValue =
+  | VariableValueContainingReferences
+  | BashExpression
+  | string;

package/src/variables/VariableValueContainingReferences.ts

@@ -0,0 +1,89 @@
+import { BashExpression } from "../bash/BashExpression";
+import type { EscapeOptions } from "../bash/bashEscape";
+import { escapeBashExpression, escapeString } from "../bash/bashEscape";
+
+export class UnresolvableReference {
+  constructor(public reference: VariableReference) {}
+  public toString() {
+    return `Unresolvable reference: ${this.reference.toString()}`;
+  }
+}
+export class VariableReference {
+  public componentName: string;
+  public variableName: string;
+
+  constructor(componentName: string, variableName: string) {
+    this.componentName = componentName;
+    this.variableName = variableName;
+  }
+
+  public toString() {
+    return `\${${this.componentName}:${this.variableName}}`;
+  }
+}
+
+type VariableValuePart =
+  | string
+  | BashExpression
+  | VariableReference
+  | UnresolvableReference;
+
+type MaybeArray<T> = T | T[];
+export class VariableValueContainingReferences {
+  public parts: VariableValuePart[];
+  constructor(
+    parts: MaybeArray<VariableValuePart | VariableValueContainingReferences>,
+  ) {
+    this.parts = (Array.isArray(parts) ? parts : [parts]).flatMap((part) =>
+      part instanceof VariableValueContainingReferences
+        ? part.parts
+        : part === ""
+          ? []
+          : [part],
+    );
+  }
+
+  public toString(
+    options: EscapeOptions = {
+      quotes: false,
+    },
+  ) {
+    return this.parts
+      .map((part) => {
+        if (typeof part === "string") {
+          return escapeString(part, options);
+        } else if (part instanceof BashExpression) {
+          return escapeBashExpression(part, options);
+        } else {
+          return part.toString();
+        }
+      })
+      .join("");
+  }
+}
+
+// regex to resolve references in catladder variables
+// those expressions have the pattern ${componentName:variableName}
+const REGEX = /\$\{(([^:}]+):)?([^}]+)}/gm;
+export const createVariableValueContainingReferencesFromString = (
+  value: string,
+  options: { componentName: string },
+) => {
+  const parts: Array<VariableValuePart> = [];
+  let match: any;
+  let lastIndex = 0;
+  while ((match = REGEX.exec(value)) !== null) {
+    const [fullMatch, _, otherComponentName, variableName] = match;
+
+    parts.push(value.slice(lastIndex, match.index));
+    parts.push(
+      new VariableReference(
+        otherComponentName || options.componentName,
+        variableName,
+      ),
+    );
+    lastIndex = REGEX.lastIndex;
+  }
+  parts.push(value.slice(lastIndex));
+  return new VariableValueContainingReferences(parts);
+};

package/src/variables/__tests__/resolveAllReferences.test.ts

@@ -0,0 +1,110 @@
+import {
+  VariableValueContainingReferences,
+  createVariableValueContainingReferencesFromString,
+} from "../VariableValueContainingReferences";
+import { resolveAllReferences } from "../resolveAllReferences";
+
+describe("replaceAllReferences", () => {
+  it("should replace all references recursivly", async () => {
+    const values = {
+      myVar: createVariableValueContainingReferencesFromString(
+        "the 2nd component has ${component2:variable1} and self reference ${variable2}",
+        {
+          componentName: "component1",
+        },
+      ),
+      myOtherVar: createVariableValueContainingReferencesFromString(
+        "the third component has ${component3:variable1}, isn't that cool?",
+        {
+          componentName: "component1",
+        },
+      ),
+      myThirdVar: createVariableValueContainingReferencesFromString(
+        "value from component3: ${component3:variable3}",
+        {
+          componentName: "component1",
+        },
+      ),
+    };
+
+    const getEnvVars = async (componentName: string) => {
+      return {
+        variable1: createVariableValueContainingReferencesFromString(
+          `i am variable1 from ${componentName}`,
+          { componentName },
+        ),
+        variable2: createVariableValueContainingReferencesFromString(
+          `foo from ${componentName}`,
+          {
+            componentName,
+          },
+        ),
+        variable3: createVariableValueContainingReferencesFromString(
+          `i am referencing \${component1:variable1}`,
+          { componentName },
+        ),
+      };
+    };
+
+    const result = await resolveAllReferences(values, getEnvVars);
+    expect(result).toEqual({
+      myVar: new VariableValueContainingReferences([
+        "the 2nd component has ",
+        "i am variable1 from component2",
+        " and self reference ",
+        "foo from component1",
+      ]),
+      myOtherVar: new VariableValueContainingReferences([
+        "the third component has ",
+        "i am variable1 from component3",
+        ", isn't that cool?",
+      ]),
+      myThirdVar: new VariableValueContainingReferences([
+        "value from component3: ",
+        "i am referencing ",
+        "i am variable1 from component1",
+      ]),
+    });
+  });
+
+  it("detects infinte loop", async () => {
+    const values = {
+      myVar: createVariableValueContainingReferencesFromString(
+        "i am referencing ${component2:variable1}",
+        {
+          componentName: "component1",
+        },
+      ),
+    };
+
+    const getEnvVars = async (componentName: string) => {
+      if (componentName === "component2") {
+        return {
+          variable1: createVariableValueContainingReferencesFromString(
+            `i am referencing \${component3:variable1}`,
+            { componentName },
+          ),
+        };
+      } else if (componentName === "component3") {
+        return {
+          variable1: createVariableValueContainingReferencesFromString(
+            `i am referencing \${component1:variable1}`,
+            { componentName },
+          ),
+        };
+      } else {
+        return {
+          variable1: createVariableValueContainingReferencesFromString(
+            `i am referencing \${component2:variable1}`,
+            { componentName },
+          ),
+        };
+      }
+    };
+
+    // expect to throw an error
+    await expect(resolveAllReferences(values, getEnvVars)).rejects.toThrow(
+      "Infinite loop detected in these variables: myVar (last reference: ${component1:variable1})",
+    );
+  });
+});

package/src/variables/__tests__/resolveAllReferencesOnce.test.ts

@@ -0,0 +1,64 @@
+import {
+  VariableReference,
+  VariableValueContainingReferences,
+  createVariableValueContainingReferencesFromString,
+} from "../VariableValueContainingReferences";
+import { resolveAllReferencesOnce } from "../resolveAllReferencesOnce";
+
+describe("resolveAllReferencesOnce", () => {
+  it("should replace all references", async () => {
+    const values = {
+      myVar: createVariableValueContainingReferencesFromString(
+        "the 2nd component has ${component2:variable1} and self reference ${variable2}",
+        { componentName: "component1" },
+      ),
+      myOtherVar: createVariableValueContainingReferencesFromString(
+        "the third component has ${component3:variable1}, isn't that cool?",
+        { componentName: "component1" },
+      ),
+      myThirdVar: createVariableValueContainingReferencesFromString(
+        "value from component3: ${component3:variable3}",
+        { componentName: "component1" },
+      ),
+    };
+
+    const getEnvVars = async (componentName: string) => {
+      return {
+        variable1: createVariableValueContainingReferencesFromString(
+          `i am variable1 from ${componentName}`,
+          { componentName },
+        ),
+        variable2: createVariableValueContainingReferencesFromString(
+          `foo from ${componentName}`,
+          {
+            componentName,
+          },
+        ),
+        variable3: createVariableValueContainingReferencesFromString(
+          `i am referencing \${component1:variable1}`,
+          { componentName },
+        ),
+      };
+    };
+
+    const result = await resolveAllReferencesOnce(values, getEnvVars);
+    expect(result).toEqual({
+      myVar: new VariableValueContainingReferences([
+        "the 2nd component has ",
+        "i am variable1 from component2",
+        " and self reference ",
+        "foo from component1",
+      ]),
+      myOtherVar: new VariableValueContainingReferences([
+        "the third component has ",
+        "i am variable1 from component3",
+        ", isn't that cool?",
+      ]),
+      myThirdVar: new VariableValueContainingReferences([
+        "value from component3: ",
+        "i am referencing ",
+        new VariableReference("component1", "variable1"),
+      ]),
+    });
+  });
+});