@catladder/pipeline 0.0.0-mongodb-tolerations-bc2a3387 → 0.0.0-multiple-gitlab-includes-1a40efc0

package/src/types/gitlab-types.ts

@@ -1,104 +1,71 @@
-export type Artifacts = {
-  paths: string[];
-  expire_in?: string;
-  when?: "always" | "on_success" | "on_failure";
-  reports?: ArtifactReports;
-};
-// Reports won't show up on MRs until https://gitlab.com/groups/gitlab-org/-/epics/8205
-export type ArtifactReports =
-  | { accessibility: string | string[] }
-  | { coverage_report: { coverage_format: string; path: string } }
-  | { codequality: string | string[] }
-  | { cyclonedx: string | string[] }
-  | { dotenv: string | string[] }
-  | { junit: string | string[] }
-  | { sast: string }
-  | { secret_detection: string }
-  | { terraform: string };
-export type GitlabJobCache = {
-  key: GitlabJobCacheKey;
-  policy?: string;
-  paths: string[];
-};
-export type GitlabJobCacheKey =
-  | string
-  | {
-      files: string[];
-      prefix?: string;
-    };
-export type GitlabRule = {
-  if?: string;
-  when?: "always" | "on_success" | "manual" | "never"; // todo
-  allow_failure?: boolean;
-};
-export type Retry = {
-  max: number;
-  when: string[];
-};
+import type { GitlabJobWithContext } from "../pipeline/gitlab/createGitlabJobs";
+import type {
+  Artifacts as GitlabCiArtifacts,
+  JobTemplate,
+  Cache as GitlabJobCache,
+  Rules,
+} from "./gitlab-ci-yml";
 
-export type GitlabJobService = {
-  name: string;
-  command?: string[];
-  entrypoint?: string[];
-  alias?: string;
-};
+export { Retry, Image as GitlabJobImage } from "./gitlab-ci-yml";
 
-export type GitlabJobImage =
-  | string
-  | {
-      name: string;
-      entrypoint?: string[];
-    };
+// Reports won't show up on MRs until https://gitlab.com/groups/gitlab-org/-/epics/8205
+export type Artifacts = GitlabCiArtifacts;
+export type { GitlabJobCache };
+export type GitlabRule = Exclude<
+  Exclude<Rules, null>[number],
+  string | string[] // we only use the object version
+>;
+export type GitlabEnvironment = Omit<
+  Exclude<JobTemplate["environment"], undefined | string>,
+  "deployment_tier"
+>;
 
-export type GitlabEnvironment = {
-  url: string;
-  name: string;
-  kubernetes?: {
-    namespace?: string;
-  };
-  action?: "stop" | "start" | "prepare" | "access"; // defaults to start
-  on_stop?: string; // other job to run on stop
-  auto_stop_in?: string;
-};
-export type GitlabJobDef = {
-  stage: string;
-  before_script?: string[];
-  script: (string | undefined)[];
-  interruptible?: boolean;
-  needs?: Array<string | { job: string; artifacts: boolean }>;
+export interface GitlabJobDef
+  extends Pick<
+    JobTemplate,
+    | "after_script"
+    | "allow_failure"
+    | "artifacts"
+    | "before_script"
+    | "coverage"
+    | "dependencies"
+    | "environment"
+    | "except"
+    | "hooks"
+    | "image"
+    | "interruptible"
+    | "needs"
+    | "only"
+    | "parallel"
+    | "release"
+    | "resource_group"
+    | "retry"
+    | "rules"
+    | "script"
+    | "services"
+    | "stage"
+    | "tags"
+    | "trigger"
+    | "variables"
+  > {
+  stage: JobTemplate["stage"];
   rules?: GitlabRule[];
   cache?: GitlabJobCache | GitlabJobCache[];
   artifacts?: Artifacts;
-  retry?: Retry;
-  services?: GitlabJobService[];
-  image?: GitlabJobImage;
-  variables?: GitlabVariables;
-  dependencies?: string[];
-  environment?: GitlabEnvironment;
-  allow_failure?: boolean;
-  parallel?: number;
-  trigger?: {
-    strategy: "depend";
-    include: Array<{
-      artifact: string;
-      job: string;
-    }>;
-  };
-};
-
-export type GitlabVariables = Record<string, string | undefined>;
+  tags?: string[];
+}
 
 /**
  * this is not precicily the type of a gitlab-pipeline
  * the jobs nee to be merge into the object.
  * Problem is, that this type cannot be represented properly with typescript, see https://github.com/microsoft/TypeScript/issues/17867
  */
-export type GitlabPipeline = {
+export interface GitlabPipeline extends Pick<JobTemplate, "variables"> {
   image: string;
-  variables: GitlabVariables;
   workflow?: {
     rules: GitlabRule[];
   };
   stages: string[];
-  jobs: Record<string, GitlabJobDef>;
-};
+  jobs: Record<string, GitlabJobWithContext>;
+  before_script?: string[];
+}

package/src/types/index.ts

@@ -1,4 +1,5 @@
 export * from "./config";
 export * from "./gitlab-types";
 export * from "./context";
+export * from "./jobDefinition";
 export * from "./pipeline";

package/src/types/jobDefinition.ts

@@ -0,0 +1,34 @@
+import type { CacheConfig } from "../build";
+import type { CatladderJob } from "./jobs";
+
+// new intermediate types. Currently not widely used, but we will built a bit  more abstraction on top of this
+
+export type AppBuildJobDefinition = Partial<
+  Omit<CatladderJob, "artifacts" | "cache">
+> & {
+  cache?: CacheConfig[];
+};
+
+export type DockerBuildJobDefinition = AppBuildJobDefinition; // currently the same
+
+export type DeployJobDefinition = Pick<
+  CatladderJob,
+  | "script"
+  | "variables"
+  | "image"
+  | "artifacts"
+  | "services"
+  | "runnerVariables"
+> & {
+  cache?: CacheConfig[];
+};
+
+export type StopJobDefinition = Pick<
+  CatladderJob,
+  "script" | "variables" | "image" | "runnerVariables"
+>;
+
+export type RollbackJobDefinition = Pick<
+  CatladderJob,
+  "script" | "variables" | "runnerVariables" | "image"
+>;

package/src/types/jobs.ts

@@ -1,13 +1,14 @@
+import type { UnspecifiedEnvVars } from "..";
 import type {
   Artifacts,
   GitlabEnvironment,
   GitlabJobCache,
-  GitlabRule,
-  GitlabVariables,
-  GitlabJobService,
+  GitlabJobDef,
   GitlabJobImage,
+  GitlabRule,
 } from "./gitlab-types";
 
+export type CatladderJobCache = GitlabJobCache;
 export const BASE_STAGES = [
   "setup",
   "test",
@@ -17,11 +18,18 @@ export const BASE_STAGES = [
   "rollback",
   "stop",
 ] as const;
-export type BaseStage = typeof BASE_STAGES[number];
+export type BaseStage = (typeof BASE_STAGES)[number];
 
 export type CatladderJobNeed =
   | string
-  | { job: string; artifacts: boolean; componentName?: string };
+  | { job: string; artifacts: boolean; componentName?: string }
+  | { job: string; artifacts: boolean; workspaceName: string };
+
+export type CatladderJobEnvironmentConfig = {
+  action?: GitlabEnvironment["action"];
+  on_stop?: GitlabEnvironment["on_stop"];
+  auto_stop_in?: string;
+};
 export type CatladderJob<S = BaseStage> = {
   /**
    * the name of the job (without any env or app prefix and suffix)
@@ -51,6 +59,7 @@ export type CatladderJob<S = BaseStage> = {
   needsStages?: {
     stage: S;
     artifacts?: boolean;
+    workspaceName?: string;
   }[];
 
   /**
@@ -61,18 +70,10 @@ export type CatladderJob<S = BaseStage> = {
    */
   needs?: Array<CatladderJobNeed>;
 
-  /**
-   * @deprecated set `componentName` to `needs` if you want to wait for a job from another component
-   */
-  needsOtherComponent?: Array<{
-    componentName: string;
-    job: string;
-    artifacts: boolean;
-  }>;
   /**
    * cache config, we use here the same shape as gitlab itself
    */
-  cache?: GitlabJobCache | GitlabJobCache[];
+  cache?: CatladderJobCache | CatladderJobCache[];
 
   /**
    * job artifacts, we also use gitlab shape here
@@ -82,7 +83,7 @@ export type CatladderJob<S = BaseStage> = {
   /**
    * additional services, mainly used for docker
    */
-  services?: GitlabJobService[];
+  services?: GitlabJobDef["services"];
 
   /**
    * image to use
@@ -92,14 +93,28 @@ export type CatladderJob<S = BaseStage> = {
   /**
    * variables to pass
    */
-  variables: GitlabVariables;
+  variables: UnspecifiedEnvVars | undefined;
+
+  /**
+   * additional vars only for the runner.
+   * Also if you use services: that require env vars, you need to set them here.
+   *
+   */
+  runnerVariables?: Record<string, string>;
 
   /**
    * whether failures are allowed
    */
   allow_failure?: boolean;
 
-  environment?: GitlabEnvironment;
+  /**
+   * gitlab environment config, subject to change
+   */
+  environment?: CatladderJobEnvironmentConfig;
+
+  /** */
+
+  when?: GitlabRule["when"];
 
   rules?: GitlabRule[];
 
@@ -109,4 +124,9 @@ export type CatladderJob<S = BaseStage> = {
    * We use the same shape as GitLab itself.
    */
   parallel?: number;
+
+  /**
+   * tags for the underlying job runner (e.g gitlab)
+   */
+  jobTags?: string[];
 };

package/src/types/utils.ts

@@ -1,7 +1,7 @@
 export type PartialDeep<T> = T extends any[]
   ? T
   : T extends Record<string, any>
-  ? {
-      [P in keyof T]?: PartialDeep<T[P]>;
-    }
-  : T;
+    ? {
+        [P in keyof T]?: PartialDeep<T[P]>;
+      }
+    : T;

package/src/utils/gitlab.ts

@@ -1,3 +1,5 @@
+import { registerGlobalScriptFunction } from "../globalScriptFunctions";
+
 export const allowFailureInScripts = (script: string[]): string[] => [
   "set +e", // disable fail job on error
   ...script,
@@ -11,7 +13,7 @@ export const repeatOnFailure = (
   command: string,
   options: {
     pauseInSeconds: number;
-  }
+  },
 ): string => {
   return `
     until ${command}
@@ -21,3 +23,28 @@ export const repeatOnFailure = (
     done
   `;
 };
+
+const start = registerGlobalScriptFunction(
+  "collapseable_section_start",
+  `local section_title="\${1}"
+  local section_description="\${2:-$section_title}"
+  echo -e "section_start:\`date +%s\`:\${section_title}[collapsed=true]\\r\\e[0K$\{section_description}"
+`,
+);
+
+const end = registerGlobalScriptFunction(
+  "collapseable_section_end",
+  `local section_title="\${1}"
+  echo -e "section_end:\`date +%s\`:\${section_title}\\r\\e[0K"
+`,
+);
+
+export const collapseableSection =
+  (name: string, header: string) =>
+  (commands: string[]): string[] => {
+    return [
+      start.invoke(`"${name}"`, `"${header}"`),
+      ...commands,
+      end.invoke(`"${name}"`),
+    ];
+  };

package/src/utils/index.ts

@@ -1,19 +1,22 @@
 import { isNil, mergeWith } from "lodash";
 
 export function notNil<TValue>(
-  value: TValue | null | undefined
+  value: TValue | null | undefined,
 ): value is TValue {
   return value !== null && value !== undefined;
 }
 
-export const ensureArray = <T>(s: T | T[]): T[] | null =>
+export const ensureArrayOrNull = <T>(s: undefined | T | T[]): T[] | null =>
   isNil(s) ? null : Array.isArray(s) ? s : [s];
 
+export const ensureArray = <T>(s: undefined | T | T[]): T[] =>
+  ensureArrayOrNull(s) ?? [];
+
 // see https://github.com/lodash/lodash/issues/5384
 export const mergeWithMergingArrays = <A, B>(obj1: A, obj2: B): A & B => {
   return mergeWith({}, obj1, obj2, (prop1, prop2) =>
     Array.isArray(prop1) && Array.isArray(prop2)
       ? [...prop1, ...prop2]
-      : undefined
+      : undefined,
   );
 };

package/src/utils/jsonParse.ts

@@ -0,0 +1,7 @@
+export const jsonParseOrThrow = (str: string): any => {
+  try {
+    return JSON.parse(str);
+  } catch (e) {
+    throw new Error(`could not parse json: ${str}`);
+  }
+};

package/src/utils/removeUndefined.ts

@@ -0,0 +1,4 @@
+export const removeUndefined = <T extends object>(obj: T): T =>
+  Object.fromEntries(
+    Object.entries(obj).filter(([, value]) => value !== undefined),
+  ) as T;

package/src/utils/writeFiles.ts

@@ -0,0 +1,52 @@
+import { writeFile } from "fs/promises";
+import { stringify } from "yaml";
+
+export const getAutoGeneratedHeader = (commentChar: string) => {
+  return [
+    "-------------------------------------------------",
+    `🐱 🔨 This file is generated by catladder`,
+    `🚨 Do not edit this file manually 🚨`,
+    "-------------------------------------------------",
+  ]
+    .map((line) => `${commentChar} ${line}`)
+    .join("\n")
+    .concat("\n");
+};
+
+export const writeGeneratedFile = async (
+  path: string,
+  content: string,
+  {
+    commentChar,
+  }: {
+    commentChar: string;
+  },
+) => {
+  await writeFile(
+    path,
+    // need to spread out the jobs, forgot why
+    [getAutoGeneratedHeader(commentChar), content].join("\n"),
+    {
+      encoding: "utf-8",
+    },
+  );
+};
+
+type StringifyOptions = Exclude<
+  Parameters<typeof stringify>[2],
+  null | undefined | string | number
+>;
+export const yamlStringifyOptions: StringifyOptions = {
+  // prevents colapsing long command statements into multiple lines
+  lineWidth: 0,
+  // represent multi line commands as single line json strings
+  doubleQuotedAsJSON: true,
+  // Better readability when bash commands most often use double quotes
+  singleQuote: true,
+};
+
+export const writeYamlfile = async (path: string, data: any) => {
+  await writeGeneratedFile(path, stringify(data, yamlStringifyOptions), {
+    commentChar: "#",
+  });
+};

package/src/variables/VariableValue.ts

@@ -0,0 +1,6 @@
+import type { BashExpression } from "../bash/BashExpression";
+import type { VariableValueContainingReferences } from "./VariableValueContainingReferences";
+export type VariableValue =
+  | VariableValueContainingReferences
+  | BashExpression
+  | string;

package/src/variables/VariableValueContainingReferences.ts

@@ -0,0 +1,89 @@
+import { BashExpression } from "../bash/BashExpression";
+import type { EscapeOptions } from "../bash/bashEscape";
+import { escapeBashExpression, escapeString } from "../bash/bashEscape";
+
+export class UnresolvableReference {
+  constructor(public reference: VariableReference) {}
+  public toString() {
+    return `Unresolvable reference: ${this.reference.toString()}`;
+  }
+}
+export class VariableReference {
+  public componentName: string;
+  public variableName: string;
+
+  constructor(componentName: string, variableName: string) {
+    this.componentName = componentName;
+    this.variableName = variableName;
+  }
+
+  public toString() {
+    return `\${${this.componentName}:${this.variableName}}`;
+  }
+}
+
+type VariableValuePart =
+  | string
+  | BashExpression
+  | VariableReference
+  | UnresolvableReference;
+
+type MaybeArray<T> = T | T[];
+export class VariableValueContainingReferences {
+  public parts: VariableValuePart[];
+  constructor(
+    parts: MaybeArray<VariableValuePart | VariableValueContainingReferences>,
+  ) {
+    this.parts = (Array.isArray(parts) ? parts : [parts]).flatMap((part) =>
+      part instanceof VariableValueContainingReferences
+        ? part.parts
+        : part === ""
+          ? []
+          : [part],
+    );
+  }
+
+  public toString(
+    options: EscapeOptions = {
+      quotes: false,
+    },
+  ) {
+    return this.parts
+      .map((part) => {
+        if (typeof part === "string") {
+          return escapeString(part, options);
+        } else if (part instanceof BashExpression) {
+          return escapeBashExpression(part, options);
+        } else {
+          return part.toString();
+        }
+      })
+      .join("");
+  }
+}
+
+// regex to resolve references in catladder variables
+// those expressions have the pattern ${componentName:variableName}
+const REGEX = /\$\{(([^:}]+):)?([^}]+)}/gm;
+export const createVariableValueContainingReferencesFromString = (
+  value: string,
+  options: { componentName: string },
+) => {
+  const parts: Array<VariableValuePart> = [];
+  let match: any;
+  let lastIndex = 0;
+  while ((match = REGEX.exec(value)) !== null) {
+    const [fullMatch, _, otherComponentName, variableName] = match;
+
+    parts.push(value.slice(lastIndex, match.index));
+    parts.push(
+      new VariableReference(
+        otherComponentName || options.componentName,
+        variableName,
+      ),
+    );
+    lastIndex = REGEX.lastIndex;
+  }
+  parts.push(value.slice(lastIndex));
+  return new VariableValueContainingReferences(parts);
+};

package/src/variables/__tests__/resolveAllReferences.test.ts

@@ -0,0 +1,110 @@
+import {
+  VariableValueContainingReferences,
+  createVariableValueContainingReferencesFromString,
+} from "../VariableValueContainingReferences";
+import { resolveAllReferences } from "../resolveAllReferences";
+
+describe("replaceAllReferences", () => {
+  it("should replace all references recursivly", async () => {
+    const values = {
+      myVar: createVariableValueContainingReferencesFromString(
+        "the 2nd component has ${component2:variable1} and self reference ${variable2}",
+        {
+          componentName: "component1",
+        },
+      ),
+      myOtherVar: createVariableValueContainingReferencesFromString(
+        "the third component has ${component3:variable1}, isn't that cool?",
+        {
+          componentName: "component1",
+        },
+      ),
+      myThirdVar: createVariableValueContainingReferencesFromString(
+        "value from component3: ${component3:variable3}",
+        {
+          componentName: "component1",
+        },
+      ),
+    };
+
+    const getEnvVars = async (componentName: string) => {
+      return {
+        variable1: createVariableValueContainingReferencesFromString(
+          `i am variable1 from ${componentName}`,
+          { componentName },
+        ),
+        variable2: createVariableValueContainingReferencesFromString(
+          `foo from ${componentName}`,
+          {
+            componentName,
+          },
+        ),
+        variable3: createVariableValueContainingReferencesFromString(
+          `i am referencing \${component1:variable1}`,
+          { componentName },
+        ),
+      };
+    };
+
+    const result = await resolveAllReferences(values, getEnvVars);
+    expect(result).toEqual({
+      myVar: new VariableValueContainingReferences([
+        "the 2nd component has ",
+        "i am variable1 from component2",
+        " and self reference ",
+        "foo from component1",
+      ]),
+      myOtherVar: new VariableValueContainingReferences([
+        "the third component has ",
+        "i am variable1 from component3",
+        ", isn't that cool?",
+      ]),
+      myThirdVar: new VariableValueContainingReferences([
+        "value from component3: ",
+        "i am referencing ",
+        "i am variable1 from component1",
+      ]),
+    });
+  });
+
+  it("detects infinte loop", async () => {
+    const values = {
+      myVar: createVariableValueContainingReferencesFromString(
+        "i am referencing ${component2:variable1}",
+        {
+          componentName: "component1",
+        },
+      ),
+    };
+
+    const getEnvVars = async (componentName: string) => {
+      if (componentName === "component2") {
+        return {
+          variable1: createVariableValueContainingReferencesFromString(
+            `i am referencing \${component3:variable1}`,
+            { componentName },
+          ),
+        };
+      } else if (componentName === "component3") {
+        return {
+          variable1: createVariableValueContainingReferencesFromString(
+            `i am referencing \${component1:variable1}`,
+            { componentName },
+          ),
+        };
+      } else {
+        return {
+          variable1: createVariableValueContainingReferencesFromString(
+            `i am referencing \${component2:variable1}`,
+            { componentName },
+          ),
+        };
+      }
+    };
+
+    // expect to throw an error
+    await expect(resolveAllReferences(values, getEnvVars)).rejects.toThrow(
+      "Infinite loop detected in these variables: myVar (last reference: ${component1:variable1})",
+    );
+  });
+});