@catladder/pipeline 0.0.0-node16-ddfae617 → 0.0.0-perf-cli-reduce-bundle-size-with-tsconfig-optimization-3556aaa4

package/src/types/gitlab-types.ts

@@ -1,74 +1,69 @@
-export type Artifacts = {
-  paths: string[];
-};
-export type GitlabJobCache = {
-  key: string;
-  policy?: string;
-  paths: string[];
-};
-export type GitlabRule = {
-  if?: string;
-  when?: "always" | "on_success" | "manual" | "never"; // todo
-  allow_failure?: boolean;
-};
-export type Retry = {
-  max: number;
-  when: string[];
-};
+import type {
+  Artifacts as GitlabCiArtifacts,
+  JobTemplate,
+  Cache,
+  Rules,
+} from "./gitlab-ci-yml";
 
-export type Service = {
-  name: string;
-  command: string[];
-};
+export { Retry, Image as GitlabJobImage } from "./gitlab-ci-yml";
 
-export type GitlabEnvironment = {
-  url: string;
-  name: string;
-  kubernetes?: {
-    namespace?: string;
-  };
-  action?: "stop" | "start" | "prepare"; // defaults to start
-  on_stop?: string; // other job to run on stop
-  auto_stop_in?: string;
-};
-export type GitlabJobDef = {
-  stage: string;
-  before_script?: string[];
-  script: (string | undefined)[];
-  interruptible?: boolean;
-  needs?: Array<string | { job: string; artifacts: boolean }>;
+// Reports won't show up on MRs until https://gitlab.com/groups/gitlab-org/-/epics/8205
+export type Artifacts = GitlabCiArtifacts;
+export type GitlabJobCache = Cache;
+export type GitlabRule = Exclude<
+  Exclude<Rules, null>[number],
+  string | string[] // we only use the object version
+>;
+export type GitlabEnvironment = Omit<
+  Exclude<JobTemplate["environment"], undefined | string>,
+  "deployment_tier"
+>;
+
+export interface GitlabJobDef
+  extends Pick<
+    JobTemplate,
+    | "after_script"
+    | "allow_failure"
+    | "artifacts"
+    | "before_script"
+    | "coverage"
+    | "dependencies"
+    | "environment"
+    | "except"
+    | "hooks"
+    | "image"
+    | "interruptible"
+    | "needs"
+    | "only"
+    | "parallel"
+    | "release"
+    | "resource_group"
+    | "retry"
+    | "rules"
+    | "script"
+    | "services"
+    | "stage"
+    | "tags"
+    | "trigger"
+    | "variables"
+  > {
+  stage: JobTemplate["stage"];
   rules?: GitlabRule[];
   cache?: GitlabJobCache | GitlabJobCache[];
   artifacts?: Artifacts;
-  retry?: Retry;
-  services?: Service[];
-  image?: string;
-  variables?: GitlabVariables;
-  dependencies?: string[];
-  environment?: GitlabEnvironment;
-  allow_failure?: boolean;
-  trigger?: {
-    strategy: "depend";
-    include: Array<{
-      artifact: string;
-      job: string;
-    }>;
-  };
-};
-
-export type GitlabVariables = Record<string, string | undefined>;
+  tags?: string[];
+}
 
 /**
  * this is not precicily the type of a gitlab-pipeline
  * the jobs nee to be merge into the object.
  * Problem is, that this type cannot be represented properly with typescript, see https://github.com/microsoft/TypeScript/issues/17867
  */
-export type GitlabPipeline = {
+export interface GitlabPipeline extends Pick<JobTemplate, "variables"> {
   image: string;
-  variables: GitlabVariables;
   workflow?: {
     rules: GitlabRule[];
   };
   stages: string[];
   jobs: Record<string, GitlabJobDef>;
-};
+}

package/src/types/jobs.ts

@@ -1,10 +1,15 @@
-import {
+import type { UnspecifiedEnvVars } from "..";
+import type {
+  BashExpression,
+  StringOrBashExpression,
+} from "../bash/BashExpression";
+import type {
   Artifacts,
-  GitlabEnvironment,
   GitlabJobCache,
   GitlabRule,
-  GitlabVariables,
-  Service,
+  GitlabJobImage,
+  GitlabJobDef,
+  GitlabEnvironment,
 } from "./gitlab-types";
 
 export const BASE_STAGES = [
@@ -13,10 +18,21 @@ export const BASE_STAGES = [
   "build",
   "deploy",
   "verify",
+  "rollback",
   "stop",
 ] as const;
-export type BaseStage = typeof BASE_STAGES[number];
+export type BaseStage = (typeof BASE_STAGES)[number];
+
+export type CatladderJobNeed =
+  | string
+  | { job: string; artifacts: boolean; componentName?: string }
+  | { job: string; artifacts: boolean; workspaceName: string };
 
+export type CatladderJobEnvironmentConfig = {
+  action?: GitlabEnvironment["action"];
+  on_stop?: GitlabEnvironment["on_stop"];
+  auto_stop_in?: string;
+};
 export type CatladderJob<S = BaseStage> = {
   /**
    * the name of the job (without any env or app prefix and suffix)
@@ -46,12 +62,25 @@ export type CatladderJob<S = BaseStage> = {
   needsStages?: {
     stage: S;
     artifacts?: boolean;
+    workspaceName?: string;
   }[];
 
   /**
-   * does this require another job?
+   * does this require another job (from the same component)?
+   *
+   * You can also require a job from another component if you set `componentName`
+   *
+   */
+  needs?: Array<CatladderJobNeed>;
+
+  /**
+   * @deprecated set `componentName` to `needs` if you want to wait for a job from another component
    */
-  needs?: Array<string | { job: string; artifacts: boolean }>;
+  needsOtherComponent?: Array<{
+    componentName: string;
+    job: string;
+    artifacts: boolean;
+  }>;
   /**
    * cache config, we use here the same shape as gitlab itself
    */
@@ -65,24 +94,50 @@ export type CatladderJob<S = BaseStage> = {
   /**
    * additional services, mainly used for docker
    */
-  services?: Service[];
+  services?: GitlabJobDef["services"];
 
   /**
    * image to use
    */
-  image?: string;
+  image?: GitlabJobImage;
 
   /**
    * variables to pass
    */
-  variables: GitlabVariables;
+  variables: UnspecifiedEnvVars | undefined;
+
+  /**
+   * additional vars only for the runner.
+   * Also if you use services: that require env vars, you need to set them here.
+   *
+   */
+  runnerVariables?: Record<string, string>;
 
   /**
    * whether failures are allowed
    */
   allow_failure?: boolean;
 
-  environment?: GitlabEnvironment;
+  /**
+   * gitlab environment config, subject to change
+   */
+  environment?: CatladderJobEnvironmentConfig;
+
+  /** */
+
+  when?: GitlabRule["when"];
 
   rules?: GitlabRule[];
+
+  /**
+   * How many instances of a job should be run in parallel.
+   * Useful for big test suites that can be split into multiple pipeline jobs.
+   * We use the same shape as GitLab itself.
+   */
+  parallel?: number;
+
+  /**
+   * tags for the underlying job runner (e.g gitlab)
+   */
+  jobTags?: string[];
 };

package/src/types/pipeline.ts

@@ -1,7 +1,11 @@
-import { GitlabJobDef, GitlabPipeline } from "./gitlab-types";
+import type { GitlabJobDef, GitlabPipeline } from "./gitlab-types";
 
 export type PipelineType = "gitlab";
 
+export type PipelineMode<T extends PipelineType> = T extends "gitlab"
+  ? "childpipeline" | "local"
+  : "local";
+
 export type PipelineJob<T extends PipelineType> = T extends "gitlab"
   ? GitlabJobDef
   : never;

package/src/types/utils.ts

@@ -0,0 +1,7 @@
+export type PartialDeep<T> = T extends any[]
+  ? T
+  : T extends Record<string, any>
+    ? {
+        [P in keyof T]?: PartialDeep<T[P]>;
+      }
+    : T;

package/src/utils/gitlab.ts

@@ -0,0 +1,33 @@
+export const allowFailureInScripts = (script: string[]): string[] => [
+  "set +e", // disable fail job on error
+  ...script,
+  "set -e", // reenable
+];
+
+export const sanitizeForBashVariable = (name: string) =>
+  name.replace(/-/g, "_");
+
+export const repeatOnFailure = (
+  command: string,
+  options: {
+    pauseInSeconds: number;
+  },
+): string => {
+  return `
+    until ${command}
+    do
+      echo "Trying again."
+      sleep ${options.pauseInSeconds}
+    done
+  `;
+};
+
+export const collapseableSection =
+  (name: string, header: string) =>
+  (commands: string[]): string[] => {
+    return [
+      `echo -e "\\e[0Ksection_start:$(date +%s):${name}[collapsed=true]\\r\\e[0K${header}"`,
+      ...commands,
+      `echo -e "\\e[0Ksection_end:$(date +%s):${name}\\r\\e[0K"`,
+    ];
+  };

package/src/utils/index.ts

@@ -1,12 +1,12 @@
 import { isNil, mergeWith } from "lodash";
 
 export function notNil<TValue>(
-  value: TValue | null | undefined
+  value: TValue | null | undefined,
 ): value is TValue {
   return value !== null && value !== undefined;
 }
 
-export const ensureArray = <T>(s: T | T[]): T[] | null =>
+export const ensureArray = <T>(s: undefined | T | T[]): T[] | null =>
   isNil(s) ? null : Array.isArray(s) ? s : [s];
 
 // see https://github.com/lodash/lodash/issues/5384
@@ -14,6 +14,6 @@ export const mergeWithMergingArrays = <A, B>(obj1: A, obj2: B): A & B => {
   return mergeWith({}, obj1, obj2, (prop1, prop2) =>
     Array.isArray(prop1) && Array.isArray(prop2)
       ? [...prop1, ...prop2]
-      : undefined
+      : undefined,
   );
 };

package/src/utils/jsonParse.ts

@@ -0,0 +1,7 @@
+export const jsonParseOrThrow = (str: string): any => {
+  try {
+    return JSON.parse(str);
+  } catch (e) {
+    throw new Error(`could not parse json: ${str}`);
+  }
+};

package/src/utils/removeUndefined.ts

@@ -0,0 +1,4 @@
+export const removeUndefined = <T extends object>(obj: T): T =>
+  Object.fromEntries(
+    Object.entries(obj).filter(([, value]) => value !== undefined),
+  ) as T;

package/src/utils/writeFiles.ts

@@ -0,0 +1,53 @@
+import { writeFile } from "fs/promises";
+import { stringify } from "yaml";
+import packageInfos from "../packageInfos";
+
+export const getAutoGeneratedHeader = (commentChar: string) => {
+  return [
+    "-------------------------------------------------",
+    `🐱 🔨 This file is generated by catladder v${packageInfos.version}`,
+    `🚨 Do not edit this file manually 🚨`,
+    "-------------------------------------------------",
+  ]
+    .map((line) => `${commentChar} ${line}`)
+    .join("\n")
+    .concat("\n");
+};
+
+export const writeGeneratedFile = async (
+  path: string,
+  content: string,
+  {
+    commentChar,
+  }: {
+    commentChar: string;
+  },
+) => {
+  await writeFile(
+    path,
+    // need to spread out the jobs, forgot why
+    [getAutoGeneratedHeader(commentChar), content].join("\n"),
+    {
+      encoding: "utf-8",
+    },
+  );
+};
+
+type StringifyOptions = Exclude<
+  Parameters<typeof stringify>[2],
+  null | undefined | string | number
+>;
+export const yamlStringifyOptions: StringifyOptions = {
+  // prevents colapsing long command statements into multiple lines
+  lineWidth: 0,
+  // represent multi line commands as single line json strings
+  doubleQuotedAsJSON: true,
+  // Better readability when bash commands most often use double quotes
+  singleQuote: true,
+};
+
+export const writeYamlfile = async (path: string, data: any) => {
+  await writeGeneratedFile(path, stringify(data, yamlStringifyOptions), {
+    commentChar: "#",
+  });
+};

package/tsconfig.json

@@ -1,12 +1,14 @@
 {
   "compilerOptions": {
-    "lib": ["es2020", "DOM"],
+    "lib": ["es2021", "DOM"],
     "strict": true,
+    "module": "CommonJS",
     "downlevelIteration": true,
     "declaration": true,
     "outDir": "dist",
+    "moduleResolution": "node",
     "esModuleInterop": true,
     "incremental": true
   },
-  "include": ["./src"]
+  "include": ["./src", "tests"]
 }

package/dist/build/base/createBuildJob.d.ts

@@ -1,7 +0,0 @@
-import { Context } from "../..";
-import { CatladderJob } from "../../types/jobs";
-export declare const createBuildJob: (context: Context, {
-  script,
-  variables,
-  ...def
-}: Partial<CatladderJob>) => CatladderJob;

package/dist/build/base/getBuildInfo.d.ts

@@ -1,2 +0,0 @@
-import { Context } from "../../types";
-export declare const getBuildInfo: (context: Context) => string[];

package/dist/build/base/getBuildInfo.js

@@ -1,12 +0,0 @@
-"use strict";
-
-exports.__esModule = true;
-exports.getBuildInfo = void 0;
-
-var getBuildInfo = function (context) {
-  var _a, _b;
-
-  return ["echo '{\"id\":\"".concat((_a = context.commitInfo) === null || _a === void 0 ? void 0 : _a.buildId, "\",\"time\":\"").concat((_b = context.commitInfo) === null || _b === void 0 ? void 0 : _b.buildTime, "\"}' > ").concat(context.componentConfig.dir, "/__build_info.json")];
-};
-
-exports.getBuildInfo = getBuildInfo;

package/dist/context/resolveReferences.test.js

@@ -1,141 +0,0 @@
-"use strict";
-
-exports.__esModule = true;
-
-var resolveReferences_1 = require("./resolveReferences");
-
-describe("resolveReferences", function () {
-  it("replaces occurences of ${componentName:VARIABLE_NAME}", function () {
-    var variables = {
-      a: "hello world",
-      b: "a replaced value looks like this: '${api:FOO}', nice!"
-    };
-    var otherVariables = {
-      api: {
-        FOO: "foo from api"
-      },
-      frontend: {
-        FOO: "foo from frontend"
-      }
-    };
-    var result = (0, resolveReferences_1.resolveReferences)(variables, function (componentName, variableName) {
-      var _a;
-
-      return (_a = otherVariables[componentName]) === null || _a === void 0 ? void 0 : _a[variableName];
-    });
-    expect(result).toEqual({
-      a: "hello world",
-      b: "a replaced value looks like this: 'foo from api', nice!"
-    });
-  });
-  it("replaces self references with structure ${VARIABLE_NAME}", function () {
-    var variables = {
-      FOO: "hello world",
-      BAR: "this: ${FOO}!"
-    };
-    var result = (0, resolveReferences_1.resolveReferences)(variables);
-    expect(result).toEqual({
-      FOO: "hello world",
-      BAR: "this: hello world!"
-    });
-  });
-  it("replaces self references mixed with other vars", function () {
-    var variables = {
-      FOO: "hello from ${api:FOO}",
-      BAR: "this: ${FOO}!"
-    };
-    var otherVariables = {
-      api: {
-        FOO: "foo from api"
-      }
-    };
-    var result = (0, resolveReferences_1.resolveReferences)(variables, function (componentName, variableName) {
-      var _a;
-
-      return (_a = otherVariables[componentName]) === null || _a === void 0 ? void 0 : _a[variableName];
-    });
-    expect(result).toEqual({
-      FOO: "hello from foo from api",
-      BAR: "this: hello from foo from api!"
-    });
-  });
-  it("keeps variables as is if not found (when null is returned)", function () {
-    var variables = {
-      a: "hello world",
-      b: "a not found value looks like this: '${api:FOO}'"
-    };
-    var otherVariables = {};
-    var result = (0, resolveReferences_1.resolveReferences)(variables, function (componentName, variableName) {
-      var _a;
-
-      return (_a = otherVariables[componentName]) === null || _a === void 0 ? void 0 : _a[variableName];
-    });
-    expect(result).toEqual({
-      a: "hello world",
-      b: "a not found value looks like this: '${api:FOO}'"
-    });
-  });
-  it("replaces mulitple levels deep", function () {
-    var variables = {
-      a: "value is ${frontend:FOO}!"
-    };
-    var otherVariables = {
-      api: {
-        FOO: "foo from api"
-      },
-      frontend: {
-        FOO: "hi, ${api:FOO}"
-      }
-    };
-    var result = (0, resolveReferences_1.resolveReferences)(variables, function (componentName, variableName) {
-      var _a, _b;
-
-      return (_b = (_a = otherVariables[componentName]) === null || _a === void 0 ? void 0 : _a[variableName]) !== null && _b !== void 0 ? _b : "";
-    });
-    expect(result).toEqual({
-      a: "value is hi, foo from api!"
-    });
-  });
-  it("prevents endless loops", function () {
-    var variables = {
-      a: "value is ${frontend:FOO}!"
-    };
-    var otherVariables = {
-      api: {
-        FOO: "api ${frontend:FOO}"
-      },
-      frontend: {
-        FOO: "frontend ${api:FOO}"
-      }
-    };
-    var result = (0, resolveReferences_1.resolveReferences)(variables, function (componentName, variableName) {
-      var _a, _b;
-
-      return (_b = (_a = otherVariables[componentName]) === null || _a === void 0 ? void 0 : _a[variableName]) !== null && _b !== void 0 ? _b : "";
-    });
-    expect(result).toEqual({
-      a: "value is frontend api ${frontend:FOO}!"
-    });
-  });
-});
-describe("translateLegacyFromComponents", function () {
-  it("translatetes the old `fromComponents` approach to ${componentName:variableName}", function () {
-    var fromComponents = {
-      api: {
-        API_FOO: "FOO",
-        API_BAR: "BAR"
-      },
-      www: {
-        WWW_X: "X",
-        WWW_Y: "Y"
-      }
-    };
-    var result = (0, resolveReferences_1.translateLegacyFromComponents)(fromComponents);
-    expect(result).toEqual({
-      API_FOO: "${api:FOO}",
-      API_BAR: "${api:BAR}",
-      WWW_X: "${www:X}",
-      WWW_Y: "${www:Y}"
-    });
-  });
-});

package/dist/deploy/base.d.ts

@@ -1,8 +0,0 @@
-import { Context } from "../types";
-import { CatladderJob } from "../types/jobs";
-export declare const DEPLOY_JOB_NAME = "\uD83D\uDE80 Deploy";
-export declare const STOP_JOB_NAME = "\uD83D\uDED1 Stop \u26A0\uFE0F";
-declare type JobWithoutScript = Omit<CatladderJob, "script">;
-export declare const getBaseDeploymentJob: (context: Context) => JobWithoutScript;
-export declare const getBaseDeploymentStopJob: (context: Context) => JobWithoutScript;
-export {};