@catladder/pipeline 0.0.0-node16-ddfae617 → 0.0.0-prettier-cbceeed9

package/src/pipeline/gitlab/createGitlabJobs.ts

@@ -0,0 +1,149 @@
+import { isObject } from "lodash";
+import { BASE_RETRY } from "../../defaults";
+import type { GitlabJobDef } from "../../types";
+import type { CatladderJob, CatladderJobNeed } from "../../types/jobs";
+import type { AllCatladderJobs } from "../createAllJobs";
+
+type AllGitlabJobs = Record<string, GitlabJobDef>;
+
+const getFullJobName = (
+  name: string,
+  componentName: string,
+  env?: string | null
+) => {
+  if (env) {
+    return `${componentName} ${name} | ${env} `;
+  }
+  return `${componentName} ${name}`;
+};
+
+const getFullReferencedJobName = (
+  referencedJobName: string,
+  componentName: string,
+  env: string,
+  allJobs: AllCatladderJobs
+) => {
+  const referencedJob = allJobs[componentName]?.[env]?.find(
+    (j) => j.name === referencedJobName
+  );
+  if (!referencedJob) {
+    throw new Error(
+      `unknown job referenced: '${referencedJobName}' from '${env}:${componentName}'`
+    );
+  }
+  const envToSet = referencedJob.envMode !== "none" ? env : null;
+  return getFullJobName(referencedJobName, componentName, envToSet);
+};
+
+const getJobName = (need: CatladderJobNeed) =>
+  isObject(need) ? need.job : need;
+
+export const makeGitlabJob = (
+  componentName: string,
+  env: string,
+  {
+    envMode,
+    needsStages,
+    needsOtherComponent,
+    name,
+    needs,
+    ...job
+  }: CatladderJob<string>,
+  allJobs: AllCatladderJobs
+): [fullName: string, job: GitlabJobDef] => {
+  const stage = envMode === "stagePerEnv" ? `${job.stage} ${env}` : job.stage;
+
+  const needsFromStages: CatladderJob["needs"] = needsStages?.flatMap((n) => {
+    const referencedComponentName = componentName;
+    const allJobNamesFromThatStage =
+      allJobs[referencedComponentName]?.[env]
+        ?.filter((j) => j.stage === n.stage)
+        ?.map((j) => j.name) ?? [];
+
+    return allJobNamesFromThatStage.map((job) => ({
+      job,
+      artifacts: n.artifacts ?? false,
+      componentName: referencedComponentName,
+    }));
+  });
+  const cleanedNeeds: CatladderJob["needs"] = [
+    ...(needs ?? []),
+    // pull in legacy needs from other component, which is now identical to needs
+    ...(needsOtherComponent ?? []),
+    ...(needsFromStages ?? []),
+  ];
+
+  const gitlabNeeds: GitlabJobDef["needs"] = cleanedNeeds
+    ?.map((n) =>
+      isObject(n)
+        ? {
+            job: getFullReferencedJobName(
+              n.job,
+              n.componentName ?? componentName,
+              env,
+              allJobs
+            ),
+            artifacts: n.artifacts,
+          }
+        : getFullReferencedJobName(n, componentName, env, allJobs)
+    ) // sort in a predictable manner for snapshot tests
+    .sort((a, b) => getJobName(a).localeCompare(getJobName(b)));
+
+  const fullJobName = getFullJobName(
+    name,
+    componentName,
+    envMode !== "none" ? env : undefined
+  );
+
+  const gitlabJob = {
+    ...job,
+    stage,
+    environment: job.environment?.on_stop
+      ? {
+          ...job.environment,
+          on_stop: getFullReferencedJobName(
+            job.environment.on_stop,
+            componentName,
+            env,
+            allJobs
+          ),
+        }
+      : job.environment,
+    // sort in a predictable manner for snapshot tests
+    needs: gitlabNeeds,
+    retry: BASE_RETRY,
+    interruptible: true,
+  };
+
+  return [fullJobName, gitlabJob];
+};
+
+export const createGitlabJobs = async (
+  allJobs: AllCatladderJobs
+): Promise<AllGitlabJobs> => {
+  return Object.keys(allJobs).reduce((accForComponents, componentName) => {
+    const componentJobs = allJobs[componentName];
+    return {
+      ...accForComponents,
+      ...Object.keys(componentJobs).reduce((accForEnvs, env) => {
+        const jobs = componentJobs[env];
+
+        return {
+          ...accForEnvs,
+          ...jobs.reduce((accForJobs, job) => {
+            const [fullJobName, gitlabJob] = makeGitlabJob(
+              componentName,
+              env,
+              job,
+              allJobs
+            );
+            return {
+              ...accForJobs,
+              [fullJobName]: gitlabJob,
+            };
+          }, {} as AllGitlabJobs),
+        };
+      }, {} as AllGitlabJobs),
+    };
+  }, {} as AllGitlabJobs);
+};

package/src/pipeline/index.ts

@@ -1,2 +1,2 @@
 export * from "./createChildPipeline";
-export * from "./createJobs";
+export * from "./createJobsForComponent";

package/src/pipeline/packageManager.ts

@@ -1,43 +1,9 @@
-import { exec } from "child-process-promise";
-import { Config, PackageManagerInfo } from "../types";
-import { pathEqual } from "path-equal";
-import memoizee from "memoizee";
-import { join } from "path";
 import { existsSync } from "fs";
+import { join } from "path";
+import { pathEqual } from "path-equal";
+import type { Config, PackageManagerInfo } from "../types";
+import { getWorkspaces, getYarnVersion } from "./yarn/yarnUtils";
 
-const execOrFail = async (cmd: string, onFail: string): Promise<string> => {
-  try {
-    return await exec(cmd).then((r) => r.stdout);
-  } catch (e) {
-    return onFail ?? null;
-  }
-};
-
-const getYarnVersion = memoizee(
-  async () => {
-    return await execOrFail("yarn --version", "");
-  },
-  { promise: true }
-);
-
-const getWorkspaces = memoizee(
-  async (isClassic: boolean): Promise<PackageManagerInfo["workspaces"]> => {
-    return isClassic
-      ? Object.values(
-          JSON.parse(
-            JSON.parse(await execOrFail("yarn workspaces --json info", "{}"))
-              ?.data ?? "{}"
-          )
-        )
-      : JSON.parse(
-          `[${(await execOrFail("yarn workspaces list --json --verbose", ""))
-            .trim()
-            .split("\n")
-            .join(",")}]`
-        );
-  },
-  { promise: true }
-);
 export const getPackageManagerInfo = async (
   config: Config,
   componentName: string

package/src/pipeline/yarn/yarnUtils.ts

@@ -0,0 +1,37 @@
+import { exec } from "child-process-promise";
+import memoizee from "memoizee";
+import type { PackageManagerInfo } from "../../types";
+
+const execOrFail = async (cmd: string, onFail: string): Promise<string> => {
+  try {
+    return await exec(cmd).then((r) => r.stdout);
+  } catch (e) {
+    return onFail ?? null;
+  }
+};
+// export for mocking
+export const getYarnVersion = memoizee(
+  async () => {
+    return await execOrFail("yarn --version", "");
+  },
+  { promise: true }
+);
+// export for mocking
+export const getWorkspaces = memoizee(
+  async (isClassic: boolean): Promise<PackageManagerInfo["workspaces"]> => {
+    return isClassic
+      ? Object.values(
+          JSON.parse(
+            JSON.parse(await execOrFail("yarn workspaces --json info", "{}"))
+              ?.data ?? "{}"
+          )
+        )
+      : JSON.parse(
+          `[${(await execOrFail("yarn workspaces list --json --verbose", ""))
+            .trim()
+            .split("\n")
+            .join(",")}]`
+        );
+  },
+  { promise: true }
+);

package/src/rules/index.ts

@@ -1,6 +1,6 @@
-import { GitlabRule } from "../types";
+import type { GitlabRule } from "../types";
 
-const NEVER_ON_RELEASE_COMMIT: GitlabRule = {
+export const RULE_NEVER_ON_RELEASE_COMMIT: GitlabRule = {
   if: "$CI_COMMIT_MESSAGE =~ /^chore\\(release\\).*/",
   when: "never",
 };
@@ -8,7 +8,7 @@ export const RULES_ALWAYS: GitlabRule[] = [
   {
     if: "$CI_COMMIT_TAG",
   },
-  NEVER_ON_RELEASE_COMMIT,
+  RULE_NEVER_ON_RELEASE_COMMIT,
   {
     if: "$CI_COMMIT_BRANCH =~ /^[0-9]+.([0-9]+|x).x$/",
   },
@@ -20,14 +20,17 @@ export const RULES_ALWAYS: GitlabRule[] = [
   },
 ];
 
+export const RULE_CONDITION_MAIN_BRANCH =
+  "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH";
+
 export const RULES_RELEASE: GitlabRule[] = [
-  NEVER_ON_RELEASE_COMMIT,
+  RULE_NEVER_ON_RELEASE_COMMIT,
   {
-    if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $AUTO_RELEASE == "true"',
+    if: RULE_CONDITION_MAIN_BRANCH + ' && $AUTO_RELEASE == "true"',
     when: "on_success",
   },
   {
-    if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH",
+    if: RULE_CONDITION_MAIN_BRANCH,
     when: "manual",
   },
   {

package/src/runner/index.ts

@@ -1,12 +1,13 @@
 import { PIPELINE_IMAGE_TAG, DOCKER_REGISTRY } from "../constants";
 
-type RunnerImageName =
+export type RunnerImageName =
   | "jobs-default"
   | "jobs-meteor"
   | "jobs-testing-chrome"
   | "kubernetes"
   | "base-pipeline"
   | "docker-build"
+  | "gcloud"
   | "semantic-release";
 export const getRunnerImage = (imageName: RunnerImageName) =>
   DOCKER_REGISTRY + "/" + imageName + ":" + PIPELINE_IMAGE_TAG;

package/src/types/config.ts

@@ -1,8 +1,9 @@
-import { BuildConfig } from "../build/types";
-import { DeployConfig } from "../deploy/types";
-import { PartialDeep } from "type-fest";
-import { CatladderJob } from "./jobs";
-import { Context } from "./context";
+import type { BuildConfig } from "../build/types";
+import type { DeployConfig } from "../deploy/types";
+
+import type { CatladderJob } from "./jobs";
+import type { Context } from "./context";
+import type { PartialDeep } from "./utils";
 export type PipelineTrigger = "mainBranch" | "mr" | "taggedRelease";
 
 /**
@@ -11,7 +12,7 @@ export type PipelineTrigger = "mainBranch" | "mr" | "taggedRelease";
  */
 export const ENV_TYPES = {
   dev: {
-    triggers: ["mainBranch", "taggedRelease"], // we also trigger dev on tagged release, so that the versions match
+    triggers: ["mainBranch"],
   },
   review: {
     triggers: ["mr"],
@@ -104,7 +105,12 @@ export type EnvConfig<E extends EnvType = EnvType> = {
   host?: string;
 } & PartialDeep<DefaultEnvConfig>;
 
-export type Env = {
+export type EnvConfigWithComponent = EnvConfig<EnvType> & ComponentConfig;
+
+export type CustomEnv = EnvConfig & {
+  type: EnvType;
+};
+export type Env<C extends ConfigProps = never> = {
   /**
    * local is a special env that is only used in local development
    */
@@ -114,13 +120,16 @@ export type Env = {
   stage?: EnvConfig<"stage"> | false;
   review?: EnvConfig<"review"> | false;
   prod?: EnvConfig<"prod"> | false;
-} & Record<string, EnvConfig | false>;
-
-export type ComponentConfig = {
+  // unfortunatly, typescript does not properly support objects with a mix of known and unknown properties.
+  // for backwards compatiblity we allow unknown props, but we cannot type it (typescript problem)
+  // however, we now support providing custom envs through a generic parameter of `Config`
+} & Record<C["CustomEnvs"], CustomEnv> &
+  Record<string, any>;
+export type ComponentConfig<C extends ConfigProps = never> = {
   /**
    * specify environment configurations
    */
-  env?: Env;
+  env?: Env<C>;
   /**
    * the directory of the component (e.g. where the package.json or similar is located). You can set "." if you only have one app.
    */
@@ -137,7 +146,11 @@ export type ComponentConfig = {
   customJobs?: CatladderJob[] | ((context: Context) => CatladderJob[]);
 } & DefaultEnvConfig;
 
-export type Config = {
+export type ConfigProps = {
+  CustomEnvs: string;
+};
+
+export type Config<C extends ConfigProps = never> = {
   /**
    * name of the customer or group
    */
@@ -155,7 +168,7 @@ export type Config = {
   /**
    * components (sub apps)
    */
-  components: Record<string, ComponentConfig>;
+  components: Record<string, ComponentConfig<C>>;
 
   /**
    * additional meta data (only for organisational purposes)

package/src/types/context.ts

@@ -1,11 +1,25 @@
-import { PipelineTrigger, ComponentConfig, Config, EnvType } from "./config";
+import type {
+  PipelineTrigger,
+  ComponentConfig,
+  Config,
+  EnvType,
+} from "./config";
 
 export type Environment = {
   host: string;
+  url: string;
+  /**
+   * the full name of the app. We use this as RELEASE_NAME in kubernetes and the service name in google cloud run
+   */
   fullName: string;
+
+  gitlabEnvironment: {
+    name: string;
+    url: string;
+  };
   shortName: string;
+  slugPrefix: string;
   slug: string;
-  url: string;
   /**
    * env vars contain all build-time env vars. secrets have to be resolved (they are stored in gitlab)
    */
@@ -39,6 +53,13 @@ export type YarnPackageManagerInfo = {
 };
 
 export type PackageManagerInfo = YarnPackageManagerInfo;
+
+export type ContextBeforeConfig = {
+  componentName: string;
+  fullConfig: Config;
+  commitInfo?: CommitInfo;
+  packageManagerInfo?: PackageManagerInfo;
+};
 export type Context = {
   componentName: string;
   componentConfig: ComponentConfig;

package/src/types/environmentContext.ts

@@ -0,0 +1,33 @@
+import type { BuildConfigGeneric, BuildConfigType } from "../build";
+import type { DeployConfigGeneric } from "../deploy";
+import type { DeployConfigType } from "../deploy";
+import type { Config, EnvConfigWithComponent, EnvType } from "./config";
+import type { CommitInfo } from "./context";
+
+export type EnvironmentContext<
+  B extends BuildConfigType,
+  D extends DeployConfigType
+> = {
+  envConfigRaw: EnvConfigWithComponent;
+  buildConfigRaw: false | BuildConfigGeneric<B>;
+  deployConfigRaw: false | DeployConfigGeneric<D>;
+  commitInfo?: CommitInfo;
+  env: string;
+  envType: EnvType;
+  componentName: string;
+  fullName: string;
+  /**
+   * the environment slug without component name.
+   */
+  environmentSlugPrefix: string;
+  /**
+   * the full environment slug, including the componentName
+   */
+  environmentSlug: string;
+  gitlabEnvironmentName: string;
+
+  /**
+   * the full catladder config
+   */
+  fullConfig: Config;
+};

package/src/types/gitlab-types.ts

@@ -1,11 +1,31 @@
 export type Artifacts = {
   paths: string[];
+  expire_in?: string;
+  when?: "always" | "on_success" | "on_failure";
+  reports?: ArtifactReports;
 };
+// Reports won't show up on MRs until https://gitlab.com/groups/gitlab-org/-/epics/8205
+export type ArtifactReports =
+  | { accessibility: string | string[] }
+  | { coverage_report: string | string[] }
+  | { codequality: string | string[] }
+  | { cyclonedx: string | string[] }
+  | { dotenv: string | string[] }
+  | { junit: string | string[] }
+  | { sast: string | string[] }
+  | { secret_detection: string | string[] }
+  | { terraform: string | string[] };
 export type GitlabJobCache = {
-  key: string;
+  key: GitlabJobCacheKey;
   policy?: string;
   paths: string[];
 };
+export type GitlabJobCacheKey =
+  | string
+  | {
+      files: string[];
+      prefix?: string;
+    };
 export type GitlabRule = {
   if?: string;
   when?: "always" | "on_success" | "manual" | "never"; // todo
@@ -16,18 +36,27 @@ export type Retry = {
   when: string[];
 };
 
-export type Service = {
+export type GitlabJobService = {
   name: string;
-  command: string[];
+  command?: string[];
+  entrypoint?: string[];
+  alias?: string;
 };
 
+export type GitlabJobImage =
+  | string
+  | {
+      name: string;
+      entrypoint?: string[];
+    };
+
 export type GitlabEnvironment = {
   url: string;
   name: string;
   kubernetes?: {
     namespace?: string;
   };
-  action?: "stop" | "start" | "prepare"; // defaults to start
+  action?: "stop" | "start" | "prepare" | "access"; // defaults to start
   on_stop?: string; // other job to run on stop
   auto_stop_in?: string;
 };
@@ -41,12 +70,13 @@ export type GitlabJobDef = {
   cache?: GitlabJobCache | GitlabJobCache[];
   artifacts?: Artifacts;
   retry?: Retry;
-  services?: Service[];
-  image?: string;
+  services?: GitlabJobService[];
+  image?: GitlabJobImage;
   variables?: GitlabVariables;
   dependencies?: string[];
   environment?: GitlabEnvironment;
   allow_failure?: boolean;
+  parallel?: number;
   trigger?: {
     strategy: "depend";
     include: Array<{

package/src/types/jobs.ts

@@ -1,10 +1,11 @@
-import {
+import type {
   Artifacts,
   GitlabEnvironment,
   GitlabJobCache,
   GitlabRule,
   GitlabVariables,
-  Service,
+  GitlabJobService,
+  GitlabJobImage,
 } from "./gitlab-types";
 
 export const BASE_STAGES = [
@@ -13,10 +14,14 @@ export const BASE_STAGES = [
   "build",
   "deploy",
   "verify",
+  "rollback",
   "stop",
 ] as const;
 export type BaseStage = typeof BASE_STAGES[number];
 
+export type CatladderJobNeed =
+  | string
+  | { job: string; artifacts: boolean; componentName?: string };
 export type CatladderJob<S = BaseStage> = {
   /**
    * the name of the job (without any env or app prefix and suffix)
@@ -49,9 +54,21 @@ export type CatladderJob<S = BaseStage> = {
   }[];
 
   /**
-   * does this require another job?
+   * does this require another job (from the same component)?
+   *
+   * You can also require a job from another component if you set `componentName`
+   *
    */
-  needs?: Array<string | { job: string; artifacts: boolean }>;
+  needs?: Array<CatladderJobNeed>;
+
+  /**
+   * @deprecated set `componentName` to `needs` if you want to wait for a job from another component
+   */
+  needsOtherComponent?: Array<{
+    componentName: string;
+    job: string;
+    artifacts: boolean;
+  }>;
   /**
    * cache config, we use here the same shape as gitlab itself
    */
@@ -65,12 +82,12 @@ export type CatladderJob<S = BaseStage> = {
   /**
    * additional services, mainly used for docker
    */
-  services?: Service[];
+  services?: GitlabJobService[];
 
   /**
    * image to use
    */
-  image?: string;
+  image?: GitlabJobImage;
 
   /**
    * variables to pass
@@ -85,4 +102,11 @@ export type CatladderJob<S = BaseStage> = {
   environment?: GitlabEnvironment;
 
   rules?: GitlabRule[];
+
+  /**
+   * How many instances of a job should be run in parallel.
+   * Useful for big test suites that can be split into multiple pipeline jobs.
+   * We use the same shape as GitLab itself.
+   */
+  parallel?: number;
 };

package/src/types/pipeline.ts

@@ -1,4 +1,4 @@
-import { GitlabJobDef, GitlabPipeline } from "./gitlab-types";
+import type { GitlabJobDef, GitlabPipeline } from "./gitlab-types";
 
 export type PipelineType = "gitlab";
 

package/src/types/utils.ts

@@ -0,0 +1,7 @@
+export type PartialDeep<T> = T extends any[]
+  ? T
+  : T extends Record<string, any>
+  ? {
+      [P in keyof T]?: PartialDeep<T[P]>;
+    }
+  : T;

package/src/utils/gitlab.ts

@@ -0,0 +1,23 @@
+export const allowFailureInScripts = (script: string[]): string[] => [
+  "set +e", // disable fail job on error
+  ...script,
+  "set -e", // reenable
+];
+
+export const sanitizeForBashVariable = (name: string) =>
+  name.replace(/-/g, "_");
+
+export const repeatOnFailure = (
+  command: string,
+  options: {
+    pauseInSeconds: number;
+  }
+): string => {
+  return `
+    until ${command} &> /dev/null
+    do
+      echo "Trying again."
+      sleep ${options.pauseInSeconds}
+    done
+  `;
+};

package/tsconfig.json

@@ -2,11 +2,13 @@
   "compilerOptions": {
     "lib": ["es2020", "DOM"],
     "strict": true,
+    "module": "CommonJS",
     "downlevelIteration": true,
     "declaration": true,
     "outDir": "dist",
+    "moduleResolution": "node",
     "esModuleInterop": true,
     "incremental": true
   },
-  "include": ["./src"]
+  "include": ["./src", "tests"]
 }

package/dist/build/base/getBuildInfo.d.ts

@@ -1,2 +0,0 @@
-import { Context } from "../../types";
-export declare const getBuildInfo: (context: Context) => string[];

package/dist/build/tests/storybook.test.d.ts

@@ -1 +0,0 @@
-export {};