@catladder/pipeline 0.0.0-more-collapse-deeb54d7 → 0.0.0-multiple-gitlab-includes-3df5566d

package/src/pipeline/packageManager.ts

@@ -1,22 +1,28 @@
 import { existsSync } from "fs";
 import { join } from "path";
 import { pathEqual } from "path-equal";
-import type { Config, PackageManagerInfo } from "../types";
-import { getWorkspaces, getYarnVersion } from "./yarn/yarnUtils";
+import type {
+  Config,
+  PackageManagerInfoBase,
+  PackageManagerInfoComponent,
+} from "../types";
+import {
+  getWorkspaces,
+  getYarnVersion,
+  getWorkspaceDependencies,
+} from "./yarn/yarnUtils";
 
-export const getPackageManagerInfo = async (
+export const getPackageManagerInfoForComponent = async (
   config: Config,
-  componentName: string
-): Promise<PackageManagerInfo> => {
+  componentName: string,
+): Promise<PackageManagerInfoComponent> => {
+  const baseInfo = await getPackageManagerInfoBase();
+  const { workspaces } = baseInfo;
   // currently only supports yarn
-  const version = await getYarnVersion();
-  if (!version) throw new Error("could not get yarn version");
-  const isClassic = version.startsWith("1");
 
   const component = config.components[componentName];
-  const workspaces = await getWorkspaces(isClassic);
   const currentWorkspace = workspaces.find((w) =>
-    pathEqual(component.dir, w.location)
+    pathEqual(component.dir, w.location),
   );
   const componentIsInWorkspace = Boolean(currentWorkspace);
   const workspaceRoot = "."; // currently we assume the root folder, later on we might support nested workspaces
@@ -38,29 +44,41 @@ export const getPackageManagerInfo = async (
 
   // get all folders that this workspace depend on
   // we will later copy them into the docker build
-  const workspaceDependencies = currentWorkspace
-    ? ([
-        ...currentWorkspace.workspaceDependencies,
-        ...currentWorkspace.mismatchedWorkspaceDependencies,
-      ]
-        .map((name) => workspaces.find((w) => w.name === name)?.location)
-        .filter(Boolean) as string[])
+  const currentWorkspaceDependencies = currentWorkspace
+    ? getWorkspaceDependencies(currentWorkspace, workspaces)
     : [];
 
   const pathsToCopyInDocker = [
-    packageJson,
-    ...(workspacePackageJson ? [workspacePackageJson] : []),
-    lockFile,
-    ...configFilePaths,
-    ...workspaceDependencies,
+    ...new Set([
+      packageJson,
+      ...(workspacePackageJson ? [workspacePackageJson] : []),
+      lockFile,
+      ...configFilePaths,
+      ...currentWorkspaceDependencies,
+    ]),
   ];
   return {
-    type: "yarn",
-    workspaces,
-    version,
-    isClassic,
+    ...baseInfo,
     currentWorkspace,
+    currentWorkspaceDependencies,
     componentIsInWorkspace,
     pathsToCopyInDocker,
   };
 };
+// TODO: memoizee
+export const getPackageManagerInfoBase =
+  async (): Promise<PackageManagerInfoBase> => {
+    // currently only supports yarn
+    const version = await getYarnVersion();
+    if (!version) throw new Error("could not get yarn version");
+    const isClassic = version.startsWith("1");
+
+    const workspaces = await getWorkspaces(isClassic);
+
+    return {
+      type: "yarn",
+      workspaces,
+      version,
+      isClassic,
+    };
+  };

package/src/pipeline/yarn/yarnUtils.ts

@@ -1,6 +1,7 @@
 import { exec } from "child-process-promise";
 import memoizee from "memoizee";
-import type { PackageManagerInfo } from "../../types";
+import type { YarnWorkspace } from "../../types";
+import { jsonParseOrThrow } from "../../utils/jsonParse";
 
 const execOrFail = async (cmd: string, onFail: string): Promise<string> => {
   try {
@@ -14,24 +15,52 @@ export const getYarnVersion = memoizee(
   async () => {
     return await execOrFail("yarn --version", "");
   },
-  { promise: true }
+  { promise: true },
 );
+
 // export for mocking
 export const getWorkspaces = memoizee(
-  async (isClassic: boolean): Promise<PackageManagerInfo["workspaces"]> => {
+  async (isClassic: boolean): Promise<Array<YarnWorkspace>> => {
     return isClassic
       ? Object.values(
-          JSON.parse(
-            JSON.parse(await execOrFail("yarn workspaces --json info", "{}"))
-              ?.data ?? "{}"
-          )
+          jsonParseOrThrow(
+            jsonParseOrThrow(
+              await execOrFail("yarn workspaces --json info", "{}"),
+            )?.data ?? "{}",
+          ),
         )
-      : JSON.parse(
+      : jsonParseOrThrow(
           `[${(await execOrFail("yarn workspaces list --json --verbose", ""))
             .trim()
             .split("\n")
-            .join(",")}]`
+            .join(",")}]`,
         );
   },
-  { promise: true }
+  { promise: true },
 );
+
+// recursivly get all workspace dependencies
+export const getWorkspaceDependencies = (
+  ws: YarnWorkspace,
+  allWorkspaces: YarnWorkspace[],
+): string[] => {
+  return ws
+    ? ([...ws.workspaceDependencies, ...ws.mismatchedWorkspaceDependencies]
+        .flatMap((location) => {
+          // we have to do this recursivly
+
+          const otherWorkspace = allWorkspaces.find(
+            (w) => w.location === location,
+          );
+
+          if (otherWorkspace) {
+            return [
+              ...getWorkspaceDependencies(otherWorkspace, allWorkspaces),
+              otherWorkspace.location,
+            ];
+          }
+          return [];
+        })
+        .filter(Boolean) as string[])
+    : [];
+};

package/src/rules/index.ts

@@ -1,33 +1,47 @@
 import type { GitlabRule } from "../types";
+export const RULE_CONDITION_MAIN_BRANCH =
+  "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH";
 
+export const RULE_IS_MAIN_BRANCH: GitlabRule = {
+  if: RULE_CONDITION_MAIN_BRANCH,
+};
 export const RULE_NEVER_ON_RELEASE_COMMIT: GitlabRule = {
   if: "$CI_COMMIT_MESSAGE =~ /^chore\\(release\\).*/",
   when: "never",
 };
+
+export const RULE_NEVER_ON_SCHEDULE: GitlabRule = {
+  if: '$CI_PIPELINE_SOURCE  == "schedule"',
+  when: "never",
+};
+
+export const RULE_IS_MERGE_REQUEST: GitlabRule = {
+  if: "$CI_MERGE_REQUEST_ID",
+};
+
+export const RULE_IS_TAGGED_RELEASE: GitlabRule = {
+  if: "$CI_COMMIT_TAG",
+};
+
 export const RULES_ALWAYS: GitlabRule[] = [
-  {
-    if: "$CI_COMMIT_TAG",
-  },
+  RULE_IS_TAGGED_RELEASE,
   RULE_NEVER_ON_RELEASE_COMMIT,
+  RULE_NEVER_ON_SCHEDULE,
   {
     if: "$CI_COMMIT_BRANCH =~ /^[0-9]+.([0-9]+|x).x$/",
   },
   {
     if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH",
   },
-  {
-    if: "$CI_MERGE_REQUEST_ID",
-  },
+  RULE_IS_MERGE_REQUEST,
 ];
 
-export const RULE_CONDITION_MAIN_BRANCH =
-  "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH";
-
 export const RULE_CONDITION_HOTFIX_BRANCH =
   "$CI_COMMIT_BRANCH =~ /^[0-9]+.([0-9]+|x).x$/";
 
 export const RULES_RELEASE: GitlabRule[] = [
   RULE_NEVER_ON_RELEASE_COMMIT,
+  RULE_NEVER_ON_SCHEDULE,
   {
     if: RULE_CONDITION_MAIN_BRANCH + ' && $AUTO_RELEASE == "true"',
     when: "on_success",
@@ -41,3 +55,16 @@ export const RULES_RELEASE: GitlabRule[] = [
     when: "manual",
   },
 ];
+
+export const RULES_MANUAL_RELEASE: GitlabRule[] = [
+  RULE_NEVER_ON_RELEASE_COMMIT,
+  RULE_NEVER_ON_SCHEDULE,
+  {
+    if: RULE_CONDITION_MAIN_BRANCH,
+    when: "manual",
+  },
+  {
+    if: RULE_CONDITION_HOTFIX_BRANCH,
+    when: "manual",
+  },
+];

package/src/runner/index.ts

@@ -5,7 +5,6 @@ export type RunnerImageName =
   | "jobs-meteor"
   | "jobs-testing-chrome"
   | "kubernetes"
-  | "base-pipeline"
   | "docker-build"
   | "gcloud"
   | "semantic-release";

package/src/types/config.ts

@@ -2,9 +2,16 @@ import type { BuildConfig } from "../build/types";
 import type { DeployConfig } from "../deploy/types";
 
 import type { CatladderJob } from "./jobs";
-import type { Context } from "./context";
+import type { ComponentContext } from "./context";
 import type { PartialDeep } from "./utils";
-export type PipelineTrigger = "mainBranch" | "mr" | "taggedRelease";
+import type { PipelineType, WorkspaceBuildConfig } from "..";
+
+export const ALL_PIPELINE_TRIGGERS = [
+  "mainBranch",
+  "mr",
+  "taggedRelease",
+] as const;
+export type PipelineTrigger = (typeof ALL_PIPELINE_TRIGGERS)[number];
 
 /**
  * all env types with their trigger.
@@ -36,7 +43,7 @@ export const ENV_TYPES = {
 export const getEnvTypesByTrigger = (trigger: PipelineTrigger) =>
   Object.entries(ENV_TYPES)
     .filter(([, e]) =>
-      (e.triggers as readonly PipelineTrigger[]).includes(trigger)
+      (e.triggers as readonly PipelineTrigger[]).includes(trigger),
     )
     .map(([e]) => e as EnvType);
 
@@ -60,19 +67,12 @@ export type EnvVars = {
    * You can reuse vars from other components in the same project using ${componentName:variableName}
    * EXAMPLE: A_VAR: "the other var is ${api:OTHER_VAR}"
    */
+
   public?: Record<string, any>;
   /**
    * secret env vars. These vars can be managed with catladder/cli
    */
   secret?: string[];
-  /**
-   * @deprecated, use ${componentName:variableName} instead
-   *
-   * With fromComponents you can inject env vars from other components.
-   */
-  fromComponents?: {
-    [otherApp: string]: Record<string, string>;
-  };
 };
 
 export type DefaultEnvConfig = {
@@ -143,18 +143,32 @@ export type ComponentConfig<C extends ConfigProps = never> = {
    * to let us know about why you need to use custom jobs.
    * This feedback will help us to generalize use cases
    */
-  customJobs?: CatladderJob[] | ((context: Context) => CatladderJob[]);
+  customJobs?:
+    | CatladderJob[]
+    | ((context: Omit<ComponentContext, "customJobs">) => CatladderJob[]);
 
   /**
-   * experimental. whether to create a .env localy and during build jobs
+   * whether to create a .env
    *
    * Be careful, this will overwrite any existing .env file!
+   *
+   * if set to true it will create .env locally and during build.
+   * Be careful: the .env file usually ends up in the build artifacts.
+   * Use this for apps and clients where env does not contain any secrets
+   *
+   *
+   * if set to "local" it will only create .env file locally.
+   * Use this in combination with @dotenvx/dotenvx or node 20 to start apps locally with .env files
+   * During build and runtime, it relies on the env vars set in the environment
+   *
+   * @defaultValue true
+   *
    */
-  dotEnv?: boolean;
+  dotEnv?: boolean | "local";
   /**
-   * experimental. whether to create a env.d.ts localy and during build jobs
+   * whether to create a env.d.ts localy and during build jobs
    *
-   * Be careful, this will overwrite any existing env.d.ts file!
+   * @defaultValue true
    */
   envDTs?: boolean;
 } & DefaultEnvConfig;
@@ -164,6 +178,11 @@ export type ConfigProps = {
 };
 
 export type Config<C extends ConfigProps = never> = {
+  /**
+   * the pipeline type to generate, defaults to gitlab
+   */
+  pipelineType?: PipelineType;
+
   /**
    * name of the customer or group
    */
@@ -178,6 +197,9 @@ export type Config<C extends ConfigProps = never> = {
    *
    */
   domainCanonical?: string;
+
+  // shared workspace Builds
+  builds?: Record<string, WorkspaceBuildConfig>;
   /**
    * components (sub apps)
    */
@@ -189,4 +211,9 @@ export type Config<C extends ConfigProps = never> = {
   meta?: {
     labels?: Record<string, string>;
   };
+
+  /**
+   * additional vars only for the runner in all jobs.
+   */
+  runnerVariables?: Record<string, string>;
 };

package/src/types/context.ts

@@ -1,18 +1,35 @@
-import type { SecretEnvVar } from "../context";
+import type { StringOrBashExpression } from "../bash/BashExpression";
+import type {
+  BuildConfigFromWorkspace,
+  BuildConfigStandalone,
+  WorkspaceBuildConfig,
+} from "../build";
+import type { PredefinedVariables, SecretEnvVar } from "../context";
+import type { DeployConfig } from "../deploy";
+import type { VariableValue } from "../variables/VariableValue";
 import type {
-  PipelineTrigger,
   ComponentConfig,
   Config,
   EnvType,
+  PipelineTrigger,
 } from "./config";
+import type { BaseStage, CatladderJob } from "./jobs";
+import type { PipelineType } from "./pipeline";
+
+export type UnspecifiedEnvVars = Record<
+  string,
+  VariableValue | undefined | null
+>;
 
-export type EnvironmentEnvVars = {
-  envVars: Record<string, string>;
+export type EnvironmentEnvVars<
+  V extends UnspecifiedEnvVars = UnspecifiedEnvVars,
+> = {
+  envVars: V & UnspecifiedEnvVars;
   secretEnvVarKeys: SecretEnvVar[];
 };
 export type EnvironmentEnvVarPart = {
-  host: string;
-  url: string;
+  host: StringOrBashExpression;
+  url: StringOrBashExpression;
 
   /**
    * vars that only are injected in certain jobs, but not elsewhere
@@ -21,64 +38,175 @@ export type EnvironmentEnvVarPart = {
     deploy: EnvironmentEnvVars;
     build: EnvironmentEnvVars;
   };
-} & EnvironmentEnvVars;
+} & EnvironmentEnvVars<PredefinedVariables>;
+// FIXME: align with EnvironmentContext
 export type Environment = {
-  host: string;
-  url: string;
+  host: StringOrBashExpression;
+  url: StringOrBashExpression;
   /**
    * the full name of the app. We use this as RELEASE_NAME in kubernetes and the service name in google cloud run
    */
-  fullName: string;
+  fullName: StringOrBashExpression;
 
-  gitlabEnvironment: {
-    name: string;
-    url: string;
-  };
-  shortName: string;
-  slugPrefix: string;
-  slug: string;
+  /**
+   * the environment slug without component name.
+   */
+  slugPrefix: StringOrBashExpression;
+  /**
+   * the review slug, if it is a review app, null otherwise
+   */
+  reviewSlug: StringOrBashExpression | null;
+  /**
+   * the full environment slug, including the componentName
+   */
+  slug: StringOrBashExpression;
 
   envType: EnvType;
 } & EnvironmentEnvVarPart;
 
-export type CommitInfo = {
-  refName: string;
-  reviewSlug: string;
-  buildTime: string;
-  buildId: string;
-  trigger: PipelineTrigger;
-  currentVersion: string;
-};
-
-type Workspace = {
+export type YarnWorkspace = {
   name: string;
   location: string;
   workspaceDependencies: string[];
   mismatchedWorkspaceDependencies: string[];
 };
-export type YarnPackageManagerInfo = {
+export type YarnPackageManagerInfoBase = {
   type: "yarn";
   version: string;
-  workspaces: Workspace[];
-  currentWorkspace?: Workspace;
+  workspaces: YarnWorkspace[];
   isClassic: boolean;
+};
+
+export type YarnPackageManagerInfoComponent = YarnPackageManagerInfoBase & {
+  currentWorkspace?: YarnWorkspace;
   componentIsInWorkspace: boolean;
   pathsToCopyInDocker: string[];
+  currentWorkspaceDependencies: string[];
 };
 
-export type PackageManagerInfo = YarnPackageManagerInfo;
+export type PackageManagerInfoComponent = YarnPackageManagerInfoComponent;
+
+/**
+ * not confuse with yarn workspaces
+ */
+export type PackageManagerInfoBase = YarnPackageManagerInfoBase;
 
 export type ContextBeforeConfig = {
   componentName: string;
   fullConfig: Config;
-  commitInfo?: CommitInfo;
-  packageManagerInfo?: PackageManagerInfo;
+  packageManagerInfo?: PackageManagerInfoComponent;
 };
-export type Context = {
-  componentName: string;
+
+export type BuildContextBase = {
+  /**
+   * the directory where the component or workspace is located
+   */
+  dir: string;
+  /**
+   * directories of all components that are part of the build.
+   * In case of a standalone build, this is just the dir of the component + yarn workspace dependencies (if mode = all)
+   *
+   * in case of a workspace this contains all components in the workspace (+ yarn workspace dependencies if mode = all)
+   */
+  getComponentDirs: (mode: "direct" | "all") => string[];
+};
+
+export type BuildContextComponentBase = BuildContextBase;
+
+export type BuildContextStandalone<
+  C extends BuildConfigStandalone = BuildConfigStandalone,
+> = BuildContextComponentBase & {
+  config: C;
+  type: "standalone";
+  buildType: C["type"];
+};
+
+export type BuildContextFromWorkspace = BuildContextComponentBase & {
+  config: BuildConfigFromWorkspace;
+  workspaceName: string;
+  buildType: WorkspaceBuildConfig["type"];
+  /**
+   * merged docker config
+   */
+  workspaceBuildConfig: WorkspaceBuildConfig;
+  type: "fromWorkspace";
+};
+
+export type BuildContextWorkspace = BuildContextBase & {
+  type: "workspace";
+  buildType: WorkspaceBuildConfig["type"];
+  config: WorkspaceBuildConfig;
+};
+
+export type BuildContextComponent =
+  | BuildContextStandalone
+  | BuildContextFromWorkspace;
+
+export type BuildContext = BuildContextComponent | BuildContextWorkspace;
+
+export type DeployContext = {
+  config: DeployConfig;
+};
+export type ComponentContext<
+  BC extends BuildContextComponent = BuildContextComponent,
+> = {
+  type: "component";
+  env: string;
+  /**
+   * the name of the component
+   */
+  name: string;
+
+  /**
+   * the merged component config.
+   *
+   * use build.config and deploy.config instead if you need something from there
+   *
+   */
   componentConfig: ComponentConfig;
+  build: BC;
+  deploy?: DeployContext | null;
   fullConfig: Config;
   environment: Environment;
-  commitInfo?: CommitInfo;
-  packageManagerInfo?: PackageManagerInfo;
+
+  trigger?: PipelineTrigger;
+  pipelineType?: PipelineType;
+
+  packageManagerInfo: PackageManagerInfoComponent;
+
+  customJobs?: CatladderJob[];
+};
+
+export type Context = ComponentContext | WorkspaceContext;
+
+export type CatladderJobWithContext<S = BaseStage> = CatladderJob<S> & {
+  context: ComponentContext;
+};
+
+export const componentContextIsStandaloneBuild = (
+  context: ComponentContext<BuildContextComponent>,
+): context is ComponentContext<BuildContextStandalone> => {
+  return context.build.type === "standalone";
+};
+
+export const componentContextHasWorkspaceBuild = (
+  context: ComponentContext<BuildContextComponent>,
+): context is ComponentContext<BuildContextFromWorkspace> => {
+  return context.build.type === "fromWorkspace";
+};
+
+export type WorkspaceContext = {
+  type: "workspace";
+  /**
+   * the name of the workspace
+   */
+  name: string;
+  fullConfig: Config;
+  workspaceConfig: WorkspaceBuildConfig;
+  packageManagerInfo: PackageManagerInfoBase;
+  components: Array<ComponentContext>;
+  build: BuildContextWorkspace;
+  trigger: PipelineTrigger;
+  pipelineType: PipelineType;
+  env: string;
 };

package/src/types/environmentContext.ts

@@ -1,33 +1,38 @@
-import type { BuildConfigGeneric, BuildConfigType } from "../build";
-import type { DeployConfigGeneric } from "../deploy";
-import type { DeployConfigType } from "../deploy";
+import type { StringOrBashExpression } from "../bash/BashExpression";
+import type { BuildConfig } from "../build";
+import type { DeployConfig } from "../deploy";
 import type { Config, EnvConfigWithComponent, EnvType } from "./config";
-import type { CommitInfo } from "./context";
+import type { PipelineType } from "./pipeline";
 
 export type EnvironmentContext<
-  B extends BuildConfigType,
-  D extends DeployConfigType
+  B extends BuildConfig = BuildConfig,
+  D extends DeployConfig = DeployConfig,
 > = {
   envConfigRaw: EnvConfigWithComponent;
-  buildConfigRaw: false | BuildConfigGeneric<B>;
-  deployConfigRaw: false | DeployConfigGeneric<D>;
-  commitInfo?: CommitInfo;
+  buildConfigRaw: false | B;
+  deployConfigRaw: false | D;
+
   env: string;
   envType: EnvType;
   componentName: string;
-  fullName: string;
+  fullName: StringOrBashExpression;
   /**
    * the environment slug without component name.
    */
-  environmentSlugPrefix: string;
+  environmentSlugPrefix: StringOrBashExpression;
+  /**
+   * the review slug, if it is a review app, null otherwise
+   */
+  reviewSlug: StringOrBashExpression | null;
   /**
    * the full environment slug, including the componentName
    */
-  environmentSlug: string;
-  gitlabEnvironmentName: string;
+  environmentSlug: StringOrBashExpression;
 
   /**
    * the full catladder config
    */
   fullConfig: Config;
+
+  pipelineType?: PipelineType;
 };

package/src/types/gitlab-ci-yml.ts

@@ -1,4 +1,3 @@
-/* eslint-disable */
 /**
  * This file was automatically generated by json-schema-to-typescript.
  * DO NOT MODIFY IT BY HAND. Instead, modify the source JSONSchema file,