@catladder/pipeline 0.0.0-refactore-catenv-command-validates-argument-78d05852 → 0.0.0-reference-database-url-1befbbee

package/src/pipeline/packageManager.ts

@@ -1,26 +1,29 @@
 import { existsSync } from "fs";
 import { join } from "path";
 import { pathEqual } from "path-equal";
-import type { Config, PackageManagerInfo } from "../types";
+import type {
+  Config,
+  PackageManagerInfoBase,
+  PackageManagerInfoComponent,
+} from "../types";
 import {
   getWorkspaces,
   getYarnVersion,
   getWorkspaceDependencies,
 } from "./yarn/yarnUtils";
+import memoizee from "memoizee";
 
-export const getPackageManagerInfo = async (
+export const getPackageManagerInfoForComponent = async (
   config: Config,
-  componentName: string
-): Promise<PackageManagerInfo> => {
+  componentName: string,
+): Promise<PackageManagerInfoComponent> => {
+  const baseInfo = await getPackageManagerInfoBase();
+  const { workspaces } = baseInfo;
   // currently only supports yarn
-  const version = await getYarnVersion();
-  if (!version) throw new Error("could not get yarn version");
-  const isClassic = version.startsWith("1");
 
   const component = config.components[componentName];
-  const workspaces = await getWorkspaces(isClassic);
   const currentWorkspace = workspaces.find((w) =>
-    pathEqual(component.dir, w.location)
+    pathEqual(component.dir, w.location),
   );
   const componentIsInWorkspace = Boolean(currentWorkspace);
   const workspaceRoot = "."; // currently we assume the root folder, later on we might support nested workspaces
@@ -47,20 +50,41 @@ export const getPackageManagerInfo = async (
     : [];
 
   const pathsToCopyInDocker = [
-    packageJson,
-    ...(workspacePackageJson ? [workspacePackageJson] : []),
-    lockFile,
-    ...configFilePaths,
-    ...currentWorkspaceDependencies,
+    ...new Set([
+      packageJson,
+      ...(workspacePackageJson ? [workspacePackageJson] : []),
+      lockFile,
+      ...configFilePaths,
+      ...currentWorkspaceDependencies,
+    ]),
   ];
   return {
-    type: "yarn",
-    workspaces,
-    version,
-    isClassic,
+    ...baseInfo,
     currentWorkspace,
     currentWorkspaceDependencies,
     componentIsInWorkspace,
     pathsToCopyInDocker,
   };
 };
+
+const _getPackageManagerInfoBase =
+  async (): Promise<PackageManagerInfoBase> => {
+    // currently only supports yarn
+
+    const version = await getYarnVersion();
+    if (!version) throw new Error("could not get yarn version");
+    const isClassic = version.startsWith("1");
+
+    const workspaces = await getWorkspaces(isClassic);
+
+    return {
+      type: "yarn",
+      workspaces,
+      version,
+      isClassic,
+    };
+  };
+
+export const getPackageManagerInfoBase = memoizee(_getPackageManagerInfoBase, {
+  promise: true,
+});

package/src/pipeline/yarn/yarnUtils.ts

@@ -1,6 +1,7 @@
 import { exec } from "child-process-promise";
 import memoizee from "memoizee";
-import type { PackageManagerInfo, YarnWorkspace } from "../../types";
+import type { YarnWorkspace } from "../../types";
+import { jsonParseOrThrow } from "../../utils/jsonParse";
 
 const execOrFail = async (cmd: string, onFail: string): Promise<string> => {
   try {
@@ -14,32 +15,34 @@ export const getYarnVersion = memoizee(
   async () => {
     return await execOrFail("yarn --version", "");
   },
-  { promise: true }
+  { promise: true },
 );
+
 // export for mocking
 export const getWorkspaces = memoizee(
-  async (isClassic: boolean): Promise<PackageManagerInfo["workspaces"]> => {
+  async (isClassic: boolean): Promise<Array<YarnWorkspace>> => {
     return isClassic
       ? Object.values(
-          JSON.parse(
-            JSON.parse(await execOrFail("yarn workspaces --json info", "{}"))
-              ?.data ?? "{}"
-          )
+          jsonParseOrThrow(
+            jsonParseOrThrow(
+              await execOrFail("yarn workspaces --json info", "{}"),
+            )?.data ?? "{}",
+          ),
         )
-      : JSON.parse(
+      : jsonParseOrThrow(
           `[${(await execOrFail("yarn workspaces list --json --verbose", ""))
             .trim()
             .split("\n")
-            .join(",")}]`
+            .join(",")}]`,
         );
   },
-  { promise: true }
+  { promise: true },
 );
 
 // recursivly get all workspace dependencies
 export const getWorkspaceDependencies = (
   ws: YarnWorkspace,
-  allWorkspaces: YarnWorkspace[]
+  allWorkspaces: YarnWorkspace[],
 ): string[] => {
   return ws
     ? ([...ws.workspaceDependencies, ...ws.mismatchedWorkspaceDependencies]
@@ -47,7 +50,7 @@ export const getWorkspaceDependencies = (
           // we have to do this recursivly
 
           const otherWorkspace = allWorkspaces.find(
-            (w) => w.location === location
+            (w) => w.location === location,
           );
 
           if (otherWorkspace) {

package/src/rules/index.ts

@@ -1,33 +1,43 @@
 import type { GitlabRule } from "../types";
+export const RULE_CONDITION_MAIN_BRANCH =
+  "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH";
+
+export const RULE_CONDITION_RELEASE_COMMIT =
+  "$CI_COMMIT_MESSAGE =~ /^chore\\(release\\).*/";
 
+export const RULE_CONDITION_NOT_RELEASE_COMMIT =
+  "$CI_COMMIT_MESSAGE !~ /^chore\\(release\\).*/";
+
+export const RULE_IS_MAIN_BRANCH: GitlabRule = {
+  if: RULE_CONDITION_MAIN_BRANCH,
+};
+export const RULE_IS_MAIN_BRANCH_AND_NOT_RELEASE_COMMIT: GitlabRule = {
+  if: RULE_CONDITION_MAIN_BRANCH + " && " + RULE_CONDITION_NOT_RELEASE_COMMIT,
+};
 export const RULE_NEVER_ON_RELEASE_COMMIT: GitlabRule = {
-  if: "$CI_COMMIT_MESSAGE =~ /^chore\\(release\\).*/",
+  if: RULE_CONDITION_RELEASE_COMMIT,
   when: "never",
 };
-export const RULES_ALWAYS: GitlabRule[] = [
-  {
-    if: "$CI_COMMIT_TAG",
-  },
-  RULE_NEVER_ON_RELEASE_COMMIT,
-  {
-    if: "$CI_COMMIT_BRANCH =~ /^[0-9]+.([0-9]+|x).x$/",
-  },
-  {
-    if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH",
-  },
-  {
-    if: "$CI_MERGE_REQUEST_ID",
-  },
-];
 
-export const RULE_CONDITION_MAIN_BRANCH =
-  "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH";
+export const RULE_NEVER_ON_SCHEDULE: GitlabRule = {
+  if: '$CI_PIPELINE_SOURCE  == "schedule"',
+  when: "never",
+};
+
+export const RULE_IS_MERGE_REQUEST: GitlabRule = {
+  if: "$CI_MERGE_REQUEST_ID",
+};
+
+export const RULE_IS_TAGGED_RELEASE: GitlabRule = {
+  if: "$CI_COMMIT_TAG",
+};
 
 export const RULE_CONDITION_HOTFIX_BRANCH =
   "$CI_COMMIT_BRANCH =~ /^[0-9]+.([0-9]+|x).x$/";
 
 export const RULES_RELEASE: GitlabRule[] = [
   RULE_NEVER_ON_RELEASE_COMMIT,
+  RULE_NEVER_ON_SCHEDULE,
   {
     if: RULE_CONDITION_MAIN_BRANCH + ' && $AUTO_RELEASE == "true"',
     when: "on_success",
@@ -41,3 +51,16 @@ export const RULES_RELEASE: GitlabRule[] = [
     when: "manual",
   },
 ];
+
+export const RULES_MANUAL_RELEASE: GitlabRule[] = [
+  RULE_NEVER_ON_RELEASE_COMMIT,
+  RULE_NEVER_ON_SCHEDULE,
+  {
+    if: RULE_CONDITION_MAIN_BRANCH,
+    when: "manual",
+  },
+  {
+    if: RULE_CONDITION_HOTFIX_BRANCH,
+    when: "manual",
+  },
+];

package/src/runner/index.ts

@@ -5,7 +5,6 @@ export type RunnerImageName =
   | "jobs-meteor"
   | "jobs-testing-chrome"
   | "kubernetes"
-  | "base-pipeline"
   | "docker-build"
   | "gcloud"
   | "semantic-release";

package/src/types/config.ts

@@ -2,9 +2,16 @@ import type { BuildConfig } from "../build/types";
 import type { DeployConfig } from "../deploy/types";
 
 import type { CatladderJob } from "./jobs";
-import type { Context } from "./context";
+import type { ComponentContext } from "./context";
 import type { PartialDeep } from "./utils";
-export type PipelineTrigger = "mainBranch" | "mr" | "taggedRelease";
+import type { PipelineType, WorkspaceBuildConfig } from "..";
+
+export const ALL_PIPELINE_TRIGGERS = [
+  "mainBranch",
+  "mr",
+  "taggedRelease",
+] as const;
+export type PipelineTrigger = (typeof ALL_PIPELINE_TRIGGERS)[number];
 
 /**
  * all env types with their trigger.
@@ -36,7 +43,7 @@ export const ENV_TYPES = {
 export const getEnvTypesByTrigger = (trigger: PipelineTrigger) =>
   Object.entries(ENV_TYPES)
     .filter(([, e]) =>
-      (e.triggers as readonly PipelineTrigger[]).includes(trigger)
+      (e.triggers as readonly PipelineTrigger[]).includes(trigger),
     )
     .map(([e]) => e as EnvType);
 
@@ -60,19 +67,12 @@ export type EnvVars = {
    * You can reuse vars from other components in the same project using ${componentName:variableName}
    * EXAMPLE: A_VAR: "the other var is ${api:OTHER_VAR}"
    */
+
   public?: Record<string, any>;
   /**
    * secret env vars. These vars can be managed with catladder/cli
    */
   secret?: string[];
-  /**
-   * @deprecated, use ${componentName:variableName} instead
-   *
-   * With fromComponents you can inject env vars from other components.
-   */
-  fromComponents?: {
-    [otherApp: string]: Record<string, string>;
-  };
 };
 
 export type DefaultEnvConfig = {
@@ -83,7 +83,7 @@ export type DefaultEnvConfig = {
   /**
    * how the app is built and its runtime
    */
-  build: BuildConfig;
+  build: BuildConfig | false;
   /**
    * environment variables
    */
@@ -94,6 +94,18 @@ export type DevLocalEnvConfig = {
   vars?: EnvVars;
   port?: number;
 };
+type AddOverrideFunction<T> = {
+  [K in keyof T]:
+    | T[K] // either accept the value
+    | ((defaultValue: T[K]) => T[K]) // or function that returns the value
+    | (T[K] extends Record<string, unknown> | undefined // unknown is important, see https://stackoverflow.com/questions/71422178/typescript-record-accepts-array-why
+        ? AddOverrideFunction<T[K]>
+        : never); // or a nested object
+};
+
+type EnvConfigWithOverride<E extends EnvType = EnvType> = AddOverrideFunction<
+  EnvConfig<E>
+>;
 export type EnvConfig<E extends EnvType = EnvType> = {
   /**
    * type of the env (stage, prod, review, dev)
@@ -107,19 +119,19 @@ export type EnvConfig<E extends EnvType = EnvType> = {
 
 export type EnvConfigWithComponent = EnvConfig<EnvType> & ComponentConfig;
 
-export type CustomEnv = EnvConfig & {
+export type CustomEnv = EnvConfigWithOverride & {
   type: EnvType;
 };
 export type Env<C extends ConfigProps = never> = {
   /**
    * local is a special env that is only used in local development
    */
-  local?: DevLocalEnvConfig;
+  local?: AddOverrideFunction<DevLocalEnvConfig>;
 
-  dev?: EnvConfig<"dev"> | false;
-  stage?: EnvConfig<"stage"> | false;
-  review?: EnvConfig<"review"> | false;
-  prod?: EnvConfig<"prod"> | false;
+  dev?: EnvConfigWithOverride<"dev"> | false;
+  stage?: EnvConfigWithOverride<"stage"> | false;
+  review?: EnvConfigWithOverride<"review"> | false;
+  prod?: EnvConfigWithOverride<"prod"> | false;
   // unfortunatly, typescript does not properly support objects with a mix of known and unknown properties.
   // for backwards compatiblity we allow unknown props, but we cannot type it (typescript problem)
   // however, we now support providing custom envs through a generic parameter of `Config`
@@ -143,18 +155,32 @@ export type ComponentConfig<C extends ConfigProps = never> = {
    * to let us know about why you need to use custom jobs.
    * This feedback will help us to generalize use cases
    */
-  customJobs?: CatladderJob[] | ((context: Context) => CatladderJob[]);
+  customJobs?:
+    | CatladderJob[]
+    | ((context: Omit<ComponentContext, "customJobs">) => CatladderJob[]);
 
   /**
-   * experimental. whether to create a .env localy and during build jobs
+   * whether to create a .env
    *
    * Be careful, this will overwrite any existing .env file!
+   *
+   * if set to true it will create .env locally and during build.
+   * Be careful: the .env file usually ends up in the build artifacts.
+   * Use this for apps and clients where env does not contain any secrets
+   *
+   *
+   * if set to "local" it will only create .env file locally.
+   * Use this in combination with @dotenvx/dotenvx or node 20 to start apps locally with .env files
+   * During build and runtime, it relies on the env vars set in the environment
+   *
+   * @defaultValue true
+   *
    */
-  dotEnv?: boolean;
+  dotEnv?: boolean | "local";
   /**
-   * experimental. whether to create a env.d.ts localy and during build jobs
+   * whether to create a env.d.ts localy and during build jobs
    *
-   * Be careful, this will overwrite any existing env.d.ts file!
+   * @defaultValue true
    */
   envDTs?: boolean;
 } & DefaultEnvConfig;
@@ -164,6 +190,11 @@ export type ConfigProps = {
 };
 
 export type Config<C extends ConfigProps = never> = {
+  /**
+   * the pipeline type to generate, defaults to gitlab
+   */
+  pipelineType?: PipelineType;
+
   /**
    * name of the customer or group
    */
@@ -178,6 +209,9 @@ export type Config<C extends ConfigProps = never> = {
    *
    */
   domainCanonical?: string;
+
+  // shared workspace Builds
+  builds?: Record<string, WorkspaceBuildConfig>;
   /**
    * components (sub apps)
    */
@@ -189,4 +223,9 @@ export type Config<C extends ConfigProps = never> = {
   meta?: {
     labels?: Record<string, string>;
   };
+
+  /**
+   * additional vars only for the runner in all jobs.
+   */
+  runnerVariables?: Record<string, string>;
 };

package/src/types/context.ts

@@ -1,19 +1,33 @@
-import type { SecretEnvVar } from "../context";
+import type { StringOrBashExpression } from "../bash/BashExpression";
+import type {
+  BuildConfigFromWorkspace,
+  BuildConfigStandalone,
+  WorkspaceBuildConfig,
+} from "../build";
+import type { PredefinedVariables, SecretEnvVar } from "../context";
+import type { DeployConfig } from "../deploy";
+import type { VariableValue } from "../variables/VariableValue";
 import type {
-  PipelineTrigger,
   ComponentConfig,
   Config,
   EnvType,
+  PipelineTrigger,
 } from "./config";
+import type { BaseStage, CatladderJob } from "./jobs";
+import type { PipelineType } from "./pipeline";
 
-export type EnvironmentEnvVars = {
-  envVars: Record<string, string>;
+export type UnspecifiedEnvVars = Record<
+  string,
+  VariableValue | undefined | null
+>;
+
+export type EnvironmentEnvVars<
+  V extends UnspecifiedEnvVars = UnspecifiedEnvVars,
+> = {
+  envVars: V & UnspecifiedEnvVars;
   secretEnvVarKeys: SecretEnvVar[];
 };
 export type EnvironmentEnvVarPart = {
-  host: string;
-  url: string;
-
   /**
    * vars that only are injected in certain jobs, but not elsewhere
    */
@@ -21,65 +35,182 @@ export type EnvironmentEnvVarPart = {
     deploy: EnvironmentEnvVars;
     build: EnvironmentEnvVars;
   };
-} & EnvironmentEnvVars;
+} & EnvironmentEnvVars<PredefinedVariables>;
+// FIXME: align with EnvironmentContext
 export type Environment = {
-  host: string;
-  url: string;
   /**
    * the full name of the app. We use this as RELEASE_NAME in kubernetes and the service name in google cloud run
    */
-  fullName: string;
+  fullName: StringOrBashExpression;
 
-  gitlabEnvironment: {
-    name: string;
-    url: string;
-  };
-  shortName: string;
-  slugPrefix: string;
-  slug: string;
+  /**
+   * the environment slug without component name.
+   */
+  slugPrefix: StringOrBashExpression;
+  /**
+   * the review slug, if it is a review app, null otherwise
+   */
+  reviewSlug: StringOrBashExpression | null;
+  /**
+   * the full environment slug, including the componentName
+   */
+  slug: StringOrBashExpression;
 
   envType: EnvType;
 } & EnvironmentEnvVarPart;
 
-export type CommitInfo = {
-  refName: string;
-  reviewSlug: string;
-  buildTime: string;
-  buildId: string;
-  trigger: PipelineTrigger;
-  currentVersion: string;
-};
-
 export type YarnWorkspace = {
   name: string;
   location: string;
   workspaceDependencies: string[];
   mismatchedWorkspaceDependencies: string[];
 };
-export type YarnPackageManagerInfo = {
+export type YarnPackageManagerInfoBase = {
   type: "yarn";
   version: string;
   workspaces: YarnWorkspace[];
-  currentWorkspace?: YarnWorkspace;
   isClassic: boolean;
+};
+
+export type YarnPackageManagerInfoComponent = YarnPackageManagerInfoBase & {
+  currentWorkspace?: YarnWorkspace;
   componentIsInWorkspace: boolean;
   pathsToCopyInDocker: string[];
   currentWorkspaceDependencies: string[];
 };
 
-export type PackageManagerInfo = YarnPackageManagerInfo;
+export type PackageManagerInfoComponent = YarnPackageManagerInfoComponent;
+
+/**
+ * not confuse with yarn workspaces
+ */
+export type PackageManagerInfoBase = YarnPackageManagerInfoBase;
 
 export type ContextBeforeConfig = {
   componentName: string;
   fullConfig: Config;
-  commitInfo?: CommitInfo;
-  packageManagerInfo?: PackageManagerInfo;
+  packageManagerInfo?: PackageManagerInfoComponent;
 };
-export type Context = {
-  componentName: string;
+
+export type BuildContextBase = {
+  /**
+   * the directory where the component or workspace is located
+   */
+  dir: string;
+  /**
+   * directories of all components that are part of the build.
+   * In case of a standalone build, this is just the dir of the component + yarn workspace dependencies (if mode = all)
+   *
+   * in case of a workspace this contains all components in the workspace (+ yarn workspace dependencies if mode = all)
+   */
+  getComponentDirs: (mode: "direct" | "all") => string[];
+};
+
+export type BuildContextComponentBase = BuildContextBase;
+
+export type BuildContextStandalone<
+  C extends BuildConfigStandalone = BuildConfigStandalone,
+> = BuildContextComponentBase & {
+  config: C;
+  type: "standalone";
+  buildType: C["type"];
+};
+
+export type BuildContextDisabled = BuildContextComponentBase & {
+  type: "disabled";
+};
+export type BuildContextFromWorkspace = BuildContextComponentBase & {
+  config: BuildConfigFromWorkspace;
+  workspaceName: string;
+  buildType: WorkspaceBuildConfig["type"];
+  /**
+   * merged docker config
+   */
+  workspaceBuildConfig: WorkspaceBuildConfig;
+  type: "fromWorkspace";
+};
+
+export type BuildContextWorkspace = BuildContextBase & {
+  type: "workspace";
+  buildType: WorkspaceBuildConfig["type"];
+  config: WorkspaceBuildConfig;
+};
+
+export type BuildContextWithBuild =
+  | BuildContextStandalone
+  | BuildContextFromWorkspace;
+
+export type BuildContextComponent =
+  | BuildContextWithBuild
+  | BuildContextDisabled;
+
+export type BuildContext = BuildContextComponent | BuildContextWorkspace;
+
+export type DeployContext = {
+  config: DeployConfig;
+};
+export type ComponentContext<
+  BC extends BuildContextComponent = BuildContextComponent,
+> = {
+  type: "component";
+  env: string;
+  /**
+   * the name of the component
+   */
+  name: string;
+
+  /**
+   * the merged component config.
+   *
+   * use build.config and deploy.config instead if you need something from there
+   *
+   */
   componentConfig: ComponentConfig;
+  build: BC;
+  deploy?: DeployContext | null;
   fullConfig: Config;
   environment: Environment;
-  commitInfo?: CommitInfo;
-  packageManagerInfo?: PackageManagerInfo;
+
+  trigger?: PipelineTrigger;
+  pipelineType?: PipelineType;
+
+  packageManagerInfo: PackageManagerInfoComponent;
+
+  customJobs?: CatladderJob[];
+};
+
+export type ComponentContextWithBuild = ComponentContext<BuildContextWithBuild>;
+
+export type Context = ComponentContext | WorkspaceContext;
+
+export type CatladderJobWithContext<S = BaseStage> = CatladderJob<S> & {
+  context: ComponentContext;
+};
+
+export const componentContextIsStandaloneBuild = (
+  context: ComponentContext<BuildContextComponent>,
+): context is ComponentContext<BuildContextStandalone> => {
+  return context.build.type === "standalone";
+};
+
+export const componentContextHasWorkspaceBuild = (
+  context: ComponentContext<BuildContextComponent>,
+): context is ComponentContext<BuildContextFromWorkspace> => {
+  return context.build.type === "fromWorkspace";
+};
+
+export type WorkspaceContext = {
+  type: "workspace";
+  /**
+   * the name of the workspace
+   */
+  name: string;
+  fullConfig: Config;
+  workspaceConfig: WorkspaceBuildConfig;
+  packageManagerInfo: PackageManagerInfoBase;
+  components: Array<ComponentContext>;
+  build: BuildContextWorkspace;
+  trigger: PipelineTrigger;
+  pipelineType: PipelineType;
+  env: string;
 };