@catladder/pipeline 1.144.1 → 1.146.0

package/src/pipeline/gitlab/gitlabReleaseJobs.ts

@@ -0,0 +1,20 @@
+import { RULES_MANUAL_RELEASE, RULES_RELEASE } from "../../rules";
+import { getRunnerImage } from "../../runner";
+
+export const getGitlabReleaseJobs = () => {
+  return {
+    ["create release"]: {
+      stage: "release",
+      image: getRunnerImage("semantic-release"),
+      script: ["semanticRelease"],
+      rules: RULES_RELEASE,
+    },
+    ["⚠️ force create release"]: {
+      stage: "release",
+      image: getRunnerImage("semantic-release"),
+      script: ["semanticRelease"],
+      needs: [],
+      rules: RULES_MANUAL_RELEASE,
+    },
+  };
+};

package/src/pipeline/index.ts

@@ -1,2 +1,4 @@
 export * from "./createChildPipeline";
 export * from "./createJobsForComponent";
+export * from "./generatePipelineFiles";
+export * from "./gitlab/gitlabReleaseJobs";

package/src/pipeline/packageManager.ts

@@ -10,7 +10,7 @@ import {
 
 export const getPackageManagerInfo = async (
   config: Config,
-  componentName: string
+  componentName: string,
 ): Promise<PackageManagerInfo> => {
   // currently only supports yarn
   const version = await getYarnVersion();
@@ -20,7 +20,7 @@ export const getPackageManagerInfo = async (
   const component = config.components[componentName];
   const workspaces = await getWorkspaces(isClassic);
   const currentWorkspace = workspaces.find((w) =>
-    pathEqual(component.dir, w.location)
+    pathEqual(component.dir, w.location),
   );
   const componentIsInWorkspace = Boolean(currentWorkspace);
   const workspaceRoot = "."; // currently we assume the root folder, later on we might support nested workspaces

package/src/pipeline/yarn/yarnUtils.ts

@@ -14,7 +14,7 @@ export const getYarnVersion = memoizee(
   async () => {
     return await execOrFail("yarn --version", "");
   },
-  { promise: true }
+  { promise: true },
 );
 // export for mocking
 export const getWorkspaces = memoizee(
@@ -23,23 +23,23 @@ export const getWorkspaces = memoizee(
       ? Object.values(
           JSON.parse(
             JSON.parse(await execOrFail("yarn workspaces --json info", "{}"))
-              ?.data ?? "{}"
-          )
+              ?.data ?? "{}",
+          ),
         )
       : JSON.parse(
           `[${(await execOrFail("yarn workspaces list --json --verbose", ""))
             .trim()
             .split("\n")
-            .join(",")}]`
+            .join(",")}]`,
         );
   },
-  { promise: true }
+  { promise: true },
 );
 
 // recursivly get all workspace dependencies
 export const getWorkspaceDependencies = (
   ws: YarnWorkspace,
-  allWorkspaces: YarnWorkspace[]
+  allWorkspaces: YarnWorkspace[],
 ): string[] => {
   return ws
     ? ([...ws.workspaceDependencies, ...ws.mismatchedWorkspaceDependencies]
@@ -47,7 +47,7 @@ export const getWorkspaceDependencies = (
           // we have to do this recursivly
 
           const otherWorkspace = allWorkspaces.find(
-            (w) => w.location === location
+            (w) => w.location === location,
           );
 
           if (otherWorkspace) {

package/src/rules/index.ts

@@ -1,5 +1,10 @@
 import type { GitlabRule } from "../types";
+export const RULE_CONDITION_MAIN_BRANCH =
+  "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH";
 
+export const RULE_IS_MAIN_BRANCH: GitlabRule = {
+  if: RULE_CONDITION_MAIN_BRANCH,
+};
 export const RULE_NEVER_ON_RELEASE_COMMIT: GitlabRule = {
   if: "$CI_COMMIT_MESSAGE =~ /^chore\\(release\\).*/",
   when: "never",
@@ -10,10 +15,16 @@ export const RULE_NEVER_ON_SCHEDULE: GitlabRule = {
   when: "never",
 };
 
+export const RULE_IS_MERGE_REQUEST: GitlabRule = {
+  if: "$CI_MERGE_REQUEST_ID",
+};
+
+export const RULE_IS_TAGGED_RELEASE: GitlabRule = {
+  if: "$CI_COMMIT_TAG",
+};
+
 export const RULES_ALWAYS: GitlabRule[] = [
-  {
-    if: "$CI_COMMIT_TAG",
-  },
+  RULE_IS_TAGGED_RELEASE,
   RULE_NEVER_ON_RELEASE_COMMIT,
   RULE_NEVER_ON_SCHEDULE,
   {
@@ -22,14 +33,9 @@ export const RULES_ALWAYS: GitlabRule[] = [
   {
     if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH",
   },
-  {
-    if: "$CI_MERGE_REQUEST_ID",
-  },
+  RULE_IS_MERGE_REQUEST,
 ];
 
-export const RULE_CONDITION_MAIN_BRANCH =
-  "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH";
-
 export const RULE_CONDITION_HOTFIX_BRANCH =
   "$CI_COMMIT_BRANCH =~ /^[0-9]+.([0-9]+|x).x$/";
 

package/src/types/config.ts

@@ -4,8 +4,14 @@ import type { DeployConfig } from "../deploy/types";
 import type { CatladderJob } from "./jobs";
 import type { Context } from "./context";
 import type { PartialDeep } from "./utils";
-import { load } from "js-yaml";
-export type PipelineTrigger = "mainBranch" | "mr" | "taggedRelease";
+import type { PipelineType } from "..";
+
+export const ALL_PIPELINE_TRIGGERS = [
+  "mainBranch",
+  "mr",
+  "taggedRelease",
+] as const;
+export type PipelineTrigger = (typeof ALL_PIPELINE_TRIGGERS)[number];
 
 /**
  * all env types with their trigger.
@@ -37,7 +43,7 @@ export const ENV_TYPES = {
 export const getEnvTypesByTrigger = (trigger: PipelineTrigger) =>
   Object.entries(ENV_TYPES)
     .filter(([, e]) =>
-      (e.triggers as readonly PipelineTrigger[]).includes(trigger)
+      (e.triggers as readonly PipelineTrigger[]).includes(trigger),
     )
     .map(([e]) => e as EnvType);
 
@@ -61,6 +67,7 @@ export type EnvVars = {
    * You can reuse vars from other components in the same project using ${componentName:variableName}
    * EXAMPLE: A_VAR: "the other var is ${api:OTHER_VAR}"
    */
+
   public?: Record<string, any>;
   /**
    * secret env vars. These vars can be managed with catladder/cli
@@ -175,6 +182,8 @@ export type ConfigProps = {
 };
 
 export type Config<C extends ConfigProps = never> = {
+  pipelineType?: PipelineType;
+
   /**
    * name of the customer or group
    */

package/src/types/context.ts

@@ -1,18 +1,28 @@
-import type { SecretEnvVar } from "../context";
+import type { StringOrBashExpression } from "../bash/BashExpression";
+import type { PredefinedVariables, SecretEnvVar } from "../context";
 import type {
   PipelineTrigger,
   ComponentConfig,
   Config,
   EnvType,
 } from "./config";
+import type { BaseStage, CatladderJob } from "./jobs";
+import type { PipelineType } from "./pipeline";
 
-export type EnvironmentEnvVars = {
-  envVars: Record<string, string>;
+export type UnspecifiedEnvVars = Record<
+  string,
+  StringOrBashExpression | undefined | null
+>;
+
+export type EnvironmentEnvVars<
+  V extends UnspecifiedEnvVars = UnspecifiedEnvVars,
+> = {
+  envVars: V & UnspecifiedEnvVars;
   secretEnvVarKeys: SecretEnvVar[];
 };
 export type EnvironmentEnvVarPart = {
-  host: string;
-  url: string;
+  host: StringOrBashExpression;
+  url: StringOrBashExpression;
 
   /**
    * vars that only are injected in certain jobs, but not elsewhere
@@ -21,35 +31,33 @@ export type EnvironmentEnvVarPart = {
     deploy: EnvironmentEnvVars;
     build: EnvironmentEnvVars;
   };
-} & EnvironmentEnvVars;
+} & EnvironmentEnvVars<PredefinedVariables>;
+// FIXME: align with EnvironmentContext
 export type Environment = {
-  host: string;
-  url: string;
+  host: StringOrBashExpression;
+  url: StringOrBashExpression;
   /**
    * the full name of the app. We use this as RELEASE_NAME in kubernetes and the service name in google cloud run
    */
-  fullName: string;
+  fullName: StringOrBashExpression;
 
-  gitlabEnvironment: {
-    name: string;
-    url?: string;
-  };
   shortName: string;
-  slugPrefix: string;
-  slug: string;
+  /**
+   * the environment slug without component name.
+   */
+  slugPrefix: StringOrBashExpression;
+  /**
+   * the review slug, if it is a review app, null otherwise
+   */
+  reviewSlug: StringOrBashExpression | null;
+  /**
+   * the full environment slug, including the componentName
+   */
+  slug: StringOrBashExpression;
 
   envType: EnvType;
 } & EnvironmentEnvVarPart;
 
-export type CommitInfo = {
-  refName: string;
-  reviewSlug: string;
-  buildTime: string;
-  buildId: string;
-  trigger: PipelineTrigger;
-  currentVersion: string;
-};
-
 export type YarnWorkspace = {
   name: string;
   location: string;
@@ -72,7 +80,6 @@ export type PackageManagerInfo = YarnPackageManagerInfo;
 export type ContextBeforeConfig = {
   componentName: string;
   fullConfig: Config;
-  commitInfo?: CommitInfo;
   packageManagerInfo?: PackageManagerInfo;
 };
 export type Context = {
@@ -80,6 +87,12 @@ export type Context = {
   componentConfig: ComponentConfig;
   fullConfig: Config;
   environment: Environment;
-  commitInfo?: CommitInfo;
+
+  trigger?: PipelineTrigger;
+  pipelineType?: PipelineType;
   packageManagerInfo?: PackageManagerInfo;
 };
+
+export type CatladderJobWithContext<S = BaseStage> = CatladderJob<S> & {
+  context: Context;
+};

package/src/types/environmentContext.ts

@@ -1,33 +1,39 @@
+import type { StringOrBashExpression } from "../bash/BashExpression";
 import type { BuildConfigGeneric, BuildConfigType } from "../build";
 import type { DeployConfigGeneric } from "../deploy";
 import type { DeployConfigType } from "../deploy";
 import type { Config, EnvConfigWithComponent, EnvType } from "./config";
-import type { CommitInfo } from "./context";
+import type { PipelineType } from "./pipeline";
 
 export type EnvironmentContext<
   B extends BuildConfigType,
-  D extends DeployConfigType
+  D extends DeployConfigType,
 > = {
   envConfigRaw: EnvConfigWithComponent;
   buildConfigRaw: false | BuildConfigGeneric<B>;
   deployConfigRaw: false | DeployConfigGeneric<D>;
-  commitInfo?: CommitInfo;
+
   env: string;
   envType: EnvType;
   componentName: string;
-  fullName: string;
+  fullName: StringOrBashExpression;
   /**
    * the environment slug without component name.
    */
-  environmentSlugPrefix: string;
+  environmentSlugPrefix: StringOrBashExpression;
+  /**
+   * the review slug, if it is a review app, null otherwise
+   */
+  reviewSlug: StringOrBashExpression | null;
   /**
    * the full environment slug, including the componentName
    */
-  environmentSlug: string;
-  gitlabEnvironmentName: string;
+  environmentSlug: StringOrBashExpression;
 
   /**
    * the full catladder config
    */
   fullConfig: Config;
+
+  pipelineType?: PipelineType;
 };

package/src/types/gitlab-ci-yml.ts

@@ -1,4 +1,3 @@
-/* eslint-disable */
 /**
  * This file was automatically generated by json-schema-to-typescript.
  * DO NOT MODIFY IT BY HAND. Instead, modify the source JSONSchema file,

package/src/types/gitlab-types.ts

@@ -8,11 +8,12 @@ import type {
 export { Retry, Image as GitlabJobImage } from "./gitlab-ci-yml";
 
 // Reports won't show up on MRs until https://gitlab.com/groups/gitlab-org/-/epics/8205
-export interface Artifacts extends Omit<GitlabCiArtifacts, "paths"> {
-  paths: string[];
-}
+export type Artifacts = GitlabCiArtifacts;
 export type GitlabJobCache = Cache;
-export type GitlabRule = Exclude<Rules, null>[number];
+export type GitlabRule = Exclude<
+  Exclude<Rules, null>[number],
+  string | string[] // we only use the object version
+>;
 export type GitlabEnvironment = Omit<
   Exclude<JobTemplate["environment"], undefined | string>,
   "deployment_tier"

package/src/types/jobs.ts

@@ -1,10 +1,15 @@
+import type { UnspecifiedEnvVars } from "..";
+import type {
+  BashExpression,
+  StringOrBashExpression,
+} from "../bash/BashExpression";
 import type {
   Artifacts,
-  GitlabEnvironment,
   GitlabJobCache,
   GitlabRule,
   GitlabJobImage,
   GitlabJobDef,
+  GitlabEnvironment,
 } from "./gitlab-types";
 
 export const BASE_STAGES = [
@@ -16,11 +21,17 @@ export const BASE_STAGES = [
   "rollback",
   "stop",
 ] as const;
-export type BaseStage = typeof BASE_STAGES[number];
+export type BaseStage = (typeof BASE_STAGES)[number];
 
 export type CatladderJobNeed =
   | string
   | { job: string; artifacts: boolean; componentName?: string };
+
+export type CatladderJobEnvironmentConfig = {
+  action?: GitlabEnvironment["action"];
+  on_stop?: GitlabEnvironment["on_stop"];
+  auto_stop_in?: string;
+};
 export type CatladderJob<S = BaseStage> = {
   /**
    * the name of the job (without any env or app prefix and suffix)
@@ -91,14 +102,28 @@ export type CatladderJob<S = BaseStage> = {
   /**
    * variables to pass
    */
-  variables: GitlabJobDef["variables"];
+  variables: UnspecifiedEnvVars | undefined;
+
+  /**
+   * additional vars only for the runner.
+   * Also if you use services: that require env vars, you need to set them here.
+   *
+   */
+  runnerVariables?: Record<string, string>;
 
   /**
    * whether failures are allowed
    */
   allow_failure?: boolean;
 
-  environment?: GitlabEnvironment;
+  /**
+   * gitlab environment config, subject to change
+   */
+  environment?: CatladderJobEnvironmentConfig;
+
+  /** */
+
+  when?: GitlabRule["when"];
 
   rules?: GitlabRule[];
 

package/src/types/pipeline.ts

@@ -2,6 +2,10 @@ import type { GitlabJobDef, GitlabPipeline } from "./gitlab-types";
 
 export type PipelineType = "gitlab";
 
+export type PipelineMode<T extends PipelineType> = T extends "gitlab"
+  ? "childpipeline" | "local"
+  : "local";
+
 export type PipelineJob<T extends PipelineType> = T extends "gitlab"
   ? GitlabJobDef
   : never;

package/src/types/utils.ts

@@ -1,7 +1,7 @@
 export type PartialDeep<T> = T extends any[]
   ? T
   : T extends Record<string, any>
-  ? {
-      [P in keyof T]?: PartialDeep<T[P]>;
-    }
-  : T;
+    ? {
+        [P in keyof T]?: PartialDeep<T[P]>;
+      }
+    : T;

package/src/utils/gitlab.ts

@@ -11,7 +11,7 @@ export const repeatOnFailure = (
   command: string,
   options: {
     pauseInSeconds: number;
-  }
+  },
 ): string => {
   return `
     until ${command}

package/src/utils/index.ts

@@ -1,7 +1,7 @@
 import { isNil, mergeWith } from "lodash";
 
 export function notNil<TValue>(
-  value: TValue | null | undefined
+  value: TValue | null | undefined,
 ): value is TValue {
   return value !== null && value !== undefined;
 }
@@ -14,6 +14,6 @@ export const mergeWithMergingArrays = <A, B>(obj1: A, obj2: B): A & B => {
   return mergeWith({}, obj1, obj2, (prop1, prop2) =>
     Array.isArray(prop1) && Array.isArray(prop2)
       ? [...prop1, ...prop2]
-      : undefined
+      : undefined,
   );
 };

package/src/utils/writeFiles.ts

@@ -0,0 +1,40 @@
+import { writeFile } from "fs/promises";
+import { stringify } from "yaml";
+import packageInfos from "../packageInfos";
+
+export const getAutoGeneratedHeader = (commentChar: string) => {
+  return [
+    "-------------------------------------------------",
+    `🐱 🔨 This file is generated by catladder v${packageInfos.version}`,
+    `🚨 Do not edit this file manually 🚨`,
+    "-------------------------------------------------",
+  ]
+    .map((line) => `${commentChar} ${line}`)
+    .join("\n")
+    .concat("\n");
+};
+
+export const writeGeneratedFile = async (
+  path: string,
+  content: string,
+  {
+    commentChar,
+  }: {
+    commentChar: string;
+  },
+) => {
+  await writeFile(
+    path,
+    // need to spread out the jobs, forgot why
+    [getAutoGeneratedHeader(commentChar), content].join("\n"),
+    {
+      encoding: "utf-8",
+    },
+  );
+};
+
+export const writeYamlfile = async (path: string, data: any) => {
+  await writeGeneratedFile(path, stringify(data), {
+    commentChar: "#",
+  });
+};

package/dist/deploy/cloudRun/createJobs/variables.d.ts

@@ -1,5 +0,0 @@
-import type { Context } from "../../../types/context";
-export declare function getDeployJobVariables(context: Context): {
-  CLOUDSDK_CORE_DISABLE_PROMPTS: string;
-  ENV_VARS: string;
-};

package/dist/deploy/cloudRun/createJobs/variables.js

@@ -1,19 +0,0 @@
-"use strict";
-
-exports.__esModule = true;
-exports.getDeployJobVariables = void 0;
-var js_yaml_1 = require("js-yaml");
-var lodash_1 = require("lodash");
-var __1 = require("../../..");
-function getDeployJobVariables(context) {
-  var allEnvVars = (0, lodash_1.omit)(context.environment.envVars, __1.GCLOUD_DEPLOY_CREDENTIALS_KEY);
-  return {
-    CLOUDSDK_CORE_DISABLE_PROMPTS: "1",
-    ENV_VARS: (0, js_yaml_1.dump)(allEnvVars, {
-      lineWidth: -1,
-      quotingType: "'",
-      forceQuotes: true
-    })
-  };
-}
-exports.getDeployJobVariables = getDeployJobVariables;

package/dist/pipeline/commitInfo/getBuildId.d.ts

@@ -1,6 +0,0 @@
-export declare const getBuildId: () => Promise<string>;
-/**
- * returns the last tagged version string Major.minor.patch
- * returns 0.0.0 if there is no tag
- */
-export declare const getCurrentVersionString: () => Promise<string>;

package/dist/pipeline/commitInfo/getCommitInfo.d.ts

@@ -1,2 +0,0 @@
-import type { CommitInfo } from "../../types";
-export declare const getBaseCommitInfo: () => Promise<Omit<CommitInfo, "trigger">>;

package/src/deploy/cloudRun/createJobs/variables.ts

@@ -1,21 +0,0 @@
-import { dump } from "js-yaml";
-import { omit } from "lodash";
-
-import type { Context } from "../../../types/context";
-
-import { GCLOUD_DEPLOY_CREDENTIALS_KEY } from "../../..";
-
-export function getDeployJobVariables(context: Context) {
-  const allEnvVars = omit(
-    context.environment.envVars,
-    GCLOUD_DEPLOY_CREDENTIALS_KEY
-  );
-  return {
-    CLOUDSDK_CORE_DISABLE_PROMPTS: "1",
-    ENV_VARS: dump(allEnvVars, {
-      lineWidth: -1,
-      quotingType: "'",
-      forceQuotes: true,
-    }),
-  };
-}

package/src/pipeline/commitInfo/getBuildId.ts

@@ -1,17 +0,0 @@
-import { exec } from "child-process-promise";
-
-export const getBuildId = async () =>
-  await exec("git describe --tags || git rev-parse HEAD").then((s) =>
-    s.stdout.trim()
-  );
-
-// thx chat gpt
-/**
- * returns the last tagged version string Major.minor.patch
- * returns 0.0.0 if there is no tag
- */
-export const getCurrentVersionString = async () => {
-  return await exec(
-    'latest_tag=$(git describe --tags --abbrev=0 2>/dev/null); if [ -z "$latest_tag" ]; then echo "0.0.0"; else echo "${latest_tag#v}"; fi | cut -d "." -f 1-3'
-  ).then((s) => s.stdout.trim());
-};

package/src/pipeline/commitInfo/getCommitInfo.ts

@@ -1,15 +0,0 @@
-import type { CommitInfo } from "../../types";
-import { getBuildId, getCurrentVersionString } from "./getBuildId";
-export const getBaseCommitInfo = async (): Promise<
-  Omit<CommitInfo, "trigger">
-> => ({
-  refName: process.env.CI_COMMIT_REF_NAME ?? "unknown",
-  reviewSlug: process.env.CI_MERGE_REQUEST_IID
-    ? `mr${process.env.CI_MERGE_REQUEST_IID}`
-    : process.env.CI_COMMIT_REF_SLUG
-    ? `hf${process.env.CI_COMMIT_REF_SLUG}`
-    : "unknown",
-  buildTime: new Date().toISOString(),
-  buildId: await getBuildId(),
-  currentVersion: await getCurrentVersionString(),
-});