@catladder/pipeline 0.0.0-refactor-gcloud-api-5bd4bbed → 0.0.0-refactor-needs-aae6e67d

package/src/deploy/cloudRun/utils/createArgsString.ts

@@ -11,9 +11,16 @@ import { isNil } from "lodash";
  * @returns
  */
 export const createArgsString = (
-  args: Record<string, string | number | true | undefined>
+  args: Record<string, string | number | true | false | undefined>
 ) =>
   Object.entries(args)
     .filter(([, value]) => !isNil(value))
-    .map(([key, value]) => `--${key}${value !== true ? `=${value}` : ""}`)
+    .map(([key, value]) => {
+      if (value === true) {
+        return `--${key}`;
+      } else if (value === false) {
+        return `--no-${key}`;
+      }
+      return `--${key}=${value}`;
+    })
     .join(" ");

package/src/deploy/kubernetes/deployJob.ts

@@ -40,7 +40,7 @@ export const createKubernetesDeployJobs = (
         quotingType: "'",
         forceQuotes: true,
       }),
-      HELM_GITLAB_CHART_NAME: "the-panter-chart",
+      HELM_GITLAB_CHART_NAME: deployConfig.chartName ?? "/helm-charts/the-panter-chart",
       HELM_ARGS: [
         ...(deployConfig.debug ? ["--debug"] : []),
         ...(deployConfig.additionalHelmArgs ?? []),

package/src/deploy/types/custom.ts

@@ -1,3 +1,4 @@
+import type { GitlabJobImage } from "../../types";
 import type { CatladderJob } from "../../types/jobs";
 import type { DeployConfigBase } from "./base";
 
@@ -26,7 +27,7 @@ export type DeployConfigCustom = {
   /**
    * image to use
    */
-  jobImage?: string;
+  jobImage?: GitlabJobImage;
 
   /**
    * customize cache for the job

package/src/deploy/types/googleCloudRun.ts

@@ -46,11 +46,16 @@ export type DeployConfigCloudRunCloudSql = {
   dbUser?: string;
 
   /**
-   * the prefix of the database, the full db name is this plus the environment slug
+   * the prefix of the database, the full db name is this plus the environment slug prefix plus the componentName
    *
    * defaults to customerName-appName
    */
   dbNamePrefix?: string | false;
+
+  /**
+   * the base name of the db, defaults to the componentName
+   */
+  dbBaseName?: string;
   /**
    * whether to delete the database if the environment is stopped
    * defaults to true for review envs, to false for every other environment
@@ -67,6 +72,8 @@ export type DeployConfigCloudRunCloudSql = {
     statement_cache_size?: number;
   };
 };
+
+type Memory = `${number}${"M" | "G" | "Mi" | "Gi"}`;
 export type DeployConfigCloudRunService = {
   /**
    * command / entrypoint, fallsback to buildConfig.startcommand
@@ -85,7 +92,7 @@ export type DeployConfigCloudRunService = {
   maxInstances?: number;
 
   /**
-   * set to true for workers
+   * set to true to allways allocate cpu, e.g. for workers. Careful that this is expensive!
    */
   noCpuThrottling?: true;
 
@@ -93,6 +100,11 @@ export type DeployConfigCloudRunService = {
    * whether to allow public access (defaults to true)
    */
   allowUnauthenticated?: boolean;
+
+  /**
+   * memory limit. Defaults to 512MB
+   */
+  memory?: Memory;
 };
 
 export type DeployConfigCloudRunJobBase = {

package/src/deploy/types/kubernetes.ts

@@ -234,6 +234,13 @@ export type DeployConfigKubernetesValues = AllowUnknownProps<{
          * It has the same environment variables as the application and an additional IS_WORKER="true" variable
          */
         worker?: KubernetesWorkerDef;
+
+        /**
+         * kubernetes resources for jobs and cronjobs
+         */
+        jobDefaults?: {
+          resources: KubernetesResourcesDef;
+        }
       }>;
 
   /**
@@ -281,4 +288,9 @@ export type DeployConfigKubernetes = {
    * values to configure the app
    */
   values?: DeployConfigKubernetesValues;
+  /**
+   * Custom Helm chart location.
+   * Not recommended to use.
+   */
+  chartName?: string
 } & DeployConfigBase;

package/src/pipeline/createJobs.ts

@@ -5,7 +5,7 @@ import { DEPLOY_TYPES } from "../deploy";
 import type { PipelineJob, PipelineType } from "../types";
 import type { Config, PipelineTrigger } from "../types/config";
 import type { CommitInfo, Context } from "../types/context";
-import type { CatladderJob, CatladderJobNeed } from "../types/jobs";
+import type { CatladderJob } from "../types/jobs";
 import { notNil } from "../utils";
 import { getBaseCommitInfo } from "./commitInfo/getCommitInfo";
 import { makeGitlabJob } from "./gitlab/makeGitlabJob";
@@ -78,28 +78,25 @@ const replaceReferences = (
   const stage =
     job.envMode === "stagePerEnv" ? `${job.stage} ${env}` : job.stage;
 
-  const needsFromNeeds: CatladderJob["needs"] = job.needs?.map((n) =>
+  const cleanedNeeds: CatladderJob["needs"] = [
+    ...(job.needs ?? []),
+    // pull in legacy needs from other component, which is now identical to needs
+    ...(job.needsOtherComponent ?? []),
+  ];
+  const needs: CatladderJob["needs"] = cleanedNeeds?.map((n) =>
     isObject(n)
       ? {
-          job: getFullReferencedJobName(n.job, componentName, env, allRawJobs),
+          job: getFullReferencedJobName(
+            n.job,
+            n.componentName ?? componentName,
+            env,
+            allRawJobs
+          ),
           artifacts: n.artifacts,
         }
       : getFullReferencedJobName(n, componentName, env, allRawJobs)
   );
-  const needsFromOtherComponents: CatladderJob["needs"] =
-    job.needsOtherComponent?.map((other) => ({
-      artifacts: other.artifacts,
-      job: getFullReferencedJobName(
-        other.job,
-        other.componentName,
-        env,
-        allRawJobs
-      ),
-    }));
-  const needs = [
-    ...(needsFromNeeds ?? []),
-    ...(needsFromOtherComponents ?? []),
-  ];
+
   return {
     ...job,
     stage,

package/src/pipeline/gitlab/makeGitlabJob.ts

@@ -16,7 +16,9 @@ export const makeGitlabJob = ({
   return {
     ...rest,
     // sort in a predictable manner for snapshot tests
-    needs: needs?.sort((a, b) => getJobName(a).localeCompare(getJobName(b))),
+    needs: needs
+      ? [...needs].sort((a, b) => getJobName(a).localeCompare(getJobName(b)))
+      : undefined,
     retry: BASE_RETRY,
     interruptible: true,
   };

package/src/types/context.ts

@@ -18,6 +18,7 @@ export type Environment = {
     url: string;
   };
   shortName: string;
+  slugPrefix: string;
   slug: string;
   /**
    * env vars contain all build-time env vars. secrets have to be resolved (they are stored in gitlab)

package/src/types/environmentContext.ts

@@ -16,6 +16,13 @@ export type EnvironmentContext<
   envType: EnvType;
   componentName: string;
   fullName: string;
+  /**
+   * the environment slug without component name.
+   */
+  environmentSlugPrefix: string;
+  /**
+   * the full environment slug, including the componentName
+   */
   environmentSlug: string;
   gitlabEnvironmentName: string;
 

package/src/types/gitlab-types.ts

@@ -2,7 +2,19 @@ export type Artifacts = {
   paths: string[];
   expire_in?: string;
   when?: "always" | "on_success" | "on_failure";
+  reports?: ArtifactReports;
 };
+// Reports won't show up on MRs until https://gitlab.com/groups/gitlab-org/-/epics/8205
+export type ArtifactReports = 
+  | { accessibility: string | string[] }
+  | { coverage_report: string | string[] }
+  | { codequality: string | string[] }
+  | { cyclonedx: string | string[] }
+  | { dotenv: string | string[] }
+  | { junit: string | string[] }
+  | { sast: string | string[] }
+  | { secret_detection: string | string[] }
+  | { terraform: string | string[] }
 export type GitlabJobCache = {
   key: GitlabJobCacheKey;
   policy?: string;
@@ -31,6 +43,11 @@ export type GitlabJobService = {
   alias?: string;
 };
 
+export type GitlabJobImage = string | {
+  name: string;
+  entrypoint?: string[];
+}
+
 export type GitlabEnvironment = {
   url: string;
   name: string;
@@ -52,7 +69,7 @@ export type GitlabJobDef = {
   artifacts?: Artifacts;
   retry?: Retry;
   services?: GitlabJobService[];
-  image?: string;
+  image?: GitlabJobImage;
   variables?: GitlabVariables;
   dependencies?: string[];
   environment?: GitlabEnvironment;

package/src/types/jobs.ts

@@ -5,6 +5,7 @@ import type {
   GitlabRule,
   GitlabVariables,
   GitlabJobService,
+  GitlabJobImage,
 } from "./gitlab-types";
 
 export const BASE_STAGES = [
@@ -18,7 +19,9 @@ export const BASE_STAGES = [
 ] as const;
 export type BaseStage = typeof BASE_STAGES[number];
 
-export type CatladderJobNeed = string | { job: string; artifacts: boolean };
+export type CatladderJobNeed =
+  | string
+  | { job: string; artifacts: boolean; componentName?: string };
 export type CatladderJob<S = BaseStage> = {
   /**
    * the name of the job (without any env or app prefix and suffix)
@@ -52,9 +55,15 @@ export type CatladderJob<S = BaseStage> = {
 
   /**
    * does this require another job (from the same component)?
+   *
+   * You can also require a job from another component if you set `componentName`
+   *
    */
   needs?: Array<CatladderJobNeed>;
 
+  /**
+   * @deprecated set `componentName` to `needs` if you want to wait for a job from another component
+   */
   needsOtherComponent?: Array<{
     componentName: string;
     job: string;
@@ -78,7 +87,7 @@ export type CatladderJob<S = BaseStage> = {
   /**
    * image to use
    */
-  image?: string;
+  image?: GitlabJobImage;
 
   /**
    * variables to pass