@catladder/pipeline 0.0.0-refactore-catenv-command-validates-argument-78d05852 → 0.0.0-reference-database-url-1befbbee

package/src/deploy/types/base.ts

@@ -26,11 +26,11 @@ export type DeployConfigBase = {
   jobVars?: EnvVars;
 
   /**
-   * additional env vars for the deploy job
+   * additional vars only for the runner.
+   * Also if you use services: that require env vars, you need to set them here.
    *
-   * @deprecated use deploy.jobVars for deploy job specific variables
    */
-  extraVars?: Record<string, string>;
+  runnerVariables?: Record<string, string>;
 };
 
 export type AllowUnknownProps<T extends Record<string, unknown>> = T &

package/src/deploy/types/custom.ts

@@ -1,5 +1,5 @@
+import type { WithCacheConfig } from "../../build";
 import type { GitlabJobImage } from "../../types";
-import type { CatladderJob } from "../../types/jobs";
 import type { DeployConfigBase } from "./base";
 
 export type DeployConfigCustom = {
@@ -28,9 +28,5 @@ export type DeployConfigCustom = {
    * image to use
    */
   jobImage?: GitlabJobImage;
-
-  /**
-   * customize cache for the job
-   */
-  jobCache?: CatladderJob["cache"];
-} & DeployConfigBase;
+} & DeployConfigBase &
+  WithCacheConfig;

package/src/deploy/types/googleCloudRun.ts

@@ -1,3 +1,4 @@
+import type { DBVariablesMode } from "../cloudRun/utils/database";
 import type { DeployConfigBase } from "./base";
 
 export type Gcloudregion =
@@ -62,6 +63,31 @@ export type DeployConfigCloudRunCloudSql = {
    */
   deleteDatabaseOnStop?: boolean;
 
+  /**
+   * format of the `DATABASE_URL` environment variable
+   *
+   * the options are:
+   * - prisma: adds [required but ignored hostname](https://www.prisma.io/docs/orm/overview/databases/postgresql#connecting-via-sockets):
+   *   `postgresql://$DB_USER:$DB_PASSWORD@localhost/$DB_NAME?host=/cloudsql/$CLOUD_SQL_INSTANCE_CONNECTION_NAME`
+   * - rails: percent-encodes the socket in the host part
+   *   `postgresql://$DB_USER:$DB_PASSWORD@%2Fcloudsql%2FprojectId%3Aregion%3Ainstancename/$DB_NAME?`
+   * - jdbc: for use with Google Cloud SQL Connector for Java
+   *   `jdbc:postgresql:///$DB_NAME?cloudSqlInstance=$CLOUD_SQL_INSTANCE_CONNECTION_NAME&socketFactory=com.google.cloud.sql.postgres.SocketFactory&user=$DB_USER&password=$DB_PASSWORD`
+   *
+   * defaults to prisma
+   */
+  dbConnectionStringFormat?: "prisma" | "rails" | "jdbc";
+
+  /**
+   * controls how variables in the connection string are handled
+   *
+   * - environment: variables like $DB_USER will be kept as environment variables to be replaced at runtime (default)
+   * - embedded: variables will be replaced with their actual values in the connection string
+   *
+   * @default "environment"
+   */
+  dbConnectionStringVariablesMode?: DBVariablesMode;
+
   /**
    * add additional query params to the database connection string
    */
@@ -74,6 +100,7 @@ export type DeployConfigCloudRunCloudSql = {
 };
 
 type Memory = `${number}${"M" | "G" | "Mi" | "Gi"}`;
+type Cpu = 1 | 2 | 4 | 6 | 8;
 export type DeployConfigCloudRunService = {
   /**
    * command / entrypoint, fallsback to buildConfig.startcommand
@@ -113,11 +140,25 @@ export type DeployConfigCloudRunService = {
    */
   ingress?: "all" | "internal" | "internal-and-cloud-load-balancing";
 
+  /**
+   * CPU limit.
+   */
+  cpu?: Cpu;
+
   /**
    * memory limit. Defaults to 512MB
    */
   memory?: Memory;
 
+  /**
+   * timeout of the service, defaults to 5 minutes.
+   *
+   * you can specify something like 10m30s. if its just a number, it will be interpreted as seconds
+   *
+   * max supported is 60min
+   */
+  timeout?: string;
+
   /**
    *
    * set the execution environment.
@@ -127,6 +168,161 @@ export type DeployConfigCloudRunService = {
    * defaults to gen1 according to gcloud
    */
   executionEnvironment?: "gen2" | "gen1";
+
+  /**
+   * Use http2 end-to-end. See https://cloud.google.com/run/docs/configuring/http2
+   *
+   * Defaults to false.
+   *
+   * Your service needs to be able to handle http2 requests.
+   * Its recommended to use http2 without tls, since cloud run handles the encryption for you. (so called "h2c" (http2 cleartext))
+   *
+   */
+  http2?: boolean;
+
+  /**
+   * Number of GPUs to use. Defaults to 0
+   *
+   * This is a preview feature and not all regions support it.
+   */
+  gpu?: number;
+
+  /**
+   * gpu type to use. Refer to https://cloud.google.com/run/docs/configuring/services/gpu#gcloud for defaults
+   */
+  gpuType?: string;
+
+  /**
+   * the image to use. Defaults to the image from the build.
+   *
+   * If you specify an image, you usually want to disable the build by setting build: false
+   */
+  image?: string;
+  /**
+   * args to pass to the command
+   */
+  args?: string[];
+
+  /**
+   * Configuration of the health check.
+   *
+   * When set to `true`, startup and liveness probes will be
+   * configured to `defaultStartupProbe` and `defaultLivenessProbe`.
+   *
+   * When not set, default Cloud Run's health check
+   * configuration is used (startup TCP probe and no liveness probe).
+   */
+  healthCheck?:
+    | true
+    | {
+        /**
+         * Startup probe configuration.
+         */
+        startupProbe: DeployConfigCloudRunProbe;
+
+        /**
+         * Liveness probe configuration.
+         * `false` disables the liveness probe.
+         */
+        livenessProbe: false | DeployConfigCloudRunProbe;
+      };
+} & DeployConfigCloudRunWithVolumes &
+  DeployConfigCloudRunNetworkConfig;
+
+export const defaultStartupProbe: DeployConfigCloudRunProbe = {
+  type: "http1",
+  path: "/__health",
+
+  initialDelaySeconds: 5,
+  timeoutSeconds: 1,
+  periodSeconds: 30,
+  failureThreshold: 3,
+};
+
+export const defaultLivenessProbe: DeployConfigCloudRunProbe = {
+  ...defaultStartupProbe,
+
+  initialDelaySeconds: 0,
+  timeoutSeconds: 10,
+  periodSeconds: 30,
+  failureThreshold: 3,
+};
+
+export type DeployConfigCloudRunProbe =
+  | DeployConfigCloudRunHttp1Probe
+  | DeployConfigCloudRunTcpProbe;
+
+export type DeployConfigCloudRunProbeShared = {
+  /**
+   * The period in seconds at which to perform the probe.
+   * For example 2 to perform the probe every 2 seconds.
+   * Specify a value **from 1 second to 240** seconds.
+   */
+  periodSeconds: number;
+
+  /**
+   * The number of seconds to wait after the container has started before performing the first probe.
+   * Specify a value **from 0 seconds to 240** seconds.
+   */
+  initialDelaySeconds: number;
+
+  /**
+   * The number of seconds to wait until the probe times out.
+   * This value cannot exceed the value specified for `period`.
+   * Specify a value **from 1 to 240**.
+   */
+  timeoutSeconds: number;
+
+  /**
+   * The number of times to retry the probe before shutting down the container.
+   */
+  failureThreshold: number;
+};
+
+export type DeployConfigCloudRunTcpProbe = DeployConfigCloudRunProbeShared & {
+  /**
+   * Cloud Run makes a TCP connection to open the TCP Socket on the specified port.
+   * If Cloud Run is unable to establish a connection, it indicates a failure.
+   */
+  type: "tcp";
+};
+
+export type DeployConfigCloudRunHttp1Probe = DeployConfigCloudRunProbeShared & {
+  /**
+   * **This will NOT work with http2 enabled services**
+   *
+   * Cloud Run makes an HTTP GET request to the service health check endpoint (for example, /ready).
+   * Any response between 200 and 400 is a success, everything else indicates failure.
+   *
+   * If a startup probe does not succeed within the specified time (failureThreshold * period), which cannot exceed 240 seconds, the container is shut down.
+   *
+   * If the HTTP startup probe succeeds within the specified time, and you have enabled liveness probe, the HTTP liveness probe is started.
+   */
+  type: "http1";
+
+  /**
+   * the route path to check for health status of the service starting with a `/`.
+   */
+  path: string;
+};
+
+export type DeployConfigCloudRunNetworkConfig = {
+  /* the vpc network, see https://cloud.google.com/sdk/gcloud/reference/run/deploy#--network  */
+  network?: string;
+
+  /**
+   * vpc subnet to use, see https://cloud.google.com/sdk/gcloud/reference/run/deploy#--subnet
+   */
+  subnet?: string;
+  /**
+   * vpc egress, see https://cloud.google.com/sdk/gcloud/reference/run/deploy#--vpc-egress
+   */
+  vpcEgress?: "all-traffic" | "private-ranges-only";
+
+  /**
+   * vpc connector
+   */
+  vpcConnector?: string;
 };
 
 export type DeployConfigCloudRunJobBase = {
@@ -135,11 +331,21 @@ export type DeployConfigCloudRunJobBase = {
    */
   command: string | string[];
 
+  /**
+   * optional, specify args. Those get overwritten by job executions if specified there
+   */
+  args?: string[];
+
   /**
    * custom image to use. Defaults to the image from the build
    */
   image?: string;
 
+  /**
+   * CPU limit of the job.
+   */
+  cpu?: Cpu;
+
   /**
    * memory limit of the job, defaults to 512Mi
    */
@@ -156,7 +362,13 @@ export type DeployConfigCloudRunJobBase = {
    * number of tasks that may run concurrently, defaults to 1
    */
   parallelism?: number;
-};
+
+  /**
+   * max number of retries of the job. Defaults to 0
+   */
+  maxRetries?: number;
+} & DeployConfigCloudRunWithVolumes &
+  DeployConfigCloudRunNetworkConfig;
 
 type Minute = string;
 type Hour = string;
@@ -165,19 +377,47 @@ type DayOfWeek = string;
 type Month = string;
 export type DeployConfigCloudRunJobWithSchedule =
   DeployConfigCloudRunJobBase & {
+    /**
+     * @deprecated use `execute` instead
+     */
     when: "schedule";
+    /**
+     * @deprecated use `run` instead
+     */
     schedule: `${Minute} ${Hour} ${DayOfMonth} ${Month} ${DayOfWeek}`;
     /**
      * Max number of retries of the cloud scheduler.
      * Note: the task itself is never retried.
      *
      * defaults to 0 (no retries)
+     *
+     *  @deprecated use `run` instead
      */
     maxRetryAttempts?: 0 | 1 | 2 | 3 | 4 | 5;
   };
 
 export type DeployConfigCloudRunJobNormal = DeployConfigCloudRunJobBase & {
-  when: "manual" | "preDeploy" | "postDeploy" | "preStop" | "postStop";
+  /**
+   * wait for completion of the job on preDeploy and postDeploy
+   *
+   * has no effect on preStop and postStop (which always wait for completion)
+   *
+   *
+   * @deprecated use `run` instead
+   */
+  waitForCompletion?: boolean;
+  /**
+   * @deprecated use `run` instead
+   */
+  when?: "manual" | "preDeploy" | "postDeploy" | "preStop" | "postStop";
+};
+
+export type DeployConfigCloudRunWithVolumes = {
+  /**
+   * adds volumes to the service. This is an experimental feature and requires beta gcloud.
+   * sets executionEnvironment to gen2 automatically, since that is required
+   */
+  volumes?: DeployConfigCloudRunVolumes;
 };
 
 export type DeployConfigCloudRunJob =
@@ -222,4 +462,121 @@ export type DeployConfigCloudRun = {
    * add cloudSql
    */
   cloudSql?: DeployConfigCloudRunCloudSql | false;
+
+  /**
+   * execute jobs on deploy or execute jobs and http requests on a schedule
+   */
+  execute?: Record<string, DeployConfigCloudRunExecute | null>;
+
+  debug?: boolean;
 } & DeployConfigBase;
+
+export type DeployConfigCloudRunVolumes = Record<
+  string,
+  DeployConfigCloudRunVolume
+>;
+
+export type DeployConfigCloudRunVolume = {
+  type: "cloud-storage";
+  bucket: string;
+  mountPath: string;
+  readonly?: boolean;
+};
+
+export type DeployConfigCloudRunExecuteJobBase = {
+  /**
+   * set to "job" to run a cloud run job
+   */
+  type: "job";
+  /**
+   * the cloud run job to run
+   */
+  job: string;
+
+  /**
+   * additional args to pass to the job
+   */
+  args?: string[];
+};
+
+type WithSchedule = {
+  /**
+   * run the job on a schedule
+   */
+  when: "schedule";
+  /**
+   * the cron schedule. See https://crontab.guru/
+   */
+  schedule: `${Minute} ${Hour} ${DayOfMonth} ${Month} ${DayOfWeek}`;
+
+  /**
+   * Max number of retries of the cloud scheduler.
+   * Note: the task itself is never retried.
+   *
+   * defaults to 0 (no retries)
+   */
+  maxRetryAttempts?: 0 | 1 | 2 | 3 | 4 | 5;
+};
+
+export type DeployConfigCloudRunExecuteJobScheduled =
+  DeployConfigCloudRunExecuteJobBase & WithSchedule;
+
+export type DeployConfigCloudRunExecuteOnDeploy =
+  DeployConfigCloudRunExecuteJobBase &
+    (
+      | {
+          /**
+           * run the job before or after the deploy
+           */
+          when: "preDeploy" | "postDeploy";
+          /**
+           * whether to wait for completion of the job
+           */
+          waitForCompletion?: boolean;
+        }
+      | {
+          /**
+           * run the job before or after the environment is stopped
+           */
+          when: "preStop" | "postStop";
+        }
+    );
+
+export type DeployConfigCloudRunExecuteJob =
+  | DeployConfigCloudRunExecuteJobScheduled
+  | DeployConfigCloudRunExecuteOnDeploy;
+
+export type DeployConfigCloudRunExecuteHttp = {
+  /**
+   * set to "http" to run a http request
+   */
+  type: "http";
+  /**
+   * the url to call
+   */
+  url: string;
+
+  /**
+   * the http-method to use. Defaults to "POST" (as specified by google cloud scheduler)
+   */
+  method?: "GET" | "POST" | "PUT" | "DELETE" | "PATCH";
+
+  /**
+   * the body to send
+   */
+  body?: string;
+
+  /**
+   * the headers to send
+   */
+  headers?: Record<string, string>;
+} & WithSchedule;
+
+export type DeployConfigCloudRunExecute =
+  | DeployConfigCloudRunExecuteJob
+  | DeployConfigCloudRunExecuteHttp;
+
+export type DeployConfigCloudRunExecuteWithSchedule = Extract<
+  DeployConfigCloudRunExecute,
+  WithSchedule
+>;

package/src/deploy/types/index.ts

@@ -21,8 +21,15 @@ export type DeployConfigGeneric<T extends DeployConfigType> = Extract<
 >;
 
 export const isOfDeployType = <T extends Array<DeployConfigType>>(
-  t: DeployConfig | false,
+  t: DeployConfig | false | null | undefined,
   ...types: T
 ): t is Extract<DeployConfig, { type: T[number] }> => {
-  return t && types.includes(t.type);
+  return t && types.includes(t.type) ? true : false;
+};
+
+export const getDeployConfigType = (
+  t: DeployConfig | false | null | undefined,
+) => {
+  if (!t) return undefined;
+  return t.type;
 };

package/src/deploy/types/kubernetes.ts

@@ -164,33 +164,6 @@ export type DeployConfigMongodbReplicaset = {
 export type DeployConfigMongodb = DeployConfigMongodbBase &
   (DeployConfigMongodbStandalone | DeployConfigMongodbReplicaset);
 
-/**
- * @deprecated use config with instanceConnectionName
- */
-type DeployConfigKubernetesValuesCloudSQLLegacy = {
-  enabled: boolean;
-  /**
-   * @deprecated use config with instanceConnectionName
-   */
-  instanceId?: string;
-  /**
-   * @deprecated use config with instanceConnectionName
-   */
-  projectId?: string;
-  /**
-   * @deprecated use config with instanceConnectionName
-   */
-  region?: string;
-  /**
-   * @deprecated use type "unmanaged".
-   *
-   * Be aware that "unmanaged" is not fully backward compatible with "legacy".
-   *
-   *
-   */
-  type?: "legacy";
-};
-
 export type DeployConfigKubernetesValuesCloudSQLUnmanaged = {
   enabled: boolean;
   instanceConnectionName: string;
@@ -220,8 +193,7 @@ export type DeployConfigKubernetesValuesCloudSQLUnmanaged = {
   dbBaseName?: string;
 };
 export type DeployConfigKubernetesValuesCloudSQL =
-  | DeployConfigKubernetesValuesCloudSQLLegacy
-  | DeployConfigKubernetesValuesCloudSQLUnmanaged;
+  DeployConfigKubernetesValuesCloudSQLUnmanaged;
 export type DeployConfigKubernetesValues = AllowUnknownProps<{
   /**
    * enable cloudsql db. Currently you have to manually set it up
@@ -358,13 +330,6 @@ export type DeployConfigKubernetesValues = AllowUnknownProps<{
   mapServiceUrlToEnv?: {
     [envVar: string]: string;
   };
-  /**
-   * Mount secrets as files in the filesystem.
-   * These secrets are still available as env vars, but will contain the path of the file instead
-   *
-   * @deprecated this will be removed in the future, because its very niche and not used
-   */
-  secretsAsFile?: string[];
 }>;
 
 export type DeployConfigKubernetes = {

package/src/deploy/utils.ts

@@ -1,9 +1,9 @@
-import type { Config, Context } from "../types";
+import type { Config, ComponentContext } from "../types";
 import type { DeployConfigKubernetesCluster } from "./types";
 import { isOfDeployType } from "./types";
 
 export const getFullKubernetesClusterName = (
-  cluster: DeployConfigKubernetesCluster
+  cluster: DeployConfigKubernetesCluster,
 ) => {
   if (cluster.type === "gcloud") {
     return `gke_${cluster.projectId}_${cluster.region}_${cluster.name}`;
@@ -12,13 +12,13 @@ export const getFullKubernetesClusterName = (
 
 export const getKubernetesNamespace = (
   config: Pick<Config, "customerName" | "appName">,
-  env: string
+  env: string,
 ) => {
   return `${config.customerName}-${config.appName}-${env}`;
 };
 
-export const contextIsStoppable = (context: Context) => {
-  const deployConfig = context.componentConfig.deploy;
+export const contextIsStoppable = (context: ComponentContext) => {
+  const deployConfig = context.deploy?.config;
   if (isOfDeployType(deployConfig, "kubernetes")) {
     return true;
   }

package/src/globalScriptFunctions/index.ts

@@ -0,0 +1,30 @@
+class GlobalScriptFunction {
+  constructor(
+    public name: string,
+    public script: string,
+  ) {}
+
+  toBashFunction() {
+    return `function ${this.name} () {
+${this.script.trim()}
+}`.trim();
+  }
+  invoke(...args: string[]) {
+    return `${this.name} ${args.join(" ")}`;
+  }
+}
+
+export const globalScriptFunctions = new Map<string, GlobalScriptFunction>();
+
+/**
+ * registers a global script function
+ * only use that in top-level scopes as it will attach the function to the global scope
+ */
+export const registerGlobalScriptFunction = (name: string, script: string) => {
+  if (globalScriptFunctions.has(name)) {
+    throw new Error(`Global script function ${name} already exists`);
+  }
+  const scriptFunction = new GlobalScriptFunction(name, script);
+  globalScriptFunctions.set(name, scriptFunction);
+  return scriptFunction;
+};

package/src/index.ts

@@ -1,4 +1,4 @@
-export * from "./pipeline/";
+export * from "./pipeline";
 export * from "./types";
 export * from "./rules";
 export * from "./runner";
@@ -6,3 +6,6 @@ export * from "./config";
 export * from "./context";
 export * from "./build";
 export * from "./deploy";
+export * from "./utils/writeFiles";
+export * from "./variables/VariableValue";
+export * from "./bash";

package/src/packageInfos.ts

@@ -0,0 +1,14 @@
+// we can't import the package.json directly, because the bundling process will inline it
+// but the version is only set during deployement, when its already bundled
+// instead we need to use require.main.path to get the path of the main module, which is actually the file in the bin
+// this will point to the root package.json
+let ownPkg = { version: "0.0.0" };
+try {
+  const path = require.main?.path + "/../package.json";
+
+  ownPkg = require(path);
+} catch (e) {
+  // ignore, currently in the gitlab-includes, this doesn't work
+}
+
+export default ownPkg;

package/src/pipeline/__tests__/__snapshots__/getPipelineStages.test.ts.snap

@@ -0,0 +1,69 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`getPipelineStages() should return all envs for SIMPLE_CONFIG1 1`] = `
+[
+  "setup",
+  "setup dev",
+  "setup review",
+  "setup stage",
+  "setup prod",
+  "setup dev2",
+  "setup review2",
+  "setup stage2",
+  "setup prod2",
+  "test",
+  "test dev",
+  "test review",
+  "test stage",
+  "test prod",
+  "test dev2",
+  "test review2",
+  "test stage2",
+  "test prod2",
+  "build",
+  "build dev",
+  "build review",
+  "build stage",
+  "build prod",
+  "build dev2",
+  "build review2",
+  "build stage2",
+  "build prod2",
+  "deploy",
+  "deploy dev",
+  "deploy review",
+  "deploy stage",
+  "deploy prod",
+  "deploy dev2",
+  "deploy review2",
+  "deploy stage2",
+  "deploy prod2",
+  "verify",
+  "verify dev",
+  "verify review",
+  "verify stage",
+  "verify prod",
+  "verify dev2",
+  "verify review2",
+  "verify stage2",
+  "verify prod2",
+  "rollback",
+  "rollback dev",
+  "rollback review",
+  "rollback stage",
+  "rollback prod",
+  "rollback dev2",
+  "rollback review2",
+  "rollback stage2",
+  "rollback prod2",
+  "stop",
+  "stop dev",
+  "stop review",
+  "stop stage",
+  "stop prod",
+  "stop dev2",
+  "stop review2",
+  "stop stage2",
+  "stop prod2",
+]
+`;