@catladder/pipeline 0.0.0-next-9a3feef4 → 0.0.0-next-e2e4f00a

package/examples/kubernetes-with-cloud-sql-legacy.test.ts

@@ -0,0 +1,11 @@
+import { createYamlLocalPipeline } from "./__utils__/helpers";
+import config from "./kubernetes-with-cloud-sql-legacy";
+
+/**
+ * This test is auto-generated.
+ * Modifications will be overwritten on every `yarn test` run!
+ */
+
+it("matches snapshot for kubernetes-with-cloud-sql-legacy local pipeline YAML", async () => {
+  expect(await createYamlLocalPipeline(config)).toMatchSnapshot();
+});

package/examples/kubernetes-with-cloud-sql-legacy.ts

@@ -0,0 +1,35 @@
+import type { Config } from "../src";
+
+const config: Config = {
+  appName: "test-app",
+  customerName: "pan",
+  components: {
+    api: {
+      dir: "api",
+      build: {
+        type: "node",
+      },
+      deploy: {
+        type: "kubernetes",
+        cluster: {
+          name: "some-cluster-name",
+          region: "europe-west6",
+          projectId: "some-project-id",
+          type: "gcloud",
+          domainCanonical: "panter.cloud",
+        },
+        values: {
+          cloudsql: {
+            enabled: true,
+          },
+        },
+      },
+    },
+  },
+};
+
+export default config;
+
+export const information = {
+  title: "K8s: With Cloud SQL Legacy",
+};

package/examples/native-app.ts

@@ -28,11 +28,9 @@ const config: Config = {
         ],
       },
       build: {
-        jobVars: {
-          public: {
-            LC_A: "L=en_US.UTF-8",
-            LANG: "en_US.UTF-8",
-          },
+        extraVars: {
+          LC_A: "L=en_US.UTF-8",
+          LANG: "en_US.UTF-8",
         },
         type: "node",
         buildCommand: [
@@ -48,11 +46,9 @@ const config: Config = {
 
       deploy: {
         type: "custom",
-        jobVars: {
-          public: {
-            LC_A: "L=en_US.UTF-8",
-            LANG: "en_US.UTF-8",
-          },
+        extraVars: {
+          LC_A: "L=en_US.UTF-8",
+          LANG: "en_US.UTF-8",
         },
         requiresDocker: false,
         script: [

package/examples/rails-k8s-with-worker.ts

@@ -44,11 +44,7 @@ const config: Config = {
           domainCanonical: "panter.cloud",
         },
         values: {
-          cloudsql: {
-            enabled: true,
-            type: "unmanaged",
-            instanceConnectionName: "myproject:europe-west6:instance-name",
-          },
+          cloudsql: { enabled: true, projectId: "some-project-id" },
           application: {
             // if deployed to Google Cloud Run use a gem like cloudtasker instead of a permanently running expensive worker
             worker: {

package/package.json

@@ -53,7 +53,7 @@
     }
   ],
   "license": "MIT",
-  "version": "0.0.0-next-9a3feef4",
+  "version": "0.0.0-next-e2e4f00a",
   "stableVersion": "0.1.0",
   "scripts": {
     "build:tsc": "yarn tsc",

package/src/build/base/createAppBuildJob.ts

@@ -40,6 +40,7 @@ export const createAppBuildJob = (
           ? {
               ...context.environment.envVars,
               ...context.environment.jobOnlyVars.build.envVars,
+              ...(context.build.config.extraVars ?? {}),
             }
           : {}),
       },

package/src/build/custom/testJob.ts

@@ -31,6 +31,7 @@ export const createCustomTestJobs = (
     variables: {
       APP_PATH: context.build.dir,
       ...context.environment.jobOnlyVars.build.envVars,
+      ...(buildConfig.extraVars ?? {}),
     },
     runnerVariables: RUNNER_CUSTOM_TEST_VARIABLES,
     services: buildConfig.jobServices,

package/src/build/node/testJob.ts

@@ -35,6 +35,7 @@ export const createNodeTestJobs = (
       ...(context.type === "component"
         ? {
             ...context.environment.jobOnlyVars.build.envVars,
+            ...(context.build.config.extraVars ?? {}),
           }
         : {}),
     },

package/src/build/rails/build.ts

@@ -47,6 +47,7 @@ export const createRailsBuildJobs = (
     dockerBuild: {
       variables: {
         ...context.environment.jobOnlyVars.build.envVars,
+        ...context.build.config.extraVars,
       },
       // custom script
       script: [

package/src/build/rails/test.ts

@@ -24,6 +24,7 @@ export const createRailsTestJobs = (
   const base: Omit<CatladderJob, "script" | "name"> = {
     variables: {
       ...context.environment.jobOnlyVars.build.envVars,
+      ...(buildConfig.extraVars ?? {}),
     },
     stage: "test",
     needs: [],

package/src/build/types.ts

@@ -82,6 +82,12 @@ export type BuildConfigBase = {
    * Can curently not reference other variables from other components
    */
   jobVars?: EnvVars;
+  /**
+   * additional env vars for the buid jobs
+   *
+   * @deprecated use jobVars or runnerVariables
+   */
+  extraVars?: Record<string, string>;
 
   /**
    * additional vars only for the runner.
@@ -226,6 +232,13 @@ export type BuildConfigDocker =
   | BuildConfigDockerBuiltIn
   | BuildConfigDockerCustom;
 
+/**
+ * @deprecated this type is no longer used. Use {@link BuildConfigDocker} instead.
+ */
+export type BuildConfigCustomDocker =
+  | BuildConfigDockerCustom
+  | BuildConfigDockerBuiltInNgninx;
+
 export type BuildConfigCustom = Omit<
   BuildConfigBase,
   "lint" | "test" | "audit"

package/src/context/__tests__/resolveReferences.test.ts

@@ -1,5 +1,8 @@
 import type { BashExpression } from "../../bash/BashExpression";
-import { resolveReferences } from "../resolveReferences";
+import {
+  resolveReferences,
+  translateLegacyFromComponents,
+} from "../resolveReferences";
 
 const unpackBashExpressions = (obj: Record<string, string | BashExpression>) =>
   Object.fromEntries(
@@ -120,3 +123,26 @@ describe("resolveReferences", () => {
     });
   });
 });
+
+describe("translateLegacyFromComponents", () => {
+  it("translatetes the old `fromComponents` approach to ${componentName:variableName}", () => {
+    const fromComponents = {
+      api: {
+        API_FOO: "FOO",
+        API_BAR: "BAR",
+      },
+      www: {
+        WWW_X: "X",
+        WWW_Y: "Y",
+      },
+    };
+
+    const result = translateLegacyFromComponents(fromComponents);
+    expect(result).toEqual({
+      API_FOO: "${api:FOO}",
+      API_BAR: "${api:BAR}",
+      WWW_X: "${www:X}",
+      WWW_Y: "${www:Y}",
+    });
+  });
+});

package/src/context/createComponentContext.ts

@@ -108,6 +108,7 @@ export const createComponentContext = async (
   const context: Omit<ComponentContext, "customJobs"> = {
     type: "component",
     name: ctx.componentName,
+    componentName: ctx.componentName,
     env: ctx.env,
     fullConfig: ctx.config,
     componentConfig,

package/src/context/getEnvironment.ts

@@ -20,7 +20,7 @@ export const getEnvironment = async (
     slugPrefix: envContext.environmentSlugPrefix,
     reviewSlug: envContext.reviewSlug,
     slug: envContext.environmentSlug,
-
+    shortName: env,
     ...variables,
   };
 };

package/src/context/getEnvironmentVariables.ts

@@ -1,3 +1,4 @@
+import { merge } from "lodash";
 import { DEPLOY_TYPES } from "../deploy";
 import type {
   ComponentContext,
@@ -12,7 +13,10 @@ import { isStandaloneBuildConfig } from "../build/types";
 import type { EnvironmentContext } from "../types/environmentContext";
 import { getBuildInfoVariables } from "./getBuildInfoVariables";
 import { getEnvironmentContext } from "./getEnvironmentContext";
-import { resolveReferences } from "./resolveReferences";
+import {
+  resolveReferences,
+  translateLegacyFromComponents,
+} from "./resolveReferences";
 import { transformJobOnlyVars } from "./transformJobOnlyVars";
 import {
   makeSecretEnvVarMapping,
@@ -39,9 +43,12 @@ type BasePredefinedVariables = ReturnType<typeof getBasePredefinedVariables>;
 
 // we export so that we have later nice autocomplete
 export type PredefinedVariables = BasePredefinedVariables & {
-  HOSTNAME?: StringOrBashExpression;
+  /**
+   * undefined in rails, Rails before 6.1 (mis)uses the `HOST` environment variable to specify the IP to bind to
+   */
+  HOST?: StringOrBashExpression;
   ROOT_URL: StringOrBashExpression;
-  HOSTNAME_INTERNAL: StringOrBashExpression;
+  HOST_INTERNAL: StringOrBashExpression;
   ROOT_URL_INTERNAL: StringOrBashExpression;
 };
 
@@ -71,8 +78,12 @@ export const getEnvironmentVariables = async (
       ...basePredefinedVariables,
       ENV_SHORT: "local",
       ROOT_URL: url,
-      HOSTNAME: host,
-      HOSTNAME_INTERNAL: host,
+      // Rails before 6.1 (mis)uses the `HOST` environment variable to specify the IP to bind to
+      ...(isStandaloneBuildConfig(buildConfigRaw) &&
+      buildConfigRaw.type === "rails"
+        ? {}
+        : { HOST: host }),
+      HOST_INTERNAL: host,
       ROOT_URL_INTERNAL: "http://" + host,
       PORT: port.toString(),
     };
@@ -83,10 +94,10 @@ export const getEnvironmentVariables = async (
         )
       : {};
 
-    const HOSTNAME_INTERNAL =
-      additionalEnvVars.HOSTNAME_INTERNAL ?? "unknown-host.example.com";
+    const HOST_INTERNAL =
+      additionalEnvVars.HOST_INTERNAL ?? "unknown-host.example.com";
 
-    host = envConfigRaw?.host ?? HOSTNAME_INTERNAL;
+    host = envConfigRaw?.host ?? HOST_INTERNAL;
     url = joinBashExpressions(["https://", host]);
 
     predefinedVariables = {
@@ -95,11 +106,12 @@ export const getEnvironmentVariables = async (
       ...(isStandaloneBuildConfig(buildConfigRaw) &&
       buildConfigRaw.type === "rails"
         ? {}
-        : { HOSTNAME: host }),
+        : { HOST: host }),
       ROOT_URL: url,
-      HOSTNAME_INTERNAL,
-
-      ROOT_URL_INTERNAL: joinBashExpressions(["https://", HOSTNAME_INTERNAL]),
+      HOST_INTERNAL,
+      /**@deprecated */
+      HOST_CANONICAL: HOST_INTERNAL, // legacy alias for HOST_INTERNAL
+      ROOT_URL_INTERNAL: joinBashExpressions(["https://", HOST_INTERNAL]),
       ...additionalEnvVars,
     };
   }
@@ -120,8 +132,17 @@ export const getEnvironmentVariables = async (
     componentName,
     secretEnvVarKeys,
   );
+  // this is deprecated, we now support: $componentname:FOO
+  const legacyFromComponents = envConfigRaw.vars?.fromComponents ?? {};
+  const publicEnvVarsRawWithLegacyFromComponents = merge(
+    {},
+    translateLegacyFromComponents(legacyFromComponents),
+    publicEnvVarsRaw,
+  );
 
-  const publicEnvVarsRawSanitized = stringifyValues(publicEnvVarsRaw);
+  const publicEnvVarsRawSanitized = stringifyValues(
+    publicEnvVarsRawWithLegacyFromComponents,
+  );
 
   const envVarsRaw = addIndexVar({
     ...predefinedVariables,

package/src/context/resolveReferences.ts

@@ -78,3 +78,16 @@ export const resolveReferences = async (
     ),
   ) as Record<string, BashExpression>;
 };
+
+export const translateLegacyFromComponents = (
+  fromComponents: Record<string, Record<string, string>>,
+) => {
+  return Object.fromEntries(
+    Object.entries(fromComponents).flatMap(([componentName, variables]) => {
+      return Object.entries(variables).map(([ourName, otherName]) => [
+        ourName,
+        "${" + componentName + ":" + otherName + "}",
+      ]);
+    }),
+  );
+};

package/src/deploy/base/deploy.ts

@@ -117,7 +117,7 @@ export const createDeployJob = (
       ...context.environment.envVars,
       ...(hasDocker ? getDockerImageVariables(context) : {}),
       ...context.environment.jobOnlyVars.deploy.envVars,
-
+      ...(deployConfig ? deployConfig.extraVars ?? {} : {}),
       ...jobDefinition.variables,
     },
     runnerVariables: {

package/src/deploy/cloudRun/index.ts

@@ -85,7 +85,7 @@ export const GCLOUD_RUN_DEPLOY_TYPE: DeployTypeDefinition<DeployConfigCloudRun>
     getAdditionalEnvVars: (ctx) => {
       const { fullName, env, componentName, deployConfigRaw } = ctx;
 
-      const HOSTNAME_INTERNAL = joinBashExpressions(
+      const HOST_INTERNAL = joinBashExpressions(
         [
           fullName,
           getBashVariable(
@@ -113,7 +113,7 @@ export const GCLOUD_RUN_DEPLOY_TYPE: DeployTypeDefinition<DeployConfigCloudRun>
           : {};
 
       return {
-        HOSTNAME_INTERNAL,
+        HOST_INTERNAL,
         ...getCloudSqlVariables(ctx),
         ...jobTriggers,
         DEPLOY_CLOUD_RUN_PROJECT_ID: deployConfigRaw

package/src/deploy/kubernetes/cloudSql/index.ts

@@ -41,7 +41,21 @@ export const createKubernetesCloudsqlBaseValues = (
     context,
     "cloudsqlProxyCredentials",
   )}`;
-  if (config.type === "unmanaged") {
+  if (config?.type !== "unmanaged") {
+    const instanceConnectionName = `${config.projectId ?? "skynet-164509"}:${
+      config.region ?? "europe-west6"
+    }:${config.instanceId ?? context.environment.envVars.KUBE_NAMESPACE}`;
+
+    return {
+      cloudsql: {
+        enabled: config.enabled,
+        dbUser: "postgres",
+        instanceConnectionName,
+        proxyCredentials,
+        fullDbName: context.environment.envVars.KUBE_APP_NAME ?? "",
+      },
+    };
+  } else if (config.type === "unmanaged") {
     return {
       cloudsql: {
         enabled: config.enabled,

package/src/deploy/kubernetes/deployJob.ts

@@ -47,6 +47,8 @@ export const createKubernetesDeployJobs = (
         ...(deployConfig.additionalHelmArgs ?? []),
       ].join(" "),
       COMPONENT_NAME: context.name,
+      /** @deprecated */
+      BUILD_ID: context.environment.envVars.BUILD_INFO_BUILD_ID,
     },
   };
 

package/src/deploy/kubernetes/index.ts

@@ -29,7 +29,7 @@ export const KUBERNETES_DEPLOY_TYPE: DeployTypeDefinition<DeployConfigKubernetes
         fullConfig.domainCanonical ||
         "panter.cloud";
 
-      const HOSTNAME_INTERNAL = joinBashExpressions(
+      const HOST_INTERNAL = joinBashExpressions(
         [
           componentSlug,
           ...(envType === "review" && reviewSlug ? [reviewSlug] : []),
@@ -44,7 +44,7 @@ export const KUBERNETES_DEPLOY_TYPE: DeployTypeDefinition<DeployConfigKubernetes
         KUBE_NAMESPACE,
         KUBE_APP_NAME,
         KUBE_APP_NAME_PREFIX,
-        HOSTNAME_INTERNAL,
+        HOST_INTERNAL,
       };
     },
   };

package/src/deploy/kubernetes/kubeValues.ts

@@ -11,6 +11,7 @@ import {
 
 import { createKubeEnv } from "./kubeEnv";
 import { createMongodbBaseConfig } from "./mongodb";
+import { processSecretsAsFiles } from "./processSecretsAsFiles";
 
 const createAppConfig = (
   context: ComponentContext,
@@ -86,12 +87,14 @@ export const createKubeValues = (context: ComponentContext) => {
       : {},
   );
 
-  const kubeValues = mergeWithMergingArrays(defaultKubeValues, {
-    jobs: removeFalsy(jobs),
-    cronjobs: removeFalsy(cronjobs),
+  const kubeValues = processSecretsAsFiles(
+    mergeWithMergingArrays(defaultKubeValues, {
+      jobs: removeFalsy(jobs),
+      cronjobs: removeFalsy(cronjobs),
 
-    ...rest,
-  });
+      ...rest,
+    }),
+  );
 
   return kubeValues;
 };

package/src/deploy/kubernetes/processSecretsAsFiles.ts

@@ -0,0 +1,35 @@
+import { omit } from "lodash";
+import type { DeployConfigKubernetesValues } from "../types";
+import type { StringOrBashExpression } from "../../bash/BashExpression";
+
+export const processSecretsAsFiles = <
+  T extends {
+    env: {
+      secret: Record<string, StringOrBashExpression>;
+    };
+  } & DeployConfigKubernetesValues,
+>(
+  values: T,
+): T => {
+  if (!values.secretsAsFile || values.secretsAsFile.length === 0) {
+    return values;
+  }
+  // the initial idea was that we "shadow" secrets that we mount as files,
+  // but there is a weird kubernetes  issue with that (https://github.com/kubernetes/kubernetes/issues/46861),
+  // so instead we rename the secrets and append _content, so that inside kuberentes, we can use the original name to contain the path and name+_content as the actual content
+  return {
+    ...values,
+    env: {
+      ...values.env,
+      secret: {
+        ...omit(values.env.secret, values.secretsAsFile),
+        ...Object.fromEntries(
+          values.secretsAsFile.map((key) => [
+            key + "_content",
+            values.env.secret[key],
+          ]),
+        ),
+      },
+    },
+  };
+};

package/src/deploy/types/base.ts

@@ -25,6 +25,12 @@ export type DeployConfigBase = {
    */
   jobVars?: EnvVars;
 
+  /**
+   * additional env vars for the deploy job
+   *
+   * @deprecated use deploy.jobVars for deploy job specific variables
+   */
+  extraVars?: Record<string, string>;
   /**
    * additional vars only for the runner.
    * Also if you use services: that require env vars, you need to set them here.

package/src/deploy/types/kubernetes.ts

@@ -164,6 +164,33 @@ export type DeployConfigMongodbReplicaset = {
 export type DeployConfigMongodb = DeployConfigMongodbBase &
   (DeployConfigMongodbStandalone | DeployConfigMongodbReplicaset);
 
+/**
+ * @deprecated use config with instanceConnectionName
+ */
+type DeployConfigKubernetesValuesCloudSQLLegacy = {
+  enabled: boolean;
+  /**
+   * @deprecated use config with instanceConnectionName
+   */
+  instanceId?: string;
+  /**
+   * @deprecated use config with instanceConnectionName
+   */
+  projectId?: string;
+  /**
+   * @deprecated use config with instanceConnectionName
+   */
+  region?: string;
+  /**
+   * @deprecated use type "unmanaged".
+   *
+   * Be aware that "unmanaged" is not fully backward compatible with "legacy".
+   *
+   *
+   */
+  type?: "legacy";
+};
+
 export type DeployConfigKubernetesValuesCloudSQLUnmanaged = {
   enabled: boolean;
   instanceConnectionName: string;
@@ -193,7 +220,8 @@ export type DeployConfigKubernetesValuesCloudSQLUnmanaged = {
   dbBaseName?: string;
 };
 export type DeployConfigKubernetesValuesCloudSQL =
-  DeployConfigKubernetesValuesCloudSQLUnmanaged;
+  | DeployConfigKubernetesValuesCloudSQLLegacy
+  | DeployConfigKubernetesValuesCloudSQLUnmanaged;
 export type DeployConfigKubernetesValues = AllowUnknownProps<{
   /**
    * enable cloudsql db. Currently you have to manually set it up
@@ -330,6 +358,13 @@ export type DeployConfigKubernetesValues = AllowUnknownProps<{
   mapServiceUrlToEnv?: {
     [envVar: string]: string;
   };
+  /**
+   * Mount secrets as files in the filesystem.
+   * These secrets are still available as env vars, but will contain the path of the file instead
+   *
+   * @deprecated this will be removed in the future, because its very niche and not used
+   */
+  secretsAsFile?: string[];
 }>;
 
 export type DeployConfigKubernetes = {

package/src/pipeline/gitlab/createGitlabJobs.ts

@@ -106,6 +106,7 @@ export const makeGitlabJob = (
     environment,
     envMode,
     needsStages,
+    needsOtherComponent,
     name,
     needs,
     jobTags,
@@ -309,7 +310,7 @@ function deduplicateNeeds(needs: GitlabJobDef["needs"]): GitlabJobDef["needs"] {
 
 function getGitlabNeedsForComponentJob(
   context: ComponentContext,
-  { needsStages, needs }: CatladderJob<string>,
+  { needsStages, needs, needsOtherComponent }: CatladderJob<string>,
   allJobs: AllCatladderJobs,
 ): GitlabJobDef["needs"] {
   const needsFromStages = needsStages?.flatMap<CatladderJobNeed>((n) => {
@@ -352,6 +353,8 @@ function getGitlabNeedsForComponentJob(
   const cleanedNeeds: CatladderJob["needs"] = [
     ...(needsFromStages ?? []),
     ...(needs ?? []),
+    // pull in legacy needs from other component, which is now identical to needs
+    ...(needsOtherComponent ?? []),
   ];
 
   return cleanedNeeds

package/src/types/config.ts

@@ -73,6 +73,14 @@ export type EnvVars = {
    * secret env vars. These vars can be managed with catladder/cli
    */
   secret?: string[];
+  /**
+   * @deprecated, use ${componentName:variableName} instead
+   *
+   * With fromComponents you can inject env vars from other components.
+   */
+  fromComponents?: {
+    [otherApp: string]: Record<string, string>;
+  };
 };
 
 export type DefaultEnvConfig = {

package/src/types/context.ts

@@ -46,7 +46,10 @@ export type Environment = {
    * the full name of the app. We use this as RELEASE_NAME in kubernetes and the service name in google cloud run
    */
   fullName: StringOrBashExpression;
-
+  /**
+   * @deprecated this is the same as context.env, use that instead
+   */
+  shortName: string;
   /**
    * the environment slug without component name.
    */
@@ -155,6 +158,10 @@ export type ComponentContext<
    * the name of the component
    */
   name: string;
+  /**
+   * @deprecated use name instead
+   */
+  componentName: string;
 
   /**
    * the merged component config.

package/src/types/jobs.ts

@@ -70,6 +70,14 @@ export type CatladderJob<S = BaseStage> = {
    */
   needs?: Array<CatladderJobNeed>;
 
+  /**
+   * @deprecated set `componentName` to `needs` if you want to wait for a job from another component
+   */
+  needsOtherComponent?: Array<{
+    componentName: string;
+    job: string;
+    artifacts: boolean;
+  }>;
   /**
    * cache config, we use here the same shape as gitlab itself
    */