@catladder/pipeline 0.0.0-more-collapse-deeb54d7 → 0.0.0-multiple-gitlab-includes-3df5566d

package/examples/kubernetes-with-cloud-sql-legacy.ts

@@ -1,33 +0,0 @@
-import type { Config } from "../src";
-import { createAllPipelines } from "./__utils__/helpers";
-const config: Config = {
-  appName: "test-app",
-  customerName: "pan",
-  components: {
-    api: {
-      dir: "api",
-      build: {
-        type: "node",
-      },
-      deploy: {
-        type: "kubernetes",
-        cluster: {
-          name: "some-cluster-name",
-          region: "europe-west6",
-          projectId: "some-project-id",
-          type: "gcloud",
-          domainCanonical: "panter.cloud",
-        },
-        values: {
-          cloudsql: {
-            enabled: true,
-          },
-        },
-      },
-    },
-  },
-};
-
-it("matches snapshot", async () => {
-  expect(await createAllPipelines(config)).toMatchSnapshot();
-});

package/scripts/bundle

@@ -1,2 +0,0 @@
-#!/bin/bash
-ncc build dist/catladder-gitlab.js -o dist/bundles/catladder-gitlab -e 'ts-node' $( ((SHOULD_MINIFY == 1)) && printf %s '-m')

package/src/context/__tests__/resolveReferences.test.ts

@@ -1,143 +0,0 @@
-import {
-  resolveReferences,
-  translateLegacyFromComponents,
-} from "../resolveReferences";
-
-describe("resolveReferences", () => {
-  it("replaces occurences of ${componentName:VARIABLE_NAME}", async () => {
-    const variables = {
-      a: "hello world",
-      b: "a replaced value looks like this: '${api:FOO}', nice!",
-    };
-    const otherVariables: Record<string, Record<string, string>> = {
-      api: {
-        FOO: "foo from api",
-      },
-      frontend: {
-        FOO: "foo from frontend",
-      },
-    };
-    const result = await resolveReferences(variables, async (componentName) => {
-      return otherVariables[componentName];
-    });
-
-    expect(result).toEqual({
-      a: "hello world",
-      b: "a replaced value looks like this: 'foo from api', nice!",
-    });
-  });
-
-  it("replaces self references with structure ${VARIABLE_NAME}", async () => {
-    const variables = {
-      FOO: "hello world",
-      BAR: "this: ${FOO}!",
-    };
-    const result = await resolveReferences(variables);
-
-    expect(result).toEqual({
-      FOO: "hello world",
-      BAR: "this: hello world!",
-    });
-  });
-
-  it("replaces self references mixed with other vars", async () => {
-    const variables = {
-      FOO: "hello from ${api:FOO}",
-      BAR: "this: ${FOO}!",
-    };
-    const otherVariables: Record<string, Record<string, string>> = {
-      api: {
-        FOO: "foo from api",
-      },
-    };
-    const result = await resolveReferences(variables, async (componentName) => {
-      return otherVariables[componentName];
-    });
-
-    expect(result).toEqual({
-      FOO: "hello from foo from api",
-      BAR: "this: hello from foo from api!",
-    });
-  });
-
-  it("keeps variables as is if not found (when null is returned)", async () => {
-    const variables = {
-      a: "hello world",
-      b: "a not found value looks like this: '${api:FOO}'",
-    };
-    const otherVariables: Record<string, Record<string, string>> = {};
-    const result = await resolveReferences(variables, async (componentName) => {
-      return otherVariables[componentName];
-    });
-
-    expect(result).toEqual({
-      a: "hello world",
-      b: "a not found value looks like this: '${api:FOO}'",
-    });
-  });
-
-  it("replaces mulitple levels deep", async () => {
-    const variables = {
-      a: "value is ${frontend:FOO}!",
-    };
-    const otherVariables: Record<string, Record<string, string>> = {
-      api: {
-        FOO: "foo from api",
-      },
-      frontend: {
-        FOO: "hi, ${api:FOO}",
-      },
-    };
-    const result = await resolveReferences(variables, async (componentName) => {
-      return otherVariables[componentName];
-    });
-
-    expect(result).toEqual({
-      a: "value is hi, foo from api!",
-    });
-  });
-
-  it("prevents endless loops", async () => {
-    const variables = {
-      a: "value is ${frontend:FOO}!",
-    };
-    const otherVariables: Record<string, Record<string, string>> = {
-      api: {
-        FOO: "api ${frontend:FOO}",
-      },
-      frontend: {
-        FOO: "frontend ${api:FOO}",
-      },
-    };
-    const result = await resolveReferences(variables, async (componentName) => {
-      return otherVariables[componentName];
-    });
-
-    expect(result).toEqual({
-      a: "value is frontend api ${frontend:FOO}!",
-    });
-  });
-});
-
-describe("translateLegacyFromComponents", () => {
-  it("translatetes the old `fromComponents` approach to ${componentName:variableName}", () => {
-    const fromComponents = {
-      api: {
-        API_FOO: "FOO",
-        API_BAR: "BAR",
-      },
-      www: {
-        WWW_X: "X",
-        WWW_Y: "Y",
-      },
-    };
-
-    const result = translateLegacyFromComponents(fromComponents);
-    expect(result).toEqual({
-      API_FOO: "${api:FOO}",
-      API_BAR: "${api:BAR}",
-      WWW_X: "${www:X}",
-      WWW_Y: "${www:Y}",
-    });
-  });
-});

package/src/context/resolveReferences.ts

@@ -1,70 +0,0 @@
-import { merge } from "lodash";
-import replaceAsync from "string-replace-async";
-
-const REGEX = /\$\{(([^:]+):)?([^}]+)}/gm;
-
-export const resolveReferences = async (
-  vars: Record<string, string>,
-  getOtherVariables?: (
-    componentName: string,
-    alreadyVisited: Record<string, Record<string, boolean>>
-  ) => Promise<Record<string, string | null>>,
-  alreadyVisitedBase: Record<string, Record<string, boolean>> = {}
-) => {
-  const replaceSingleValue = async (
-    value: string,
-    alreadyVisited: Record<string, Record<string, boolean>> = alreadyVisitedBase
-  ): Promise<string> => {
-    if (REGEX.test(value)) {
-      return await replaceAsync(
-        value,
-        REGEX,
-        async (match, _, componentName, variableName) => {
-          if (alreadyVisited[componentName]?.[variableName]) {
-            return match; // prevent endless loop
-          }
-          const newAlreadyVisited = merge({}, alreadyVisited, {
-            [componentName]: {
-              [variableName]: true,
-            },
-          });
-          const result = componentName
-            ? (await getOtherVariables?.(componentName, newAlreadyVisited).then(
-                (r) => r?.[variableName]
-              )) ?? null
-            : vars[variableName]; // is self reference
-
-          const replaced =
-            result !== null && result !== undefined
-              ? await replaceSingleValue(result, newAlreadyVisited)
-              : match;
-
-          return replaced;
-        }
-      );
-    } else {
-      return value;
-    }
-  };
-
-  return Object.fromEntries(
-    await Promise.all(
-      Object.entries(vars).map(async ([key, value]) => {
-        return [key, await replaceSingleValue(value)];
-      })
-    )
-  ) as Record<string, string>;
-};
-
-export const translateLegacyFromComponents = (
-  fromComponents: Record<string, Record<string, string>>
-) => {
-  return Object.fromEntries(
-    Object.entries(fromComponents).flatMap(([componentName, variables]) => {
-      return Object.entries(variables).map(([ourName, otherName]) => [
-        ourName,
-        "${" + componentName + ":" + otherName + "}",
-      ]);
-    })
-  );
-};

package/src/deploy/cloudRun/deployJob.ts

@@ -1,301 +0,0 @@
-import { dump } from "js-yaml";
-import { merge, omit } from "lodash";
-import { GCLOUD_DEPLOY_CREDENTIALS_KEY } from ".";
-import { getDockerJobBaseProps, gitlabDockerLogin } from "../../build/docker";
-import { getLabels } from "../../context/getLabels";
-import { getRunnerImage } from "../../runner";
-import type { Context } from "../../types/context";
-import type { CatladderJob } from "../../types/jobs";
-import { allowFailureInScripts, collapseableSection } from "../../utils/gitlab";
-import { createDeployementJobs } from "../base";
-import {
-  getDependencyTrackDeleteScript,
-  getDependencyTrackUploadScript,
-} from "../sbom";
-import type {
-  DeployConfigCloudRun,
-  DeployConfigCloudRunJob,
-  DeployConfigCloudRunJobWithSchedule,
-  DeployConfigCloudRunService,
-} from "../types";
-import { isOfDeployType } from "../types";
-import { createArgsString } from "./utils/createArgsString";
-import {
-  getDatabaseCreateScript,
-  getDatabaseDeleteScript,
-} from "./utils/database";
-import { gcloudServiceAccountLoginCommands } from "./utils/gcloudServiceAccountLoginCommands";
-import { getCloudRunJobName } from "./utils/jobName";
-
-const setExtraVarsScripts = (deployConfig: DeployConfigCloudRun) => [
-  `export GCLOUD_PROJECT_NUMBER=$(gcloud projects describe ${deployConfig.projectId} --format="value(projectNumber)")`,
-  `echo "GCLOUD_PROJECT_NUMBER: $GCLOUD_PROJECT_NUMBER"`,
-];
-
-export const createGoogleCloudRunDeployJobs = (
-  context: Context
-): CatladderJob[] => {
-  const deployConfig = context.componentConfig.deploy;
-  if (deployConfig === false) {
-    return [];
-  }
-  if (!isOfDeployType(deployConfig, "google-cloudrun")) {
-    // should not happen
-    throw new Error("deploy config is wrong");
-  }
-
-  const fullAppName = `${context.fullConfig.customerName}-${context.fullConfig.appName}`;
-  const dockerUrl = `${deployConfig.region}-docker.pkg.dev/${deployConfig.projectId}/catladder-deploy/${fullAppName}`;
-  const gcloudImagePath = [
-    dockerUrl,
-    context.environment.shortName,
-    context.componentName,
-
-    ...(context.environment.envType === "review"
-      ? [context.commitInfo?.reviewSlug]
-      : []),
-  ];
-  const gcloudImageName = `${gcloudImagePath.join("/")}:$DOCKER_IMAGE_TAG`;
-
-  const pushImageToArtifactsRegistry = [
-    gitlabDockerLogin,
-    `gcloud auth configure-docker ${deployConfig.region}-docker.pkg.dev`,
-    `docker pull $DOCKER_IMAGE:$DOCKER_IMAGE_TAG`,
-    `docker tag $DOCKER_IMAGE:$DOCKER_IMAGE_TAG ${gcloudImageName}`,
-    `docker push ${gcloudImageName}`,
-  ];
-  const allEnvVars = omit(
-    context.environment.envVars,
-    GCLOUD_DEPLOY_CREDENTIALS_KEY
-  );
-
-  const labelsString = Object.entries(getLabels(context))
-    .map(([key, value]) => `${key}=${value}`)
-    .join(",");
-
-  const commonArgs = {
-    project: deployConfig.projectId,
-    region: deployConfig.region,
-  };
-
-  const commonDeployArgs = {
-    image: gcloudImageName,
-    ...commonArgs,
-    "set-cloudsql-instances": deployConfig.cloudSql
-      ? deployConfig.cloudSql.instanceConnectionName
-      : undefined,
-    labels: labelsString,
-  };
-
-  const serviceName = context.environment.fullName.toLowerCase();
-
-  const getFullJobName = (name: string) =>
-    getCloudRunJobName(context.environment.fullName, name);
-
-  const jobsWithNames = Object.entries(deployConfig.jobs ?? {})
-    // filter out disabled jobs
-    .filter((entry): entry is [string, DeployConfigCloudRunJob] =>
-      Boolean(entry[1])
-    )
-    .map(([name, job]) => ({
-      jobName: getFullJobName(name),
-      job,
-    }));
-  const jobsWithSchedule = jobsWithNames
-    .filter(
-      (
-        entry
-      ): entry is {
-        jobName: string;
-        job: DeployConfigCloudRunJobWithSchedule;
-      } => entry.job.when === "schedule"
-    )
-    .map(({ job, jobName }) => ({
-      job,
-      jobName,
-      schedulerName: jobName + "-scheduler",
-    }));
-
-  const getServiceDeployScript = (
-    service: DeployConfigCloudRunService | true | undefined,
-    nameSuffix?: string
-  ) => {
-    const customConfig = service !== true ? service : undefined;
-    const command =
-      service !== true
-        ? service?.command ?? context.componentConfig.build.startCommand
-        : undefined;
-
-    const argsString = createArgsString({
-      // command as empty string resets it to default (uses the image's entrypoint)
-      command: command ? '"' + command.split(" ").join(",") + '"' : '""',
-      ...commonDeployArgs,
-      "env-vars-file": "____envvars.yaml",
-      "min-instances": customConfig?.minInstances ?? 0,
-      "max-instances": customConfig?.maxInstances ?? 100,
-      "cpu-throttling": customConfig?.noCpuThrottling !== true,
-      memory: customConfig?.memory,
-      "allow-unauthenticated": customConfig?.allowUnauthenticated ?? true,
-      "cpu-boost": true,
-    });
-
-    return `gcloud run deploy ${serviceName}${nameSuffix ?? ""} ${argsString}`;
-  };
-
-  const getJobCreateScriptsForJob = (
-    jobName: string,
-    job: DeployConfigCloudRunJob
-  ) => {
-    // due to some oversight from google, jobs create does not accept `--env-vars-file` 🤦
-    // lucky, update on the other hand accepts it... so let's just imediatly update it
-
-    // also we cannot upsert a job, so we have to create it and catch the error and then update
-    const commonDeployArgsString = createArgsString({
-      command: '"' + job.command.split(" ").join(",") + '"',
-      ...commonDeployArgs,
-      memory: job.memory || "512Mi",
-      "task-timeout": job.timeout || "10m",
-    });
-
-    const argsString = `${jobName} ${commonDeployArgsString}`;
-    return [
-      ...allowFailureInScripts([`gcloud beta run jobs create ${argsString}`]),
-      `gcloud beta run jobs update ${argsString} --env-vars-file=____envvars.yaml`,
-    ];
-  };
-
-  const commonArgsString = createArgsString(commonArgs);
-
-  const getJobCreateScripts = () =>
-    jobsWithNames
-      .map(({ job, jobName }) => getJobCreateScriptsForJob(jobName, job))
-      .flat();
-
-  const getJobRunScriptForJob = (jobName: string) => {
-    return `gcloud beta run jobs execute ${jobName} ${commonArgsString}`;
-  };
-
-  const getJobRunScripts = (when: DeployConfigCloudRunJob["when"]) =>
-    jobsWithNames
-      .filter(({ job }) => job.when === when)
-      .map(({ jobName }) => getJobRunScriptForJob(jobName));
-
-  const getCreateScheduleScripts = () => {
-    return jobsWithSchedule
-      .map(({ job, jobName, schedulerName }) => {
-        const argsString = createArgsString({
-          project: deployConfig.projectId,
-          location: deployConfig.region,
-          schedule: `"${job.schedule}"`,
-          "max-retry-attempts": job.maxRetryAttempts ?? 0,
-
-          uri: `"https://${deployConfig.region}-run.googleapis.com/apis/run.googleapis.com/v1/namespaces/${deployConfig.projectId}/jobs/${jobName}:run"`,
-          "http-method": "POST",
-          "oauth-service-account-email":
-            "$GCLOUD_PROJECT_NUMBER-compute@developer.gserviceaccount.com",
-        });
-        const commonArgs = `http ${schedulerName} ${argsString}`;
-        return [
-          ...allowFailureInScripts([
-            `gcloud scheduler jobs create ${commonArgs}`,
-          ]),
-          `gcloud scheduler jobs update ${commonArgs}`,
-        ];
-      })
-      .flat();
-  };
-
-  const getDeleteSchedulesScripts = () => {
-    const argsString = createArgsString({
-      project: deployConfig.projectId,
-      location: deployConfig.region,
-    });
-    return jobsWithSchedule
-      .map(({ schedulerName }) => {
-        return [`gcloud scheduler jobs delete ${schedulerName} ${argsString}`];
-      })
-      .flat();
-  };
-
-  const getDeleteJobsScripts = () =>
-    jobsWithNames.flatMap(({ jobName }) => [
-      // first delete all job executions. Otherwise delete might fail if one of those is still running
-      `gcloud beta run jobs executions list ${commonArgsString} --job ${jobName} --format="value(name)" | xargs -I {} gcloud beta run jobs executions delete {}  --quiet ${commonArgsString}`,
-      `gcloud beta run jobs delete ${jobName} ${commonArgsString}`,
-    ]);
-
-  const deployScripts = [
-    ...collapseableSection(
-      "prepare",
-      "Prepare..."
-    )([
-      ...gcloudServiceAccountLoginCommands(context),
-      ...setExtraVarsScripts(deployConfig),
-    ]),
-    ...collapseableSection(
-      "pushToArtifactRegistry",
-      "Pushing image to artifacts registry"
-    )(pushImageToArtifactsRegistry),
-    ...collapseableSection(
-      "deploy",
-      "Deploy to cloud run"
-    )([
-      `echo "$ENV_VARS" > ____envvars.yaml`, // TODO: split secrets out
-      ...(deployConfig.cloudSql
-        ? getDatabaseCreateScript(context, deployConfig) // we create the db, so that we can also delete it afterwards
-        : []),
-      ...getCreateScheduleScripts(),
-      ...getJobCreateScripts(),
-      ...getJobRunScripts("preDeploy"),
-
-      ...(deployConfig.service !== false
-        ? [getServiceDeployScript(deployConfig.service)]
-        : []),
-      ...Object.entries(deployConfig.additionalServices ?? {}).map(
-        ([name, service]) => getServiceDeployScript(service, "-" + name)
-      ),
-      ...getJobRunScripts("postDeploy"),
-    ]),
-    `docker image rm ${gcloudImageName}`,
-    ...getDependencyTrackUploadScript(context),
-  ];
-
-  const stopScripts = [
-    ...gcloudServiceAccountLoginCommands(context),
-    ...(deployConfig.service !== false
-      ? [`gcloud run services delete ${serviceName} ${commonArgsString}`]
-      : []),
-    ...Object.entries(deployConfig.additionalServices ?? {}).map(
-      ([name]) =>
-        `gcloud run services delete ${serviceName}-${name} ${commonArgsString}`
-    ),
-    ...getDeleteSchedulesScripts(),
-    ...getDeleteJobsScripts(),
-    ...(deployConfig.cloudSql && deployConfig.cloudSql.deleteDatabaseOnStop
-      ? getDatabaseDeleteScript(context, deployConfig)
-      : []),
-    ...getDependencyTrackDeleteScript(context),
-  ];
-  return createDeployementJobs(context, {
-    deploy: merge(getDockerJobBaseProps(context), {
-      artifacts: { paths: ["____envvars.yaml"] },
-      variables: {
-        CLOUDSDK_CORE_DISABLE_PROMPTS: "1",
-        ENV_VARS: dump(allEnvVars, {
-          lineWidth: -1,
-          quotingType: "'",
-          forceQuotes: true,
-        }),
-      },
-      image: getRunnerImage("gcloud"),
-      script: deployScripts,
-    }),
-    stop: {
-      image: getRunnerImage("gcloud"),
-      variables: {
-        CLOUDSDK_CORE_DISABLE_PROMPTS: "1",
-      },
-      script: allowFailureInScripts(stopScripts),
-    },
-  });
-};

package/src/deploy/kubernetes/processSecretsAsFiles.ts

@@ -1,34 +0,0 @@
-import { omit } from "lodash";
-import type { DeployConfigKubernetesValues } from "../types";
-
-export const processSecretsAsFiles = <
-  T extends {
-    env: {
-      secret: Record<string, string>;
-    };
-  } & DeployConfigKubernetesValues
->(
-  values: T
-): T => {
-  if (!values.secretsAsFile || values.secretsAsFile.length === 0) {
-    return values;
-  }
-  // the initial idea was that we "shadow" secrets that we mount as files,
-  // but there is a weird kubernetes  issue with that (https://github.com/kubernetes/kubernetes/issues/46861),
-  // so instead we rename the secrets and append _content, so that inside kuberentes, we can use the original name to contain the path and name+_content as the actual content
-  return {
-    ...values,
-    env: {
-      ...values.env,
-      secret: {
-        ...omit(values.env.secret, values.secretsAsFile),
-        ...Object.fromEntries(
-          values.secretsAsFile.map((key) => [
-            key + "_content",
-            values.env.secret[key],
-          ])
-        ),
-      },
-    },
-  };
-};

package/src/pipeline/commitInfo/getBuildId.ts

@@ -1,17 +0,0 @@
-import { exec } from "child-process-promise";
-
-export const getBuildId = async () =>
-  await exec("git describe --tags || git rev-parse HEAD").then((s) =>
-    s.stdout.trim()
-  );
-
-// thx chat gpt
-/**
- * returns the last tagged version string Major.minor.patch
- * returns 0.0.0 if there is no tag
- */
-export const getCurrentVersionString = async () => {
-  return await exec(
-    'latest_tag=$(git describe --tags --abbrev=0 2>/dev/null); if [ -z "$latest_tag" ]; then echo "0.0.0"; else echo "${latest_tag#v}"; fi | cut -d "." -f 1-3'
-  ).then((s) => s.stdout.trim());
-};

package/src/pipeline/commitInfo/getCommitInfo.ts

@@ -1,15 +0,0 @@
-import type { CommitInfo } from "../../types";
-import { getBuildId, getCurrentVersionString } from "./getBuildId";
-export const getBaseCommitInfo = async (): Promise<
-  Omit<CommitInfo, "trigger">
-> => ({
-  refName: process.env.CI_COMMIT_REF_NAME ?? "unknown",
-  reviewSlug: process.env.CI_MERGE_REQUEST_IID
-    ? `mr${process.env.CI_MERGE_REQUEST_IID}`
-    : process.env.CI_COMMIT_REF_SLUG
-    ? `hf${process.env.CI_COMMIT_REF_SLUG}`
-    : "unknown",
-  buildTime: new Date().toISOString(),
-  buildId: await getBuildId(),
-  currentVersion: await getCurrentVersionString(),
-});

package/src/pipeline/createChildPipeline.ts

@@ -1,45 +0,0 @@
-import { getAllEnvsInAllComponents } from "../config";
-import { RULES_ALWAYS } from "../rules";
-import { getRunnerImage } from "../runner";
-import type { GitlabPipeline, Pipeline, PipelineType } from "../types";
-import type { Config, PipelineTrigger } from "../types/config";
-import { BASE_STAGES } from "../types/jobs";
-import { createAllJobs } from "./createAllJobs";
-import { createGitlabJobs } from "./gitlab/createGitlabJobs";
-
-export const createChildPipeline = async <T extends PipelineType>(
-  type: T,
-  trigger: PipelineTrigger,
-  config: Config
-): Promise<Pipeline<T>> => {
-  const jobs = await createAllJobs(config, trigger);
-
-  // while technically not required, we group different envs in its own stage
-  // each job from `createJobs` that is defined as `envMode: "stagePerEnv"` will have `deploy dev`, etc. instead of just `deploy`
-  // this is just so that it looks nicer in gitlab and makes running mutliple manual tasks more easy to use
-
-  const allEnvs = getAllEnvsInAllComponents(config);
-  const stages = BASE_STAGES.reduce<string[]>(
-    (acc, baseStage) => [
-      ...acc,
-      baseStage,
-      ...allEnvs.map((e) => `${baseStage} ${e}`),
-    ],
-    []
-  );
-  if (type === "gitlab") {
-    const pipeline: GitlabPipeline = {
-      image: getRunnerImage("jobs-default"), // default image
-      variables: {
-        FF_USE_FASTZIP: "true",
-      },
-      workflow: {
-        rules: RULES_ALWAYS,
-      },
-      stages,
-      jobs: await createGitlabJobs(jobs),
-    };
-    return pipeline as Pipeline<T>;
-  }
-  throw new Error(`${type} is not supported`);
-};