@catladder/cli 0.0.0-test7 → 0.0.0-update-meteor-53480fed

package/src/apps/cli/verify/migration/fromv2.ts

@@ -0,0 +1,262 @@
+import type {
+  BuildConfig,
+  ComponentConfig,
+  Config,
+  DeployConfigKubernetes,
+  Env,
+} from "@catladder/pipeline";
+import {
+  existsSync,
+  lstatSync,
+  readdirSync,
+  readFile,
+  writeFile,
+} from "fs-extra";
+import { isEmpty } from "lodash";
+import { join } from "path";
+import type Vorpal from "vorpal";
+import {
+  getGitlabCi,
+  getGitlabCiFilePath,
+} from "../../../../config/getProjectConfig";
+import { readYaml } from "../../../../utils/files";
+import { syncBitwarden } from "../../../../utils/passwordstore";
+import { getGitRoot } from "../../../../utils/projects";
+import { writeConfig } from "../../config/writeConfig";
+import { migrateSecrets } from "./migrateSecrets";
+import type { OldGitlabCiFile } from "./oldGitlabCi";
+import { detectBuildConfig, isOldInclude } from "./oldGitlabCi";
+export const LEGACY_ENVS = [
+  "dev-local",
+  "dev",
+  "review",
+  "stage",
+  "prod",
+] as const;
+
+const arrayToRecord = (arr: { name: string }[]): Record<string, any> => {
+  if (!arr) return undefined;
+  if (!Array.isArray(arr)) return arr;
+  return Object.fromEntries(arr.map(({ name, ...rest }) => [name, rest]));
+};
+
+const transformVars = (rawVars: {
+  public: Record<string, any>;
+  secret: Record<string, any>;
+  fromCommponents: any;
+}) => {
+  if (rawVars.fromCommponents) {
+    console.warn("cant transform legacy fromCommponents");
+  }
+
+  return {
+    public: rawVars.public,
+    secret: rawVars.secret ? Object.keys(rawVars.secret) : undefined,
+  };
+};
+
+const getLegacyMonorepoSubCiFiles = (dir: string) => {
+  return readdirSync(dir)
+    .filter((file) => lstatSync(file).isDirectory())
+    .map((dir) => ({
+      dir,
+      ci: join(dir, ".gitlab-ci.yml"),
+    }))
+    .filter(({ ci }) => existsSync(ci));
+};
+
+const transformValues = (
+  valuesIn: Record<string, any>
+): DeployConfigKubernetes["values"] => {
+  if (isEmpty(valuesIn)) {
+    return undefined;
+  }
+  delete valuesIn?.application?.host;
+  delete valuesIn?.application?.command;
+
+  return {
+    ...valuesIn,
+    jobs: arrayToRecord(valuesIn?.jobs),
+    cronjobs: arrayToRecord(valuesIn?.cronjobs),
+  };
+};
+
+export const isV2 = async () => {
+  const gitlabCi = await getGitlabCi<OldGitlabCiFile>();
+  return isOldInclude(gitlabCi);
+};
+export const migrateV2 = async (vorpal: Vorpal) => {
+  const gitlabCi = await getGitlabCi<OldGitlabCiFile>();
+  const gitRoot = await getGitRoot();
+
+  const {
+    CUSTOMER_NAME,
+    APP_NAME,
+    COMPONENT_NAME = "web",
+    APP_DIR = ".",
+    STAGING_ENABLED,
+  } = gitlabCi.variables;
+
+  vorpal
+    .command("migrate")
+
+    .action(async function () {
+      this.log("");
+      this.log("⚠️ this project uses legacy catladder (v2)");
+      this.log("");
+      this.log(
+        "😼 I can migrate the project for you. This contains the following steps:"
+      );
+      this.log("");
+      this.log(
+        " - migrate the config from values-*.yml to catladder config file"
+      );
+      this.log(
+        " - migrate the secrets from bitwarden to gitlab (this will trash the entries in bitwarden)"
+      );
+      this.log("");
+      this.log(
+        "☝ make sure that you checked in your current state in case something goes wrong."
+      );
+      this.log(
+        "☝ secrets in bitwarden are deleted, but can be restored within 30 days."
+      );
+      this.log("");
+      const { shouldContinue } = await this.prompt({
+        default: true,
+        message: "Migrate project now? 🤔",
+        name: "shouldContinue",
+        type: "confirm",
+      });
+
+      if (shouldContinue) {
+        vorpal.log("");
+        vorpal.log("");
+        vorpal.log("💪😼 ok, let's go...");
+        vorpal.log("");
+        vorpal.log("");
+
+        const createComponent = async (
+          dir: string,
+          ciFile: OldGitlabCiFile
+        ): Promise<ComponentConfig> => {
+          const { env, ...baseValues } =
+            (await readYaml(dir + "/values.yml")) ?? {};
+
+          const startCommand = Array.isArray(baseValues?.application?.command)
+            ? baseValues?.application?.command.join(" ")
+            : baseValues?.application?.command;
+          return {
+            vars: env ? transformVars(env) : undefined,
+            dir: dir,
+            build: {
+              type: detectBuildConfig(ciFile),
+              startCommand: startCommand,
+            } as BuildConfig,
+            deploy: {
+              type: "kubernetes",
+              values: transformValues(baseValues),
+              cluster: {
+                type: "gcloud",
+                name: "ch-production",
+                projectId: "skynet-swiss",
+                region: "europe-west6-a",
+                domainCanonical: "panter.swiss",
+              },
+            },
+            env: await LEGACY_ENVS.reduce<Promise<Env>>(
+              async (acc, envName) => {
+                if (envName === "stage" && !STAGING_ENABLED) {
+                  return {
+                    ...(await acc),
+                    [envName]: false,
+                  };
+                }
+                const newEnvName = envName === "dev-local" ? "local" : envName;
+
+                const envValues =
+                  (await readYaml(dir + `/values-${envName}.yml`)) ?? {};
+                const { env, ...rest } = envValues;
+                const host = rest?.application?.host;
+                const values = transformValues(rest);
+                return {
+                  ...(await acc),
+                  [newEnvName]: {
+                    host: host,
+                    vars: env ? transformVars(env) : undefined,
+                    deploy: values
+                      ? {
+                          values: values,
+                        }
+                      : undefined,
+                  },
+                };
+              },
+              Promise.resolve({})
+            ),
+          };
+        };
+        let components: Record<string, ComponentConfig>;
+        if (detectBuildConfig(gitlabCi) === "monorepo") {
+          components = await getLegacyMonorepoSubCiFiles(gitRoot).reduce<
+            Promise<Record<string, ComponentConfig>>
+          >(async (acc, el) => {
+            return {
+              ...(await acc),
+              [el.dir]: await createComponent(el.dir, await readYaml(el.ci)),
+            };
+          }, Promise.resolve({}));
+        } else {
+          components = {
+            [COMPONENT_NAME]: await createComponent(APP_DIR, gitlabCi),
+          };
+        }
+        // we only have one component
+        const config: Config = {
+          customerName: CUSTOMER_NAME,
+          appName: APP_NAME,
+          components,
+        };
+
+        const comment =
+          "Old migrated config:\n\n" +
+          (await readFile(await getGitlabCiFilePath(), {
+            encoding: "utf-8",
+          }));
+
+        this.log("-------------------");
+        this.log("migrate config");
+        await writeConfig(this, config, {
+          endComment: comment,
+        });
+        this.log("write gitlab-ci.yml");
+        await writeFile(
+          await getGitlabCiFilePath(),
+          "include: https://git.panter.ch/api/v4/projects/catladder%2Fcatladder/packages/generic/ci-includes/main/gitlab-ci.yml",
+          { encoding: "utf-8" }
+        );
+
+        this.log("-------------------");
+        this.log("migrate secrets");
+        await syncBitwarden();
+        for (const env of LEGACY_ENVS) {
+          if (env === "stage" && !STAGING_ENABLED) return;
+          await migrateSecrets(this, config, env);
+        }
+
+        this.log("-------------------");
+
+        this.log("done!");
+        this.log("");
+        this.log("You can remove the values*.yml files now.");
+
+        this.log("-------------------");
+      } else {
+        this.log(
+          "☝ if you want to use catladder in legacy mode, install @panter/catladder globally and invoke catladder-legacy"
+        );
+      }
+    });
+
+  vorpal.exec("migrate");
+};

package/src/apps/cli/verify/migration/migrateSecrets.ts

@@ -0,0 +1,47 @@
+import type { Config } from "@catladder/pipeline";
+import { getEnvironment } from "@catladder/pipeline";
+import { load } from "js-yaml";
+import { pick } from "lodash";
+import type { CommandInstance } from "vorpal";
+import { upsertAllVariables } from "../../../../utils/gitlab";
+import { readPass, trashItem } from "../../../../utils/passwordstore";
+import type { LEGACY_ENVS } from "./fromv2";
+
+const getPassPath = (newConfig: Config, env: string) => {
+  return `${newConfig.customerName}/${newConfig.appName}/${env}/secrets.yml`;
+};
+
+export const migrateSecrets = async (
+  vorpal: CommandInstance,
+  newConfig: Config,
+  oldEnv: typeof LEGACY_ENVS[number]
+) => {
+  const newEnv = oldEnv === "dev-local" ? "local" : oldEnv;
+  const path = getPassPath(newConfig, oldEnv);
+  try {
+    const yamlstring = await readPass(path);
+    const secrets = load(yamlstring);
+
+    Object.keys(newConfig.components).forEach(async (componentName) => {
+      const environment = await getEnvironment(
+        newConfig,
+        componentName,
+        newEnv
+      );
+      await upsertAllVariables(
+        vorpal,
+        pick(
+          secrets,
+          environment.secretEnvVarKeys
+            .filter((k) => !k.hidden)
+            .map((k) => k.key)
+        ),
+        newEnv,
+        componentName
+      );
+    });
+    await trashItem(path);
+  } catch (e) {
+    console.warn(`could not migrate secrets for env '${oldEnv}': ${e}`);
+  }
+};

package/src/apps/cli/verify/migration/oldGitlabCi.ts

@@ -0,0 +1,48 @@
+import type { BuildConfig } from "@catladder/pipeline";
+
+export type OldGitlabCiFile = {
+  variables: {
+    CUSTOMER_NAME: string;
+    APP_NAME: string;
+    COMPONENT_NAME?: string;
+    APP_DIR?: string;
+    CLUSTER_NAME?: string;
+    STAGING_ENABLED?: string;
+  };
+  include: {
+    project: string;
+    ref: string;
+    file: string;
+  }[];
+};
+
+export const isOldInclude = (gitlabCi: OldGitlabCiFile) => {
+  return gitlabCi.include[0]?.project === "catladder/gitlab-ci";
+};
+
+export const detectBuildConfig = (
+  gitlabCi: OldGitlabCiFile
+): BuildConfig["type"] | "monorepo" => {
+  if (!isOldInclude(gitlabCi)) {
+    throw new Error("unsupported gitlab-ci file");
+  }
+
+  const firstInclude = gitlabCi.include[0];
+
+  if (firstInclude.file === "monorepo.yml") return "monorepo";
+
+  if (firstInclude.file === "node-kubernetes.yml") {
+    return "node";
+  }
+
+  if (firstInclude.file === "static-js-kubernetes.yml") {
+    return "node-static";
+  }
+  if (firstInclude.file === "meteor-kubernetes.yml") {
+    return "meteor";
+  }
+
+  if (firstInclude.file === "rails-kubernetes.yml") {
+    return "rails";
+  }
+};

package/src/catenv.ts

@@ -0,0 +1,10 @@
+import catenv from "./apps/catenv/catenv";
+
+import { parseChoice } from "./config/parseChoice";
+
+const choice = process.argv[2] ? parseChoice(process.argv[2]) : null;
+
+catenv(choice).then(() => {
+  // we have to exit manually, because we have some file watches
+  process.exit();
+});

package/src/cli.ts

@@ -0,0 +1,3 @@
+import cli from "./apps/cli/cli";
+
+cli();

package/src/config/constants.ts

@@ -1,5 +1,2 @@
 export const GRAFANA_PROXY_LOCAL_PORT = 9090;
 export const GRAFANA_PROXY_TARGET_PORT = 9090;
-
-export const GOOGLE_PROJECT = "skynet-164509";
-export const GOOGLE_CLOUD_SQL_PASS_PATH = "cloud-sql-proxy-credentials";

package/src/config/getProjectConfig.ts

@@ -0,0 +1,200 @@
+import type { Config, EnvironmentEnvVars } from "@catladder/pipeline";
+import {
+  readConfigSync,
+  getAllEnvs,
+  getEnvironment as _getEnvironment,
+  createContext,
+  getSecretVarName,
+} from "@catladder/pipeline";
+
+import type { CommandInstance } from "vorpal";
+import { getAllVariables, getVariableValueByRawName } from "../utils/gitlab";
+
+import { getGitRoot } from "../utils/projects";
+import { readYaml } from "../utils/files";
+import { watch } from "fs";
+
+export { parseChoice } from "./parseChoice";
+
+let currentConfig: Config | null = null;
+
+// reload the config on change
+const reloadConfigAndObserve = async () => {
+  const gitRoot = await getGitRoot();
+  const result = readConfigSync(gitRoot);
+  if (!result) {
+    // can't do anything, there is no config
+    return;
+  }
+  const { config, path } = result;
+  const watcher = watch(path, () => {
+    watcher.close();
+    reloadConfigAndObserve();
+  });
+  currentConfig = config;
+};
+
+export const getProjectConfig = async () => {
+  if (!currentConfig) {
+    // initially
+    await reloadConfigAndObserve();
+  }
+  return currentConfig as Config;
+};
+
+export const getGitlabCiFilePath = async () => {
+  const gitRoot = await getGitRoot();
+  return gitRoot + "/.gitlab-ci.yml";
+};
+export const getGitlabCi = async <T = Record<string, any>>() => {
+  try {
+    return readYaml(await getGitlabCiFilePath()) as Promise<T>;
+  } catch (e) {
+    // ignore
+    return null;
+  }
+};
+
+export const getProjectComponents = async () => {
+  const config = await getProjectConfig();
+  if (!config) return [];
+  return Object.keys(config.components);
+};
+
+export const getPipelineContextByChoice = async (
+  env: string,
+  componentName: string
+) => {
+  const config = await getProjectConfig();
+  return await createContext(config, componentName, env);
+};
+export const getAllComponentsWithAllEnvsFlat = async (): Promise<
+  Array<{ env: string; componentName: string }>
+> => {
+  const config = await getProjectConfig();
+  if (!config) {
+    return [];
+  }
+  return Object.keys(config.components).flatMap((componentName) =>
+    getAllEnvs(config, componentName).map((env) => ({ env, componentName }))
+  );
+};
+
+export const getAllComponentsWithAllEnvsHierarchical = async (): Promise<{
+  [componentName: string]: string[];
+}> => {
+  const config = await getProjectConfig();
+  if (!config) {
+    return {};
+  }
+
+  return Object.fromEntries(
+    Object.keys(config.components).map((componentName) => [
+      componentName,
+      getAllEnvs(config, componentName),
+    ])
+  );
+};
+
+export const getAllPipelineContexts = async () => {
+  return Promise.all(
+    (await getAllComponentsWithAllEnvsFlat())
+      .filter((c) => c.env !== "local")
+      .map(({ env, componentName }) =>
+        getPipelineContextByChoice(env, componentName)
+      )
+  );
+};
+
+export const getEnvironment = async (env: string, componentName: string) => {
+  const config = await getProjectConfig();
+
+  return _getEnvironment(config, componentName, env);
+};
+
+export const getGitlabVar = async (
+  vorpal: CommandInstance,
+  env: string,
+  componentName: string,
+  variableName: string
+) => {
+  const rawVariableName = getSecretVarName(env, componentName, variableName);
+  return await getVariableValueByRawName(vorpal, rawVariableName);
+};
+
+const resolveSecrets = async (
+  vorpal: CommandInstance | null,
+  varSets: EnvironmentEnvVars[]
+): Promise<Record<string, string>> => {
+  const allVariablesInGitlab = await getAllVariables(vorpal);
+
+  return Object.fromEntries(
+    varSets.flatMap((set) =>
+      Object.entries(set.envVars)
+        .map(([key, value]) => {
+          const secretKey = set.secretEnvVarKeys.find((k) => k.key === key);
+
+          if (secretKey) {
+            if (secretKey.hidden) {
+              return null;
+            }
+            for (const variable of allVariablesInGitlab) {
+              value = value.replace(
+                new RegExp("\\$" + variable.key, "g"),
+                variable.value
+              );
+            }
+            return [key, value];
+          }
+          return [key, value];
+        })
+        .filter(Boolean)
+    )
+  );
+};
+
+export const getEnvVarsResolved = async (
+  vorpal: CommandInstance | null,
+  env: string,
+  componentName: string | null
+) => {
+  if (!componentName) {
+    return {};
+  }
+  try {
+    const envionment = await getEnvironment(env, componentName);
+
+    // in the pipeline the secrets alreadyy exists  and bash will expand them
+    // but here we need to manually load them
+    return resolveSecrets(vorpal, [
+      {
+        envVars: envionment.envVars,
+        secretEnvVarKeys: envionment.secretEnvVarKeys,
+      },
+    ]);
+  } catch (e) {
+    // env is disabled
+    return {};
+  }
+};
+
+/**
+ *
+ * is used to get job only vars that should also be editable locally with catladder.
+ */
+export const getJobOnlyEnvVarsResolved = async (
+  vorpal: CommandInstance,
+  env: string,
+  componentName: string
+) => {
+  try {
+    const envionment = await getEnvironment(env, componentName);
+    return resolveSecrets(vorpal, [
+      envionment.jobOnlyVars.build,
+      envionment.jobOnlyVars.deploy,
+    ]);
+  } catch (e) {
+    // env is disabled
+    return {};
+  }
+};

package/src/config/parseChoice.ts

@@ -0,0 +1,4 @@
+export const parseChoice = (envComponent: string) => {
+  const [env, componentName] = envComponent.split(":").map((x) => x || null);
+  return { env: env as string, componentName };
+};

package/src/gcloud/artifactsRegistry.ts

@@ -0,0 +1,16 @@
+import type { Gcloudregion } from "@catladder/pipeline";
+import { exec } from "child-process-promise";
+
+export const upsertGcloudArtifactsRegistry = async (config: {
+  projectId: string;
+  region: Gcloudregion;
+}) => {
+  try {
+    await exec(
+      `gcloud artifacts repositories create catladder-deploy --project="${config.projectId}" --repository-format=docker --location=${config.region}`
+    );
+  } catch (e) {
+    // probably already exists
+    //
+  }
+};

package/src/gcloud/cloudRun/getCloudRunDomainSuffix.ts

@@ -0,0 +1,35 @@
+import type { DeployConfigCloudRun } from "@catladder/pipeline";
+import { exec } from "child-process-promise";
+
+const getSuffixFromService = (service: any) => {
+  const fullUrl = service.status.url;
+  const name = service.metadata.name;
+
+  return fullUrl.replace("https://" + name + "-", "");
+};
+export const getCloudRunDomainSuffix = async (config: DeployConfigCloudRun) => {
+  /* google cloud run domains are partially predictable, they have a fixed suffix that depends on the region and project, but we don't know it beforehand
+
+  So what we do is either:
+   - get one service and extract its suffix
+   - create a dummy project, extract the suffix and save that as a gitlab variable
+  
+  
+  */
+
+  const dummyServiceName = "cl-dummy-service-delete-me";
+
+  const existingServices = await exec(
+    `gcloud run services list --format=json  --project="${config.projectId}"  --region=${config.region} --limit=1`
+  ).then((r) => JSON.parse(r.stdout));
+
+  if (existingServices.length > 0) {
+    return getSuffixFromService(existingServices[0]);
+  }
+
+  const result = await exec(
+    `gcloud run deploy ${dummyServiceName} --region=${config.region} --allow-unauthenticated --project ${config.projectId} --image=us-docker.pkg.dev/cloudrun/container/hello --format=json`
+  ).then((r) => JSON.parse(r.stdout));
+
+  return await getSuffixFromService(result);
+};

package/src/gcloud/cloudRun/openCloudRunDashboards.ts

@@ -0,0 +1,22 @@
+import type { Context } from "@catladder/pipeline";
+import { isOfDeployType } from "@catladder/pipeline";
+import type { CommandInstance } from "vorpal";
+import { openGoogleCloudDashboard } from "../openDashboard";
+
+export const openGoogleCloudRunDashboard = async (
+  instance: CommandInstance,
+  context: Context
+) => {
+  if (!isOfDeployType(context.componentConfig.deploy, "google-cloudrun")) {
+    throw new Error("deploy type is not google-cloudrun ");
+  }
+  const { fullName } = context.environment;
+  const { region, projectId } = context.componentConfig.deploy;
+  await openGoogleCloudDashboard(
+    instance,
+    `run/detail/${region}/${fullName}/metrics`,
+    {
+      project: projectId,
+    }
+  );
+};