@catladder/cli 0.0.0-test7 → 0.0.0-waitfor-deploy-e64a33ba

package/src/apps/cli/config/writeConfig.ts

@@ -0,0 +1,71 @@
+import { Config } from "@catladder/pipeline";
+import { spawn, exec } from "child-process-promise";
+import { writeFile } from "fs-extra";
+import { dump } from "js-yaml";
+import { format } from "prettier";
+import { CommandInstance } from "vorpal";
+import { getGitRoot } from "../../../utils/projects";
+
+export const writeConfig = async (
+  vorpal: CommandInstance,
+  config: Config,
+  options?: {
+    endComment?: string;
+  }
+) => {
+  const gitRoot = await getGitRoot();
+  const TS = "typescript (recommended)";
+  const { configType } = await vorpal.prompt({
+    type: "list",
+    name: "configType",
+    choices: [TS, "yaml"],
+    message: "In which format do you want the config? 🤔",
+  });
+  vorpal.log("");
+  if (configType === TS) {
+    const file = gitRoot + "/catladder.ts";
+    const content = format(
+      `
+      import type { Config } from "@catladder/pipeline";
+      
+      const config: Config = ${JSON.stringify(config)};
+
+      export default config;
+
+     
+      ${options?.endComment ? `/*${options.endComment}*/` : ""}
+      
+      `,
+      {
+        parser: "babel",
+      }
+    );
+
+    await writeFile(file, content, {
+      encoding: "utf-8",
+    });
+    vorpal.log("adding type @catladder/pipeline....");
+    await spawn("yarn add --non-interactive @catladder/pipeline -DW", {
+      shell: true,
+    });
+    await exec("git add " + file);
+  } else {
+    const file = gitRoot + "/catladder.yml";
+    const content = dump(config);
+
+    await writeFile(
+      file,
+      content +
+        "\n\n" +
+        (options.endComment
+          ? "# " + options.endComment.split("\n").join("\n# ")
+          : ""),
+      {
+        encoding: "utf-8",
+      }
+    );
+    await exec("git add " + file);
+  }
+  vorpal.log("done!");
+  vorpal.log("");
+};

package/src/apps/cli/verify/index.ts

@@ -0,0 +1,30 @@
+import Vorpal from "vorpal";
+import {
+  getGitlabCi,
+  getProjectConfig,
+} from "../../../config/getProjectConfig";
+import { hasGitlabToken, setupGitlabToken } from "../../../utils/gitlab";
+import { isV2, migrateV2 } from "./migration/fromv2";
+
+export const verify = async (vorpal: Vorpal) => {
+  // check if has all settings
+
+  if (!(await hasGitlabToken())) {
+    vorpal.command("setup-gitlab-token").action(async function () {
+      await setupGitlabToken(this);
+    });
+    vorpal.exec("setup-gitlab-token");
+  }
+
+  const projectConfig = await getProjectConfig();
+  try {
+    const gitlabCi = getGitlabCi();
+
+    if (gitlabCi && !projectConfig && (await isV2())) {
+      vorpal.log("no project config, needs migration");
+      await migrateV2(vorpal);
+    }
+  } catch (e) {
+    // no gitroot, ignore
+  }
+};

package/src/apps/cli/verify/migration/fromv2.ts

@@ -0,0 +1,264 @@
+import {
+  BuildConfig,
+  ComponentConfig,
+  Config,
+  DeployConfigKubernetes,
+  Env,
+} from "@catladder/pipeline";
+import {
+  existsSync,
+  lstatSync,
+  readdirSync,
+  readFile,
+  writeFile,
+} from "fs-extra";
+import { isEmpty } from "lodash";
+import { join } from "path";
+import Vorpal from "vorpal";
+import {
+  getGitlabCi,
+  getGitlabCiFilePath,
+} from "../../../../config/getProjectConfig";
+import { readYaml } from "../../../../utils/files";
+import { syncBitwarden } from "../../../../utils/passwordstore";
+import { getGitRoot } from "../../../../utils/projects";
+import { writeConfig } from "../../config/writeConfig";
+import { migrateSecrets } from "./migrateSecrets";
+import {
+  detectBuildConfig,
+  isOldInclude,
+  OldGitlabCiFile,
+} from "./oldGitlabCi";
+export const LEGACY_ENVS = [
+  "dev-local",
+  "dev",
+  "review",
+  "stage",
+  "prod",
+] as const;
+
+const arrayToRecord = (arr: { name: string }[]): Record<string, any> => {
+  if (!arr) return undefined;
+  return Object.fromEntries(arr.map(({ name, ...rest }) => [name, rest]));
+};
+
+const transformVars = (rawVars: {
+  public: Record<string, any>;
+  secret: Record<string, any>;
+  fromCommponents: any;
+}) => {
+  if (rawVars.fromCommponents) {
+    console.warn("cant transform legacy fromCommponents");
+  }
+
+  return {
+    public: rawVars.public,
+    secret: rawVars.secret ? Object.keys(rawVars.secret) : undefined,
+  };
+};
+
+const getLegacyMonorepoSubCiFiles = (dir: string) => {
+  return readdirSync(dir)
+    .filter((file) => lstatSync(file).isDirectory())
+    .map((dir) => ({
+      dir,
+      ci: join(dir, ".gitlab-ci.yml"),
+    }))
+    .filter(({ ci }) => existsSync(ci));
+};
+
+const transformValues = (
+  valuesIn: Record<string, any>
+): DeployConfigKubernetes["values"] => {
+  if (isEmpty(valuesIn)) {
+    return undefined;
+  }
+  delete valuesIn?.application?.host;
+  delete valuesIn?.application?.command;
+
+  return {
+    ...valuesIn,
+    jobs: arrayToRecord(valuesIn?.jobs),
+    cronjobs: arrayToRecord(valuesIn?.cronjobs),
+  };
+};
+
+export const isV2 = async () => {
+  const gitlabCi = await getGitlabCi<OldGitlabCiFile>();
+  return isOldInclude(gitlabCi);
+};
+export const migrateV2 = async (vorpal: Vorpal) => {
+  const gitlabCi = await getGitlabCi<OldGitlabCiFile>();
+  const gitRoot = await getGitRoot();
+
+  const {
+    CUSTOMER_NAME,
+    APP_NAME,
+    COMPONENT_NAME = "web",
+    APP_DIR = ".",
+    STAGING_ENABLED,
+  } = gitlabCi.variables;
+
+  vorpal
+    .command("migrate")
+
+    .action(async function () {
+      this.log("");
+      this.log("⚠️ this project uses legacy catladder (v2)");
+      this.log("");
+      this.log(
+        "😼 I can migrate the project for you. This contains the following steps:"
+      );
+      this.log("");
+      this.log(
+        " - migrate the config from values-*.yml to catladder config file"
+      );
+      this.log(
+        " - migrate the secrets from bitwarden to gitlab (this will trash the entries in bitwarden)"
+      );
+      this.log("");
+      this.log(
+        "☝ make sure that you checked in your current state in case something goes wrong."
+      );
+      this.log(
+        "☝ secrets in bitwarden are deleted, but can be restored within 30 days."
+      );
+      this.log("");
+      const { shouldContinue } = await this.prompt({
+        default: true,
+        message: "Migrate project now? 🤔",
+        name: "shouldContinue",
+        type: "confirm",
+      });
+
+      if (shouldContinue) {
+        vorpal.log("");
+        vorpal.log("");
+        vorpal.log("💪😼 ok, let's go...");
+        vorpal.log("");
+        vorpal.log("");
+
+        const createComponent = async (
+          dir: string,
+          ciFile: OldGitlabCiFile
+        ): Promise<ComponentConfig> => {
+          const { env, ...baseValues } =
+            (await readYaml(dir + "/values.yml")) ?? {};
+
+          const startCommand = Array.isArray(baseValues?.application?.command)
+            ? baseValues?.application?.command.join(" ")
+            : baseValues?.application?.command;
+          return {
+            vars: env ? transformVars(env) : undefined,
+            dir: dir,
+            build: {
+              type: detectBuildConfig(ciFile),
+              startCommand: startCommand,
+            } as BuildConfig,
+            deploy: {
+              type: "kubernetes",
+              values: transformValues(baseValues),
+              cluster: {
+                type: "gcloud",
+                name: "ch-production",
+                projectId: "skynet-swiss",
+                region: "europe-west6-a",
+                domainCanonical: "panter.swiss",
+              },
+            },
+            env: await LEGACY_ENVS.reduce<Promise<Env>>(
+              async (acc, envName) => {
+                if (envName === "stage" && !STAGING_ENABLED) {
+                  return {
+                    ...(await acc),
+                    [envName]: false,
+                  };
+                }
+                const newEnvName = envName === "dev-local" ? "local" : envName;
+
+                const envValues =
+                  (await readYaml(dir + `/values-${envName}.yml`)) ?? {};
+                const { env, ...rest } = envValues;
+                const host = rest?.application?.host;
+                const values = transformValues(rest);
+                return {
+                  ...(await acc),
+                  [newEnvName]: {
+                    host: host,
+                    vars: env ? transformVars(env) : undefined,
+                    deploy: values
+                      ? {
+                          values: values,
+                        }
+                      : undefined,
+                  },
+                };
+              },
+              Promise.resolve({})
+            ),
+          };
+        };
+        let components: Record<string, ComponentConfig>;
+        if (detectBuildConfig(gitlabCi) === "monorepo") {
+          components = await getLegacyMonorepoSubCiFiles(gitRoot).reduce<
+            Promise<Record<string, ComponentConfig>>
+          >(async (acc, el) => {
+            return {
+              ...(await acc),
+              [el.dir]: await createComponent(el.dir, await readYaml(el.ci)),
+            };
+          }, Promise.resolve({}));
+        } else {
+          components = {
+            [COMPONENT_NAME]: await createComponent(APP_DIR, gitlabCi),
+          };
+        }
+        // we only have one component
+        const config: Config = {
+          customerName: CUSTOMER_NAME,
+          appName: APP_NAME,
+          components,
+        };
+
+        const comment =
+          "Old migrated config:\n\n" +
+          (await readFile(await getGitlabCiFilePath(), {
+            encoding: "utf-8",
+          }));
+
+        this.log("-------------------");
+        this.log("migrate config");
+        await writeConfig(this, config, {
+          endComment: comment,
+        });
+        this.log("write gitlab-ci.yml");
+        await writeFile(
+          await getGitlabCiFilePath(),
+          "include: https://git.panter.ch/api/v4/projects/catladder%2Fcatladder/packages/generic/ci-includes/main/gitlab-ci.yml",
+          { encoding: "utf-8" }
+        );
+
+        this.log("-------------------");
+        this.log("migrate secrets");
+        await syncBitwarden();
+        for (const env of LEGACY_ENVS) {
+          if (env === "stage" && !STAGING_ENABLED) return;
+          await migrateSecrets(this, config, env);
+        }
+
+        this.log("-------------------");
+
+        this.log("done!");
+        this.log("");
+        this.log("You can remove the values*.yml files now.");
+
+        this.log("-------------------");
+      } else {
+        this.log(
+          "☝ if you want to use catladder in legacy mode, install @panter/catladder globally and invoke catladder-legacy"
+        );
+      }
+    });
+
+  vorpal.exec("migrate");
+};

package/src/apps/cli/verify/migration/migrateSecrets.ts

@@ -0,0 +1,37 @@
+import { Config, getEnvironment } from "@catladder/pipeline";
+import { load } from "js-yaml";
+import { pick } from "lodash";
+import { CommandInstance } from "vorpal";
+import { upsertAllVariables } from "../../../../utils/gitlab";
+import { readPass, trashItem } from "../../../../utils/passwordstore";
+import { LEGACY_ENVS } from "./fromv2";
+
+const getPassPath = (newConfig: Config, env: string) => {
+  return `${newConfig.customerName}/${newConfig.appName}/${env}/secrets.yml`;
+};
+
+export const migrateSecrets = async (
+  vorpal: CommandInstance,
+  newConfig: Config,
+  oldEnv: typeof LEGACY_ENVS[number]
+) => {
+  const newEnv = oldEnv === "dev-local" ? "local" : oldEnv;
+  const path = getPassPath(newConfig, oldEnv);
+  try {
+    const yamlstring = await readPass(path);
+    const secrets = load(yamlstring);
+
+    Object.keys(newConfig.components).forEach(async (componentName) => {
+      const environment = getEnvironment(newConfig, componentName, newEnv);
+      await upsertAllVariables(
+        vorpal,
+        pick(secrets, environment.secretEnvVarKeys),
+        newEnv,
+        componentName
+      );
+    });
+    await trashItem(path);
+  } catch (e) {
+    console.warn(`could not migrate secrets for env '${oldEnv}': ${e}`);
+  }
+};

package/src/apps/cli/verify/migration/oldGitlabCi.ts

@@ -0,0 +1,48 @@
+import { BuildConfig } from "@catladder/pipeline";
+
+export type OldGitlabCiFile = {
+  variables: {
+    CUSTOMER_NAME: string;
+    APP_NAME: string;
+    COMPONENT_NAME?: string;
+    APP_DIR?: string;
+    CLUSTER_NAME?: string;
+    STAGING_ENABLED?: string;
+  };
+  include: {
+    project: string;
+    ref: string;
+    file: string;
+  }[];
+};
+
+export const isOldInclude = (gitlabCi: OldGitlabCiFile) => {
+  return gitlabCi.include[0]?.project === "catladder/gitlab-ci";
+};
+
+export const detectBuildConfig = (
+  gitlabCi: OldGitlabCiFile
+): BuildConfig["type"] | "monorepo" => {
+  if (!isOldInclude(gitlabCi)) {
+    throw new Error("unsupported gitlab-ci file");
+  }
+
+  const firstInclude = gitlabCi.include[0];
+
+  if (firstInclude.file === "monorepo.yml") return "monorepo";
+
+  if (firstInclude.file === "node-kubernetes.yml") {
+    return "node";
+  }
+
+  if (firstInclude.file === "static-js-kubernetes.yml") {
+    return "node-static";
+  }
+  if (firstInclude.file === "meteor-kubernetes.yml") {
+    return "meteor";
+  }
+
+  if (firstInclude.file === "rails-kubernetes.yml") {
+    throw new Error("rails is not yet implemented");
+  }
+};

package/src/catenv.ts

@@ -0,0 +1,11 @@
+import catenv from "./apps/catenv/catenv";
+import { $, argv } from "zx";
+import { parseChoice } from "./config/getProjectConfig";
+
+const choice = argv._[0] ? parseChoice(argv._[0]) : null;
+
+$.verbose = false;
+catenv(choice).then(() => {
+  // we have to exit manually, because we have some file watches
+  process.exit();
+});

package/src/cli.ts

@@ -0,0 +1,9 @@
+import updateNotifier from "update-notifier";
+import cli from "./apps/cli/cli";
+import packageInfos from "./packageInfos";
+import { $ } from "zx";
+$.verbose = false;
+updateNotifier({
+  pkg: packageInfos,
+}).notify();
+cli();

package/src/config/constants.ts

@@ -1,5 +1,4 @@
 export const GRAFANA_PROXY_LOCAL_PORT = 9090;
 export const GRAFANA_PROXY_TARGET_PORT = 9090;
 
-export const GOOGLE_PROJECT = "skynet-164509";
 export const GOOGLE_CLOUD_SQL_PASS_PATH = "cloud-sql-proxy-credentials";

package/src/config/getProjectConfig.ts

@@ -0,0 +1,160 @@
+import {
+  readConfigSync,
+  getAllEnvs,
+  getEnvironment as _getEnvironment,
+  createContext,
+  getSecretVarName,
+  Config,
+} from "@catladder/pipeline";
+
+import { CommandInstance } from "vorpal";
+import { getAllVariables, getVariableValueByRawName } from "../utils/gitlab";
+
+import { getGitRoot } from "../utils/projects";
+import { readYaml } from "../utils/files";
+import { watch } from "fs";
+
+let currentConfig: Config = null;
+
+// reload the config on change
+const reloadConfigAndObserve = async () => {
+  const gitRoot = await getGitRoot();
+  const result = readConfigSync(gitRoot);
+  if (!result) {
+    // can't do anything, there is no config
+    return;
+  }
+  const { config, path } = result;
+  const watcher = watch(path, () => {
+    watcher.close();
+    reloadConfigAndObserve();
+  });
+  currentConfig = config;
+};
+
+export const getProjectConfig = async () => {
+  if (!currentConfig) {
+    // initially
+    await reloadConfigAndObserve();
+  }
+  return currentConfig;
+};
+
+export const getGitlabCiFilePath = async () => {
+  const gitRoot = await getGitRoot();
+  return gitRoot + "/.gitlab-ci.yml";
+};
+export const getGitlabCi = async <T = Record<string, any>>() => {
+  try {
+    return readYaml(await getGitlabCiFilePath()) as Promise<T>;
+  } catch (e) {
+    // ignore
+    return null;
+  }
+};
+
+export const getProjectComponents = async () => {
+  const config = await getProjectConfig();
+  if (!config) return [];
+  return Object.keys(config.components);
+};
+
+export const parseChoice = (envComponent: string) => {
+  const [env, componentName] = envComponent.split(":").map((x) => x || null);
+  return { env, componentName };
+};
+
+export const getPipelineContextByChoice = async (
+  env: string,
+  componentName: string
+) => {
+  const config = await getProjectConfig();
+  return createContext(config, componentName, env);
+};
+export const getAllComponentsWithAllEnvsFlat = async (): Promise<
+  Array<{ env: string; componentName: string }>
+> => {
+  const config = await getProjectConfig();
+  if (!config) {
+    return [];
+  }
+  return Object.keys(config.components).flatMap((componentName) =>
+    getAllEnvs(config, componentName).map((env) => ({ env, componentName }))
+  );
+};
+
+export const getAllComponentsWithAllEnvsHierarchical = async (): Promise<{
+  [componentName: string]: string[];
+}> => {
+  const config = await getProjectConfig();
+  if (!config) {
+    return {};
+  }
+
+  return Object.fromEntries(
+    Object.keys(config.components).map((componentName) => [
+      componentName,
+      getAllEnvs(config, componentName),
+    ])
+  );
+};
+
+export const getAllPipelineContexts = async () => {
+  return Promise.all(
+    (await getAllComponentsWithAllEnvsFlat())
+      .filter((c) => c.env !== "local")
+      .map(({ env, componentName }) =>
+        getPipelineContextByChoice(env, componentName)
+      )
+  );
+};
+
+export const getEnvironment = async (env: string, componentName: string) => {
+  const config = await getProjectConfig();
+
+  return _getEnvironment(config, componentName, env);
+};
+
+export const getGitlabVar = async (
+  vorpal: CommandInstance,
+  env: string,
+  componentName: string,
+  variableName: string
+) => {
+  const rawVariableName = getSecretVarName(env, componentName, variableName);
+  return await getVariableValueByRawName(vorpal, rawVariableName);
+};
+
+const resolveSecrets = async (
+  vorpal: CommandInstance,
+  allEnvVars: Record<string, string>
+) => {
+  const allVariablesInGitlab = await getAllVariables(vorpal);
+
+  return Object.fromEntries(
+    Object.entries(allEnvVars).map(([key, value]) => {
+      const containsSecret = String(value)?.includes?.("$CL_");
+      if (containsSecret) {
+        for (const variable of allVariablesInGitlab) {
+          value = value.replace(
+            new RegExp("\\$" + variable.key, "g"),
+            variable.value
+          );
+        }
+        return [key, value];
+      }
+      return [key, value];
+    })
+  );
+};
+
+export const getEnvVars = async (
+  vorpal: CommandInstance,
+  env: string,
+  componentName: string
+) => {
+  const envionment = await getEnvironment(env, componentName);
+  // in the pipeline the secrets alreadyy exists  and bash will expand them
+  // but here we need to manually load them
+  return resolveSecrets(vorpal, envionment.envVars);
+};

package/src/k8sApi/index.ts

@@ -5,13 +5,29 @@ import {
   BatchV1beta1Api,
 } from "@kubernetes/client-node";
 
-const kc = new KubeConfig();
+const getKubeConfig = () => {
+  const kc = new KubeConfig();
+  kc.loadFromDefault();
+  return kc;
+};
+/**
+ * get kubernetes client. avoid reusing the instance when context get changed
+ * @returns kuberenetes client
+ */
+export const getk8sApi = () => {
+  const kc = getKubeConfig();
 
-kc.loadFromDefault();
+  return kc.makeApiClient(CoreV1Api);
+};
 
-const k8sApi = kc.makeApiClient(CoreV1Api);
+export const getk8sApiBatch = () => {
+  const kc = getKubeConfig();
 
-export const k8sApiBatch = kc.makeApiClient(BatchV1Api);
-export const k8sApiBatchBeta = kc.makeApiClient(BatchV1beta1Api);
+  return kc.makeApiClient(BatchV1Api);
+};
 
-export default k8sApi;
+export const getk8sApiBatchBeta = () => {
+  const kc = getKubeConfig();
+
+  return kc.makeApiClient(BatchV1beta1Api);
+};

package/src/types/types.ts

@@ -1,5 +1,3 @@
-export type Env = "dev-local" | "prod" | "dev" | "stage" | "review";
-
 export type KubernetesSecretName = string;
 export interface ISecrets {
   [key: string]: KubernetesSecretName;