@catladder/cli 0.0.1-bla → 1.0.3

package/src/apps/{shell → cli}/commands/project/index.ts

@@ -22,15 +22,16 @@ import commandPauseProject from "./commandPauseProject";
 import commandOpenGrafana from "./commandOpenGrafana";
 import commandPortForward from "./commandPortForward";
 import commandTriggerCronjob from "./commandTriggerCronjob";
-import commandVariables from "./commandVariables";
+
 import commandOpenGrafanaPod from "./commandOpenGrafanaPod";
+import commandReloadConfig from "./commandReloadConfig";
 
-export default (vorpal: Vorpal) => {
+export default async (vorpal: Vorpal) => {
+  commandReloadConfig(vorpal);
   commandInitProject(vorpal);
   commandInitGitlab(vorpal);
   commandEnvVars(vorpal);
 
-  commandVariables(vorpal);
   commandNamespace(vorpal);
   commandListPods(vorpal);
   commandPauseProject(vorpal);

package/src/apps/cli/commands/project/utils/autocompletions.ts

@@ -0,0 +1,26 @@
+import { getAllEnvs, getAllEnvsInAllComponents } from "@catladder/pipeline";
+import {
+  getAllComponentsWithAllEnvs,
+  getProjectConfig,
+} from "../../../../../config/getProjectConfig";
+
+export const allEnvs = async () => {
+  const config = await getProjectConfig();
+  if (!config) {
+    return [];
+  }
+  return getAllEnvsInAllComponents(config);
+};
+
+export const envAndComponents = async () => {
+  const allEnvAndcomponents = await getAllComponentsWithAllEnvs();
+
+  return allEnvAndcomponents.reduce<string[]>(
+    (acc, { env, componentName }) => [...acc, env + ":" + componentName],
+    []
+  );
+};
+
+export const allEnvsAndAllComponents = async () => {
+  return [...(await allEnvs()), ...(await envAndComponents())];
+};

package/src/apps/cli/commands/project/utils/ensureCluster.ts

@@ -0,0 +1,37 @@
+import { isOfDeployType } from "@catladder/pipeline";
+import {
+  getPipelineContextByChoice,
+  parseChoice,
+} from "../../../../../config/getProjectConfig";
+import {
+  connectToCluster,
+  getCurrentConnectedClusterName,
+} from "../../../../../utils/cluster";
+export default async function (envComponent: string) {
+  const { env, componentName } = parseChoice(envComponent);
+  const context = await getPipelineContextByChoice(env, componentName);
+  if (!isOfDeployType(context.componentConfig.deploy, "kubernetes")) {
+    throw new Error("can't ensure cluster for non-kubernetes deployments");
+  }
+  const cluster = context.componentConfig.deploy.cluster || "production";
+  const connectedClusterName = await getCurrentConnectedClusterName();
+
+  if (cluster !== connectedClusterName) {
+    this.log(
+      `you are currently connected to cluster '${connectedClusterName}'`
+    );
+    this.log(`but the project requires cluster '${cluster}'`);
+    const { shouldContinue } = await this.prompt({
+      type: "confirm",
+      name: "shouldContinue",
+      default: true,
+      message: `Do you want to connect to '${cluster}'?`,
+    });
+    if (!shouldContinue) {
+      this.log("abort");
+    } else {
+      await connectToCluster(cluster);
+      this.log(`connected to cluster '${cluster}'`);
+    }
+  }
+}

package/src/apps/{shell → cli}/commands/project/utils/ensureNamespace.ts

@@ -1,10 +1,9 @@
 import { V1Namespace, V1ObjectMeta } from "@kubernetes/client-node";
 import k8sApi from "../../../../../k8sApi";
-import { Env } from "../../../../../types/types";
 import { getProjectNamespace } from "../../../../../utils/projects/index";
 
-export default async function (env: Env) {
-  const namespace = await getProjectNamespace(env);
+export default async function (envComponent: string) {
+  const namespace = await getProjectNamespace(envComponent);
   try {
     await k8sApi.readNamespace(namespace);
   } catch (e) {

package/src/apps/cli/commands/project/utils/showProjectBanner.ts

@@ -0,0 +1,11 @@
+import Vorpal from "vorpal";
+import { getProjectConfig } from "../../../../../config/getProjectConfig";
+
+export const showProjectBanner = async (vorpal: Vorpal) => {
+  const projectConfig = await getProjectConfig();
+  if (projectConfig) {
+    vorpal.log("project: " + projectConfig.appName);
+    vorpal.log("customer: " + projectConfig.customerName);
+    vorpal.log("");
+  }
+};

package/src/apps/{shell → cli}/commands/theStuffThatReallyMatters/index.ts

@@ -2,31 +2,31 @@ import { spawn } from "child-process-promise";
 import fetch from "node-fetch";
 import Vorpal from "vorpal";
 
-export default (vorpal: Vorpal) => {
-  vorpal.command("dadjoke", "something for jonas.").action(async function() {
+export default async (vorpal: Vorpal) => {
+  vorpal.command("dadjoke", "something for jonas.").action(async function () {
     const result = await fetch("https://icanhazdadjoke.com/", {
       headers: {
-        Accept: "text/plain"
-      }
+        Accept: "text/plain",
+      },
     });
     const text = await result.text();
     this.log("");
     this.log(text);
     this.log("");
   });
-  vorpal.command("pantobot", "the allknowing oracle").action(async function() {
+  vorpal.command("pantobot", "the allknowing oracle").action(async function () {
     const { message } = await this.prompt({
       type: "string",
       name: "message",
 
-      message: "Your question, unworthy?: "
+      message: "Your question, unworthy?: ",
     });
     const result = await fetch(
       `https://paas.panter.cloud/api/pantobot/${message}`,
       {
         headers: {
-          Accept: "text/plain"
-        }
+          Accept: "text/plain",
+        },
       }
     );
     const text = await result.text();
@@ -37,15 +37,15 @@ export default (vorpal: Vorpal) => {
   let starwarsPromise: any;
   vorpal
     .command("starwars", "Long time ago... in a galaxy far far away...")
-    .action(async function() {
+    .action(async function () {
       starwarsPromise = spawn("telnet", ["towel.blinkenlights.nl"], {
-        stdio: ["pipe", "inherit", "pipe"]
+        stdio: ["pipe", "inherit", "pipe"],
       });
       await starwarsPromise;
     })
     // we need to close it properly, because telnet towel.blinkenlights.nl has no way to cancel and stop itself
     // this is also why we need to only inherit the stdout and not the stdin
-    .cancel(async function() {
+    .cancel(async function () {
       starwarsPromise.childProcess.kill();
     });
 };

package/src/apps/cli/config/writeConfig.ts

@@ -0,0 +1,69 @@
+import { Config } from "@catladder/pipeline";
+import { spawn, exec } from "child-process-promise";
+import { writeFile } from "fs-extra";
+import { dump } from "js-yaml";
+import { format } from "prettier";
+import { CommandInstance } from "vorpal";
+import { getGitRoot } from "../../../utils/projects";
+
+export const writeConfig = async (
+  vorpal: CommandInstance,
+  config: Config,
+  options?: {
+    endComment?: string;
+  }
+) => {
+  const gitRoot = await getGitRoot();
+  const TS = "typescript (recommended)";
+  const { configType } = await vorpal.prompt({
+    type: "list",
+    name: "configType",
+    choices: [TS, "yaml"],
+    message: "In which format do you want the config? 🤔",
+  });
+  vorpal.log("");
+  if (configType === TS) {
+    const content = format(
+      `
+      import type { Config } from "@catladder/pipeline";
+      
+      const config: Config = ${JSON.stringify(config)};
+
+      export default config;
+
+     
+      ${options?.endComment ? `/*${options.endComment}*/` : ""}
+      
+      `,
+      {
+        parser: "babel",
+      }
+    );
+
+    await writeFile(gitRoot + "/catladder.ts", content, {
+      encoding: "utf-8",
+    });
+    vorpal.log("adding type @catladder/pipeline....");
+    await spawn("yarn add @catladder/pipeline -DW", {
+      shell: true,
+    });
+    await exec("git add " + gitRoot + "/catladder.ts");
+  } else {
+    const content = dump(config);
+
+    await writeFile(
+      gitRoot + "/catladder.yml",
+      content +
+        "\n\n" +
+        (options.endComment
+          ? "# " + options.endComment.split("\n").join("\n# ")
+          : ""),
+      {
+        encoding: "utf-8",
+      }
+    );
+    await exec("git add " + gitRoot + "/catladder.ts");
+  }
+  vorpal.log("done!");
+  vorpal.log("");
+};

package/src/apps/cli/verify/index.ts

@@ -0,0 +1,34 @@
+import Vorpal from "vorpal";
+import {
+  getGitlabCi,
+  getProjectConfig,
+} from "../../../config/getProjectConfig";
+import { hasGitlabToken, setupGitlabToken } from "../../../utils/gitlab";
+import { migrateV2 } from "./migration/fromv2";
+
+export const verify = async (vorpal: Vorpal) => {
+  // check if has all settings
+
+  if (!(await hasGitlabToken())) {
+    vorpal.command("setup-gitlab-token").action(async function () {
+      await setupGitlabToken(this);
+    });
+    vorpal.exec("setup-gitlab-token");
+  }
+
+  const projectConfig = await getProjectConfig();
+  try {
+    const gitlabCi = getGitlabCi();
+
+    if (!gitlabCi) {
+      vorpal.log("not initialized");
+    }
+
+    if (!projectConfig) {
+      vorpal.log("no project config, needs migration");
+      await migrateV2(vorpal);
+    }
+  } catch (e) {
+    // no gitroot, ignore
+  }
+};

package/src/apps/cli/verify/migration/fromv2.ts

@@ -0,0 +1,241 @@
+import {
+  BuildConfig,
+  ComponentConfig,
+  Config,
+  DeployConfigKubernetes,
+  Env,
+} from "@catladder/pipeline";
+import {
+  existsSync,
+  lstatSync,
+  readdirSync,
+  readFile,
+  writeFile,
+} from "fs-extra";
+import { isEmpty } from "lodash";
+import { join } from "path";
+import Vorpal from "vorpal";
+import {
+  getGitlabCi,
+  getGitlabCiFilePath,
+} from "../../../../config/getProjectConfig";
+import { readYaml } from "../../../../utils/files";
+import { syncBitwarden } from "../../../../utils/passwordstore";
+import { getGitRoot } from "../../../../utils/projects";
+import { writeConfig } from "../../config/writeConfig";
+import { migrateSecrets } from "./migrateSecrets";
+import { detectBuildConfig, OldGitlabCiFile } from "./oldGitlabCi";
+export const LEGACY_ENVS = [
+  "dev-local",
+  "dev",
+  "review",
+  "stage",
+  "prod",
+] as const;
+
+const arrayToRecord = (arr: { name: string }[]): Record<string, any> => {
+  if (!arr) return undefined;
+  return Object.fromEntries(arr.map(({ name, ...rest }) => [name, rest]));
+};
+
+const transformVars = (rawVars: {
+  public: Record<string, any>;
+  secret: Record<string, any>;
+  fromCommponents: any;
+}) => {
+  if (rawVars.fromCommponents) {
+    console.warn("cant transform legacy fromCommponents");
+  }
+
+  return {
+    public: rawVars.public,
+    secret: rawVars.secret ? Object.keys(rawVars.secret) : undefined,
+  };
+};
+
+const getLegacyMonorepoSubCiFiles = (dir: string) => {
+  return readdirSync(dir)
+    .filter((file) => lstatSync(file).isDirectory())
+    .map((dir) => ({
+      dir,
+      ci: join(dir, ".gitlab-ci.yml"),
+    }))
+    .filter(({ ci }) => existsSync(ci));
+};
+
+const transformValues = (
+  valuesIn: Record<string, any>
+): DeployConfigKubernetes["values"] => {
+  if (isEmpty(valuesIn)) {
+    return undefined;
+  }
+  delete valuesIn?.application?.hostname;
+  delete valuesIn?.application?.command;
+
+  return {
+    ...valuesIn,
+    jobs: arrayToRecord(valuesIn?.jobs),
+    cronjobs: arrayToRecord(valuesIn?.cronjobs),
+  };
+};
+export const migrateV2 = async (vorpal: Vorpal) => {
+  const gitlabCi = await getGitlabCi<OldGitlabCiFile>();
+  const gitRoot = await getGitRoot();
+
+  const {
+    CUSTOMER_NAME,
+    APP_NAME,
+    COMPONENT_NAME,
+    APP_DIR = ".",
+    CLUSTER_NAME,
+    STAGING_ENABLED,
+  } = gitlabCi.variables;
+
+  vorpal
+    .command("migrate")
+
+    .action(async function () {
+      this.log("");
+      this.log("⚠️ this project uses legacy catladder (v2)");
+      this.log("");
+      this.log(
+        "😼 I can migrate the project for you. This contains the following steps:"
+      );
+      this.log("");
+      this.log(
+        " - migrate the config from values-*.yml to catladder config file"
+      );
+      this.log(
+        " - migrate the secrets from bitwarden to gitlab (this will trash the entries in bitwarden)"
+      );
+      this.log("");
+      this.log(
+        "☝ make sure that you checked in your current state in case something goes wrong."
+      );
+      this.log(
+        "☝ secrets in bitwarden are deleted, but can be restored within 30 days."
+      );
+      this.log("");
+      const { shouldContinue } = await this.prompt({
+        default: true,
+        message: "Migrate project now? 🤔",
+        name: "shouldContinue",
+        type: "confirm",
+      });
+      vorpal.log("");
+      vorpal.log("");
+      vorpal.log("💪😼 ok, let's go...");
+      vorpal.log("");
+      vorpal.log("");
+
+      if (shouldContinue) {
+        const createComponent = async (
+          dir: string,
+          ciFile: OldGitlabCiFile
+        ): Promise<ComponentConfig> => {
+          const { env, ...baseValues } =
+            (await readYaml(dir + "/values.yml")) ?? {};
+
+          const startCommand = baseValues?.application?.command?.join(" ");
+          return {
+            vars: env ? transformVars(env) : undefined,
+            dir: dir,
+            build: {
+              type: detectBuildConfig(ciFile),
+              startCommand: startCommand,
+            } as BuildConfig,
+            deploy: {
+              type: "kubernetes",
+              values: transformValues(baseValues),
+              cluster: CLUSTER_NAME,
+            },
+            env: await LEGACY_ENVS.reduce<Promise<Env>>(
+              async (acc, envName) => {
+                if (envName === "stage" && !STAGING_ENABLED) {
+                  return {
+                    ...(await acc),
+                    [envName]: false,
+                  };
+                }
+                const newEnvName = envName === "dev-local" ? "local" : envName;
+
+                const envValues =
+                  (await readYaml(dir + `/values-${envName}.yml`)) ?? {};
+                const { env, ...rest } = envValues;
+                const hostname = rest?.application?.hostname;
+                const values = transformValues(rest);
+                return {
+                  ...(await acc),
+                  [newEnvName]: {
+                    hostname: hostname,
+                    vars: env ? transformVars(env) : undefined,
+                    deploy: values
+                      ? {
+                          values: values,
+                        }
+                      : undefined,
+                  },
+                };
+              },
+              Promise.resolve({})
+            ),
+          };
+        };
+        let components: Record<string, ComponentConfig>;
+        if (detectBuildConfig(gitlabCi) === "monorepo") {
+          components = await getLegacyMonorepoSubCiFiles(gitRoot).reduce<
+            Promise<Record<string, ComponentConfig>>
+          >(async (acc, el) => {
+            return {
+              ...(await acc),
+              [el.dir]: await createComponent(el.dir, await readYaml(el.ci)),
+            };
+          }, Promise.resolve({}));
+        } else {
+          components = {
+            [COMPONENT_NAME]: await createComponent(APP_DIR, gitlabCi),
+          };
+        }
+        // we only have one component
+        const config: Config = {
+          customerName: CUSTOMER_NAME,
+          appName: APP_NAME,
+          components,
+        };
+
+        const comment =
+          "Old migrated config:\n\n" +
+          (await readFile(await getGitlabCiFilePath(), {
+            encoding: "utf-8",
+          }));
+
+        this.log("-------------------");
+        this.log("migrate config");
+        await writeConfig(this, config, {
+          endComment: comment,
+        });
+        this.log("write gitlab-ci.yml");
+        await writeFile(
+          await getGitlabCiFilePath(),
+          "include: https://git.panter.ch/api/v4/projects/catladder%2Fcatladder/packages/generic/ci-includes/main/gitlab-ci.yml",
+          { encoding: "utf-8" }
+        );
+
+        this.log("-------------------");
+        this.log("migrate secrets");
+        await syncBitwarden();
+        for (const env of LEGACY_ENVS) {
+          if (env === "stage" && !STAGING_ENABLED) return;
+          await migrateSecrets(this, config, env);
+        }
+
+        this.log("-------------------");
+
+        this.log("done!");
+
+        this.log("-------------------");
+      }
+    });
+
+  vorpal.exec("migrate");
+};

package/src/apps/cli/verify/migration/migrateSecrets.ts

@@ -0,0 +1,37 @@
+import { Config, getEnvironment } from "@catladder/pipeline";
+import { load } from "js-yaml";
+import { pick } from "lodash";
+import { CommandInstance } from "vorpal";
+import { upsertAllVariables } from "../../../../utils/gitlab";
+import { readPass, trashItem } from "../../../../utils/passwordstore";
+import { LEGACY_ENVS } from "./fromv2";
+
+const getPassPath = (newConfig: Config, env: string) => {
+  return `${newConfig.customerName}/${newConfig.appName}/${env}/secrets.yml`;
+};
+
+export const migrateSecrets = async (
+  vorpal: CommandInstance,
+  newConfig: Config,
+  oldEnv: typeof LEGACY_ENVS[number]
+) => {
+  const newEnv = oldEnv === "dev-local" ? "local" : oldEnv;
+  const path = getPassPath(newConfig, oldEnv);
+  try {
+    const yamlstring = await readPass(path);
+    const secrets = load(yamlstring);
+
+    Object.keys(newConfig.components).forEach(async (componentName) => {
+      const environment = getEnvironment(newConfig, componentName, newEnv);
+      await upsertAllVariables(
+        vorpal,
+        pick(secrets, environment.secretEnvVarKeys),
+        newEnv,
+        componentName
+      );
+    });
+    await trashItem(path);
+  } catch (e) {
+    console.warn(`could not migrate secrets for env '${oldEnv}': ${e}`);
+  }
+};

package/src/apps/cli/verify/migration/oldGitlabCi.ts

@@ -0,0 +1,43 @@
+import { BuildConfig } from "@catladder/pipeline";
+
+export type OldGitlabCiFile = {
+  variables: {
+    CUSTOMER_NAME: string;
+    APP_NAME: string;
+    COMPONENT_NAME?: string;
+    APP_DIR?: string;
+    CLUSTER_NAME?: string;
+    STAGING_ENABLED?: string;
+  };
+  include: {
+    project: string;
+    ref: string;
+    file: string;
+  }[];
+};
+
+export const detectBuildConfig = (
+  gitlabCi: OldGitlabCiFile
+): BuildConfig["type"] | "monorepo" => {
+  const firstInclude = gitlabCi.include[0];
+  if (!firstInclude || gitlabCi.include[0]?.project !== "catladder/gitlab-ci") {
+    throw new Error("unsupported gitlab-ci file");
+  }
+
+  if (firstInclude.file === "monorepo.yml") return "monorepo";
+
+  if (firstInclude.file === "node-kubernetes.yml") {
+    return "node";
+  }
+
+  if (firstInclude.file === "static-js-kubernetes.yml") {
+    return "node-static";
+  }
+  if (firstInclude.file === "meteor-kubernetes.yml") {
+    return "meteor";
+  }
+
+  if (firstInclude.file === "rails-kubernetes.yml") {
+    throw new Error("rails is not yet implemented");
+  }
+};

package/src/catenv.ts

@@ -0,0 +1,3 @@
+import catenv from "./apps/catenv/catenv";
+
+catenv();

package/src/cli.ts

@@ -0,0 +1,8 @@
+import updateNotifier from "update-notifier";
+import cli from "./apps/cli/cli";
+import packageInfos from "./packageInfos";
+
+updateNotifier({
+  pkg: packageInfos,
+}).notify();
+cli();