@catladder/cli 0.0.0-test7 → 0.0.0-update-meteor-53480fed

package/src/apps/cli/commands/project/setup/setupCloudRun.ts

@@ -0,0 +1,84 @@
+import type { Context } from "@catladder/pipeline";
+import {
+  GCLOUD_DEPLOY_CREDENTIALS_KEY,
+  GCLOUD_RUN_CANONICAL_HOST_SUFFIX,
+  isOfDeployType,
+} from "@catladder/pipeline";
+
+import type { CommandInstance } from "vorpal";
+import { upsertGcloudArtifactsRegistry } from "../../../../../gcloud/artifactsRegistry";
+import { getCloudRunDomainSuffix } from "../../../../../gcloud/cloudRun/getCloudRunDomainSuffix";
+import { enableGCloudServices } from "../../../../../gcloud/enableServices";
+import { upsertGcloudServiceAccountAndSaveSecret } from "../../../../../gcloud/serviceAccounts";
+import { upsertAllVariables } from "../../../../../utils/gitlab";
+
+export const setupCloudRun = async (
+  instance: CommandInstance,
+  context: Context
+) => {
+  if (!isOfDeployType(context.componentConfig.deploy, "google-cloudrun")) {
+    throw new Error("deploy config is not of type 'google-cloudrun'");
+  }
+
+  const config = context.componentConfig.deploy;
+
+  // enable services
+
+  instance.log("enable required servies...");
+  await enableGCloudServices(
+    [
+      "run.googleapis.com",
+      "artifactregistry.googleapis.com",
+      "cloudscheduler.googleapis.com", // for scheduling jobs
+      "cloudresourcemanager.googleapis.com", // only required to get google cloud project number
+      ...(config.cloudSql
+        ? ["sqladmin.googleapis.com", "sql-component.googleapis.com"]
+        : []),
+    ],
+    config
+  );
+  instance.log("upsert artifacts registry...");
+  await upsertGcloudArtifactsRegistry(config);
+
+  instance.log("upsert deploy service account...");
+  await upsertGcloudServiceAccountAndSaveSecret(
+    instance,
+    context,
+    {
+      projectId: config.projectId,
+      displayName: "Catladder gcloud deploy",
+      description: "This service account deploys to google cloud",
+      name: "cl-d",
+      // TODO: run.admin is a bit much, would be better to follow https://stackoverflow.com/questions/55788714/deploying-to-cloud-run-with-a-custom-service-account-failed-with-iam-serviceacco
+      roles: [
+        "roles/artifactregistry.repoAdmin",
+        "roles/run.admin",
+        "roles/iam.serviceAccountUser",
+        "roles/cloudscheduler.admin", // for scheduling
+        ...(config.cloudSql ? ["roles/cloudsql.admin"] : []),
+      ],
+    },
+    GCLOUD_DEPLOY_CREDENTIALS_KEY
+  );
+
+  // gcloud run automatically gives us a hostname
+  // but the pipeline need to know it before deployment
+  // so we get this through some magic and add this as a ci/cd variable
+  // the pipeline than can use this to construct the canonical host
+  instance.log(
+    "get service domain suffix... that might take a while initially"
+  );
+  const suffix = await getCloudRunDomainSuffix(config);
+  instance.log("domain suffix: " + suffix);
+
+  await upsertAllVariables(
+    instance,
+    {
+      [GCLOUD_RUN_CANONICAL_HOST_SUFFIX]: suffix,
+    },
+    context.environment.shortName,
+    context.componentName,
+    false, // backup
+    false // masked
+  );
+};

package/src/apps/cli/commands/project/setup/setupContext.ts

@@ -0,0 +1,44 @@
+import type { Context } from "@catladder/pipeline";
+import { isOfDeployType } from "@catladder/pipeline";
+import type { CommandInstance } from "vorpal";
+import { setupCloudRun } from "./setupCloudRun";
+
+import { setupKubernetes } from "./setupKubernetes";
+
+export const setupContext = async (
+  instance: CommandInstance,
+  context: Context
+) => {
+  instance.log("");
+  instance.log(
+    "=================================================================================="
+  );
+
+  instance.log(
+    "🐱 🔧 setting up " +
+      context.environment.shortName +
+      ":" +
+      context.componentName +
+      "..."
+  );
+  instance.log("");
+  if (isOfDeployType(context.componentConfig.deploy, "google-cloudrun")) {
+    await setupCloudRun(instance, context);
+  }
+
+  const deployConfig = context.componentConfig.deploy;
+  if (isOfDeployType(deployConfig, "kubernetes")) {
+    await setupKubernetes(instance, context);
+  }
+
+  instance.log("");
+  instance.log(
+    "✅ " +
+      context.environment.shortName +
+      ":" +
+      context.componentName +
+      " done!"
+  );
+
+  instance.log("");
+};

package/src/apps/cli/commands/project/setup/setupKubernetes.ts

@@ -0,0 +1,140 @@
+import type { Context } from "@catladder/pipeline";
+import {
+  getFullKubernetesClusterName,
+  isOfDeployType,
+} from "@catladder/pipeline";
+import type { CommandInstance } from "vorpal";
+import { exec } from "child-process-promise";
+import { connectToCluster } from "../../../../../utils/cluster";
+import { upsertAllVariables } from "../../../../../utils/gitlab";
+import ensureNamespace from "../utils/ensureNamespace";
+
+export const setupKubernetes = async (
+  instance: CommandInstance,
+  context: Context
+) => {
+  const deployConfig = context.componentConfig.deploy;
+  if (!isOfDeployType(deployConfig, "kubernetes")) {
+    throw new Error("cannot run setupKubernetes on non-kubernetes deployments");
+  }
+
+  const fullName = getFullKubernetesClusterName(deployConfig.cluster);
+  instance.log(`cluster: ${fullName}`);
+
+  await connectToCluster(fullName);
+  instance.log("");
+  instance.log("ensuring namespace ...");
+  const namespace = await ensureNamespace(context);
+  instance.log("Namespace " + namespace + " created / updated!");
+  instance.log("");
+  //$.verbose = true;
+
+  // we name the service account and the role and the role binding with the same name
+  // we currently create one per component to better separate them
+  instance.log("ensuring service accounts...");
+  const serviceAccountName = `cl-${context.componentName}-deploy`;
+  const KUBE_URL = await exec(
+    `TERM=dumb kubectl cluster-info | grep -E 'Kubernetes master|Kubernetes control plane' | awk '/http/ {print $NF}'`
+  ).then((s) => s.stdout.trim());
+
+  // first upsert service acount in the ns
+  try {
+    await exec(
+      `kubectl delete serviceaccount --namespace ${namespace} ${serviceAccountName}`
+    );
+    await exec(
+      `kubectl delete rolebinding --namespace ${namespace} ${serviceAccountName}`
+    );
+    await exec(
+      `kubectl delete role --namespace ${namespace} ${serviceAccountName}`
+    );
+  } catch (e) {
+    // ignore
+  }
+
+  await exec(
+    `kubectl create serviceaccount --namespace ${namespace} ${serviceAccountName}`
+  );
+
+  // upsert role in the ns
+
+  await exec(`cat <<EOF | kubectl apply -f -
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  namespace: ${namespace}
+  name: ${serviceAccountName}
+rules:
+- apiGroups: ["", "extensions", "apps", "networking.k8s.io", "batch", "autoscaling", "rbac.authorization.k8s.io","snapshot.storage.k8s.io"]
+  resources: ["deployments", "replicasets", "statefulsets", "pods", "secrets", "configmaps", "services", "ingresses", "serviceaccounts", "roles", "rolebindings", "jobs", "cronjobs", "horizontalpodautoscalers", "persistentvolumeclaims", "volumesnapshots"]
+  verbs: ["*"]
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ${serviceAccountName}
+  namespace: ${namespace}
+subjects:
+  - kind: ServiceAccount
+    name: ${serviceAccountName}
+    namespace:  ${namespace}
+roleRef:
+  kind: Role
+  name: ${serviceAccountName}
+  apiGroup: rbac.authorization.k8s.io
+EOF
+`);
+
+  const secretName = `${serviceAccountName}-secret`;
+  // create token for the service account
+  await exec(`
+kubectl apply -f - <<EOF
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ${secretName}
+  namespace: ${namespace}
+  annotations:
+    kubernetes.io/service-account.name: ${serviceAccountName}
+type: kubernetes.io/service-account-token
+EOF
+  `);
+
+  const KUBE_CA_PEM = await exec(
+    `kubectl get secret ${secretName} --namespace ${namespace} -o jsonpath="{['data']['ca\\.crt']}"`
+  ).then((c) => c.stdout.trim());
+  const KUBE_TOKEN = await exec(
+    `kubectl get secret ${secretName} --namespace ${namespace} -o jsonpath="{['data']['token']}" | base64 --decode`
+  ).then((c) => c.stdout.trim());
+
+  const vars = {
+    KUBE_TOKEN,
+    KUBE_CA_PEM,
+    KUBE_URL,
+  };
+
+  const missing = Object.entries(vars).filter((e) => !e[1]);
+  if (missing.length > 0) {
+    throw new Error(
+      "could not setup credentials. Missing vars: " +
+        missing.map((m) => m[0]).join(", ") +
+        ". Check whether your local kubectl is still working for '" +
+        fullName +
+        "'"
+    );
+  }
+
+  instance.log("service accounts created / updated!");
+
+  instance.log("");
+  instance.log("pusing secrets to gitlab...");
+
+  await upsertAllVariables(
+    instance,
+    vars,
+    context.environment.shortName,
+    context.componentName,
+    false // no backup
+  );
+  instance.log("done!");
+};

package/src/apps/cli/commands/project/setup/setupTopic.ts

@@ -0,0 +1,22 @@
+import type { CommandInstance } from "vorpal";
+import { doGitlabRequest, getProjectInfo } from "../../../../../utils/gitlab";
+
+const catladderTopic = "catladder"
+
+export const setupTopic = async (instance: CommandInstance) => {
+  const { id: projectId } = await getProjectInfo(
+    instance
+  );
+
+  const { topics } = await doGitlabRequest(
+    instance,
+    `projects/${projectId}`
+  );
+
+  await doGitlabRequest(
+    instance,
+    `projects/${projectId}`,
+    { topics: [catladderTopic, ...topics] },
+    "PUT"
+  );
+};

package/src/apps/cli/commands/project/utils/autocompletions.ts

@@ -0,0 +1,26 @@
+import { getAllEnvs, getAllEnvsInAllComponents } from "@catladder/pipeline";
+import {
+  getAllComponentsWithAllEnvsFlat,
+  getProjectConfig,
+} from "../../../../../config/getProjectConfig";
+
+export const allEnvs = async () => {
+  const config = await getProjectConfig();
+  if (!config) {
+    return [];
+  }
+  return getAllEnvsInAllComponents(config);
+};
+
+export const envAndComponents = async () => {
+  const allEnvAndcomponents = await getAllComponentsWithAllEnvsFlat();
+
+  return allEnvAndcomponents.reduce<string[]>(
+    (acc, { env, componentName }) => [...acc, env + ":" + componentName],
+    []
+  );
+};
+
+export const allEnvsAndAllComponents = async () => {
+  return [...(await allEnvs()), ...(await envAndComponents())];
+};

package/src/apps/cli/commands/project/utils/ensureCluster.ts

@@ -0,0 +1,42 @@
+import {
+  isOfDeployType,
+  getFullKubernetesClusterName,
+} from "@catladder/pipeline";
+import {
+  getPipelineContextByChoice,
+  parseChoice,
+} from "../../../../../config/getProjectConfig";
+import {
+  connectToCluster,
+  getCurrentConnectedClusterName,
+} from "../../../../../utils/cluster";
+export default async function (envComponent: string) {
+  const { env, componentName } = parseChoice(envComponent);
+  const context = await getPipelineContextByChoice(env, componentName);
+  if (!isOfDeployType(context.componentConfig.deploy, "kubernetes")) {
+    throw new Error("can't ensure cluster for non-kubernetes deployments");
+  }
+  const cluster = getFullKubernetesClusterName(
+    context.componentConfig.deploy.cluster
+  );
+  const connectedClusterName = await getCurrentConnectedClusterName();
+
+  if (cluster !== connectedClusterName) {
+    this.log(
+      `you are currently connected to cluster '${connectedClusterName}'`
+    );
+    this.log(`but the project requires cluster '${cluster}'`);
+    const { shouldContinue } = await this.prompt({
+      type: "confirm",
+      name: "shouldContinue",
+      default: true,
+      message: `Do you want to connect to '${cluster}'?`,
+    });
+    if (!shouldContinue) {
+      this.log("abort");
+    } else {
+      await connectToCluster(cluster);
+      this.log(`connected to cluster '${cluster}'`);
+    }
+  }
+}

package/src/apps/cli/commands/project/utils/ensureNamespace.ts

@@ -0,0 +1,58 @@
+import type { Context} from "@catladder/pipeline";
+import { getKubernetesNamespace } from "@catladder/pipeline";
+import type { V1ObjectMeta } from "@kubernetes/client-node";
+import { V1Namespace } from "@kubernetes/client-node";
+import { getk8sApi } from "../../../../../k8sApi";
+
+export default async function (context: Context) {
+  const namespace = getKubernetesNamespace(
+    context.fullConfig,
+    context.environment.shortName
+  );
+  const namespaceBody = new V1Namespace();
+  const metadata: V1ObjectMeta = {
+    name: namespace,
+    labels: {
+      customerName: context.fullConfig.customerName,
+      appName: context.fullConfig.appName,
+      environment: context.environment.shortName,
+      components: Object.keys(context.fullConfig.components).join("_"), // limited chars available...
+      buildTypes: Object.values(context.fullConfig.components)
+        .map((config) => config.build.type)
+        .join("_"), // limited chars available...
+      ...Object.fromEntries(
+        Object.entries(context.fullConfig.components).map(
+          ([componentName, config]) => [
+            "buildType_" + componentName,
+            config.build.type,
+          ]
+        )
+      ),
+      ...(context.fullConfig.meta?.labels ?? {}),
+    },
+  };
+
+  namespaceBody.metadata = metadata;
+  const k8sApi = getk8sApi();
+  try {
+    await k8sApi.readNamespace(namespace);
+
+    await k8sApi.patchNamespace(
+      namespace,
+      namespaceBody,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      { headers: { "content-type": "application/merge-patch+json" } } // see https://github.com/kubernetes-client/javascript/issues/443
+    ); // update meta data
+  } catch (e) {
+    if (e.response?.body?.reason === "NotFound") {
+      await k8sApi.createNamespace(namespaceBody);
+    } else {
+      console.error(e.response?.body);
+      throw e;
+    }
+  }
+  return namespace;
+}

package/src/apps/cli/commands/project/utils/showProjectBanner.ts

@@ -0,0 +1,11 @@
+import type Vorpal from "vorpal";
+import { getProjectConfig } from "../../../../../config/getProjectConfig";
+
+export const showProjectBanner = async (vorpal: Vorpal) => {
+  const projectConfig = await getProjectConfig();
+  if (projectConfig) {
+    vorpal.log("project: " + projectConfig.appName);
+    vorpal.log("customer: " + projectConfig.customerName);
+    vorpal.log("");
+  }
+};

package/src/apps/cli/commands/theStuffThatReallyMatters/index.ts

@@ -0,0 +1,32 @@
+import { spawn } from "child-process-promise";
+import fetch from "node-fetch";
+import type Vorpal from "vorpal";
+
+export default async (vorpal: Vorpal) => {
+  vorpal.command("dadjoke", "something for jonas.").action(async function () {
+    const result = await fetch("https://icanhazdadjoke.com/", {
+      headers: {
+        Accept: "text/plain",
+      },
+    });
+    const text = await result.text();
+    this.log("");
+    this.log(text);
+    this.log("");
+  });
+
+  let starwarsPromise: any;
+  vorpal
+    .command("starwars", "Long time ago... in a galaxy far far away...")
+    .action(async function () {
+      starwarsPromise = spawn("telnet", ["2001:7b8:666:ffff::1:42"], {
+        stdio: ["pipe", "inherit", "pipe"],
+      });
+      await starwarsPromise;
+    })
+    // we need to close it properly, because telnet towel.blinkenlights.nl has no way to cancel and stop itself
+    // this is also why we need to only inherit the stdout and not the stdin
+    .cancel(async function () {
+      starwarsPromise.childProcess.kill();
+    });
+};

package/src/apps/cli/config/writeConfig.ts

@@ -0,0 +1,71 @@
+import type { Config } from "@catladder/pipeline";
+import { spawn, exec } from "child-process-promise";
+import { writeFile } from "fs-extra";
+import { dump } from "js-yaml";
+import { format } from "prettier";
+import type { CommandInstance } from "vorpal";
+import { getGitRoot } from "../../../utils/projects";
+
+export const writeConfig = async (
+  vorpal: CommandInstance,
+  config: Config,
+  options?: {
+    endComment?: string;
+  }
+) => {
+  const gitRoot = await getGitRoot();
+  const TS = "typescript (recommended)";
+  const { configType } = await vorpal.prompt({
+    type: "list",
+    name: "configType",
+    choices: [TS, "yaml"],
+    message: "In which format do you want the config? 🤔",
+  });
+  vorpal.log("");
+  if (configType === TS) {
+    const file = gitRoot + "/catladder.ts";
+    const content = format(
+      `
+      import type { Config } from "@catladder/pipeline";
+      
+      const config: Config = ${JSON.stringify(config)};
+
+      export default config;
+
+     
+      ${options?.endComment ? `/*${options.endComment}*/` : ""}
+      
+      `,
+      {
+        parser: "babel",
+      }
+    );
+
+    await writeFile(file, content, {
+      encoding: "utf-8",
+    });
+    vorpal.log("adding type @catladder/pipeline....");
+    await spawn("yarn add --non-interactive @catladder/pipeline -DW", {
+      shell: true,
+    });
+    await exec("git add " + file);
+  } else {
+    const file = gitRoot + "/catladder.yml";
+    const content = dump(config);
+
+    await writeFile(
+      file,
+      content +
+        "\n\n" +
+        (options.endComment
+          ? "# " + options.endComment.split("\n").join("\n# ")
+          : ""),
+      {
+        encoding: "utf-8",
+      }
+    );
+    await exec("git add " + file);
+  }
+  vorpal.log("done!");
+  vorpal.log("");
+};

package/src/apps/cli/verify/index.ts

@@ -0,0 +1,30 @@
+import type Vorpal from "vorpal";
+import {
+  getGitlabCi,
+  getProjectConfig,
+} from "../../../config/getProjectConfig";
+import { hasGitlabToken, setupGitlabToken } from "../../../utils/gitlab";
+import { isV2, migrateV2 } from "./migration/fromv2";
+
+export const verify = async (vorpal: Vorpal) => {
+  // check if has all settings
+
+  if (!(await hasGitlabToken())) {
+    vorpal.command("setup-gitlab-token").action(async function () {
+      await setupGitlabToken(this);
+    });
+    vorpal.exec("setup-gitlab-token");
+  }
+
+  const projectConfig = await getProjectConfig();
+  try {
+    const gitlabCi = getGitlabCi();
+
+    if (gitlabCi && !projectConfig && (await isV2())) {
+      vorpal.log("no project config, needs migration");
+      await migrateV2(vorpal);
+    }
+  } catch (e) {
+    // no gitroot, ignore
+  }
+};