@catladder/cli 0.0.0-fix-migrate-string-cmd-508a469f → 0.0.0-fix-values-as-files-73f39da5

package/src/apps/cli/commands/project/commandDeletePods.ts

@@ -1,5 +1,5 @@
 import Vorpal from "vorpal";
-import k8sApi from "../../../../k8sApi";
+import { getk8sApi } from "../../../../k8sApi";
 
 import { logError } from "../../../../utils/log";
 import {
@@ -40,6 +40,7 @@ export default async (vorpal: Vorpal) =>
         message: "Continue ? 🤔  ",
       });
       this.log("");
+      const k8sApi = getk8sApi();
       if (shouldContinue) {
         for (const podName of selectedPodNames) {
           await k8sApi.deleteNamespacedPod(podName, namespace, "true");

package/src/apps/cli/commands/project/commandSetup.ts

@@ -0,0 +1,12 @@
+import Vorpal from "vorpal";
+import { setupProject } from "./setup";
+
+export default async (vorpal: Vorpal) =>
+  vorpal
+    .command(
+      "project-setup",
+      "Initializes all environments and creates requires resources, service accounts, etc."
+    )
+    .action(async function () {
+      await setupProject(this);
+    });

package/src/apps/cli/commands/project/commandTriggerCronjob.ts

@@ -1,7 +1,7 @@
 import { V1Job, V1ObjectMeta } from "@kubernetes/client-node";
 
 import Vorpal from "vorpal";
-import { k8sApiBatch, k8sApiBatchBeta } from "../../../../k8sApi";
+import { getk8sApiBatch, getk8sApiBatchBeta } from "../../../../k8sApi";
 import { logError } from "../../../../utils/log";
 import { namespaceAutoCompletion } from "../general/namespaceAutoCompletion";
 
@@ -12,7 +12,7 @@ import ensureCluster from "./utils/ensureCluster";
 async function triggerCronjob(namespace: string) {
   const {
     body: { items: jobs },
-  } = await k8sApiBatchBeta.listNamespacedCronJob(namespace);
+  } = await getk8sApiBatchBeta().listNamespacedCronJob(namespace);
 
   const jobNames = jobs.map((j) => j.metadata.name);
 
@@ -34,7 +34,7 @@ async function triggerCronjob(namespace: string) {
   job.metadata = metadata as V1ObjectMeta;
   job.spec = jobSpec;
   try {
-    const result = await k8sApiBatch.createNamespacedJob(namespace, job);
+    const result = await getk8sApiBatch().createNamespacedJob(namespace, job);
 
     this.log("");
     this.log(`yeah, you got a job, man. 😺 ${result.body.metadata.name}`);

package/src/apps/cli/commands/project/index.ts

@@ -8,8 +8,8 @@ import commandEnvVars from "./commandEnvVars";
 import commandGetMyTotalWorktime from "./commandGetMyTotalWorktime";
 import commandGetShell from "./commandGetShell";
 import commandGitlabCi from "./commandGitlabCi";
-import commandInitGitlab from "./commandInitGitlab";
-import commandInitProject from "./commandInitProject";
+import commandSetup from "./commandSetup";
+//import commandInitProject from "./commandInitProject.old";
 import commandListPods from "./commandListPods";
 import commandMigrateHelm3 from "./commandMigrateHelm3";
 import commandNamespace from "./commandNamespace";
@@ -24,12 +24,10 @@ import commandPortForward from "./commandPortForward";
 import commandTriggerCronjob from "./commandTriggerCronjob";
 
 import commandOpenGrafanaPod from "./commandOpenGrafanaPod";
-import commandReloadConfig from "./commandReloadConfig";
 
 export default async (vorpal: Vorpal) => {
-  commandReloadConfig(vorpal);
-  commandInitProject(vorpal);
-  commandInitGitlab(vorpal);
+  commandSetup(vorpal);
+
   commandEnvVars(vorpal);
 
   commandNamespace(vorpal);

package/src/apps/cli/commands/project/setup/index.ts

@@ -0,0 +1,42 @@
+import { CommandInstance } from "vorpal";
+import { getAllPipelineContexts } from "../../../../../config/getProjectConfig";
+import { projectConfigSecrets } from "../commandConfigSecrets";
+import { setupAccessTokens } from "./setupAccessTokens";
+import { setupContext } from "./setupContext";
+
+export const setupProject = async (instance: CommandInstance) => {
+  const allContext = await getAllPipelineContexts();
+
+  for (const context of allContext) {
+    await setupContext(instance, context);
+  }
+  await setupAccessTokens(instance);
+  instance.log("");
+  const { configSecrets } = await instance.prompt({
+    default: true,
+    message: "Before deployments work, you need to config secrets. Do it now?",
+    name: "configSecrets",
+    type: "confirm",
+  });
+  instance.log("");
+  if (configSecrets) {
+    await projectConfigSecrets(instance);
+  } else {
+    instance.log(
+      "👆 don't forget to config secret using `project-config-secrets`"
+    );
+  }
+  instance.log("");
+  instance.log("gitlab is ready! 🥂");
+  instance.log("\n");
+  instance.log("do not forget to make sure that:");
+  [
+    "you have __health route in place",
+    "lint and test are defined",
+    "secrets are configured (call project-config-secret)",
+    "eat your vegetables",
+    "be awesome 🤩",
+  ].forEach((tip) => instance.log(` - ${tip}`));
+  instance.log("\n");
+  instance.log("\n");
+};

package/src/apps/cli/commands/project/setup/setupAccessTokens.ts

@@ -0,0 +1,72 @@
+import open from "open";
+import { CommandInstance } from "vorpal";
+import { doGitlabRequest, getProjectInfo } from "../../../../../utils/gitlab";
+
+export const setupAccessTokens = async (instance: CommandInstance) => {
+  const { id: projectId, web_url: projectWebUrl } = await getProjectInfo(
+    instance
+  );
+  try {
+    await doGitlabRequest(instance, `projects/${projectId}/variables/GL_TOKEN`);
+  } catch (e) {
+    if (e.message !== "not found") {
+      throw e;
+    }
+    // not found
+
+    instance.log(
+      "I need add a GL_TOKEN to the project, so that semantic release will work\n"
+    );
+    instance.log(
+      "👉 Please please create a project access token in gitlab and copy its value into clipboard\n\n - name: something like 'semantic-release'\n - expires: leave empty\n - scopes: api, read_repository"
+    );
+    instance.log("\n");
+
+    const { understood } = await instance.prompt({
+      default: true,
+      message: "Understood and open gitlab now? 🤔",
+      name: "understood",
+      type: "confirm",
+    });
+    if (!understood) {
+      instance.log("continuing anyway...");
+    }
+    open(`${projectWebUrl}/-/settings/access_tokens`);
+
+    instance.log("\n");
+
+    instance.log("Enter your copied token now: ");
+
+    instance.log("\n");
+    const { GL_TOKEN } = await instance.prompt({
+      type: "password",
+      name: "GL_TOKEN",
+      message: "Access Token: ",
+    });
+    await doGitlabRequest(instance, `projects/${projectId}/variables`, {
+      key: "GL_TOKEN",
+      value: GL_TOKEN,
+    });
+  }
+
+  const deploy_tokens = await doGitlabRequest(
+    instance,
+    `projects/${projectId}/deploy_tokens`
+  );
+
+  if (
+    !deploy_tokens.find(
+      (v: { name: string }) => v.name === "gitlab-deploy-token"
+    )
+  ) {
+    instance.log(
+      "I will setup the 'GitLab Deploy Token', so Kubernetes can pull images from this project."
+    );
+
+    await doGitlabRequest(instance, `projects/${projectId}/deploy_tokens`, {
+      id: projectId,
+      name: "gitlab-deploy-token",
+      scopes: ["read_registry"],
+    });
+  }
+};

package/src/apps/cli/commands/project/setup/setupContext.ts

@@ -0,0 +1,37 @@
+import { Context, isOfDeployType } from "@catladder/pipeline";
+import { CommandInstance } from "vorpal";
+import { setupKubernetes } from "./setupKubernetes";
+
+export const setupContext = async (
+  instance: CommandInstance,
+  context: Context
+) => {
+  instance.log("");
+  instance.log(
+    "=================================================================================="
+  );
+
+  instance.log(
+    "🐱 🔧 setting up " +
+      context.environment.shortName +
+      ":" +
+      context.componentName +
+      "..."
+  );
+  instance.log("");
+  const deployConfig = context.componentConfig.deploy;
+  if (isOfDeployType(deployConfig, "kubernetes")) {
+    await setupKubernetes(instance, context);
+  }
+
+  instance.log("");
+  instance.log(
+    "✅ " +
+      context.environment.shortName +
+      ":" +
+      context.componentName +
+      " done!"
+  );
+
+  instance.log("");
+};

package/src/apps/cli/commands/project/setup/setupKubernetes.ts

@@ -0,0 +1,112 @@
+import {
+  Context,
+  getFullKubernetesClusterName,
+  isOfDeployType,
+} from "@catladder/pipeline";
+import { CommandInstance } from "vorpal";
+import { $ } from "zx";
+import { connectToCluster } from "../../../../../utils/cluster";
+import { upsertAllVariables } from "../../../../../utils/gitlab";
+import ensureNamespace from "../utils/ensureNamespace";
+
+export const setupKubernetes = async (
+  instance: CommandInstance,
+  context: Context
+) => {
+  const deployConfig = context.componentConfig.deploy;
+  if (!isOfDeployType(deployConfig, "kubernetes")) {
+    throw new Error("cannot run setupKubernetes on non-kubernetes deployments");
+  }
+
+  const fullName = getFullKubernetesClusterName(deployConfig.cluster);
+  instance.log(`cluster: ${fullName}`);
+
+  await connectToCluster(fullName);
+  instance.log("");
+  instance.log("ensuring namespace ...");
+  const namespace = await ensureNamespace(context);
+  instance.log("Namespace " + namespace + " created / updated!");
+  instance.log("");
+  //$.verbose = true;
+
+  // we name the service account and the role and the role binding with the same name
+  // we currently create one per component to better separate them
+  instance.log("ensuring service accounts...");
+  const serviceAccountName = `cl-${context.componentName}-deploy`;
+  const KUBE_URL =
+    await $`TERM=dumb kubectl cluster-info | grep -E 'Kubernetes master|Kubernetes control plane' | awk '/http/ {print $NF}'`.then(
+      (s) => s.stdout.trim()
+    );
+
+  // first upsert service acount in the ns
+  try {
+    await $`kubectl delete serviceaccount --namespace ${namespace} ${serviceAccountName}`;
+    await $`kubectl delete rolebinding --namespace ${namespace} ${serviceAccountName}`;
+    await $`kubectl delete role --namespace ${namespace} ${serviceAccountName}`;
+  } catch (e) {
+    // ignore
+  }
+
+  await $`kubectl create serviceaccount --namespace ${namespace} ${serviceAccountName}`;
+
+  // upsert role in the ns
+
+  await $`cat <<EOF | kubectl apply -f -
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  namespace: ${namespace}
+  name: ${serviceAccountName}
+rules:
+- apiGroups: ["", "extensions", "apps", "networking.k8s.io", "batch"]
+  resources: ["deployments", "replicasets", "statefulsets", "pods", "secrets", "configmaps", "services", "ingresses", "serviceaccounts", "jobs", "cronjobs"]
+  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] # You can also use ["*"]
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ${serviceAccountName}
+  namespace: ${namespace}
+subjects:
+  - kind: ServiceAccount
+    name: ${serviceAccountName}
+    namespace:  ${namespace}
+roleRef:
+  kind: Role
+  name: ${serviceAccountName}
+  apiGroup: rbac.authorization.k8s.io
+EOF
+`;
+
+  // get token name
+  const tokenName =
+    await $`kubectl get serviceaccount --namespace ${namespace} ${serviceAccountName} -o jsonpath='{.secrets[0].name}'`;
+
+  const KUBE_CA_PEM =
+    await $`kubectl get secret ${tokenName} --namespace ${namespace} -o jsonpath="{['data']['ca\\.crt']}"`.then(
+      (c) => c.stdout.trim()
+    );
+  const KUBE_TOKEN =
+    await $`kubectl get secret ${tokenName} --namespace ${namespace} -o jsonpath="{['data']['token']}" | base64 --decode`.then(
+      (c) => c.stdout.trim()
+    );
+
+  const vars = {
+    KUBE_TOKEN,
+    KUBE_CA_PEM,
+    KUBE_URL,
+  };
+
+  instance.log("service accounts created / updated!");
+
+  instance.log("");
+  instance.log("pusing secrets to gitlab...");
+
+  await upsertAllVariables(
+    instance,
+    vars,
+    context.environment.shortName,
+    context.componentName
+  );
+  instance.log("done!");
+};

package/src/apps/cli/commands/project/utils/autocompletions.ts

@@ -1,6 +1,6 @@
 import { getAllEnvs, getAllEnvsInAllComponents } from "@catladder/pipeline";
 import {
-  getAllComponentsWithAllEnvs,
+  getAllComponentsWithAllEnvsFlat,
   getProjectConfig,
 } from "../../../../../config/getProjectConfig";
 
@@ -13,7 +13,7 @@ export const allEnvs = async () => {
 };
 
 export const envAndComponents = async () => {
-  const allEnvAndcomponents = await getAllComponentsWithAllEnvs();
+  const allEnvAndcomponents = await getAllComponentsWithAllEnvsFlat();
 
   return allEnvAndcomponents.reduce<string[]>(
     (acc, { env, componentName }) => [...acc, env + ":" + componentName],

package/src/apps/cli/commands/project/utils/ensureNamespace.ts

@@ -1,6 +1,6 @@
 import { Context, getKubernetesNamespace } from "@catladder/pipeline";
 import { V1Namespace, V1ObjectMeta } from "@kubernetes/client-node";
-import k8sApi from "../../../../../k8sApi";
+import { getk8sApi } from "../../../../../k8sApi";
 
 export default async function (context: Context) {
   const namespace = getKubernetesNamespace(
@@ -31,6 +31,7 @@ export default async function (context: Context) {
   };
 
   namespaceBody.metadata = metadata;
+  const k8sApi = getk8sApi();
   try {
     await k8sApi.readNamespace(namespace);
 

package/src/apps/cli/config/writeConfig.ts

@@ -45,7 +45,7 @@ export const writeConfig = async (
       encoding: "utf-8",
     });
     vorpal.log("adding type @catladder/pipeline....");
-    await spawn("yarn add @catladder/pipeline -DW", {
+    await spawn("yarn add --non-interactive @catladder/pipeline -DW", {
       shell: true,
     });
     await exec("git add " + file);

package/src/apps/cli/verify/migration/fromv2.ts

@@ -94,7 +94,7 @@ export const migrateV2 = async (vorpal: Vorpal) => {
   const {
     CUSTOMER_NAME,
     APP_NAME,
-    COMPONENT_NAME,
+    COMPONENT_NAME = "web",
     APP_DIR = ".",
     STAGING_ENABLED,
   } = gitlabCi.variables;
@@ -147,7 +147,7 @@ export const migrateV2 = async (vorpal: Vorpal) => {
 
           const startCommand = Array.isArray(baseValues?.application?.command)
             ? baseValues?.application?.command.join(" ")
-            : baseValues?.application?.command
+            : baseValues?.application?.command;
           return {
             vars: env ? transformVars(env) : undefined,
             dir: dir,
@@ -249,6 +249,8 @@ export const migrateV2 = async (vorpal: Vorpal) => {
         this.log("-------------------");
 
         this.log("done!");
+        this.log("");
+        this.log("You can remove the values*.yml files now.");
 
         this.log("-------------------");
       } else {

package/src/config/getProjectConfig.ts

@@ -3,29 +3,42 @@ import {
   getAllEnvs,
   getEnvironment as _getEnvironment,
   createContext,
+  getSecretVarName,
+  Config,
 } from "@catladder/pipeline";
 
 import { CommandInstance } from "vorpal";
-import { getAllVariables } from "../utils/gitlab";
-import memoizee from "memoizee";
+import { getAllVariables, getVariableValueByRawName } from "../utils/gitlab";
+
 import { getGitRoot } from "../utils/projects";
 import { readYaml } from "../utils/files";
-// currently cant change
-
-export const getProjectConfig = memoizee(
-  async () => {
-    try {
-      const gitRoot = await getGitRoot();
-      return readConfigSync(gitRoot);
-    } catch (e) {
-      // ignore
-      return null;
-    }
-  },
-  { promise: true }
-);
-
-export const reloadConfig = () => getProjectConfig.clear();
+import { watch } from "fs";
+
+let currentConfig: Config = null;
+
+// reload the config on change
+const reloadConfigAndObserve = async () => {
+  const gitRoot = await getGitRoot();
+  const result = readConfigSync(gitRoot);
+  if (!result) {
+    // can't do anything, there is no config
+    return;
+  }
+  const { config, path } = result;
+  const watcher = watch(path, () => {
+    watcher.close();
+    reloadConfigAndObserve();
+  });
+  currentConfig = config;
+};
+
+export const getProjectConfig = async () => {
+  if (!currentConfig) {
+    // initially
+    await reloadConfigAndObserve();
+  }
+  return currentConfig;
+};
 
 export const getGitlabCiFilePath = async () => {
   const gitRoot = await getGitRoot();
@@ -58,21 +71,37 @@ export const getPipelineContextByChoice = async (
   const config = await getProjectConfig();
   return createContext(config, componentName, env);
 };
-export const getAllComponentsWithAllEnvs = async () => {
+export const getAllComponentsWithAllEnvsFlat = async (): Promise<
+  Array<{ env: string; componentName: string }>
+> => {
   const config = await getProjectConfig();
   if (!config) {
     return [];
   }
-  return Promise.all(
-    Object.keys(config.components).flatMap((componentName) =>
-      getAllEnvs(config, componentName).map((env) => ({ env, componentName }))
-    )
+  return Object.keys(config.components).flatMap((componentName) =>
+    getAllEnvs(config, componentName).map((env) => ({ env, componentName }))
+  );
+};
+
+export const getAllComponentsWithAllEnvsHierarchical = async (): Promise<{
+  [componentName: string]: string[];
+}> => {
+  const config = await getProjectConfig();
+  if (!config) {
+    return {};
+  }
+
+  return Object.fromEntries(
+    Object.keys(config.components).map((componentName) => [
+      componentName,
+      getAllEnvs(config, componentName),
+    ])
   );
 };
 
 export const getAllPipelineContexts = async () => {
   return Promise.all(
-    (await getAllComponentsWithAllEnvs())
+    (await getAllComponentsWithAllEnvsFlat())
       .filter((c) => c.env !== "local")
       .map(({ env, componentName }) =>
         getPipelineContextByChoice(env, componentName)
@@ -82,9 +111,20 @@ export const getAllPipelineContexts = async () => {
 
 export const getEnvironment = async (env: string, componentName: string) => {
   const config = await getProjectConfig();
+
   return _getEnvironment(config, componentName, env);
 };
 
+export const getGitlabVar = async (
+  vorpal: CommandInstance,
+  env: string,
+  componentName: string,
+  variableName: string
+) => {
+  const rawVariableName = getSecretVarName(env, componentName, variableName);
+  return await getVariableValueByRawName(vorpal, rawVariableName);
+};
+
 const resolveSecrets = async (
   vorpal: CommandInstance,
   allEnvVars: Record<string, string>
@@ -93,16 +133,15 @@ const resolveSecrets = async (
 
   return Object.fromEntries(
     Object.entries(allEnvVars).map(([key, value]) => {
-      const isSecret = String(value)?.startsWith?.("$CL_");
-      if (isSecret) {
-        // secrets have CL_XXXX structure
-        const found = allVariablesInGitlab.find((v) => "$" + v.key === value);
-
-        if (found) {
-          return [key, found.value];
-        } else {
-          return [key, ""];
+      const containsSecret = String(value)?.includes?.("$CL_");
+      if (containsSecret) {
+        for (const variable of allVariablesInGitlab) {
+          value = value.replace(
+            new RegExp("\\$" + variable.key, "g"),
+            variable.value
+          );
         }
+        return [key, value];
       }
       return [key, value];
     })
@@ -115,6 +154,7 @@ export const getEnvVars = async (
   componentName: string
 ) => {
   const envionment = await getEnvironment(env, componentName);
-
+  // in the pipeline the secrets alreadyy exists  and bash will expand them
+  // but here we need to manually load them
   return resolveSecrets(vorpal, envionment.envVars);
 };

package/src/k8sApi/index.ts

@@ -5,17 +5,29 @@ import {
   BatchV1beta1Api,
 } from "@kubernetes/client-node";
 
-const kc = new KubeConfig();
-
-export const reload = () => {
+const getKubeConfig = () => {
+  const kc = new KubeConfig();
   kc.loadFromDefault();
+  return kc;
 };
+/**
+ * get kubernetes client. avoid reusing the instance when context get changed
+ * @returns kuberenetes client
+ */
+export const getk8sApi = () => {
+  const kc = getKubeConfig();
 
-reload();
+  return kc.makeApiClient(CoreV1Api);
+};
 
-const k8sApi = kc.makeApiClient(CoreV1Api);
+export const getk8sApiBatch = () => {
+  const kc = getKubeConfig();
 
-export const k8sApiBatch = kc.makeApiClient(BatchV1Api);
-export const k8sApiBatchBeta = kc.makeApiClient(BatchV1beta1Api);
+  return kc.makeApiClient(BatchV1Api);
+};
 
-export default k8sApi;
+export const getk8sApiBatchBeta = () => {
+  const kc = getKubeConfig();
+
+  return kc.makeApiClient(BatchV1beta1Api);
+};

package/src/utils/cluster.ts

@@ -1,5 +1,4 @@
 import { $ } from "zx";
-import { reload } from "../k8sApi";
 
 export const getCurrentContext = async () =>
   (await $`kubectl config current-context`).stdout.trim();
@@ -10,5 +9,4 @@ export const getCurrentConnectedClusterName = async () => {
 
 export const connectToCluster = async (fullname: string) => {
   await $`kubectl config use-context ${fullname}`;
-  reload();
 };

package/src/utils/gitlab.ts

@@ -73,6 +73,9 @@ export const doGitlabRequest = async <T = any>(
   if (result.status >= 200 && result.status < 400) {
     return result.json();
   }
+  if (result.status === 404) {
+    throw new Error("not found");
+  }
 
   throw new Error(
     `Could not send request to gitlab api ${path}: ${result.status} "${
@@ -125,6 +128,14 @@ export const getAllVariables = memoizee(
   { promise: true }
 );
 
+export const getVariableValueByRawName = async (
+  vorpal: CommandInstance,
+  rawName: string
+) => {
+  const allVariables = await getAllVariables(vorpal);
+  return allVariables.find((v) => v.key === rawName)?.value;
+};
+
 const maskableRegex = new RegExp("^[a-zA-Z0-9_+=/@:.~-]{8,}$"); // SEE https://gitlab.com/gitlab-org/gitlab-foss/-/blob/master/spec/frontend/ci_variable_list/components/ci_variable_modal_spec.js#L20
 const isMaskable = (value: string): boolean => maskableRegex.test(value);