@catladder/cli 0.0.0-vars-5d04a62d → 0.0.0-wait-for-completion-88d93ae7

package/src/apps/cli/commands/project/setup/setupKubernetes.ts

@@ -1,23 +1,54 @@
+import type { ComponentContext } from "@catladder/pipeline";
 import {
-  Context,
   getFullKubernetesClusterName,
   isOfDeployType,
 } from "@catladder/pipeline";
-import { CommandInstance } from "vorpal";
-import { $ } from "zx";
+import type { CommandInstance } from "vorpal";
+import { exec } from "child-process-promise";
 import { connectToCluster } from "../../../../../utils/cluster";
-import { upsertAllVariables } from "../../../../../utils/gitlab";
+import {
+  doGitlabRequest,
+  getProjectInfo,
+  upsertAllVariables,
+} from "../../../../../utils/gitlab";
 import ensureNamespace from "../utils/ensureNamespace";
 
 export const setupKubernetes = async (
   instance: CommandInstance,
-  context: Context
+  context: ComponentContext,
 ) => {
-  const deployConfig = context.componentConfig.deploy;
+  const deployConfig = context.deploy?.config;
   if (!isOfDeployType(deployConfig, "kubernetes")) {
     throw new Error("cannot run setupKubernetes on non-kubernetes deployments");
   }
 
+  const { id: projectId } = await getProjectInfo(instance);
+  const deploy_tokens = await doGitlabRequest(
+    instance,
+    `projects/${projectId}/deploy_tokens`,
+  );
+
+  if (
+    !deploy_tokens.find(
+      (v: { name: string }) => v.name === "gitlab-deploy-token",
+    )
+  ) {
+    instance.log(
+      "I will setup the 'GitLab Deploy Token', so Kubernetes can pull images from this project.",
+    );
+
+    await doGitlabRequest(
+      instance,
+      `projects/${projectId}/deploy_tokens`,
+      {
+        id: projectId,
+        name: "gitlab-deploy-token",
+        scopes: ["read_registry"],
+      },
+      "POST",
+    );
+  }
+
   const fullName = getFullKubernetesClusterName(deployConfig.cluster);
   instance.log(`cluster: ${fullName}`);
 
@@ -32,35 +63,42 @@ export const setupKubernetes = async (
   // we name the service account and the role and the role binding with the same name
   // we currently create one per component to better separate them
   instance.log("ensuring service accounts...");
-  const serviceAccountName = `cl-${context.componentName}-deploy`;
-  const KUBE_URL =
-    await $`TERM=dumb kubectl cluster-info | grep -E 'Kubernetes master|Kubernetes control plane' | awk '/http/ {print $NF}'`.then(
-      (s) => s.stdout.trim()
-    );
+  const serviceAccountName = `cl-${context.name}-deploy`;
+  const KUBE_URL = await exec(
+    `TERM=dumb kubectl cluster-info | grep -E 'Kubernetes master|Kubernetes control plane' | awk '/http/ {print $NF}'`,
+  ).then((s) => s.stdout.trim());
 
   // first upsert service acount in the ns
   try {
-    await $`kubectl delete serviceaccount --namespace ${namespace} ${serviceAccountName}`;
-    await $`kubectl delete rolebinding --namespace ${namespace} ${serviceAccountName}`;
-    await $`kubectl delete role --namespace ${namespace} ${serviceAccountName}`;
+    await exec(
+      `kubectl delete serviceaccount --namespace ${namespace} ${serviceAccountName}`,
+    );
+    await exec(
+      `kubectl delete rolebinding --namespace ${namespace} ${serviceAccountName}`,
+    );
+    await exec(
+      `kubectl delete role --namespace ${namespace} ${serviceAccountName}`,
+    );
   } catch (e) {
     // ignore
   }
 
-  await $`kubectl create serviceaccount --namespace ${namespace} ${serviceAccountName}`;
+  await exec(
+    `kubectl create serviceaccount --namespace ${namespace} ${serviceAccountName}`,
+  );
 
   // upsert role in the ns
 
-  await $`cat <<EOF | kubectl apply -f -
+  await exec(`cat <<EOF | kubectl apply -f -
 kind: Role
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   namespace: ${namespace}
   name: ${serviceAccountName}
 rules:
-- apiGroups: ["", "extensions", "apps", "networking.k8s.io", "batch"]
-  resources: ["deployments", "replicasets", "statefulsets", "pods", "secrets", "configmaps", "services", "ingresses", "serviceaccounts", "jobs", "cronjobs"]
-  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] # You can also use ["*"]
+- apiGroups: ["", "extensions", "apps", "networking.k8s.io", "batch", "autoscaling", "rbac.authorization.k8s.io","snapshot.storage.k8s.io"]
+  resources: ["deployments", "replicasets", "statefulsets", "pods", "secrets", "configmaps", "services", "ingresses", "serviceaccounts", "roles", "rolebindings", "jobs", "cronjobs", "horizontalpodautoscalers", "persistentvolumeclaims", "volumesnapshots"]
+  verbs: ["*"]
 ---
 kind: RoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
@@ -76,20 +114,29 @@ roleRef:
   name: ${serviceAccountName}
   apiGroup: rbac.authorization.k8s.io
 EOF
-`;
+`);
 
-  // get token name
-  const tokenName =
-    await $`kubectl get serviceaccount --namespace ${namespace} ${serviceAccountName} -o jsonpath='{.secrets[0].name}'`;
+  const secretName = `${serviceAccountName}-secret`;
+  // create token for the service account
+  await exec(`
+kubectl apply -f - <<EOF
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ${secretName}
+  namespace: ${namespace}
+  annotations:
+    kubernetes.io/service-account.name: ${serviceAccountName}
+type: kubernetes.io/service-account-token
+EOF
+  `);
 
-  const KUBE_CA_PEM =
-    await $`kubectl get secret ${tokenName} --namespace ${namespace} -o jsonpath="{['data']['ca\\.crt']}"`.then(
-      (c) => c.stdout.trim()
-    );
-  const KUBE_TOKEN =
-    await $`kubectl get secret ${tokenName} --namespace ${namespace} -o jsonpath="{['data']['token']}" | base64 --decode`.then(
-      (c) => c.stdout.trim()
-    );
+  const KUBE_CA_PEM = await exec(
+    `kubectl get secret ${secretName} --namespace ${namespace} -o jsonpath="{['data']['ca\\.crt']}"`,
+  ).then((c) => c.stdout.trim());
+  const KUBE_TOKEN = await exec(
+    `kubectl get secret ${secretName} --namespace ${namespace} -o jsonpath="{['data']['token']}" | base64 --decode`,
+  ).then((c) => c.stdout.trim());
 
   const vars = {
     KUBE_TOKEN,
@@ -97,6 +144,17 @@ EOF
     KUBE_URL,
   };
 
+  const missing = Object.entries(vars).filter((e) => !e[1]);
+  if (missing.length > 0) {
+    throw new Error(
+      "could not setup credentials. Missing vars: " +
+        missing.map((m) => m[0]).join(", ") +
+        ". Check whether your local kubectl is still working for '" +
+        fullName +
+        "'",
+    );
+  }
+
   instance.log("service accounts created / updated!");
 
   instance.log("");
@@ -105,8 +163,9 @@ EOF
   await upsertAllVariables(
     instance,
     vars,
-    context.environment.shortName,
-    context.componentName
+    context.env,
+    context.name,
+    false, // no backup
   );
   instance.log("done!");
 };

package/src/apps/cli/commands/project/setup/setupTopic.ts

@@ -0,0 +1,17 @@
+import type { CommandInstance } from "vorpal";
+import { doGitlabRequest, getProjectInfo } from "../../../../../utils/gitlab";
+
+const catladderTopic = "catladder";
+
+export const setupTopic = async (instance: CommandInstance) => {
+  const { id: projectId } = await getProjectInfo(instance);
+
+  const { topics } = await doGitlabRequest(instance, `projects/${projectId}`);
+
+  await doGitlabRequest(
+    instance,
+    `projects/${projectId}`,
+    { topics: [catladderTopic, ...topics] },
+    "PUT",
+  );
+};

package/src/apps/cli/commands/project/utils/autocompletions.ts

@@ -17,7 +17,7 @@ export const envAndComponents = async () => {
 
   return allEnvAndcomponents.reduce<string[]>(
     (acc, { env, componentName }) => [...acc, env + ":" + componentName],
-    []
+    [],
   );
 };
 

package/src/apps/cli/commands/project/utils/ensureCluster.ts

@@ -13,17 +13,15 @@ import {
 export default async function (envComponent: string) {
   const { env, componentName } = parseChoice(envComponent);
   const context = await getPipelineContextByChoice(env, componentName);
-  if (!isOfDeployType(context.componentConfig.deploy, "kubernetes")) {
+  if (!isOfDeployType(context.deploy?.config, "kubernetes")) {
     throw new Error("can't ensure cluster for non-kubernetes deployments");
   }
-  const cluster = getFullKubernetesClusterName(
-    context.componentConfig.deploy.cluster
-  );
+  const cluster = getFullKubernetesClusterName(context.deploy?.config.cluster);
   const connectedClusterName = await getCurrentConnectedClusterName();
 
   if (cluster !== connectedClusterName) {
     this.log(
-      `you are currently connected to cluster '${connectedClusterName}'`
+      `you are currently connected to cluster '${connectedClusterName}'`,
     );
     this.log(`but the project requires cluster '${cluster}'`);
     const { shouldContinue } = await this.prompt({

package/src/apps/cli/commands/project/utils/ensureNamespace.ts

@@ -1,31 +1,20 @@
-import { Context, getKubernetesNamespace } from "@catladder/pipeline";
-import { V1Namespace, V1ObjectMeta } from "@kubernetes/client-node";
+import type { ComponentContext } from "@catladder/pipeline";
+import { getKubernetesNamespace } from "@catladder/pipeline";
+import type { V1ObjectMeta } from "@kubernetes/client-node";
+import { V1Namespace } from "@kubernetes/client-node";
 import { getk8sApi } from "../../../../../k8sApi";
 
-export default async function (context: Context) {
-  const namespace = getKubernetesNamespace(
-    context.fullConfig,
-    context.environment.shortName
-  );
+export default async function (context: ComponentContext) {
+  const namespace = getKubernetesNamespace(context.fullConfig, context.env);
   const namespaceBody = new V1Namespace();
   const metadata: V1ObjectMeta = {
     name: namespace,
     labels: {
       customerName: context.fullConfig.customerName,
       appName: context.fullConfig.appName,
-      environment: context.environment.shortName,
+      environment: context.env,
       components: Object.keys(context.fullConfig.components).join("_"), // limited chars available...
-      buildTypes: Object.values(context.fullConfig.components)
-        .map((config) => config.build.type)
-        .join("_"), // limited chars available...
-      ...Object.fromEntries(
-        Object.entries(context.fullConfig.components).map(
-          ([componentName, config]) => [
-            "buildType_" + componentName,
-            config.build.type,
-          ]
-        )
-      ),
+
       ...(context.fullConfig.meta?.labels ?? {}),
     },
   };
@@ -42,7 +31,7 @@ export default async function (context: Context) {
       undefined,
       undefined,
       undefined,
-      { headers: { "content-type": "application/merge-patch+json" } } // see https://github.com/kubernetes-client/javascript/issues/443
+      { headers: { "content-type": "application/merge-patch+json" } }, // see https://github.com/kubernetes-client/javascript/issues/443
     ); // update meta data
   } catch (e) {
     if (e.response?.body?.reason === "NotFound") {

package/src/apps/cli/commands/project/utils/showProjectBanner.ts

@@ -1,4 +1,4 @@
-import Vorpal from "vorpal";
+import type Vorpal from "vorpal";
 import { getProjectConfig } from "../../../../../config/getProjectConfig";
 
 export const showProjectBanner = async (vorpal: Vorpal) => {

package/src/apps/cli/commands/theStuffThatReallyMatters/index.ts

@@ -1,6 +1,6 @@
 import { spawn } from "child-process-promise";
 import fetch from "node-fetch";
-import Vorpal from "vorpal";
+import type Vorpal from "vorpal";
 
 export default async (vorpal: Vorpal) => {
   vorpal.command("dadjoke", "something for jonas.").action(async function () {
@@ -19,7 +19,7 @@ export default async (vorpal: Vorpal) => {
   vorpal
     .command("starwars", "Long time ago... in a galaxy far far away...")
     .action(async function () {
-      starwarsPromise = spawn("telnet", ["towel.blinkenlights.nl"], {
+      starwarsPromise = spawn("telnet", ["2001:7b8:666:ffff::1:42"], {
         stdio: ["pipe", "inherit", "pipe"],
       });
       await starwarsPromise;

package/src/apps/cli/utils/getGoogleAuthUserNumber.ts

@@ -8,7 +8,7 @@ const KEY = "googleAuthUserNumber";
 export const getGoogleAuthUserNumber = async function () {
   if (!(await hasPreference(KEY))) {
     this.log(
-      "Please type in your google auth user number (0 if you have only one google account or maybe 1 if you have multiple)"
+      "Please type in your google auth user number (0 if you have only one google account or maybe 1 if you have multiple)",
     );
 
     const { authUserNumber } = await this.prompt({

package/src/apps/cli/verify/index.ts

@@ -1,10 +1,6 @@
-import Vorpal from "vorpal";
-import {
-  getGitlabCi,
-  getProjectConfig,
-} from "../../../config/getProjectConfig";
+import type Vorpal from "vorpal";
+
 import { hasGitlabToken, setupGitlabToken } from "../../../utils/gitlab";
-import { isV2, migrateV2 } from "./migration/fromv2";
 
 export const verify = async (vorpal: Vorpal) => {
   // check if has all settings
@@ -15,16 +11,4 @@ export const verify = async (vorpal: Vorpal) => {
     });
     vorpal.exec("setup-gitlab-token");
   }
-
-  const projectConfig = await getProjectConfig();
-  try {
-    const gitlabCi = getGitlabCi();
-
-    if (gitlabCi && !projectConfig && (await isV2())) {
-      vorpal.log("no project config, needs migration");
-      await migrateV2(vorpal);
-    }
-  } catch (e) {
-    // no gitroot, ignore
-  }
 };

package/src/catci.ts

@@ -0,0 +1,3 @@
+import { runCatCi } from "./apps/catci/catci";
+
+runCatCi();

package/src/catenv.ts

@@ -1,4 +1,20 @@
 import catenv from "./apps/catenv/catenv";
-import { $ } from "zx";
-$.verbose = false;
-catenv();
+
+import { parseChoice } from "./config/parseChoice";
+
+const args = process.argv.slice(2);
+
+const helpFlags = ["--help", "-h", "help"];
+if (args.some((arg) => helpFlags.includes(arg))) {
+  const docLink =
+    "https://git.panter.ch/catladder/catladder/-/blob/main/docs/1_VARS.md";
+  console.log(
+    `\nUsage: catenv [env|env:component]\n\nEnv variable and catenv documentation:\n${docLink}`,
+  );
+  process.exit(0);
+}
+
+catenv(args[0] ? parseChoice(args[0]) : null).then(() => {
+  // we have to exit manually, because we have some file watches
+  process.exit();
+});

package/src/cli.ts

@@ -1,9 +1,3 @@
-import updateNotifier from "update-notifier";
 import cli from "./apps/cli/cli";
-import packageInfos from "./packageInfos";
-import { $ } from "zx";
-$.verbose = false;
-updateNotifier({
-  pkg: packageInfos,
-}).notify();
+
 cli();

package/src/config/constants.ts

@@ -1,4 +1,2 @@
 export const GRAFANA_PROXY_LOCAL_PORT = 9090;
 export const GRAFANA_PROXY_TARGET_PORT = 9090;
-
-export const GOOGLE_CLOUD_SQL_PASS_PATH = "cloud-sql-proxy-credentials";

package/src/config/getProjectConfig.ts

@@ -1,43 +1,52 @@
+import type {
+  Config,
+  EnvironmentEnvVars,
+  VariableValue,
+} from "@catladder/pipeline";
 import {
-  readConfigSync,
+  readConfig,
   getAllEnvs,
   getEnvironment as _getEnvironment,
-  createContext,
+  createComponentContext,
+  getSecretVarName,
 } from "@catladder/pipeline";
 
-import { CommandInstance } from "vorpal";
-import { getAllVariables } from "../utils/gitlab";
-import memoizee from "memoizee";
+import type { CommandInstance } from "vorpal";
+import { getAllVariables, getVariableValueByRawName } from "../utils/gitlab";
+
 import { getGitRoot } from "../utils/projects";
-import { readYaml } from "../utils/files";
-// currently cant change
-
-export const getProjectConfig = memoizee(
-  async () => {
-    try {
-      const gitRoot = await getGitRoot();
-      return readConfigSync(gitRoot);
-    } catch (e) {
-      // ignore
-      return null;
-    }
-  },
-  { promise: true }
-);
-
-export const reloadConfig = () => getProjectConfig.clear();
-
-export const getGitlabCiFilePath = async () => {
+
+import { watch } from "fs";
+
+export { parseChoice } from "./parseChoice";
+
+let currentConfig: Config | null = null;
+
+// reload the config on change
+const reloadConfigAndObserve = async () => {
   const gitRoot = await getGitRoot();
-  return gitRoot + "/.gitlab-ci.yml";
+  if (!gitRoot) {
+    return;
+  }
+  const result = await readConfig(gitRoot);
+  if (!result) {
+    // can't do anything, there is no config
+    return;
+  }
+  const { config, path } = result;
+  const watcher = watch(path, () => {
+    watcher.close();
+    reloadConfigAndObserve();
+  });
+  currentConfig = config;
 };
-export const getGitlabCi = async <T = Record<string, any>>() => {
-  try {
-    return readYaml(await getGitlabCiFilePath()) as Promise<T>;
-  } catch (e) {
-    // ignore
-    return null;
+
+export const getProjectConfig = async () => {
+  if (!currentConfig) {
+    // initially
+    await reloadConfigAndObserve();
   }
+  return currentConfig as Config;
 };
 
 export const getProjectComponents = async () => {
@@ -46,17 +55,16 @@ export const getProjectComponents = async () => {
   return Object.keys(config.components);
 };
 
-export const parseChoice = (envComponent: string) => {
-  const [env, componentName] = envComponent.split(":");
-  return { env, componentName };
-};
-
 export const getPipelineContextByChoice = async (
   env: string,
-  componentName: string
+  componentName: string,
 ) => {
   const config = await getProjectConfig();
-  return createContext(config, componentName, env);
+  return await createComponentContext({
+    config,
+    componentName,
+    env,
+  });
 };
 export const getAllComponentsWithAllEnvsFlat = async (): Promise<
   Array<{ env: string; componentName: string }>
@@ -66,7 +74,7 @@ export const getAllComponentsWithAllEnvsFlat = async (): Promise<
     return [];
   }
   return Object.keys(config.components).flatMap((componentName) =>
-    getAllEnvs(config, componentName).map((env) => ({ env, componentName }))
+    getAllEnvs(config, componentName).map((env) => ({ env, componentName })),
   );
 };
 
@@ -82,7 +90,7 @@ export const getAllComponentsWithAllEnvsHierarchical = async (): Promise<{
     Object.keys(config.components).map((componentName) => [
       componentName,
       getAllEnvs(config, componentName),
-    ])
+    ]),
   );
 };
 
@@ -91,46 +99,87 @@ export const getAllPipelineContexts = async () => {
     (await getAllComponentsWithAllEnvsFlat())
       .filter((c) => c.env !== "local")
       .map(({ env, componentName }) =>
-        getPipelineContextByChoice(env, componentName)
-      )
+        getPipelineContextByChoice(env, componentName),
+      ),
   );
 };
 
 export const getEnvironment = async (env: string, componentName: string) => {
   const config = await getProjectConfig();
-  return _getEnvironment(config, componentName, env);
+
+  return _getEnvironment({ config, componentName, env });
 };
 
-const resolveSecrets = async (
+export const getGitlabVar = async (
   vorpal: CommandInstance,
-  allEnvVars: Record<string, string>
+  env: string,
+  componentName: string,
+  variableName: string,
 ) => {
+  const rawVariableName = getSecretVarName(env, componentName, variableName);
+  return await getVariableValueByRawName(vorpal, rawVariableName);
+};
+
+const resolveSecrets = async (
+  vorpal: CommandInstance | null,
+  varSets: EnvironmentEnvVars[],
+): Promise<Record<string, string>> => {
   const allVariablesInGitlab = await getAllVariables(vorpal);
 
   return Object.fromEntries(
-    Object.entries(allEnvVars).map(([key, value]) => {
-      const containsSecret = String(value)?.includes?.("$CL_");
-      if (containsSecret) {
-        for (const variable of allVariablesInGitlab) {
-          value = value.replace(
-            new RegExp("\\$" + variable.key, "g"),
-            variable.value
-          );
-        }
-        return [key, value];
-      }
-      return [key, value];
-    })
+    varSets.flatMap(({ envVars, secretEnvVarKeys }) =>
+      Object.entries(envVars)
+        .filter(
+          ([key, value]) =>
+            value !== undefined &&
+            value !== null &&
+            !secretEnvVarKeys.find((k) => k.key === key)?.hidden,
+        )
+        .map(([key, value]) => [
+          key,
+          allVariablesInGitlab.reduce(
+            (acc, curr) =>
+              acc.replace(new RegExp("\\$" + curr.key, "g"), curr.value),
+            `${value}`,
+          ),
+        ]),
+    ),
   );
 };
 
-export const getEnvVars = async (
-  vorpal: CommandInstance,
+export const getEnvVarsResolved = async (
+  vorpal: CommandInstance | null,
   env: string,
-  componentName: string
+  componentName: string | null,
 ) => {
+  if (!componentName) {
+    return {};
+  }
+
   const envionment = await getEnvironment(env, componentName);
+
   // in the pipeline the secrets alreadyy exists  and bash will expand them
   // but here we need to manually load them
-  return resolveSecrets(vorpal, envionment.envVars);
+  return resolveSecrets(vorpal, [
+    {
+      envVars: envionment.envVars,
+      secretEnvVarKeys: envionment.secretEnvVarKeys,
+    },
+  ]);
+};
+
+/**
+ *
+ * is used to get job only vars that should also be editable locally with catladder.
+ */
+export const getJobOnlyEnvVarsResolved = async (
+  vorpal: CommandInstance,
+  env: string,
+  componentName: string,
+) => {
+  const envionment = await getEnvironment(env, componentName);
+  return resolveSecrets(vorpal, [
+    envionment.jobOnlyVars.build,
+    envionment.jobOnlyVars.deploy,
+  ]);
 };

package/src/config/parseChoice.ts

@@ -0,0 +1,4 @@
+export const parseChoice = (envComponent: string) => {
+  const [env, componentName] = envComponent.split(":").map((x) => x || null);
+  return { env: env as string, componentName };
+};

package/src/gcloud/artifactsRegistry.ts

@@ -0,0 +1,16 @@
+import type { Gcloudregion } from "@catladder/pipeline";
+import { exec } from "child-process-promise";
+
+export const upsertGcloudArtifactsRegistry = async (config: {
+  projectId: string;
+  region: Gcloudregion;
+}) => {
+  try {
+    await exec(
+      `gcloud artifacts repositories create catladder-deploy --project="${config.projectId}" --repository-format=docker --location=${config.region}`,
+    );
+  } catch (e) {
+    // probably already exists
+    //
+  }
+};