@catladder/cli 0.0.0-node16-ddfae617 → 0.0.0-pipeline-variables-ed2d2565

package/src/apps/cli/commands/project/setup/setupKubernetes.ts

@@ -1,10 +1,10 @@
+import type { Context } from "@catladder/pipeline";
 import {
-  Context,
   getFullKubernetesClusterName,
   isOfDeployType,
 } from "@catladder/pipeline";
-import { CommandInstance } from "vorpal";
-import { $ } from "zx";
+import type { CommandInstance } from "vorpal";
+import { exec } from "child-process-promise";
 import { connectToCluster } from "../../../../../utils/cluster";
 import { upsertAllVariables } from "../../../../../utils/gitlab";
 import ensureNamespace from "../utils/ensureNamespace";
@@ -33,34 +33,41 @@ export const setupKubernetes = async (
   // we currently create one per component to better separate them
   instance.log("ensuring service accounts...");
   const serviceAccountName = `cl-${context.componentName}-deploy`;
-  const KUBE_URL =
-    await $`TERM=dumb kubectl cluster-info | grep -E 'Kubernetes master|Kubernetes control plane' | awk '/http/ {print $NF}'`.then(
-      (s) => s.stdout.trim()
-    );
+  const KUBE_URL = await exec(
+    `TERM=dumb kubectl cluster-info | grep -E 'Kubernetes master|Kubernetes control plane' | awk '/http/ {print $NF}'`
+  ).then((s) => s.stdout.trim());
 
   // first upsert service acount in the ns
   try {
-    await $`kubectl delete serviceaccount --namespace ${namespace} ${serviceAccountName}`;
-    await $`kubectl delete rolebinding --namespace ${namespace} ${serviceAccountName}`;
-    await $`kubectl delete role --namespace ${namespace} ${serviceAccountName}`;
+    await exec(
+      `kubectl delete serviceaccount --namespace ${namespace} ${serviceAccountName}`
+    );
+    await exec(
+      `kubectl delete rolebinding --namespace ${namespace} ${serviceAccountName}`
+    );
+    await exec(
+      `kubectl delete role --namespace ${namespace} ${serviceAccountName}`
+    );
   } catch (e) {
     // ignore
   }
 
-  await $`kubectl create serviceaccount --namespace ${namespace} ${serviceAccountName}`;
+  await exec(
+    `kubectl create serviceaccount --namespace ${namespace} ${serviceAccountName}`
+  );
 
   // upsert role in the ns
 
-  await $`cat <<EOF | kubectl apply -f -
+  await exec(`cat <<EOF | kubectl apply -f -
 kind: Role
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   namespace: ${namespace}
   name: ${serviceAccountName}
 rules:
-- apiGroups: ["", "extensions", "apps", "networking.k8s.io", "batch", "autoscaling"]
-  resources: ["deployments", "replicasets", "statefulsets", "pods", "secrets", "configmaps", "services", "ingresses", "serviceaccounts", "jobs", "cronjobs", "horizontalpodautoscalers"]
-  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] # You can also use ["*"]
+- apiGroups: ["", "extensions", "apps", "networking.k8s.io", "batch", "autoscaling", "rbac.authorization.k8s.io","snapshot.storage.k8s.io"]
+  resources: ["deployments", "replicasets", "statefulsets", "pods", "secrets", "configmaps", "services", "ingresses", "serviceaccounts", "roles", "rolebindings", "jobs", "cronjobs", "horizontalpodautoscalers", "persistentvolumeclaims", "volumesnapshots"]
+  verbs: ["*"]
 ---
 kind: RoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
@@ -76,20 +83,29 @@ roleRef:
   name: ${serviceAccountName}
   apiGroup: rbac.authorization.k8s.io
 EOF
-`;
+`);
 
-  // get token name
-  const tokenName =
-    await $`kubectl get serviceaccount --namespace ${namespace} ${serviceAccountName} -o jsonpath='{.secrets[0].name}'`;
+  const secretName = `${serviceAccountName}-secret`;
+  // create token for the service account
+  await exec(`
+kubectl apply -f - <<EOF
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ${secretName}
+  namespace: ${namespace}
+  annotations:
+    kubernetes.io/service-account.name: ${serviceAccountName}
+type: kubernetes.io/service-account-token
+EOF
+  `);
 
-  const KUBE_CA_PEM =
-    await $`kubectl get secret ${tokenName} --namespace ${namespace} -o jsonpath="{['data']['ca\\.crt']}"`.then(
-      (c) => c.stdout.trim()
-    );
-  const KUBE_TOKEN =
-    await $`kubectl get secret ${tokenName} --namespace ${namespace} -o jsonpath="{['data']['token']}" | base64 --decode`.then(
-      (c) => c.stdout.trim()
-    );
+  const KUBE_CA_PEM = await exec(
+    `kubectl get secret ${secretName} --namespace ${namespace} -o jsonpath="{['data']['ca\\.crt']}"`
+  ).then((c) => c.stdout.trim());
+  const KUBE_TOKEN = await exec(
+    `kubectl get secret ${secretName} --namespace ${namespace} -o jsonpath="{['data']['token']}" | base64 --decode`
+  ).then((c) => c.stdout.trim());
 
   const vars = {
     KUBE_TOKEN,
@@ -97,6 +113,17 @@ EOF
     KUBE_URL,
   };
 
+  const missing = Object.entries(vars).filter((e) => !e[1]);
+  if (missing.length > 0) {
+    throw new Error(
+      "could not setup credentials. Missing vars: " +
+        missing.map((m) => m[0]).join(", ") +
+        ". Check whether your local kubectl is still working for '" +
+        fullName +
+        "'"
+    );
+  }
+
   instance.log("service accounts created / updated!");
 
   instance.log("");
@@ -106,7 +133,8 @@ EOF
     instance,
     vars,
     context.environment.shortName,
-    context.componentName
+    context.componentName,
+    false // no backup
   );
   instance.log("done!");
 };

package/src/apps/cli/commands/project/setup/setupTopic.ts

@@ -0,0 +1,22 @@
+import type { CommandInstance } from "vorpal";
+import { doGitlabRequest, getProjectInfo } from "../../../../../utils/gitlab";
+
+const catladderTopic = "catladder"
+
+export const setupTopic = async (instance: CommandInstance) => {
+  const { id: projectId } = await getProjectInfo(
+    instance
+  );
+
+  const { topics } = await doGitlabRequest(
+    instance,
+    `projects/${projectId}`
+  );
+
+  await doGitlabRequest(
+    instance,
+    `projects/${projectId}`,
+    { topics: [catladderTopic, ...topics] },
+    "PUT"
+  );
+};

package/src/apps/cli/commands/project/utils/ensureNamespace.ts

@@ -1,5 +1,7 @@
-import { Context, getKubernetesNamespace } from "@catladder/pipeline";
-import { V1Namespace, V1ObjectMeta } from "@kubernetes/client-node";
+import type { Context} from "@catladder/pipeline";
+import { getKubernetesNamespace } from "@catladder/pipeline";
+import type { V1ObjectMeta } from "@kubernetes/client-node";
+import { V1Namespace } from "@kubernetes/client-node";
 import { getk8sApi } from "../../../../../k8sApi";
 
 export default async function (context: Context) {

package/src/apps/cli/commands/project/utils/showProjectBanner.ts

@@ -1,4 +1,4 @@
-import Vorpal from "vorpal";
+import type Vorpal from "vorpal";
 import { getProjectConfig } from "../../../../../config/getProjectConfig";
 
 export const showProjectBanner = async (vorpal: Vorpal) => {

package/src/apps/cli/commands/theStuffThatReallyMatters/index.ts

@@ -1,6 +1,6 @@
 import { spawn } from "child-process-promise";
 import fetch from "node-fetch";
-import Vorpal from "vorpal";
+import type Vorpal from "vorpal";
 
 export default async (vorpal: Vorpal) => {
   vorpal.command("dadjoke", "something for jonas.").action(async function () {
@@ -19,7 +19,7 @@ export default async (vorpal: Vorpal) => {
   vorpal
     .command("starwars", "Long time ago... in a galaxy far far away...")
     .action(async function () {
-      starwarsPromise = spawn("telnet", ["towel.blinkenlights.nl"], {
+      starwarsPromise = spawn("telnet", ["2001:7b8:666:ffff::1:42"], {
         stdio: ["pipe", "inherit", "pipe"],
       });
       await starwarsPromise;

package/src/apps/cli/config/writeConfig.ts

@@ -1,9 +1,9 @@
-import { Config } from "@catladder/pipeline";
+import type { Config } from "@catladder/pipeline";
 import { spawn, exec } from "child-process-promise";
 import { writeFile } from "fs-extra";
 import { dump } from "js-yaml";
 import { format } from "prettier";
-import { CommandInstance } from "vorpal";
+import type { CommandInstance } from "vorpal";
 import { getGitRoot } from "../../../utils/projects";
 
 export const writeConfig = async (

package/src/apps/cli/verify/index.ts

@@ -1,4 +1,4 @@
-import Vorpal from "vorpal";
+import type Vorpal from "vorpal";
 import {
   getGitlabCi,
   getProjectConfig,

package/src/apps/cli/verify/migration/fromv2.ts

@@ -1,4 +1,4 @@
-import {
+import type {
   BuildConfig,
   ComponentConfig,
   Config,
@@ -14,7 +14,7 @@ import {
 } from "fs-extra";
 import { isEmpty } from "lodash";
 import { join } from "path";
-import Vorpal from "vorpal";
+import type Vorpal from "vorpal";
 import {
   getGitlabCi,
   getGitlabCiFilePath,
@@ -24,11 +24,8 @@ import { syncBitwarden } from "../../../../utils/passwordstore";
 import { getGitRoot } from "../../../../utils/projects";
 import { writeConfig } from "../../config/writeConfig";
 import { migrateSecrets } from "./migrateSecrets";
-import {
-  detectBuildConfig,
-  isOldInclude,
-  OldGitlabCiFile,
-} from "./oldGitlabCi";
+import type { OldGitlabCiFile } from "./oldGitlabCi";
+import { detectBuildConfig, isOldInclude } from "./oldGitlabCi";
 export const LEGACY_ENVS = [
   "dev-local",
   "dev",
@@ -39,6 +36,7 @@ export const LEGACY_ENVS = [
 
 const arrayToRecord = (arr: { name: string }[]): Record<string, any> => {
   if (!arr) return undefined;
+  if (!Array.isArray(arr)) return arr;
   return Object.fromEntries(arr.map(({ name, ...rest }) => [name, rest]));
 };
 

package/src/apps/cli/verify/migration/migrateSecrets.ts

@@ -1,10 +1,11 @@
-import { Config, getEnvironment } from "@catladder/pipeline";
+import type { Config } from "@catladder/pipeline";
+import { getEnvironment } from "@catladder/pipeline";
 import { load } from "js-yaml";
 import { pick } from "lodash";
-import { CommandInstance } from "vorpal";
+import type { CommandInstance } from "vorpal";
 import { upsertAllVariables } from "../../../../utils/gitlab";
 import { readPass, trashItem } from "../../../../utils/passwordstore";
-import { LEGACY_ENVS } from "./fromv2";
+import type { LEGACY_ENVS } from "./fromv2";
 
 const getPassPath = (newConfig: Config, env: string) => {
   return `${newConfig.customerName}/${newConfig.appName}/${env}/secrets.yml`;
@@ -22,10 +23,19 @@ export const migrateSecrets = async (
     const secrets = load(yamlstring);
 
     Object.keys(newConfig.components).forEach(async (componentName) => {
-      const environment = getEnvironment(newConfig, componentName, newEnv);
+      const environment = await getEnvironment(
+        newConfig,
+        componentName,
+        newEnv
+      );
       await upsertAllVariables(
         vorpal,
-        pick(secrets, environment.secretEnvVarKeys),
+        pick(
+          secrets,
+          environment.secretEnvVarKeys
+            .filter((k) => !k.hidden)
+            .map((k) => k.key)
+        ),
         newEnv,
         componentName
       );

package/src/apps/cli/verify/migration/oldGitlabCi.ts

@@ -1,4 +1,4 @@
-import { BuildConfig } from "@catladder/pipeline";
+import type { BuildConfig } from "@catladder/pipeline";
 
 export type OldGitlabCiFile = {
   variables: {
@@ -43,6 +43,6 @@ export const detectBuildConfig = (
   }
 
   if (firstInclude.file === "rails-kubernetes.yml") {
-    throw new Error("rails is not yet implemented");
+    return "rails";
   }
 };

package/src/catenv.ts

@@ -1,7 +1,10 @@
 import catenv from "./apps/catenv/catenv";
-import { $ } from "zx";
-$.verbose = false;
-catenv().then(() => {
+
+import { parseChoice } from "./config/parseChoice";
+
+const choice = process.argv[2] ? parseChoice(process.argv[2]) : null;
+
+catenv(choice).then(() => {
   // we have to exit manually, because we have some file watches
   process.exit();
 });

package/src/cli.ts

@@ -1,9 +1,3 @@
-import updateNotifier from "update-notifier";
 import cli from "./apps/cli/cli";
-import packageInfos from "./packageInfos";
-import { $ } from "zx";
-$.verbose = false;
-updateNotifier({
-  pkg: packageInfos,
-}).notify();
+
 cli();

package/src/config/constants.ts

@@ -1,4 +1,2 @@
 export const GRAFANA_PROXY_LOCAL_PORT = 9090;
 export const GRAFANA_PROXY_TARGET_PORT = 9090;
-
-export const GOOGLE_CLOUD_SQL_PASS_PATH = "cloud-sql-proxy-credentials";

package/src/config/getProjectConfig.ts

@@ -1,20 +1,22 @@
+import type { Config, EnvironmentEnvVars } from "@catladder/pipeline";
 import {
   readConfigSync,
   getAllEnvs,
   getEnvironment as _getEnvironment,
   createContext,
   getSecretVarName,
-  Config,
 } from "@catladder/pipeline";
 
-import { CommandInstance } from "vorpal";
+import type { CommandInstance } from "vorpal";
 import { getAllVariables, getVariableValueByRawName } from "../utils/gitlab";
 
 import { getGitRoot } from "../utils/projects";
 import { readYaml } from "../utils/files";
 import { watch } from "fs";
 
-let currentConfig: Config = null;
+export { parseChoice } from "./parseChoice";
+
+let currentConfig: Config | null = null;
 
 // reload the config on change
 const reloadConfigAndObserve = async () => {
@@ -37,7 +39,7 @@ export const getProjectConfig = async () => {
     // initially
     await reloadConfigAndObserve();
   }
-  return currentConfig;
+  return currentConfig as Config;
 };
 
 export const getGitlabCiFilePath = async () => {
@@ -59,17 +61,12 @@ export const getProjectComponents = async () => {
   return Object.keys(config.components);
 };
 
-export const parseChoice = (envComponent: string) => {
-  const [env, componentName] = envComponent.split(":");
-  return { env, componentName };
-};
-
 export const getPipelineContextByChoice = async (
   env: string,
   componentName: string
 ) => {
   const config = await getProjectConfig();
-  return createContext(config, componentName, env);
+  return await createContext(config, componentName, env);
 };
 export const getAllComponentsWithAllEnvsFlat = async (): Promise<
   Array<{ env: string; componentName: string }>
@@ -126,35 +123,78 @@ export const getGitlabVar = async (
 };
 
 const resolveSecrets = async (
-  vorpal: CommandInstance,
-  allEnvVars: Record<string, string>
+  vorpal: CommandInstance | null,
+  varSets: EnvironmentEnvVars[]
 ) => {
   const allVariablesInGitlab = await getAllVariables(vorpal);
 
   return Object.fromEntries(
-    Object.entries(allEnvVars).map(([key, value]) => {
-      const containsSecret = String(value)?.includes?.("$CL_");
-      if (containsSecret) {
-        for (const variable of allVariablesInGitlab) {
-          value = value.replace(
-            new RegExp("\\$" + variable.key, "g"),
-            variable.value
-          );
-        }
-        return [key, value];
-      }
-      return [key, value];
-    })
+    varSets.flatMap((set) =>
+      Object.entries(set.envVars)
+        .map(([key, value]) => {
+          const secretKey = set.secretEnvVarKeys.find((k) => k.key === key);
+
+          if (secretKey) {
+            if (secretKey.hidden) {
+              return null;
+            }
+            for (const variable of allVariablesInGitlab) {
+              value = value.replace(
+                new RegExp("\\$" + variable.key, "g"),
+                variable.value
+              );
+            }
+            return [key, value];
+          }
+          return [key, value];
+        })
+        .filter(Boolean)
+    )
   );
 };
 
-export const getEnvVars = async (
+export const getEnvVarsResolved = async (
+  vorpal: CommandInstance | null,
+  env: string,
+  componentName: string | null
+) => {
+  if (!componentName) {
+    return {};
+  }
+  try {
+    const envionment = await getEnvironment(env, componentName);
+
+    // in the pipeline the secrets alreadyy exists  and bash will expand them
+    // but here we need to manually load them
+    return resolveSecrets(vorpal, [
+      {
+        envVars: envionment.envVars,
+        secretEnvVarKeys: envionment.secretEnvVarKeys,
+      },
+    ]);
+  } catch (e) {
+    // env is disabled
+    return {};
+  }
+};
+
+/**
+ *
+ * is used to get job only vars that should also be editable locally with catladder.
+ */
+export const getJobOnlyEnvVarsResolved = async (
   vorpal: CommandInstance,
   env: string,
   componentName: string
 ) => {
-  const envionment = await getEnvironment(env, componentName);
-  // in the pipeline the secrets alreadyy exists  and bash will expand them
-  // but here we need to manually load them
-  return resolveSecrets(vorpal, envionment.envVars);
+  try {
+    const envionment = await getEnvironment(env, componentName);
+    return resolveSecrets(vorpal, [
+      envionment.jobOnlyVars.build,
+      envionment.jobOnlyVars.deploy,
+    ]);
+  } catch (e) {
+    // env is disabled
+    return {};
+  }
 };

package/src/config/parseChoice.ts

@@ -0,0 +1,4 @@
+export const parseChoice = (envComponent: string) => {
+  const [env, componentName] = envComponent.split(":").map((x) => x || null);
+  return { env: env as string, componentName };
+};

package/src/gcloud/artifactsRegistry.ts

@@ -0,0 +1,16 @@
+import type { Gcloudregion } from "@catladder/pipeline";
+import { exec } from "child-process-promise";
+
+export const upsertGcloudArtifactsRegistry = async (config: {
+  projectId: string;
+  region: Gcloudregion;
+}) => {
+  try {
+    await exec(
+      `gcloud artifacts repositories create catladder-deploy --project="${config.projectId}" --repository-format=docker --location=${config.region}`
+    );
+  } catch (e) {
+    // probably already exists
+    //
+  }
+};

package/src/gcloud/cloudRun/getCloudRunDomainSuffix.ts

@@ -0,0 +1,35 @@
+import type { DeployConfigCloudRun } from "@catladder/pipeline";
+import { exec } from "child-process-promise";
+
+const getSuffixFromService = (service: any) => {
+  const fullUrl = service.status.url;
+  const name = service.metadata.name;
+
+  return fullUrl.replace("https://" + name + "-", "");
+};
+export const getCloudRunDomainSuffix = async (config: DeployConfigCloudRun) => {
+  /* google cloud run domains are partially predictable, they have a fixed suffix that depends on the region and project, but we don't know it beforehand
+
+  So what we do is either:
+   - get one service and extract its suffix
+   - create a dummy project, extract the suffix and save that as a gitlab variable
+  
+  
+  */
+
+  const dummyServiceName = "cl-dummy-service-delete-me";
+
+  const existingServices = await exec(
+    `gcloud run services list --format=json  --project="${config.projectId}"  --region=${config.region} --limit=1`
+  ).then((r) => JSON.parse(r.stdout));
+
+  if (existingServices.length > 0) {
+    return getSuffixFromService(existingServices[0]);
+  }
+
+  const result = await exec(
+    `gcloud run deploy ${dummyServiceName} --region=${config.region} --allow-unauthenticated --project ${config.projectId} --image=us-docker.pkg.dev/cloudrun/container/hello --format=json`
+  ).then((r) => JSON.parse(r.stdout));
+
+  return await getSuffixFromService(result);
+};

package/src/gcloud/cloudRun/openCloudRunDashboards.ts

@@ -0,0 +1,22 @@
+import type { Context } from "@catladder/pipeline";
+import { isOfDeployType } from "@catladder/pipeline";
+import type { CommandInstance } from "vorpal";
+import { openGoogleCloudDashboard } from "../openDashboard";
+
+export const openGoogleCloudRunDashboard = async (
+  instance: CommandInstance,
+  context: Context
+) => {
+  if (!isOfDeployType(context.componentConfig.deploy, "google-cloudrun")) {
+    throw new Error("deploy type is not google-cloudrun ");
+  }
+  const { fullName } = context.environment;
+  const { region, projectId } = context.componentConfig.deploy;
+  await openGoogleCloudDashboard(
+    instance,
+    `run/detail/${region}/${fullName}/metrics`,
+    {
+      project: projectId,
+    }
+  );
+};

package/src/gcloud/cloudSql/startProxy.ts

@@ -0,0 +1,74 @@
+import { spawn } from "child-process-promise";
+import commandExists from "command-exists-promise";
+
+export const ERROR_NOT_INSTALLED = "cloud-sql-proxy not installed";
+
+export type CloudSqlBackgroundProxy = {
+  stop: () => void;
+};
+
+export type CloudSqlProxyOptions = {
+  instanceName: string;
+  localPort: number;
+};
+
+const getProxyCommandSpawnArgs = async ({
+  localPort,
+  instanceName,
+}: CloudSqlProxyOptions) => {
+  const commandString = (await commandExists("cloud-sql-proxy"))
+    ? `cloud-sql-proxy --port ${localPort} ${instanceName}`
+    : (await commandExists(
+        "cloud_sql_proxy" // v1
+      ))
+    ? `cloud_sql_proxy -instances ${instanceName}=tcp:${localPort}`
+    : null;
+  if (!commandString) {
+    throw new Error(ERROR_NOT_INSTALLED);
+  }
+
+  const [cmd, ...args] = commandString.split(" ");
+  return { cmd, args };
+};
+
+export const startCloudSqlProxyInCurrentShell = async (
+  opts: CloudSqlProxyOptions
+) => {
+  const { cmd, args } = await getProxyCommandSpawnArgs(opts);
+
+  await spawn(cmd, args, {
+    stdio: "inherit",
+    shell: true,
+  });
+};
+
+export const startCloudSqlProxyInBackground = async (
+  opts: CloudSqlProxyOptions
+): Promise<CloudSqlBackgroundProxy> => {
+  const { cmd, args } = await getProxyCommandSpawnArgs(opts);
+
+  const proxyPromise = spawn(cmd, args, { shell: "bash" });
+
+  // wait until it starts
+  await spawn(
+    `echo -n "Waiting for proxy"
+      until echo > /dev/tcp/localhost/${opts.localPort}; do
+        sleep 0.2
+        echo -n "."
+      done 2>/dev/null`,
+    [],
+    { shell: "bash" }
+  );
+  const stop = () => {
+    proxyPromise.catch(() => {
+      // ignore
+    });
+    proxyPromise.childProcess.kill();
+  };
+  // stop if catladder i stopped
+  process.on("beforeExit", stop);
+
+  return {
+    stop,
+  };
+};

package/src/gcloud/enableServices.ts

@@ -0,0 +1,14 @@
+import { exec } from "child-process-promise";
+
+export const enableGCloudServices = async (
+  services: string[],
+  config: {
+    projectId: string;
+  }
+) => {
+  for (const service of services) {
+    await exec(
+      `gcloud services enable ${service} --project=${config.projectId}`
+    );
+  }
+};