@catladder/cli 1.105.1 → 1.106.0

package/package.json

@@ -24,7 +24,7 @@
     "node": ">=12.0.0"
   },
   "devDependencies": {
-    "@catladder/pipeline": "1.105.1",
+    "@catladder/pipeline": "1.106.0",
     "@kubernetes/client-node": "^0.16.2",
     "@tsconfig/node14": "^1.0.1",
     "@types/common-tags": "^1.8.0",
@@ -57,5 +57,5 @@
     "typescript": "^4.5.4",
     "vorpal": "^1.12.0"
   },
-  "version": "1.105.1"
+  "version": "1.106.0"
 }

package/src/apps/catenv/printVariables.ts

@@ -13,7 +13,7 @@ const getAllVariablesToPrint = async (config: Config, choice?: Choice) => {
     choice
   );
 
-  let variables = {};
+  let variables: Record<string, string> = {};
   if (currentComponent) {
     variables = await getEnvVarsResolved(null, env, currentComponent);
   } else {
@@ -23,6 +23,7 @@ const getAllVariablesToPrint = async (config: Config, choice?: Choice) => {
     variables = await Object.keys(config.components).reduce(
       async (acc, componentName) => {
         const subappvars = await getEnvVarsResolved(null, env, componentName);
+        delete subappvars["_ALL_ENV_VAR_KEYS"];
         return {
           ...(await acc),
           ...subappvars,
@@ -38,6 +39,7 @@ const getAllVariablesToPrint = async (config: Config, choice?: Choice) => {
       {}
     );
   }
+
   return variables;
 };
 

package/src/apps/catenv/writeDotEnvFiles.ts

@@ -32,6 +32,7 @@ export const writeDotEnvFiles = async (config: Config, choice?: Choice) => {
 
   for (const componentName of componentsToActuallyWriteDotEnvNow) {
     const variables = await getEnvVarsResolved(null, env, componentName);
+    delete variables["_ALL_ENV_VAR_KEYS"];
     const componentDir = getComponentFullPath(gitRoot, config, componentName);
     const filePath = join(componentDir, ".env");
     // many .dotenv don't like multiline values, so we sanitize them here

package/src/apps/cli/commands/cloudSQL/commandRestoreDb.ts

@@ -1,29 +1,8 @@
 import { spawn } from "child-process-promise";
-import type Vorpal from "vorpal";
 
-const createProxy = async (instance: string, port: number) => {
-  const proxy = spawn(
-    `cloud_sql_proxy -instances ${instance}=tcp:${port}`,
-    [],
-    { shell: "bash" }
-  );
-  // wait until it starts
-  await spawn(
-    `echo -n "Waiting for proxy"
-      until echo > /dev/tcp/localhost/${port}; do
-        sleep 0.2
-        echo -n "."
-      done 2>/dev/null`,
-    [],
-    { shell: "bash" }
-  );
-  const stop = () => proxy.childProcess.kill();
-  process.on("beforeExit", stop);
-
-  return {
-    stop,
-  };
-};
+import type Vorpal from "vorpal";
+import type { CloudSqlBackgroundProxy } from "../../../../gcloud/cloudSql/startProxy";
+import { startCloudSqlProxyInBackground } from "../../../../gcloud/cloudSql/startProxy";
 
 export default async (vorpal: Vorpal) =>
   vorpal
@@ -39,9 +18,9 @@ export default async (vorpal: Vorpal) =>
         message: "Source instance (connection string or 'local')? 🤔 ",
       });
 
-      let sourceProxy: { stop: () => void };
+      let sourceProxy: CloudSqlBackgroundProxy;
 
-      let targetProxy: { stop: () => void };
+      let targetProxy: CloudSqlBackgroundProxy;
 
       let sourcePort: number;
       let targetPort: number;
@@ -57,7 +36,10 @@ export default async (vorpal: Vorpal) =>
         sourcePort = sourceLocalPort;
       } else {
         sourcePort = 54399;
-        sourceProxy = await createProxy(sourceInstance, sourcePort);
+        sourceProxy = await startCloudSqlProxyInBackground({
+          instanceName: sourceInstance,
+          localPort: sourcePort,
+        });
       }
 
       const { sourceUsername } = await this.prompt({
@@ -86,7 +68,7 @@ export default async (vorpal: Vorpal) =>
         type: "input",
         name: "targetInstance",
 
-        message: "Targe INSTANCE (connection string or 'local')? 🤔  ",
+        message: "Target INSTANCE (connection string or 'local')? 🤔  ",
       });
 
       if (targetInstance === "local") {
@@ -100,7 +82,10 @@ export default async (vorpal: Vorpal) =>
         targetPort = targetLocalPort;
       } else {
         targetPort = 54499;
-        targetProxy = await createProxy(targetInstance, targetPort);
+        targetProxy = await startCloudSqlProxyInBackground({
+          instanceName: targetInstance,
+          localPort: targetPort,
+        });
       }
 
       const { targetUsername } = await this.prompt({

package/src/apps/cli/commands/project/commandCloudSqlProxy.ts

@@ -3,7 +3,6 @@ import {
   createKubernetesCloudsqlBaseValues,
   isOfDeployType,
 } from "@catladder/pipeline";
-import { spawn } from "child-process-promise";
 import type Vorpal from "vorpal";
 import type { CommandInstance } from "vorpal";
 import {
@@ -12,6 +11,7 @@ import {
   parseChoice,
 } from "../../../../config/getProjectConfig";
 import { envAndComponents } from "./utils/autocompletions";
+import { startCloudSqlProxyInCurrentShell } from "../../../../gcloud/cloudSql/startProxy";
 
 type ProxyInfo = {
   instanceName: string;
@@ -70,14 +70,11 @@ export default async (vorpal: Vorpal) =>
         `DATABASE_JDBC_URL=jdbc:postgresql://localhost:${localPort}/${DB_NAME}?schema=public&user=${DB_USER}&password=${DB_PASSWORD}`
       );
       this.log("");
-      await spawn(
-        "cloud_sql_proxy",
-        ["-instances", `${instanceName}=tcp:${localPort}`],
-        {
-          stdio: "inherit",
-          shell: true,
-        }
-      );
+
+      await startCloudSqlProxyInCurrentShell({
+        instanceName,
+        localPort,
+      });
     });
 
 const getProxyInfoForKubernetes = async (

package/src/config/getProjectConfig.ts

@@ -1,4 +1,4 @@
-import type { Config } from "@catladder/pipeline";
+import type { Config, EnvironmentEnvVars } from "@catladder/pipeline";
 import {
   readConfigSync,
   getAllEnvs,
@@ -124,24 +124,32 @@ export const getGitlabVar = async (
 
 const resolveSecrets = async (
   vorpal: CommandInstance | null,
-  allEnvVars: Record<string, string>
+  varSets: EnvironmentEnvVars[]
 ) => {
   const allVariablesInGitlab = await getAllVariables(vorpal);
 
   return Object.fromEntries(
-    Object.entries(allEnvVars).map(([key, value]) => {
-      const containsSecret = String(value)?.includes?.("$CL_");
-      if (containsSecret) {
-        for (const variable of allVariablesInGitlab) {
-          value = value.replace(
-            new RegExp("\\$" + variable.key, "g"),
-            variable.value
-          );
-        }
-        return [key, value];
-      }
-      return [key, value];
-    })
+    varSets.flatMap((set) =>
+      Object.entries(set.envVars)
+        .map(([key, value]) => {
+          const secretKey = set.secretEnvVarKeys.find((k) => k.key === key);
+
+          if (secretKey) {
+            if (secretKey.hidden) {
+              return null;
+            }
+            for (const variable of allVariablesInGitlab) {
+              value = value.replace(
+                new RegExp("\\$" + variable.key, "g"),
+                variable.value
+              );
+            }
+            return [key, value];
+          }
+          return [key, value];
+        })
+        .filter(Boolean)
+    )
   );
 };
 
@@ -155,9 +163,15 @@ export const getEnvVarsResolved = async (
   }
   try {
     const envionment = await getEnvironment(env, componentName);
+
     // in the pipeline the secrets alreadyy exists  and bash will expand them
     // but here we need to manually load them
-    return resolveSecrets(vorpal, envionment.envVars);
+    return resolveSecrets(vorpal, [
+      {
+        envVars: envionment.envVars,
+        secretEnvVarKeys: envionment.secretEnvVarKeys,
+      },
+    ]);
   } catch (e) {
     // env is disabled
     return {};
@@ -175,10 +189,10 @@ export const getJobOnlyEnvVarsResolved = async (
 ) => {
   try {
     const envionment = await getEnvironment(env, componentName);
-    return resolveSecrets(vorpal, {
-      ...envionment.jobOnlyVars.build.envVars,
-      ...envionment.jobOnlyVars.deploy.envVars,
-    });
+    return resolveSecrets(vorpal, [
+      envionment.jobOnlyVars.build,
+      envionment.jobOnlyVars.deploy,
+    ]);
   } catch (e) {
     // env is disabled
     return {};

package/src/gcloud/cloudSql/startProxy.ts

@@ -0,0 +1,74 @@
+import { spawn } from "child-process-promise";
+import commandExists from "command-exists-promise";
+
+export const ERROR_NOT_INSTALLED = "cloud-sql-proxy not installed";
+
+export type CloudSqlBackgroundProxy = {
+  stop: () => void;
+};
+
+export type CloudSqlProxyOptions = {
+  instanceName: string;
+  localPort: number;
+};
+
+const getProxyCommandSpawnArgs = async ({
+  localPort,
+  instanceName,
+}: CloudSqlProxyOptions) => {
+  const commandString = (await commandExists("cloud-sql-proxy"))
+    ? `cloud-sql-proxy --port ${localPort} ${instanceName}`
+    : (await commandExists(
+        "cloud_sql_proxy" // v1
+      ))
+    ? `cloud_sql_proxy -instances ${instanceName}=tcp:${localPort}`
+    : null;
+  if (!commandString) {
+    throw new Error(ERROR_NOT_INSTALLED);
+  }
+
+  const [cmd, ...args] = commandString.split(" ");
+  return { cmd, args };
+};
+
+export const startCloudSqlProxyInCurrentShell = async (
+  opts: CloudSqlProxyOptions
+) => {
+  const { cmd, args } = await getProxyCommandSpawnArgs(opts);
+
+  await spawn(cmd, args, {
+    stdio: "inherit",
+    shell: true,
+  });
+};
+
+export const startCloudSqlProxyInBackground = async (
+  opts: CloudSqlProxyOptions
+): Promise<CloudSqlBackgroundProxy> => {
+  const { cmd, args } = await getProxyCommandSpawnArgs(opts);
+
+  const proxyPromise = spawn(cmd, args, { shell: "bash" });
+
+  // wait until it starts
+  await spawn(
+    `echo -n "Waiting for proxy"
+      until echo > /dev/tcp/localhost/${opts.localPort}; do
+        sleep 0.2
+        echo -n "."
+      done 2>/dev/null`,
+    [],
+    { shell: "bash" }
+  );
+  const stop = () => {
+    proxyPromise.catch(() => {
+      // ignore
+    });
+    proxyPromise.childProcess.kill();
+  };
+  // stop if catladder i stopped
+  process.on("beforeExit", stop);
+
+  return {
+    stop,
+  };
+};

package/src/types/child-process-promise.d.ts

@@ -13,7 +13,7 @@ declare module "child-process-promise" {
   type Stdio = "inherit" | "pipe";
   type SpawnOptions = {
     stdio?: Stdio | Stdio[];
-    shell?: boolean;
+    shell?: boolean | string;
     env?: Record<string, string>;
   };
 
@@ -31,6 +31,7 @@ declare module "child-process-promise" {
   ): Promise<unknown> & {
     childProcess: {
       stdout: ReadStream;
+      kill: () => void;
     };
   };
 }