@dataformer/env-service 2.3.0 → 2.4.0

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @dataformer/env-service
 
+## 2.4.0
+
+### Minor Changes
+
+- fix circular dependency with LoggerService, EnvService, and SecretManagerClient
+
 ## 2.3.0
 
 ### Minor Changes

package/dist/index.js

@@ -1,24 +1,60 @@
 // src/index.ts
 import readline from "readline";
 import path from "path";
-import { SecretManagerClient } from "@dataformer/secret-manager-client";
-var secretManager = null;
+import { SecretManagerServiceClient } from "@google-cloud/secret-manager";
+import { execSync } from "child_process";
+function getGcpProjectIdFromGcloud() {
+  try {
+    const cmd = "gcloud config get-value core/project";
+    const projectId2 = execSync(cmd, { encoding: "utf8", stdio: "pipe" }).trim();
+    if (projectId2 && projectId2 !== "(unset)") {
+      return projectId2;
+    }
+    return null;
+  } catch {
+    return null;
+  }
+}
+function getProjectId() {
+  if (process.env.GCP_PROJECT_ID) {
+    return process.env.GCP_PROJECT_ID;
+  }
+  const gcloudProjectId = getGcpProjectIdFromGcloud();
+  if (gcloudProjectId) {
+    return gcloudProjectId;
+  }
+  throw new Error(
+    "\u{1F6A8} GCP project ID is required for Secret Manager.\n\nCI/CD DEPLOYMENT (REQUIRED):\n  Set GCP_PROJECT_ID as environment variable\n  Example: GCP_PROJECT_ID=dataformer-prod\n\nLOCAL DEVELOPMENT:\n  Set your gcloud project: gcloud config set project YOUR_PROJECT_ID\n  Verify with: gcloud config get-value project"
+  );
+}
+var secretManagerClient = null;
 var secretManagerInitialized = false;
-async function getSecretManager() {
+var projectId = null;
+async function getSecretManagerClient() {
   if (secretManagerInitialized) {
-    return secretManager;
+    return secretManagerClient && projectId ? { client: secretManagerClient, projectId } : null;
   }
   try {
-    secretManager = await SecretManagerClient.create();
+    projectId = getProjectId();
+    secretManagerClient = new SecretManagerServiceClient({ projectId });
     secretManagerInitialized = true;
-    return secretManager;
+    return { client: secretManagerClient, projectId };
   } catch (error) {
-    console.error("[EnvService.ts] Failed to initialize SecretManagerClient:", error.message);
+    console.error("[EnvService.ts] Failed to initialize SecretManagerServiceClient:", error.message);
     console.warn("[EnvService.ts] Secret Manager functionality will be unavailable.");
     secretManagerInitialized = true;
     return null;
   }
 }
+function buildSecretName(projectId2, baseSecretId) {
+  return `${projectId2}_${baseSecretId}`;
+}
+function buildSecretPath(projectId2, secretName) {
+  return `projects/${projectId2}/secrets/${secretName}`;
+}
+function buildSecretVersionPath(projectId2, secretName, version = "latest") {
+  return `projects/${projectId2}/secrets/${secretName}/versions/${version}`;
+}
 var rl = readline.createInterface({
   input: process.stdin,
   output: process.stdout
@@ -40,12 +76,26 @@ function obfuscateCred(cred) {
   }
 }
 async function getEnvVar(baseSecretKey) {
-  const sm = await getSecretManager();
-  if (!sm) {
+  const smClient = await getSecretManagerClient();
+  if (!smClient) {
     console.warn(`[EnvService.ts] SecretManager not available for getEnvVar (key: ${baseSecretKey})`);
     return null;
   }
-  return sm.getSecret(baseSecretKey);
+  try {
+    const secretName = buildSecretName(smClient.projectId, baseSecretKey);
+    const secretVersionPath = buildSecretVersionPath(smClient.projectId, secretName);
+    const [response] = await smClient.client.accessSecretVersion({
+      name: secretVersionPath
+    });
+    const secretValue = response.payload?.data?.toString();
+    return secretValue || null;
+  } catch (error) {
+    if (error.code === 5) {
+      return null;
+    }
+    console.warn(`[EnvService.ts] Failed to get secret ${baseSecretKey}: ${error.message}`);
+    return null;
+  }
 }
 async function getCustomSearchApiKey() {
   const apiKey = await getEnvVar("CUSTOM_SEARCH_API_KEY");
@@ -67,15 +117,15 @@ async function getGcpProjectId() {
     return envProjectId;
   }
   try {
-    const { execSync } = await import("child_process");
-    const projectId = execSync("gcloud config get-value project", {
+    const { execSync: execSync2 } = await import("child_process");
+    const projectId2 = execSync2("gcloud config get-value project", {
       encoding: "utf8",
       stdio: ["ignore", "pipe", "ignore"]
     }).trim();
-    if (!projectId || projectId === "(unset)") {
+    if (!projectId2 || projectId2 === "(unset)") {
       throw new Error("No GCP project configured. Run `gcloud config set project YOUR_PROJECT_ID`");
     }
-    return projectId;
+    return projectId2;
   } catch (error) {
     throw new Error(
       "GCP_PROJECT_ID not found. Either:\n1. Set GCP_PROJECT_ID environment variable (for deployed environments), or\n2. Configure gcloud: `gcloud config set project YOUR_PROJECT_ID` (for local development)"
@@ -83,17 +133,58 @@ async function getGcpProjectId() {
   }
 }
 async function setEnvVar(baseSecretKey, value) {
-  const sm = await getSecretManager();
-  if (!sm) {
+  const smClient = await getSecretManagerClient();
+  if (!smClient) {
     console.warn(`[EnvService.ts] SecretManager not available for setEnvVar (key: ${baseSecretKey})`);
     return false;
   }
-  return sm.setSecret(baseSecretKey, value);
+  try {
+    const secretName = buildSecretName(smClient.projectId, baseSecretKey);
+    const addVersion = async () => {
+      const secretPath = buildSecretPath(smClient.projectId, secretName);
+      await smClient.client.addSecretVersion({
+        parent: secretPath,
+        payload: {
+          data: Buffer.from(value, "utf8")
+        }
+      });
+      console.log(`\u2705 Secret '${baseSecretKey}' saved in GCP project: ${smClient.projectId}`);
+    };
+    try {
+      await addVersion();
+      return true;
+    } catch (err) {
+      if (err.code === 5) {
+        await smClient.client.createSecret({
+          parent: `projects/${smClient.projectId}`,
+          secretId: secretName,
+          secret: {
+            replication: {
+              automatic: {}
+            }
+          }
+        });
+        console.log(`[EnvService.ts] Created secret resource: ${secretName}`);
+        await addVersion();
+        return true;
+      }
+      console.warn(`[EnvService.ts] setEnvVar failed: ${err.message}`);
+      return false;
+    }
+  } catch (error) {
+    console.warn(`[EnvService.ts] Failed to set secret ${baseSecretKey}: ${error.message}`);
+    return false;
+  }
 }
 var setLogToConsole = async () => {
   const logToConsole = await askQuestion("Use consoleLogger to log to console? (yes, no)", "no");
   if (logToConsole) {
-    await setEnvVar("LOG_TO_CONSOLE", logToConsole);
+    const normalizedValue = logToConsole.toLowerCase();
+    if (normalizedValue === "yes" || normalizedValue === "no") {
+      await setEnvVar("LOG_TO_CONSOLE", normalizedValue);
+    } else {
+      console.warn(`Invalid LOG_TO_CONSOLE value: '${logToConsole}'. Only 'yes' or 'no' are valid.`);
+    }
   }
 };
 var setPostgresUser = async () => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dataformer/env-service",
-  "version": "2.3.0",
+  "version": "2.4.0",
   "description": "Environment service for Dataformer",
   "type": "module",
   "main": "dist/index.js",
@@ -12,8 +12,7 @@
     }
   },
   "dependencies": {
-    "@google-cloud/secret-manager": "^5.4.0",
-    "@dataformer/secret-manager-client": "^2.3.0"
+    "@google-cloud/secret-manager": "^5.4.0"
   },
   "devDependencies": {
     "tsup": "^8.0.0",

package/src/index.ts

@@ -39,29 +39,83 @@
 
 import readline from 'readline';
 import path from 'path';
-import { SecretManagerClient } from '@dataformer/secret-manager-client';
+import { SecretManagerServiceClient } from '@google-cloud/secret-manager';
+import { execSync } from 'child_process';
 
-// SecretManager client will be initialized lazily on first use
-let secretManager: SecretManagerClient | null = null;
+// Helper functions for GCP project ID detection
+function getGcpProjectIdFromGcloud(): string | null {
+  try {
+    const cmd = 'gcloud config get-value core/project';
+    const projectId = execSync(cmd, { encoding: 'utf8', stdio: 'pipe' }).trim();
+    if (projectId && projectId !== '(unset)') {
+      return projectId;
+    }
+    return null;
+  } catch {
+    return null;
+  }
+}
+
+function getProjectId(): string {
+  // Project ID resolution priority:
+  // 1. GCP_PROJECT_ID env var (REQUIRED for CI/CD)
+  // 2. gcloud config project (for local development)
+  if (process.env.GCP_PROJECT_ID) {
+    return process.env.GCP_PROJECT_ID;
+  }
+  
+  const gcloudProjectId = getGcpProjectIdFromGcloud();
+  if (gcloudProjectId) {
+    return gcloudProjectId;
+  }
+  
+  throw new Error(
+    '🚨 GCP project ID is required for Secret Manager.\n\n' +
+    'CI/CD DEPLOYMENT (REQUIRED):\n' +
+    '  Set GCP_PROJECT_ID as environment variable\n' +
+    '  Example: GCP_PROJECT_ID=dataformer-prod\n\n' +
+    'LOCAL DEVELOPMENT:\n' +
+    '  Set your gcloud project: gcloud config set project YOUR_PROJECT_ID\n' +
+    '  Verify with: gcloud config get-value project'
+  );
+}
+
+// Secret Manager direct implementation
+let secretManagerClient: SecretManagerServiceClient | null = null;
 let secretManagerInitialized = false;
+let projectId: string | null = null;
 
-async function getSecretManager(): Promise<SecretManagerClient | null> {
+async function getSecretManagerClient(): Promise<{ client: SecretManagerServiceClient; projectId: string } | null> {
   if (secretManagerInitialized) {
-    return secretManager;
+    return secretManagerClient && projectId ? { client: secretManagerClient, projectId } : null;
   }
   
   try {
-    secretManager = await SecretManagerClient.create();
+    projectId = getProjectId();
+    secretManagerClient = new SecretManagerServiceClient({ projectId });
     secretManagerInitialized = true;
-    return secretManager;
+    return { client: secretManagerClient, projectId };
   } catch (error) {
-    console.error('[EnvService.ts] Failed to initialize SecretManagerClient:', (error as Error).message);
+    console.error('[EnvService.ts] Failed to initialize SecretManagerServiceClient:', (error as Error).message);
     console.warn('[EnvService.ts] Secret Manager functionality will be unavailable.');
     secretManagerInitialized = true; // Prevent retrying
     return null;
   }
 }
 
+// Helper functions for secret operations
+function buildSecretName(projectId: string, baseSecretId: string): string {
+  return `${projectId}_${baseSecretId}`;
+}
+
+function buildSecretPath(projectId: string, secretName: string): string {
+  return `projects/${projectId}/secrets/${secretName}`;
+}
+
+function buildSecretVersionPath(projectId: string, secretName: string, version: string = 'latest'): string {
+  return `projects/${projectId}/secrets/${secretName}/versions/${version}`;
+}
+
 const rl = readline.createInterface({
   input: process.stdin,
   output: process.stdout,
@@ -87,12 +141,29 @@ function obfuscateCred(cred: string | null): string {
 
 // --- Generic Environment Variable Accessors ---
 async function getEnvVar(baseSecretKey: string): Promise<string | null> {
-  const sm = await getSecretManager();
-  if (!sm) {
+  const smClient = await getSecretManagerClient();
+  if (!smClient) {
     console.warn(`[EnvService.ts] SecretManager not available for getEnvVar (key: ${baseSecretKey})`);
     return null;
   }
-  return sm.getSecret(baseSecretKey);
+  
+  try {
+    const secretName = buildSecretName(smClient.projectId, baseSecretKey);
+    const secretVersionPath = buildSecretVersionPath(smClient.projectId, secretName);
+    
+    const [response] = await smClient.client.accessSecretVersion({
+      name: secretVersionPath,
+    });
+    
+    const secretValue = response.payload?.data?.toString();
+    return secretValue || null;
+  } catch (error: any) {
+    if (error.code === 5) { // NOT_FOUND
+      return null;
+    }
+    console.warn(`[EnvService.ts] Failed to get secret ${baseSecretKey}: ${error.message}`);
+    return null;
+  }
 }
 
 export async function getCustomSearchApiKey(): Promise<string> {
@@ -141,21 +212,67 @@ export async function getGcpProjectId(): Promise<string> {
 }
 
 async function setEnvVar(baseSecretKey: string, value: string): Promise<boolean> {
-  const sm = await getSecretManager();
-  if (!sm) {
+  const smClient = await getSecretManagerClient();
+  if (!smClient) {
     console.warn(`[EnvService.ts] SecretManager not available for setEnvVar (key: ${baseSecretKey})`);
     return false; // Indicate failure
   }
-  // TODO: need to get change iamPolicy permissions
-  // const currentUserOnly = true;
-  return sm.setSecret(baseSecretKey, value);
+  
+  try {
+    const secretName = buildSecretName(smClient.projectId, baseSecretKey);
+    
+    // Try to add a version first
+    const addVersion = async (): Promise<void> => {
+      const secretPath = buildSecretPath(smClient.projectId, secretName);
+      await smClient.client.addSecretVersion({
+        parent: secretPath,
+        payload: {
+          data: Buffer.from(value, 'utf8'),
+        },
+      });
+      console.log(`✅ Secret '${baseSecretKey}' saved in GCP project: ${smClient.projectId}`);
+    };
+
+    try {
+      await addVersion();
+      return true;
+    } catch (err: any) {
+      if (err.code === 5) { // NOT_FOUND → create then retry
+        // Create the secret resource first
+        await smClient.client.createSecret({
+          parent: `projects/${smClient.projectId}`,
+          secretId: secretName,
+          secret: {
+            replication: {
+              automatic: {},
+            },
+          },
+        });
+        console.log(`[EnvService.ts] Created secret resource: ${secretName}`);
+        
+        await addVersion();
+        return true;
+      }
+      console.warn(`[EnvService.ts] setEnvVar failed: ${err.message}`);
+      return false;
+    }
+  } catch (error: any) {
+    console.warn(`[EnvService.ts] Failed to set secret ${baseSecretKey}: ${error.message}`);
+    return false;
+  }
 }
 
 // --- Specific Setters (Now use generic setEnvVar) ---
 const setLogToConsole = async (): Promise<void> => {
     const logToConsole = await askQuestion('Use consoleLogger to log to console? (yes, no)', 'no');
     if (logToConsole) {
-      await setEnvVar('LOG_TO_CONSOLE', logToConsole);
+      // Normalize to lowercase and validate - only 'yes' or 'no' are valid
+      const normalizedValue = logToConsole.toLowerCase();
+      if (normalizedValue === 'yes' || normalizedValue === 'no') {
+        await setEnvVar('LOG_TO_CONSOLE', normalizedValue);
+      } else {
+        console.warn(`Invalid LOG_TO_CONSOLE value: '${logToConsole}'. Only 'yes' or 'no' are valid.`);
+      }
     }
 };
 
@@ -315,12 +432,12 @@ const getProjectRoot = (): string => {
 };
 
 const _getResolvedGcpProjectId = async (): Promise<string | null> => {
-  const sm = await getSecretManager();
-  if (!sm) {
+  const smClient = await getSecretManagerClient();
+  if (!smClient) {
     console.warn('[EnvService.ts] SecretManager not available for _getResolvedGcpProjectId');
     return null;
   }
-  return sm.getResolvedProjectId();
+  return smClient.projectId;
 };
 
 const printEnv = async (): Promise<void> => {