@aifabrix/builder 2.41.0 → 2.42.0

package/lib/utils/paths.js

@@ -1,13 +1,11 @@
 /**
  * Path Utilities for AI Fabrix Builder
- *
- * Centralized helpers for resolving filesystem locations with support for
- * AIFABRIX_HOME override. Defaults to ~/.aifabrix when not specified.
- *
+ * Centralized helpers for resolving filesystem locations with AIFABRIX_HOME override.
  * @fileoverview Path resolution utilities with environment overrides
  * @author AI Fabrix Team
  * @version 2.0.0
  */
+/* eslint-disable max-lines -- Central path resolution; resolveIntegrationAppKeyFromCwd for datasource commands */
 
 'use strict';
 
@@ -279,6 +277,76 @@ function getIntegrationBuilderBaseDir() {
   return cwd;
 }
 
+/**
+ * Returns the integration root directory (used for listing apps).
+ * @returns {string} Absolute path to integration/ directory
+ */
+function getIntegrationRoot() {
+  return path.join(getIntegrationBuilderBaseDir(), 'integration');
+}
+
+/**
+ * Returns the builder root directory. Uses AIFABRIX_BUILDER_DIR when set, else project/cwd + builder.
+ * @returns {string} Absolute path to builder/ directory
+ */
+function getBuilderRoot() {
+  const envDir = process.env.AIFABRIX_BUILDER_DIR && typeof process.env.AIFABRIX_BUILDER_DIR === 'string'
+    ? process.env.AIFABRIX_BUILDER_DIR.trim()
+    : null;
+  if (envDir) {
+    return path.resolve(envDir);
+  }
+  return path.join(getIntegrationBuilderBaseDir(), 'builder');
+}
+
+/**
+ * Lists app names (directories) under integration root. Excludes dot-prefixed entries.
+ * Returns [] if root does not exist.
+ * @returns {string[]} Sorted list of app directory names
+ */
+function listIntegrationAppNames() {
+  const root = getIntegrationRoot();
+  if (!fs.existsSync(root)) {
+    return [];
+  }
+  const stat = fs.statSync(root);
+  if (!stat || typeof stat.isDirectory !== 'function' || !stat.isDirectory()) {
+    return [];
+  }
+  const entries = fs.readdirSync(root);
+  return entries
+    .filter(name => !name.startsWith('.'))
+    .filter(name => {
+      const fullPath = path.join(root, name);
+      return fs.statSync(fullPath).isDirectory();
+    })
+    .sort();
+}
+
+/**
+ * Lists app names (directories) under builder root. Excludes dot-prefixed entries.
+ * Returns [] if root does not exist.
+ * @returns {string[]} Sorted list of app directory names
+ */
+function listBuilderAppNames() {
+  const root = getBuilderRoot();
+  if (!fs.existsSync(root)) {
+    return [];
+  }
+  const stat = fs.statSync(root);
+  if (!stat || typeof stat.isDirectory !== 'function' || !stat.isDirectory()) {
+    return [];
+  }
+  const entries = fs.readdirSync(root);
+  return entries
+    .filter(name => !name.startsWith('.'))
+    .filter(name => {
+      const fullPath = path.join(root, name);
+      return fs.statSync(fullPath).isDirectory();
+    })
+    .sort();
+}
+
 /**
  * Gets the integration folder path for external systems.
  * @param {string} appName - Application name
@@ -470,6 +538,14 @@ async function getResolveAppPath(appName) {
   return { appPath: result.appPath, envOnly: false };
 }
 
+/** Resolve appKey when cwd is inside integration/<appKey>/. */
+function resolveIntegrationAppKeyFromCwd() {
+  const integrationNorm = path.resolve(path.join(getIntegrationBuilderBaseDir(), 'integration'));
+  const cwd = path.resolve(process.cwd());
+  if (cwd !== integrationNorm && !cwd.startsWith(integrationNorm + path.sep)) return null;
+  return path.relative(integrationNorm, cwd).split(path.sep)[0] || null;
+}
+
 module.exports = {
   getAifabrixHome,
   getConfigDirForPaths,
@@ -479,11 +555,16 @@ module.exports = {
   getProjectRoot,
   getIntegrationPath,
   getBuilderPath,
+  getIntegrationRoot,
+  getBuilderRoot,
+  listIntegrationAppNames,
+  listBuilderAppNames,
   resolveBuildContext,
   getDeployJsonPath,
   resolveApplicationConfigPath,
   detectAppType,
   getResolveAppPath,
+  resolveIntegrationAppKeyFromCwd,
   clearProjectRootCache
 };
 

package/lib/utils/secrets-canonical.js

@@ -0,0 +1,93 @@
+/**
+ * Canonical secrets path and YAML helpers
+ *
+ * @fileoverview Read/merge canonical secrets from config path
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const yaml = require('js-yaml');
+const config = require('../core/config');
+
+/**
+ * Read a YAML file and return parsed object
+ * @function readYamlAtPath
+ * @param {string} filePath - Absolute file path
+ * @returns {Object} Parsed YAML object
+ */
+function readYamlAtPath(filePath) {
+  const content = fs.readFileSync(filePath, 'utf8');
+  return yaml.load(content);
+}
+
+/**
+ * Merge a single secret value from canonical into result
+ * @function mergeSecretValue
+ * @param {Object} result - Result object to merge into
+ * @param {string} key - Secret key
+ * @param {*} canonicalValue - Value from canonical secrets
+ */
+function mergeSecretValue(result, key, canonicalValue) {
+  const currentValue = result[key];
+  // Fill missing, empty, or undefined values
+  if (!(key in result) || currentValue === undefined || currentValue === null || currentValue === '') {
+    result[key] = canonicalValue;
+    return;
+  }
+  // Only replace values that are encrypted (have secure:// prefix)
+  // Plaintext values (no secure://) are used as-is
+  if (typeof currentValue === 'string' && typeof canonicalValue === 'string') {
+    if (currentValue.startsWith('secure://')) {
+      result[key] = canonicalValue;
+    }
+  }
+}
+
+/**
+ * Apply canonical secrets path override if configured and file exists
+ * @async
+ * @function applyCanonicalSecretsOverride
+ * @param {Object} currentSecrets - Current secrets map
+ * @returns {Promise<Object>} Possibly overridden secrets
+ */
+async function applyCanonicalSecretsOverride(currentSecrets) {
+  let mergedSecrets = currentSecrets || {};
+  try {
+    const canonicalPath = await config.getSecretsPath();
+    if (!canonicalPath) {
+      return mergedSecrets;
+    }
+    const resolvedCanonical = path.isAbsolute(canonicalPath)
+      ? canonicalPath
+      : path.resolve(process.cwd(), canonicalPath);
+    if (!fs.existsSync(resolvedCanonical)) {
+      return mergedSecrets;
+    }
+    const configSecrets = readYamlAtPath(resolvedCanonical);
+    if (!configSecrets || typeof configSecrets !== 'object') {
+      return mergedSecrets;
+    }
+    // Apply canonical secrets as a fallback source:
+    // - Do NOT override any existing keys from user/build
+    // - Add only missing keys from canonical path
+    // - Also fill in empty/undefined values from canonical path
+    // - Replace encrypted values (secure://) with canonical plaintext
+    const result = { ...mergedSecrets };
+    for (const [key, canonicalValue] of Object.entries(configSecrets)) {
+      mergeSecretValue(result, key, canonicalValue);
+    }
+    mergedSecrets = result;
+  } catch {
+    // ignore and fall through
+  }
+  return mergedSecrets;
+}
+
+module.exports = {
+  readYamlAtPath,
+  applyCanonicalSecretsOverride
+};

package/lib/utils/secrets-generator.js

@@ -208,6 +208,25 @@ function saveSecretsFile(resolvedPath, secrets) {
 
 const YAML_DUMP_OPTS = { indent: 2, lineWidth: 120, noRefs: true, sortKeys: false };
 
+/**
+ * Merges secret keys into the secrets file (load existing, merge, overwrite file).
+ * Use when setting or updating keys so that existing keys are updated in place instead of duplicated.
+ * Creates the file if it does not exist. Tolerant of duplicate keys in existing file (last wins when loading).
+ *
+ * @function mergeSecretsIntoFile
+ * @param {string} resolvedPath - Path to secrets file
+ * @param {Object} secrets - Key-value object to merge (overwrites existing same keys)
+ * @throws {Error} If write fails
+ */
+function mergeSecretsIntoFile(resolvedPath, secrets) {
+  if (!secrets || typeof secrets !== 'object' || Object.keys(secrets).length === 0) {
+    return;
+  }
+  const existing = loadExistingSecrets(resolvedPath);
+  const merged = { ...existing, ...secrets };
+  saveSecretsFile(resolvedPath, merged);
+}
+
 /**
  * Appends secret keys to the end of the secrets file without modifying existing content (preserves comments and structure).
  * Creates the file if it does not exist. For existing files, new keys are appended.
@@ -329,6 +348,7 @@ module.exports = {
   loadYamlTolerantOfDuplicateKeys,
   loadExistingSecrets,
   saveSecretsFile,
+  mergeSecretsIntoFile,
   appendSecretsToFile,
   generateMissingSecrets,
   createDefaultSecrets

package/lib/utils/secrets-helpers.js

@@ -18,6 +18,7 @@ const { loadEnvConfig } = require('./env-config-loader');
 const { updateContainerPortInEnvFile } = require('./env-ports');
 const { buildEnvVarMap } = require('./env-map');
 const { getLocalPortFromPath } = require('./port-resolver');
+const { readYamlAtPath, applyCanonicalSecretsOverride } = require('./secrets-canonical');
 
 /**
  * Interpolate ${VAR} occurrences with values from envVars map
@@ -42,25 +43,84 @@ function isCommentOrEmptyLine(line) {
   return t === '' || t.startsWith('#');
 }
 
+/** Regex for kv:// path (allows slashes, e.g. kv://hubspot/clientId) */
+const KV_REF_PATTERN = /kv:\/\/([a-zA-Z0-9_\-/]+)/g;
+
+/**
+ * Find object key that matches part case-insensitively.
+ * @param {Object} obj - Object to search
+ * @param {string} part - Key to match (e.g. 'clientid')
+ * @returns {string|undefined} Actual key in obj or undefined
+ */
+function findKeyCaseInsensitive(obj, part) {
+  if (!obj || typeof obj !== 'object' || part === null || part === undefined) return undefined;
+  const lower = String(part).toLowerCase();
+  for (const key of Object.keys(obj)) {
+    if (key.toLowerCase() === lower) return key;
+  }
+  return undefined;
+}
+
+/**
+ * Resolve value by walking path parts (nested keys).
+ * @param {Object} secrets - Root secrets object
+ * @param {string[]} parts - Path parts (e.g. ['hubspot', 'clientId'])
+ * @returns {*} Value or undefined
+ */
+function getValueByNestedPath(secrets, parts) {
+  let value = secrets;
+  for (const part of parts) {
+    if (!value || typeof value !== 'object') return undefined;
+    const key = part in value ? part : findKeyCaseInsensitive(value, part);
+    value = key !== undefined ? value[key] : undefined;
+    if (value === undefined) return undefined;
+  }
+  return value;
+}
+
 /**
- * Collect missing kv:// secrets referenced in content (skips commented and empty lines)
+ * Get secret value by path. Supports flat key (hubspot/clientId), nested object (hubspot.clientId),
+ * and case-insensitive matching (clientid matches clientId). Path-style and hyphen-style are distinct:
+ * hubspot/clientid and hubspot-clientid are different keys.
+ * @param {Object} secrets - Secrets object (may be nested)
+ * @param {string} pathStr - Path after kv:// (e.g. 'hubspot/clientId' or 'hubspot/clientid')
+ * @returns {*} Value or undefined if not found
+ */
+function getValueByPath(secrets, pathStr) {
+  if (!secrets || typeof secrets !== 'object' || !pathStr) {
+    return undefined;
+  }
+  const direct = secrets[pathStr];
+  if (direct !== undefined) return direct;
+  const flatKey = findKeyCaseInsensitive(secrets, pathStr);
+  if (flatKey !== undefined) return secrets[flatKey];
+  if (!pathStr.includes('/')) return undefined;
+  return getValueByNestedPath(secrets, pathStr.split('/'));
+}
+
+/**
+ * Collect missing kv:// secrets referenced in content (skips commented and empty lines).
+ * Supports path-style refs (e.g. kv://hubspot/clientId). Returns unique refs.
  * @function collectMissingSecrets
  * @param {string} content - Text content
- * @param {Object} secrets - Available secrets
- * @returns {string[]} Array of missing kv://<key> references
+ * @param {Object} secrets - Available secrets (flat or nested)
+ * @returns {string[]} Array of missing kv://<path> references (unique)
  */
 function collectMissingSecrets(content, secrets) {
-  const kvPattern = /kv:\/\/([a-zA-Z0-9-_]+)/g;
+  const seen = new Set();
   const missing = [];
   const lines = content.split('\n');
   for (const line of lines) {
     if (isCommentOrEmptyLine(line)) continue;
     let match;
-    kvPattern.lastIndex = 0;
-    while ((match = kvPattern.exec(line)) !== null) {
-      const secretKey = match[1];
-      if (!(secretKey in secrets)) {
-        missing.push(`kv://${secretKey}`);
+    KV_REF_PATTERN.lastIndex = 0;
+    while ((match = KV_REF_PATTERN.exec(line)) !== null) {
+      const pathStr = match[1];
+      if (seen.has(pathStr)) continue;
+      seen.add(pathStr);
+      const value = getValueByPath(secrets, pathStr);
+      if (value === undefined || value === null) {
+        missing.push(`kv://${pathStr}`);
       }
     }
   }
@@ -91,26 +151,26 @@ function formatMissingSecretsFileInfo(secretsFilePaths) {
 }
 
 /**
- * Replace kv:// references with actual values (skips commented and empty lines)
+ * Replace kv:// references with actual values (skips commented and empty lines).
+ * Supports path-style refs (e.g. kv://hubspot/clientId) and nested secrets.
  * @function replaceKvInContent
  * @param {string} content - Text content containing kv:// references
- * @param {Object} secrets - Secrets map
+ * @param {Object} secrets - Secrets map (flat or nested)
  * @param {Object} envVars - Environment variables map for nested interpolation
  * @returns {string} Content with kv:// references replaced
  */
 function replaceKvInContent(content, secrets, envVars) {
-  const kvPattern = /kv:\/\/([a-zA-Z0-9-_]+)/g;
   const lines = content.split('\n');
   const result = lines.map(line => {
     if (isCommentOrEmptyLine(line)) return line;
-    return line.replace(kvPattern, (match, secretKey) => {
-      let value = secrets[secretKey];
+    return line.replace(KV_REF_PATTERN, (match, pathStr) => {
+      let value = getValueByPath(secrets, pathStr);
       if (typeof value === 'string') {
         value = value.replace(/\$\{([A-Z_]+)\}/g, (m, envVar) => {
           return envVars[envVar] || m;
         });
       }
-      return value;
+      return value !== null && value !== undefined ? String(value) : match;
     });
   });
   return result.join('\n');
@@ -377,80 +437,6 @@ async function adjustLocalEnvPortsInContent(envContent, variablesPath) {
   return updated;
 }
 
-/**
- * Read a YAML file and return parsed object
- * @function readYamlAtPath
- * @param {string} filePath - Absolute file path
- * @returns {Object} Parsed YAML object
- */
-function readYamlAtPath(filePath) {
-  const content = fs.readFileSync(filePath, 'utf8');
-  return yaml.load(content);
-}
-
-/**
- * Merge a single secret value from canonical into result
- * @function mergeSecretValue
- * @param {Object} result - Result object to merge into
- * @param {string} key - Secret key
- * @param {*} canonicalValue - Value from canonical secrets
- */
-function mergeSecretValue(result, key, canonicalValue) {
-  const currentValue = result[key];
-  // Fill missing, empty, or undefined values
-  if (!(key in result) || currentValue === undefined || currentValue === null || currentValue === '') {
-    result[key] = canonicalValue;
-    return;
-  }
-  // Only replace values that are encrypted (have secure:// prefix)
-  // Plaintext values (no secure://) are used as-is
-  if (typeof currentValue === 'string' && typeof canonicalValue === 'string') {
-    if (currentValue.startsWith('secure://')) {
-      result[key] = canonicalValue;
-    }
-  }
-}
-
-/**
- * Apply canonical secrets path override if configured and file exists
- * @async
- * @function applyCanonicalSecretsOverride
- * @param {Object} currentSecrets - Current secrets map
- * @returns {Promise<Object>} Possibly overridden secrets
- */
-async function applyCanonicalSecretsOverride(currentSecrets) {
-  let mergedSecrets = currentSecrets || {};
-  try {
-    const canonicalPath = await config.getSecretsPath();
-    if (!canonicalPath) {
-      return mergedSecrets;
-    }
-    const resolvedCanonical = path.isAbsolute(canonicalPath)
-      ? canonicalPath
-      : path.resolve(process.cwd(), canonicalPath);
-    if (!fs.existsSync(resolvedCanonical)) {
-      return mergedSecrets;
-    }
-    const configSecrets = readYamlAtPath(resolvedCanonical);
-    if (!configSecrets || typeof configSecrets !== 'object') {
-      return mergedSecrets;
-    }
-    // Apply canonical secrets as a fallback source:
-    // - Do NOT override any existing keys from user/build
-    // - Add only missing keys from canonical path
-    // - Also fill in empty/undefined values from canonical path
-    // - Replace encrypted values (secure://) with canonical plaintext
-    const result = { ...mergedSecrets };
-    for (const [key, canonicalValue] of Object.entries(configSecrets)) {
-      mergeSecretValue(result, key, canonicalValue);
-    }
-    mergedSecrets = result;
-  } catch {
-    // ignore and fall through
-  }
-  return mergedSecrets;
-}
-
 /**
  * Ensure secrets map is non-empty or throw a friendly guidance error
  * @function ensureNonEmptySecrets

package/lib/utils/test-log-writer.js

@@ -0,0 +1,56 @@
+/**
+ * Test log writer - writes debug logs to integration/<appKey>/logs/
+ * Sanitization (tokens, secrets) is done by dataplane before responses are returned.
+ *
+ * @fileoverview Write test request/response logs for debugging
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+const fs = require('fs').promises;
+const path = require('path');
+
+/**
+ * Prepare object for JSON serialization (handles circular refs)
+ * @param {*} obj - Object to prepare
+ * @param {Set} [seen] - Set of seen object references (for circular refs)
+ * @returns {*} Copy safe for JSON.stringify
+ */
+function sanitizeForLog(obj, seen = new Set()) {
+  if (obj === null || obj === undefined || typeof obj !== 'object') return obj;
+  if (seen.has(obj)) return '[Circular]';
+  seen.add(obj);
+  if (Array.isArray(obj)) return obj.map(item => sanitizeForLog(item, seen));
+  const out = {};
+  for (const [key, value] of Object.entries(obj)) {
+    out[key] = sanitizeForLog(value, seen);
+  }
+  return out;
+}
+
+/**
+ * Write test log to integration/<appKey>/logs/<logType>-<timestamp>.json
+ * @async
+ * @param {string} appKey - Application key (used for path)
+ * @param {Object} data - Log data (request, response) - will be sanitized
+ * @param {string} [logType] - Log type prefix (default: test-integration)
+ * @param {string} [integrationBaseDir] - Base dir for integration (default: cwd/integration)
+ * @returns {Promise<string>} Path to written file
+ * @throws {Error} If write fails
+ */
+async function writeTestLog(appKey, data, logType = 'test-integration', integrationBaseDir) {
+  const baseDir = integrationBaseDir || path.join(process.cwd(), 'integration');
+  const logsDir = path.join(baseDir, appKey, 'logs');
+  await fs.mkdir(logsDir, { recursive: true });
+  const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+  const filename = `${logType}-${timestamp}.json`;
+  const filePath = path.join(logsDir, filename);
+  const sanitized = sanitizeForLog(data);
+  await fs.writeFile(filePath, JSON.stringify(sanitized, null, 2), 'utf8');
+  return filePath;
+}
+
+module.exports = {
+  sanitizeForLog,
+  writeTestLog
+};

package/lib/utils/token-manager.js

@@ -247,48 +247,29 @@ async function tryClientTokenAuth(environment, appName, controllerUrl) {
     const clientToken = await getOrRefreshClientToken(environment, appName, controllerUrl);
     if (clientToken && clientToken.token) {
       return {
-        type: 'bearer',
+        type: 'client-token',
         token: clientToken.token,
         controller: clientToken.controller
       };
     }
   } catch {
-    // Client token unavailable; getDeploymentAuth will try client credentials next (no warning here to avoid misleading output when env credentials succeed)
-  }
-  return null;
-}
-
-/**
- * Tries to get client credentials for deployment auth
- * @async
- * @function tryClientCredentialsAuth
- * @param {string} appName - Application name
- * @param {string} controllerUrl - Controller URL
- * @returns {Promise<Object|null>} Auth config with client credentials or null
- */
-async function tryClientCredentialsAuth(appName, controllerUrl) {
-  const credentials = await loadClientCredentials(appName);
-  if (credentials && credentials.clientId && credentials.clientSecret) {
-    return {
-      type: 'client-credentials',
-      clientId: credentials.clientId,
-      clientSecret: credentials.clientSecret,
-      controller: controllerUrl
-    };
+    // Client token unavailable; getDeploymentAuth will try exchanging credentials for token (no warning here to avoid misleading output when refresh succeeds)
   }
   return null;
 }
 
 /**
  * Get deployment authentication configuration with priority:
- * 1. Device token (Bearer) - for user-level audit tracking (preferred)
- * 2. Client token (Bearer) - for application-level authentication
- * 3. Client credentials (x-client-id/x-client-secret) - direct credential authentication
+ * 1. Device token → type 'bearer' (user token) → send as Authorization: Bearer
+ * 2. Client token → type 'client-token' (application token) → send as x-client-token header
+ * 3. When no token available: if client credentials exist, exchange for client token and return type 'client-token'.
+ *
+ * x-client-id/x-client-secret are used only at the token-issuing endpoint (e.g. POST /api/v1/auth/token).
  *
  * @param {string} controllerUrl - Controller URL
  * @param {string} environment - Environment key
  * @param {string} appName - Application name
- * @returns {Promise<{type: 'bearer'|'client-credentials', token?: string, clientId?: string, clientSecret?: string, controller: string}>} Auth configuration
+ * @returns {Promise<{type: 'bearer'|'client-token', token: string, controller: string}>} Auth config: bearer = user token, client-token = app token (x-client-token header)
  * @throws {Error} If no authentication method is available
  */
 async function getDeploymentAuth(controllerUrl, environment, appName) {
@@ -306,10 +287,21 @@ async function getDeploymentAuth(controllerUrl, environment, appName) {
     return clientTokenAuth;
   }
 
-  // Priority 3: Use client credentials directly
-  const credentialsAuth = await tryClientCredentialsAuth(appName, controllerUrl);
-  if (credentialsAuth) {
-    return credentialsAuth;
+  // Priority 3: Exchange client credentials for a token (never return client-credentials for app endpoints)
+  const credentials = await loadClientCredentials(appName);
+  if (credentials && credentials.clientId && credentials.clientSecret) {
+    try {
+      const refreshed = await refreshClientToken(environment, appName, controllerUrl);
+      if (refreshed && refreshed.token) {
+        return {
+          type: 'client-token',
+          token: refreshed.token,
+          controller: controllerUrl
+        };
+      }
+    } catch {
+      // Refresh failed; fall through to throw below
+    }
   }
 
   throw new Error(`No authentication method available. Run 'aifabrix login' for device token, or add credentials to ~/.aifabrix/secrets.local.yaml as '${appName}-client-idKeyVault' and '${appName}-client-secretKeyVault'`);
@@ -337,7 +329,7 @@ async function extractClientCredentials(authConfig, appKey, envKey, _options = {
     };
   }
 
-  if (authConfig.type === 'bearer') {
+  if (authConfig.type === 'bearer' || authConfig.type === 'client-token') {
     if (authConfig.clientId && authConfig.clientSecret) {
       return {
         clientId: authConfig.clientId,