@aifabrix/builder 2.40.2 → 2.42.0

package/lib/utils/external-system-test-helpers.js

@@ -12,7 +12,8 @@
 const fs = require('fs').promises;
 const path = require('path');
 const { testDatasourceViaPipeline } = require('../api/pipeline.api');
-const { requireBearerForDataplanePipeline } = require('./token-manager');
+
+/** Pipeline test endpoints accept client credentials; do not enforce Bearer-only */
 
 /**
  * Retry API call with exponential backoff
@@ -40,26 +41,31 @@ async function retryApiCall(fn, maxRetries = 3, backoffMs = 1000) {
 }
 
 /**
- * Calls pipeline test endpoint using centralized API client
+ * Calls pipeline test endpoint using centralized API client.
+ * Pipeline test accepts Bearer, API_KEY, or client credentials (x-client-id/x-client-secret) for CI/CD.
  * @async
  * @param {Object} params - Function parameters
  * @param {string} params.systemKey - System key
  * @param {string} params.datasourceKey - Datasource key
  * @param {Object} params.payloadTemplate - Test payload template
  * @param {string} params.dataplaneUrl - Dataplane URL
- * @param {Object} params.authConfig - Authentication configuration
+ * @param {Object} params.authConfig - Authentication configuration (token or client credentials)
  * @param {number} [params.timeout] - Request timeout in milliseconds (default: 30000)
+ * @param {boolean} [params.includeDebug] - Include debug output in response
  * @returns {Promise<Object>} Test response
  */
-async function callPipelineTestEndpoint({ systemKey, datasourceKey, payloadTemplate, dataplaneUrl, authConfig, timeout = 30000 }) {
-  requireBearerForDataplanePipeline(authConfig);
+async function callPipelineTestEndpoint({ systemKey, datasourceKey, payloadTemplate, dataplaneUrl, authConfig, timeout = 30000, includeDebug = false }) {
+  const testData = { payloadTemplate };
+  if (includeDebug) {
+    testData.includeDebug = true;
+  }
   const response = await retryApiCall(async() => {
     return await testDatasourceViaPipeline({
       dataplaneUrl,
       systemKey,
       datasourceKey,
       authConfig,
-      testData: { payloadTemplate },
+      testData,
       options: { timeout }
     });
   });
@@ -67,6 +73,11 @@ async function callPipelineTestEndpoint({ systemKey, datasourceKey, payloadTempl
   if (!response.success || !response.data) {
     throw new Error(`Test endpoint failed: ${response.error || response.formattedError || 'Unknown error'}`);
   }
+  // When 200 with success: false in body, pass through; caller interprets via data.success
+  if (response.data?.success === false) {
+    const errMsg = response.data?.error || response.data?.formattedError || 'Test failed';
+    throw new Error(`Test endpoint failed: ${errMsg}`);
+  }
 
   return response.data.data || response.data;
 }
@@ -114,16 +125,18 @@ function determinePayloadTemplate(datasource, datasourceKey, customPayload) {
  * @param {string} params.dataplaneUrl - Dataplane URL
  * @param {Object} params.authConfig - Authentication configuration
  * @param {number} params.timeout - Request timeout
+ * @param {boolean} [params.includeDebug] - Include debug in response
  * @returns {Promise<Object>} Test result
  */
-async function testSingleDatasource({ systemKey, datasourceKey, payloadTemplate, dataplaneUrl, authConfig, timeout }) {
+async function testSingleDatasource({ systemKey, datasourceKey, payloadTemplate, dataplaneUrl, authConfig, timeout, includeDebug = false }) {
   const testResponse = await callPipelineTestEndpoint({
     systemKey,
     datasourceKey,
     payloadTemplate,
     dataplaneUrl,
     authConfig,
-    timeout
+    timeout,
+    includeDebug
   });
 
   return {

package/lib/utils/external-system-validators.js

@@ -9,6 +9,7 @@
  */
 
 const Ajv = require('ajv');
+const addFormats = require('ajv-formats');
 
 /**
  * Validates field mapping expression syntax (pipe-based DSL)
@@ -263,6 +264,7 @@ function validateMetadataSchema(datasource, testPayload) {
 
   try {
     const ajv = new Ajv({ allErrors: true, strict: false });
+    addFormats(ajv);
     const validate = ajv.compile(datasource.metadataSchema);
     const valid = validate(payloadTemplate);
 
@@ -319,6 +321,7 @@ function validateAgainstSchema(data, schema) {
     allowUnionTypes: true,
     validateSchema: false
   });
+  addFormats(ajv);
   // Remove $schema for draft-2020-12 to avoid AJV issues
   const schemaCopy = { ...schema };
   if (schemaCopy.$schema && schemaCopy.$schema.includes('2020-12')) {

package/lib/utils/file-upload.js

@@ -1,29 +1,14 @@
 /**
  * @fileoverview File upload utilities for multipart/form-data requests
+ * All API calls go via ApiClient (lib/api/index.js); no duplicate auth logic.
  * @author AI Fabrix Team
  * @version 2.0.0
  */
 
 const fs = require('fs').promises;
 const path = require('path');
-const { makeApiCall, authenticatedApiCall } = require('./api');
+const { ApiClient } = require('../api');
 
-/**
- * Upload a file using multipart/form-data
- * @async
- * @function uploadFile
- * @param {string} url - API endpoint URL
- * @param {string} filePath - Path to file to upload
- * @param {string} fieldName - Form field name for the file (default: 'file')
- * @param {Object} [authConfig] - Authentication configuration
- * @param {string} [authConfig.type] - Auth type ('bearer' | 'client-credentials')
- * @param {string} [authConfig.token] - Bearer token
- * @param {string} [authConfig.clientId] - Client ID
- * @param {string} [authConfig.clientSecret] - Client secret
- * @param {Object} [additionalFields] - Additional form fields to include
- * @returns {Promise<Object>} API response
- * @throws {Error} If file upload fails
- */
 /**
  * Validates file exists
  * @async
@@ -63,47 +48,32 @@ async function buildFormData(filePath, fieldName, additionalFields) {
 }
 
 /**
- * Builds authentication headers
- * @function buildAuthHeaders
- * @param {Object} authConfig - Authentication configuration
- * @returns {Object} Headers object
+ * Upload a file using multipart/form-data via ApiClient (single place for auth and API calls).
+ * @async
+ * @function uploadFile
+ * @param {string} url - Full API endpoint URL (e.g. https://dataplane.example.com/api/v1/wizard/parse-openapi)
+ * @param {string} filePath - Path to file to upload
+ * @param {string} fieldName - Form field name for the file (default: 'file')
+ * @param {Object} [authConfig] - Authentication configuration (token-only for app endpoints)
+ * @param {string} [authConfig.type] - Auth type ('bearer' | 'client-token')
+ * @param {string} [authConfig.token] - Token (Bearer user token or x-client-token application token)
+ * @param {Object} [additionalFields] - Additional form fields to include
+ * @returns {Promise<Object>} API response
+ * @throws {Error} If file upload fails
  */
-function buildAuthHeaders(authConfig) {
-  const headers = {};
-  if (authConfig.type === 'bearer' && authConfig.token) {
-    headers['Authorization'] = `Bearer ${authConfig.token}`;
-  } else if (authConfig.type === 'client-credentials') {
-    if (authConfig.clientId) {
-      headers['x-client-id'] = authConfig.clientId;
-    }
-    if (authConfig.clientSecret) {
-      headers['x-client-secret'] = authConfig.clientSecret;
-    }
-  }
-  return headers;
-}
-
 async function uploadFile(url, filePath, fieldName = 'file', authConfig = {}, additionalFields = {}) {
   await validateFileExists(filePath);
 
-  const formData = await buildFormData(filePath, fieldName, additionalFields);
-  const headers = buildAuthHeaders(authConfig);
-
-  const options = {
-    method: 'POST',
-    headers,
-    body: formData
-  };
+  const parsed = new URL(url);
+  const baseUrl = parsed.origin;
+  const endpointPath = parsed.pathname + parsed.search;
 
-  // Use authenticatedApiCall if bearer token, otherwise makeApiCall
-  if (authConfig.type === 'bearer' && authConfig.token) {
-    return await authenticatedApiCall(url, options, authConfig.token);
-  }
+  const formData = await buildFormData(filePath, fieldName, additionalFields);
+  const client = new ApiClient(baseUrl, authConfig);
 
-  return await makeApiCall(url, options);
+  return await client.postFormData(endpointPath, formData);
 }
 
 module.exports = {
   uploadFile
 };
-

package/lib/utils/help-builder.js

@@ -45,6 +45,13 @@ const CATEGORIES = [
       { name: 'wizard' },
       { name: 'build', term: 'build <app>' },
       { name: 'run', term: 'run <app>' },
+      { name: 'shell', term: 'shell <app>' },
+      { name: 'test', term: 'test <app>' },
+      { name: 'install', term: 'install <app>' },
+      { name: 'test-e2e', term: 'test-e2e <app>' },
+      { name: 'lint', term: 'lint <app>' },
+      { name: 'logs', term: 'logs <app>' },
+      { name: 'stop', term: 'stop <app>' },
       { name: 'dockerfile', term: 'dockerfile <app>' }
     ]
   },
@@ -66,7 +73,10 @@ const CATEGORIES = [
     name: 'Application & Datasource Management',
     commands: [
       { name: 'app' },
-      { name: 'datasource' }
+      { name: 'datasource' },
+      { name: 'credential' },
+      { name: 'deployment' },
+      { name: 'service-user' }
     ]
   },
   {
@@ -75,6 +85,8 @@ const CATEGORIES = [
       { name: 'resolve', term: 'resolve <app>' },
       { name: 'json', term: 'json <app>' },
       { name: 'split-json', term: 'split-json <app>' },
+      { name: 'convert', term: 'convert <app>' },
+      { name: 'show', term: 'show <appKey>' },
       { name: 'validate', term: 'validate <appOrFile>' },
       { name: 'diff', term: 'diff <file1> <file2>' }
     ]
@@ -83,7 +95,9 @@ const CATEGORIES = [
     name: 'External Systems',
     commands: [
       { name: 'download', term: 'download <system-key>' },
+      { name: 'upload', term: 'upload <system-key>' },
       { name: 'delete', term: 'delete <system-key>' },
+      { name: 'repair', term: 'repair <app>' },
       { name: 'test', term: 'test <app>' },
       { name: 'test-integration', term: 'test-integration <app>' }
     ]
@@ -92,7 +106,7 @@ const CATEGORIES = [
     name: 'Developer & Secrets',
     commands: [
       { name: 'dev' },
-      { name: 'secrets' },
+      { name: 'secret' },
       { name: 'secure' }
     ]
   }

package/lib/utils/infra-status.js

@@ -17,9 +17,53 @@ const containerUtils = require('./infra-containers');
 
 const execAsync = promisify(exec);
 
+/**
+ * Builds services config map from ports and config flags.
+ * @param {Object} ports - Port configuration
+ * @param {Object} cfg - Config (pgadmin, redisCommander, traefik)
+ * @returns {Object} Map of serviceName -> { port, url }
+ */
+function buildServicesConfig(ports, cfg) {
+  const services = {
+    postgres: { port: ports.postgres, url: `localhost:${ports.postgres}` },
+    redis: { port: ports.redis, url: `localhost:${ports.redis}` }
+  };
+  if (cfg.pgadmin !== false) services.pgadmin = { port: ports.pgadmin, url: `http://localhost:${ports.pgadmin}` };
+  if (cfg.redisCommander !== false) {
+    services['redis-commander'] = { port: ports.redisCommander, url: `http://localhost:${ports.redisCommander}` };
+  }
+  if (cfg.traefik) {
+    services.traefik = {
+      port: `${ports.traefikHttp}, ${ports.traefikHttps}`,
+      url: `http://localhost:${ports.traefikHttp}, https://localhost:${ports.traefikHttps}`
+    };
+  }
+  return services;
+}
+
+/**
+ * Gets status for a single service.
+ * @param {string} serviceName - Service name
+ * @param {Object} serviceConfig - { port, url }
+ * @param {string} devId - Developer ID
+ * @returns {Promise<Object>} Status entry
+ */
+async function getServiceStatus(serviceName, serviceConfig, devId) {
+  try {
+    const containerName = await containerUtils.findContainer(serviceName, devId, { strict: true });
+    const rawStatus = containerName
+      ? (await execAsync(`docker inspect --format='{{.State.Status}}' ${containerName}`)).stdout.trim().replace(/['"]/g, '')
+      : 'not running';
+    return { status: rawStatus, port: serviceConfig.port, url: serviceConfig.url };
+  } catch {
+    return { status: 'not running', port: serviceConfig.port, url: serviceConfig.url };
+  }
+}
+
 /**
  * Gets the status of infrastructure services
- * Returns detailed information about running containers
+ * Returns detailed information about running containers.
+ * Only includes pgAdmin, Redis Commander, and Traefik when enabled in config.
  *
  * @async
  * @function getInfraStatus
@@ -31,51 +75,13 @@ const execAsync = promisify(exec);
  */
 async function getInfraStatus() {
   const devId = await config.getDeveloperId();
-  // Convert string developer ID to number for getDevPorts
-  const devIdNum = parseInt(devId, 10);
-  const ports = devConfig.getDevPorts(devIdNum);
-  const services = {
-    postgres: { port: ports.postgres, url: `localhost:${ports.postgres}` },
-    redis: { port: ports.redis, url: `localhost:${ports.redis}` },
-    pgadmin: { port: ports.pgadmin, url: `http://localhost:${ports.pgadmin}` },
-    'redis-commander': { port: ports.redisCommander, url: `http://localhost:${ports.redisCommander}` },
-    traefik: {
-      port: `${ports.traefikHttp}, ${ports.traefikHttps}`,
-      url: `http://localhost:${ports.traefikHttp}, https://localhost:${ports.traefikHttps}`
-    }
-  };
-
+  const cfg = await config.getConfig();
+  const ports = devConfig.getDevPorts(parseInt(devId, 10));
+  const services = buildServicesConfig(ports, cfg);
   const status = {};
-
-  for (const [serviceName, serviceConfig] of Object.entries(services)) {
-    try {
-      // Strict: only this developer's infra (no fallback to dev 0), so status reflects reality
-      const containerName = await containerUtils.findContainer(serviceName, devId, { strict: true });
-      if (containerName) {
-        const { stdout } = await execAsync(`docker inspect --format='{{.State.Status}}' ${containerName}`);
-        // Normalize status value (trim whitespace and remove quotes)
-        const normalizedStatus = stdout.trim().replace(/['"]/g, '');
-        status[serviceName] = {
-          status: normalizedStatus,
-          port: serviceConfig.port,
-          url: serviceConfig.url
-        };
-      } else {
-        status[serviceName] = {
-          status: 'not running',
-          port: serviceConfig.port,
-          url: serviceConfig.url
-        };
-      }
-    } catch (error) {
-      status[serviceName] = {
-        status: 'not running',
-        port: serviceConfig.port,
-        url: serviceConfig.url
-      };
-    }
+  for (const [name, svc] of Object.entries(services)) {
+    status[name] = await getServiceStatus(name, svc, devId);
   }
-
   return status;
 }
 
@@ -186,8 +192,37 @@ async function getAppStatus() {
   return apps;
 }
 
+/**
+ * Lists app container names for a developer (excludes infra containers).
+ * Used by down-infra to stop/remove all app-related containers on the same network.
+ * When includeExited is true, includes stopped/exited containers (e.g. db-init one-offs).
+ *
+ * @async
+ * @function listAppContainerNamesForDeveloper
+ * @param {string} devId - Developer ID
+ * @param {Object} [options] - Options
+ * @param {boolean} [options.includeExited=false] - If true, use docker ps -a to include exited containers
+ * @returns {Promise<string[]>} Container names (e.g. aifabrix-myapp, aifabrix-keycloak-db-init)
+ */
+async function listAppContainerNamesForDeveloper(devId, options = {}) {
+  const devIdNum = parseInt(devId, 10);
+  const filterPattern = devIdNum === 0 ? 'aifabrix-' : `aifabrix-dev${devId}-`;
+  const infraContainers = getInfraContainerNames(devIdNum, devId);
+  const includeExited = !!options.includeExited;
+  try {
+    const allFlag = includeExited ? ' -a' : '';
+    const { stdout } = await execAsync(`docker ps${allFlag} --filter "name=${filterPattern}" --format "{{.Names}}"`);
+    const names = (stdout || '').trim().split('\n').filter(Boolean);
+    return names.filter(n => !infraContainers.includes(n));
+  } catch {
+    return [];
+  }
+}
+
 module.exports = {
   getInfraStatus,
-  getAppStatus
+  getAppStatus,
+  extractAppName,
+  listAppContainerNamesForDeveloper
 };
 

package/lib/utils/local-secrets.js

@@ -13,11 +13,12 @@ const path = require('path');
 const yaml = require('js-yaml');
 const logger = require('../utils/logger');
 const pathsUtil = require('./paths');
+const { mergeSecretsIntoFile } = require('./secrets-generator');
 
 /**
  * Saves a secret to ~/.aifabrix/secrets.local.yaml
  * Uses paths.getAifabrixHome() to respect config.yaml aifabrix-home override
- * Merges with existing secrets without overwriting other keys
+ * Merges the key into the file (updates in place if key already exists, e.g. after rotate-secret)
  *
  * @async
  * @function saveLocalSecret
@@ -39,48 +40,12 @@ async function saveLocalSecret(key, value) {
   }
 
   const secretsPath = path.join(pathsUtil.getAifabrixHome(), 'secrets.local.yaml');
-  const secretsDir = path.dirname(secretsPath);
-
-  // Create directory if needed
-  if (!fs.existsSync(secretsDir)) {
-    fs.mkdirSync(secretsDir, { recursive: true, mode: 0o700 });
-  }
-
-  // Load existing secrets
-  let existingSecrets = {};
-  if (fs.existsSync(secretsPath)) {
-    try {
-      const content = fs.readFileSync(secretsPath, 'utf8');
-      existingSecrets = yaml.load(content) || {};
-      if (typeof existingSecrets !== 'object') {
-        existingSecrets = {};
-      }
-    } catch (error) {
-      logger.warn(`Warning: Could not read existing secrets file: ${error.message}`);
-      existingSecrets = {};
-    }
-  }
-
-  // Merge with new secret
-  const updatedSecrets = {
-    ...existingSecrets,
-    [key]: value
-  };
-
-  // Save to file
-  const yamlContent = yaml.dump(updatedSecrets, {
-    indent: 2,
-    lineWidth: 120,
-    noRefs: true,
-    sortKeys: false
-  });
-
-  fs.writeFileSync(secretsPath, yamlContent, { mode: 0o600 });
+  mergeSecretsIntoFile(secretsPath, { [key]: value });
 }
 
 /**
  * Saves a secret to a specified secrets file path
- * Merges with existing secrets without overwriting other keys
+ * Merges the key into the file (updates in place if key already exists)
  *
  * @async
  * @function saveSecret
@@ -134,11 +99,11 @@ function resolveAndPrepareSecretsPath(secretsPath) {
 
 /**
  * Loads existing secrets from file
- * @function loadExistingSecrets
+ * @function _loadExistingSecrets
  * @param {string} resolvedPath - Resolved secrets path
  * @returns {Object} Existing secrets object
  */
-function loadExistingSecrets(resolvedPath) {
+function _loadExistingSecrets(resolvedPath) {
   if (!fs.existsSync(resolvedPath)) {
     return {};
   }
@@ -157,17 +122,7 @@ async function saveSecret(key, value, secretsPath) {
   validateSaveSecretParams(key, value, secretsPath);
 
   const resolvedPath = resolveAndPrepareSecretsPath(secretsPath);
-  const existingSecrets = loadExistingSecrets(resolvedPath);
-
-  const updatedSecrets = { ...existingSecrets, [key]: value };
-  const yamlContent = yaml.dump(updatedSecrets, {
-    indent: 2,
-    lineWidth: 120,
-    noRefs: true,
-    sortKeys: false
-  });
-
-  fs.writeFileSync(resolvedPath, yamlContent, { mode: 0o600 });
+  mergeSecretsIntoFile(resolvedPath, { [key]: value });
 }
 
 /**