@aifabrix/builder 2.46.0 → 2.47.0

package/lib/datasource/capability/run-capability-diff.js

@@ -6,8 +6,8 @@
  * @version 2.0.0
  */
 
-const fs = require('fs');
 const path = require('path');
+const { readDatasourceConfigFile } = require('./run-capability-copy');
 const { resolveValidateInputPath } = require('../validate');
 const { normalizeCapabilityKey } = require('./capability-key');
 const { extractCapabilitySliceForDiff } = require('./capability-diff-slice');
@@ -108,8 +108,8 @@ function resolveProfileKeys(opts) {
 function runCapabilityDiff(opts) {
   const pathA = resolveValidateInputPath(opts.fileA.trim());
   const pathB = resolveValidateInputPath(opts.fileB.trim());
-  const docA = JSON.parse(fs.readFileSync(pathA, 'utf8'));
-  const docB = JSON.parse(fs.readFileSync(pathB, 'utf8'));
+  const docA = readDatasourceConfigFile(pathA);
+  const docB = readDatasourceConfigFile(pathB);
 
   const { capA, capB } = resolveCapabilityKeys(opts);
   const { profA, profB } = resolveProfileKeys(opts);

package/lib/datasource/capability/run-capability-dimension.js

@@ -8,7 +8,6 @@
 
 'use strict';
 
-const fs = require('fs');
 const { resolveControllerUrl } = require('../../utils/controller-url');
 const { normalizeControllerUrl } = require('../../core/config');
 const { getOrRefreshDeviceToken } = require('../../utils/token-manager');
@@ -19,7 +18,7 @@ const {
 } = require('../validate');
 const { applyCapabilityDimension } = require('./dimension-operations');
 const { validateDimensionSemantics } = require('./dimension-validate');
-const { writeBackup, atomicWriteJson } = require('./run-capability-copy');
+const { writeBackup, atomicWriteJson, readDatasourceConfigFile } = require('./run-capability-copy');
 const { tryResolveDatasourceKeyToLocalPath, readJsonFile } = require('../../resolvers/datasource-resolver');
 const { tryFetchDatasourceConfig } = require('../../resolvers/manifest-resolver');
 
@@ -231,8 +230,7 @@ function applyBindingAndValidateSchema({ parsed, opts, semanticWarnings, remoteF
 async function runCapabilityDimension(rawOpts) {
   const opts = normalizeRunOptions(rawOpts);
   const resolvedPath = resolveValidateInputPath(opts.fileOrKey);
-  const raw = fs.readFileSync(resolvedPath, 'utf8');
-  const parsed = JSON.parse(raw);
+  const parsed = readDatasourceConfigFile(resolvedPath);
 
   const systemKeyForAuth = resolveSystemKeyForAuth(parsed);
   const { remoteTargetsByKey, remoteFetchMeta } = await loadRemoteTargetsByKey({

package/lib/datasource/capability/run-capability-edit.js

@@ -6,7 +6,6 @@
  * @version 2.0.0
  */
 
-const fs = require('fs');
 const { execSync } = require('child_process');
 const inquirer = require('inquirer');
 const {
@@ -19,7 +18,8 @@ const {
   resolveSingleOpsKey,
   resolveProfileKeyForLogical
 } = require('./capability-resolve');
-const { writeBackup, atomicWriteJson } = require('./run-capability-copy');
+const { writeBackup, atomicWriteJson, readDatasourceConfigFile } = require('./run-capability-copy');
+const { listCapabilityKeys } = require('./capability-manifest');
 
 /**
  * @typedef {object} RunCapabilityEditOpts
@@ -35,10 +35,7 @@ const { writeBackup, atomicWriteJson } = require('./run-capability-copy');
  * @returns {string[]}
  */
 function collectCapabilityKeys(doc) {
-  const s = new Set();
-  if (Array.isArray(doc.capabilities)) {
-    doc.capabilities.forEach((c) => s.add(String(c)));
-  }
+  const s = new Set(listCapabilityKeys(doc.capabilities));
   if (doc.openapi?.operations) {
     Object.keys(doc.openapi.operations).forEach((k) => s.add(k));
   }
@@ -351,7 +348,7 @@ async function runCapabilityEdit(opts) {
     throw new Error('capability edit requires an interactive terminal (TTY). Use a terminal or SSH with -t.');
   }
   const resolvedPath = resolveValidateInputPath(opts.fileOrKey.trim());
-  const doc = JSON.parse(fs.readFileSync(resolvedPath, 'utf8'));
+  const doc = readDatasourceConfigFile(resolvedPath);
   const { capKey, section, profileKey } = await resolveEditTargets(doc, opts);
   const initial = getEditableJson(doc, capKey, section, profileKey);
   const body = await promptEditorJson(initial);

package/lib/datasource/capability/run-capability-relate.js

@@ -6,14 +6,13 @@
  * @version 2.0.0
  */
 
-const fs = require('fs');
 const {
   resolveValidateInputPath,
   validateDatasourceParsed
 } = require('../validate');
 const { applyCapabilityRelate } = require('./relate-operations');
 const { validateRelateSemantics } = require('./relate-validate');
-const { writeBackup, atomicWriteJson } = require('./run-capability-copy');
+const { writeBackup, atomicWriteJson, readDatasourceConfigFile } = require('./run-capability-copy');
 const { tryResolveDatasourceKeyToLocalPath, readJsonFile } = require('../../resolvers/datasource-resolver');
 const { tryFetchDatasourceConfig } = require('../../resolvers/manifest-resolver');
 
@@ -134,8 +133,7 @@ function applyRelateAndValidateSchema(parsed, opts, semanticWarnings, remoteVali
  */
 async function runCapabilityRelate(opts) {
   const resolvedPath = resolveValidateInputPath(opts.fileOrKey.trim());
-  const raw = fs.readFileSync(resolvedPath, 'utf8');
-  const parsed = JSON.parse(raw);
+  const parsed = readDatasourceConfigFile(resolvedPath);
 
   const targetDocLocal = loadTargetDocLocal(opts.targetDatasource);
   const systemKeyForAuth = resolveSystemKeyForAuth(parsed);

package/lib/datasource/capability/run-capability-remove.js

@@ -6,14 +6,13 @@
  * @version 2.0.0
  */
 
-const fs = require('fs');
 const {
   resolveValidateInputPath,
   validateDatasourceParsed
 } = require('../validate');
 const { normalizeCapabilityKey } = require('./capability-key');
 const { applyCapabilityRemove } = require('./remove-operations');
-const { writeBackup, atomicWriteJson } = require('./run-capability-copy');
+const { writeBackup, atomicWriteJson, readDatasourceConfigFile } = require('./run-capability-copy');
 
 /**
  * @param {{ valid: boolean, errors: string[] }} validation
@@ -67,8 +66,7 @@ function makeRemoveResult(resolvedPath, capability, result, validation, dryRun,
  */
 async function runCapabilityRemove(opts) {
   const resolvedPath = resolveValidateInputPath(opts.fileOrKey.trim());
-  const raw = fs.readFileSync(resolvedPath, 'utf8');
-  const parsed = JSON.parse(raw);
+  const parsed = readDatasourceConfigFile(resolvedPath);
 
   const capability = normalizeCapabilityKey(opts.capability, 'Capability (--capability)');
 

package/lib/datasource/capability/validate-capability-slice.js

@@ -7,6 +7,7 @@
  */
 
 const { normalizeCapabilityKey } = require('./capability-key');
+const { capabilitiesIncludes } = require('./capability-manifest');
 
 /**
  * Lists structural locations where a capability key should appear.
@@ -18,7 +19,7 @@ const { normalizeCapabilityKey } = require('./capability-key');
 function checkCapabilitySlices(parsed, rawKey) {
   const key = normalizeCapabilityKey(rawKey, 'capability');
   const missing = [];
-  if (!Array.isArray(parsed.capabilities) || !parsed.capabilities.includes(key)) {
+  if (!capabilitiesIncludes(parsed.capabilities, key)) {
     missing.push(`capabilities[] (missing "${key}")`);
   }
   if (!parsed.openapi?.operations?.[key]) {

package/lib/datasource/datasource-identifier.js

@@ -38,7 +38,8 @@ function datasourceIdentifierMatchesFile(identifier, fileRef, parsed) {
   if (stem === id) {
     return true;
   }
-  if (`${stem}.json` === id.toLowerCase()) {
+  const lowerId = id.toLowerCase();
+  if (`${stem}.json` === lowerId || `${stem}.yaml` === lowerId || `${stem}.yml` === lowerId) {
     return true;
   }
   return false;

package/lib/datasource/datasource-validate-display.js

@@ -30,7 +30,7 @@ function logHeaderAndIdentity(result, trimmed, showMapping) {
   const { resolvedPath, summary } = result;
   logger.log('');
   logger.log(sectionTitle('Datasource validation'));
-  logger.log(metadata('Offline — JSON schema and integration wiring'));
+  logger.log(metadata('Offline — YAML/JSON schema and integration wiring'));
   logger.log('');
   if (summary) {
     logger.log(`  ${headerKeyValue('Key:', summary.key)}`);

package/lib/datasource/field-reference-validator.js

@@ -10,6 +10,8 @@
  * @version 2.0.0
  */
 
+const { validateOrphanExposedProfiles } = require('./orphan-profiles-validator');
+
 /**
  * Set of attribute names plus root dimension keys valid for field references.
  * Used for primaryKey (schema allows dimension keys or attributes) and other paths (attributes only).
@@ -40,16 +42,34 @@ function checkPrimaryKey(parsed, attributesAndDimensions, errors) {
   });
 }
 
+/** @returns {Set<string>} */
+function profileReferenceSet(parsed) {
+  if (parsed?.entityType === 'serviceContract') {
+    const schema = parsed?.exposed?.schema;
+    if (schema && typeof schema === 'object' && !Array.isArray(schema)) {
+      return new Set(
+        Object.keys(schema).filter(key => typeof key === 'string' && key !== '')
+      );
+    }
+  }
+  return new Set(Object.keys(parsed?.fieldMappings?.attributes ?? {}));
+}
+
 /** @param {string[]} errors */
-function checkExposedProfiles(parsed, attrSet, errors) {
+function checkExposedProfiles(parsed, _attrSet, errors) {
   const profiles = parsed?.exposed?.profiles;
   if (!profiles || typeof profiles !== 'object' || Array.isArray(profiles)) return;
+  const profileRefSet = profileReferenceSet(parsed);
+  const referenceLabel =
+    parsed?.entityType === 'serviceContract'
+      ? 'exposed.schema'
+      : 'fieldMappings.attributes';
   Object.entries(profiles).forEach(([profileName, fields]) => {
     if (!Array.isArray(fields)) return;
     fields.forEach((field, idx) => {
-      if (typeof field === 'string' && !attrSet.has(field)) {
+      if (typeof field === 'string' && !profileRefSet.has(field)) {
         errors.push(
-          `exposed.profiles.${profileName}[${idx}]: field '${field}' does not exist in fieldMappings.attributes. Add the attribute or remove the reference.`
+          `exposed.profiles.${profileName}[${idx}]: field '${field}' does not exist in ${referenceLabel}. Add the attribute or remove the reference.`
         );
       }
     });
@@ -121,6 +141,7 @@ function validateFieldReferences(parsed) {
 
   checkPrimaryKey(parsed, attributesAndDimensions, errors);
   checkExposedProfiles(parsed, attrSet, errors);
+  errors.push(...validateOrphanExposedProfiles(parsed));
   if (normalizedAttributes.length === 0) return errors;
 
   checkIndexingAndValidation(parsed, normalizedAttributes, errors);

package/lib/datasource/orphan-profiles-validator.js

@@ -0,0 +1,98 @@
+/**
+ * Validate exposed.profiles keys against capabilities (346 §13 — no orphan profiles).
+ *
+ * @fileoverview Orphan exposed profile validation for datasource manifests
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const { listCapabilityKeys } = require('./capability/capability-manifest');
+
+const FORBIDDEN_ORPHAN_PROFILE_KEYS = new Set(['minimal', 'analytics', 'metadata']);
+
+/**
+ * @param {string} name
+ * @returns {string}
+ */
+function normalizeProfileKey(name) {
+  return String(name).trim().toLowerCase();
+}
+
+/**
+ * @param {object} parsed
+ * @returns {Set<string>}
+ */
+function operationKeysFromConfig(parsed) {
+  const keys = new Set();
+  const openapiOps = parsed?.openapi?.operations;
+  if (openapiOps && typeof openapiOps === 'object') {
+    Object.keys(openapiOps).forEach((k) => keys.add(normalizeProfileKey(k)));
+  }
+  const cipOps = parsed?.execution?.cip?.operations;
+  if (cipOps && typeof cipOps === 'object') {
+    Object.keys(cipOps).forEach((k) => keys.add(normalizeProfileKey(k)));
+  }
+  return keys;
+}
+
+/**
+ * @param {object} parsed
+ * @returns {Set<string>}
+ */
+function allowedExposedProfileKeys(parsed) {
+  const allowed = new Set(['default']);
+  const capabilityKeys = listCapabilityKeys(parsed?.capabilities).map(normalizeProfileKey);
+  if (capabilityKeys.length > 0) {
+    capabilityKeys.forEach((k) => allowed.add(k));
+  } else {
+    operationKeysFromConfig(parsed).forEach((k) => allowed.add(k));
+  }
+  return allowed;
+}
+
+/**
+ * @param {object} parsed - Parsed datasource manifest
+ * @returns {string[]} Error messages; empty when valid
+ */
+function validateOrphanExposedProfiles(parsed) {
+  const errors = [];
+  const profiles = parsed?.exposed?.profiles;
+  if (!profiles || typeof profiles !== 'object' || Array.isArray(profiles)) {
+    return errors;
+  }
+
+  const allowed = allowedExposedProfileKeys(parsed);
+  const orphans = Object.keys(profiles).filter((name) => {
+    const normalized = normalizeProfileKey(name);
+    return FORBIDDEN_ORPHAN_PROFILE_KEYS.has(normalized) || !allowed.has(normalized);
+  });
+
+  if (orphans.length === 0) {
+    return errors;
+  }
+
+  const decorative = orphans.filter((name) =>
+    FORBIDDEN_ORPHAN_PROFILE_KEYS.has(normalizeProfileKey(name))
+  );
+  if (decorative.length > 0) {
+    errors.push(
+      `exposed.profiles: orphan exposed profiles are forbidden: ${decorative.join(', ')}. ` +
+        'Use exposed.profiles.default and one profile per capabilities[].key only.'
+    );
+    return errors;
+  }
+
+  errors.push(
+    `exposed.profiles: orphan profile keys must match capabilities[].key or default: ${orphans.join(', ')}. ` +
+      `Allowed profile keys: ${[...allowed].sort().join(', ')}.`
+  );
+  return errors;
+}
+
+module.exports = {
+  FORBIDDEN_ORPHAN_PROFILE_KEYS,
+  allowedExposedProfileKeys,
+  validateOrphanExposedProfiles
+};

package/lib/datasource/validate.js

@@ -10,6 +10,7 @@
 
 const path = require('path');
 const fs = require('fs');
+const { loadConfigFile } = require('../utils/config-format');
 const pathsUtil = require('../utils/paths');
 const {
   datasourceIdentifierMatchesFile,
@@ -22,7 +23,16 @@ const { validateFieldReferences } = require('./field-reference-validator');
 const { validateAbac } = require('./abac-validator');
 const { buildDatasourceValidateSummary } = require('./datasource-validate-summary');
 
-const EXCLUDE_JSON_NAMES = new Set(['application.json', 'rbac.json', 'package.json', 'package-lock.json']);
+const DATASOURCE_CONFIG_EXTENSIONS = new Set(['.json', '.yaml', '.yml']);
+const EXCLUDE_CONFIG_NAMES = new Set([
+  'application.json',
+  'application.yaml',
+  'application.yml',
+  'rbac.json',
+  'rbac.yaml',
+  'package.json',
+  'package-lock.json'
+]);
 
 /**
  * Resolve identifier to an existing file path, or null.
@@ -64,11 +74,14 @@ function pickIntegrationAppForDatasourceKey(datasourceKey, appNames) {
  * @param {string} fileName
  * @returns {boolean}
  */
-function shouldScanJsonDatasourceCandidate(fileName) {
+function shouldScanDatasourceConfigCandidate(fileName) {
   const lower = fileName.toLowerCase();
-  if (!lower.endsWith('.json')) return false;
-  if (EXCLUDE_JSON_NAMES.has(fileName)) return false;
+  const ext = path.extname(lower);
+  if (!DATASOURCE_CONFIG_EXTENSIONS.has(ext)) return false;
+  if (EXCLUDE_CONFIG_NAMES.has(fileName)) return false;
   if (lower.endsWith('-deploy.json') || lower.endsWith('-system.json')) return false;
+  if (lower.endsWith('-deploy.yaml') || lower.endsWith('-deploy.yml')) return false;
+  if (lower.endsWith('-system.yaml') || lower.endsWith('-system.yml')) return false;
   return true;
 }
 
@@ -86,13 +99,13 @@ function isDatasourceNamedJson(fileName) {
  * @param {string} datasourceKey
  * @returns {string[]}
  */
-function collectJsonPathsMatchingDatasourceKey(dir, scanOrder, datasourceKey) {
+function collectConfigPathsMatchingDatasourceKey(dir, scanOrder, datasourceKey) {
   const hits = [];
   for (const name of scanOrder) {
     const full = path.join(dir, name);
     let parsed;
     try {
-      parsed = JSON.parse(fs.readFileSync(full, 'utf8'));
+      parsed = loadConfigFile(full);
     } catch {
       continue;
     }
@@ -115,14 +128,14 @@ function resolvePathFromIntegrationFolderScan(appName, datasourceKey) {
     throw new Error(`Integration folder not found: ${dir}`);
   }
   const entries = fs.readdirSync(dir);
-  const jsonFiles = entries.filter(shouldScanJsonDatasourceCandidate);
-  const prefer = jsonFiles.filter(isDatasourceNamedJson);
-  const scanOrder = [...prefer, ...jsonFiles.filter((f) => !prefer.includes(f))];
-  const hits = collectJsonPathsMatchingDatasourceKey(dir, scanOrder, datasourceKey);
+  const configFiles = entries.filter(shouldScanDatasourceConfigCandidate);
+  const prefer = configFiles.filter(isDatasourceNamedJson);
+  const scanOrder = [...prefer, ...configFiles.filter((f) => !prefer.includes(f))];
+  const hits = collectConfigPathsMatchingDatasourceKey(dir, scanOrder, datasourceKey);
   if (hits.length === 0) {
     throw new Error(
-      `No datasource JSON matching "${datasourceKey}" under integration/${appName}/. ` +
-        `Checked: ${scanOrder.length ? scanOrder.join(', ') : '(no JSON candidates)'}`
+      `No datasource config matching "${datasourceKey}" under integration/${appName}/. ` +
+        `Checked: ${scanOrder.length ? scanOrder.join(', ') : '(no JSON/YAML candidates)'}`
     );
   }
   if (hits.length > 1) {
@@ -167,7 +180,8 @@ function resolveValidateInputPath(identifier) {
   }
   const asFile = tryResolveAsExistingFile(trimmed);
   if (asFile) return asFile;
-  if (trimmed.toLowerCase().endsWith('.json')) {
+  const ext = path.extname(trimmed).toLowerCase();
+  if (DATASOURCE_CONFIG_EXTENSIONS.has(ext)) {
     throw new Error(`File not found: ${trimmed}`);
   }
   return resolvePathFromIntegrationDatasourceKey(trimmed);
@@ -187,6 +201,18 @@ function resolveValidateInputPath(identifier) {
  * const result = await validateDatasourceFile('./hubspot-deal.json');
  * // Returns: { valid: true, errors: [], warnings: [], resolvedPath: '...' }
  */
+function buildParseFailureResult(filePath, error) {
+  const message =
+    error && error.message ? String(error.message) : 'Invalid config file';
+  return {
+    valid: false,
+    errors: [message],
+    warnings: [],
+    resolvedPath: filePath,
+    summary: null
+  };
+}
+
 async function validateDatasourceFile(filePathOrKey) {
   if (!filePathOrKey || typeof filePathOrKey !== 'string') {
     throw new Error('File path is required and must be a string');
@@ -194,19 +220,11 @@ async function validateDatasourceFile(filePathOrKey) {
 
   const filePath = resolveValidateInputPath(filePathOrKey.trim());
 
-  const content = fs.readFileSync(filePath, 'utf8');
   let parsed;
-
   try {
-    parsed = JSON.parse(content);
+    parsed = loadConfigFile(filePath);
   } catch (error) {
-    return {
-      valid: false,
-      errors: [`Invalid JSON syntax: ${error.message}`],
-      warnings: [],
-      resolvedPath: filePath,
-      summary: null
-    };
+    return buildParseFailureResult(filePath, error);
   }
 
   const summary = buildDatasourceValidateSummary(parsed);

package/lib/external-system/download-helpers.js

@@ -8,8 +8,90 @@
  * @version 2.0.0
  */
 
+const {
+  formatSuccessLine,
+  formatSuccessParagraph,
+  sectionTitle,
+  headerKeyValue
+} = require('../utils/cli-test-layout-chalk');
+const logger = require('../utils/logger');
 const { generateExternalReadmeContent } = require('../utils/external-readme');
 
+/**
+ * Validates system type from downloaded application
+ * @param {Object} application - External system configuration
+ * @returns {string} System type (openapi, mcp, custom)
+ */
+function validateSystemType(application) {
+  if (!application || typeof application !== 'object') {
+    throw new Error('Application configuration is required');
+  }
+  const validTypes = ['openapi', 'mcp', 'custom'];
+  const systemType = application.type;
+  if (!systemType || !validTypes.includes(systemType)) {
+    throw new Error(`Invalid system type: ${systemType}. Must be one of: ${validTypes.join(', ')}`);
+  }
+  return systemType;
+}
+
+function validateApplicationConfig(application) {
+  if (!application || typeof application !== 'object') {
+    throw new Error('Application configuration is required');
+  }
+  if (!application.key || typeof application.key !== 'string') {
+    throw new Error('Application key is required');
+  }
+}
+
+function validateDatasourceConfig(datasource, applicationKey) {
+  if (!datasource.key || typeof datasource.key !== 'string') {
+    throw new Error('Datasource key is required for all datasources');
+  }
+  if (!datasource.systemKey || typeof datasource.systemKey !== 'string') {
+    throw new Error('Datasource systemKey is required for all datasources');
+  }
+  if (datasource.systemKey !== applicationKey) {
+    throw new Error(
+      `Datasource systemKey (${datasource.systemKey}) does not match application key (${applicationKey})`
+    );
+  }
+}
+
+function validateDownloadedData(application, dataSources) {
+  validateApplicationConfig(application);
+  if (!Array.isArray(dataSources)) {
+    throw new Error('DataSources must be an array');
+  }
+  for (const datasource of dataSources) {
+    validateDatasourceConfig(datasource, application.key);
+  }
+}
+
+function logDownloadCommandHeader(systemKey) {
+  logger.log('');
+  logger.log(sectionTitle('Download'));
+  logger.log(headerKeyValue('System:', systemKey));
+}
+
+function displayDownloadSuccess(systemKey, finalPath, datasourceCount) {
+  logger.log(
+    formatSuccessParagraph(
+      `Downloaded ${systemKey} (${datasourceCount} datasource${datasourceCount === 1 ? '' : 's'})`
+    )
+  );
+  logger.log(headerKeyValue('Location:', finalPath));
+}
+
+async function runConvertToJsonIfRequested(systemKey) {
+  const { runConvert } = require('../commands/convert');
+  try {
+    await runConvert(systemKey, { format: 'json', force: true });
+    logger.log(formatSuccessLine('Converted component files to JSON'));
+  } catch (convertErr) {
+    throw new Error(`Download succeeded but convert to JSON failed: ${convertErr.message}`);
+  }
+}
+
 /**
  * Generates application.yaml content for downloaded system
  * @param {string} systemKey - System key
@@ -90,6 +172,11 @@ function generateReadme(systemKey, application, dataSources) {
 }
 
 module.exports = {
+  validateSystemType,
+  validateDownloadedData,
+  logDownloadCommandHeader,
+  displayDownloadSuccess,
+  runConvertToJsonIfRequested,
   generateVariablesYaml,
   generateReadme
 };