@aifabrix/builder 2.45.5 → 2.47.0

package/lib/commands/repair-datasource-test-payload.js

@@ -0,0 +1,259 @@
+/**
+ * Schema-aware testPayload repair (payloadTemplate / expectedResult).
+ *
+ * @fileoverview Helpers for repair-datasource testPayload generation
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+'use strict';
+
+/**
+ * @param {string|string[]|undefined} type - JSON Schema type
+ * @returns {string}
+ */
+function primarySchemaType(type) {
+  if (Array.isArray(type)) {
+    const nonNull = type.filter(t => t !== 'null');
+    return nonNull[0] || 'string';
+  }
+  return type || 'string';
+}
+
+/**
+ * @param {Object|null|undefined} schema - JSON Schema node
+ * @param {string} key - Property name
+ * @returns {Object|null}
+ */
+function getSchemaProperty(schema, key) {
+  if (!schema || typeof schema !== 'object') return null;
+  const props = schema.properties;
+  if (props && typeof props === 'object' && Object.prototype.hasOwnProperty.call(props, key)) {
+    return props[key];
+  }
+  if (schema.additionalProperties === true) {
+    return { type: 'string' };
+  }
+  if (schema.additionalProperties && typeof schema.additionalProperties === 'object') {
+    return schema.additionalProperties;
+  }
+  return null;
+}
+
+/**
+ * @param {Object|null|undefined} rootSchema - metadataSchema root
+ * @param {string[]} pathParts - Dot path under raw payload (no raw. prefix)
+ * @returns {Object|null}
+ */
+function getSchemaNodeAtPath(rootSchema, pathParts) {
+  if (!rootSchema || !pathParts.length) return rootSchema;
+  let node = rootSchema;
+  for (const part of pathParts) {
+    if (!part) continue;
+    const next = getSchemaProperty(node, part);
+    if (!next) return null;
+    node = next;
+  }
+  return node;
+}
+
+/**
+ * @param {Object|null|undefined} node - JSON Schema node
+ * @returns {*}
+ */
+function placeholderFromSchemaNode(node) {
+  if (!node || typeof node !== 'object') return '';
+  const type = primarySchemaType(node.type);
+  if (node.format === 'date-time') return '2025-01-15T10:30:00Z';
+  if (node.format === 'date') return '2025-01-15';
+  if (node.format === 'uuid') return '00000000-0000-4000-8000-000000000001';
+  if (type === 'boolean') return false;
+  if (type === 'integer' || type === 'number') return 0;
+  if (type === 'array') return [];
+  if (type === 'object') {
+    if (node.properties && typeof node.properties === 'object') {
+      const out = {};
+      for (const [k, sub] of Object.entries(node.properties)) {
+        out[k] = placeholderFromSchemaNode(sub);
+      }
+      return out;
+    }
+    return {};
+  }
+  return '';
+}
+
+/**
+ * @param {string} type - Legacy attribute type
+ * @returns {*}
+ */
+function placeholderForType(type) {
+  if (type === 'number' || type === 'integer') return 0;
+  if (type === 'boolean') return false;
+  if (type === 'array') return [];
+  if (type === 'object') return {};
+  return '';
+}
+
+/**
+ * @param {Object} parsed - Datasource config
+ * @param {string[]} pathParts - Path under raw payload
+ * @param {Object} [attrConfig] - Attribute config (optional type fallback)
+ * @returns {*}
+ */
+function resolvePlaceholderForPath(parsed, pathParts, attrConfig) {
+  const node = getSchemaNodeAtPath(parsed?.metadataSchema, pathParts);
+  if (node) return placeholderFromSchemaNode(node);
+  return placeholderForType(attrConfig?.type || 'string');
+}
+
+/**
+ * @param {Object} obj - Target object (mutated)
+ * @param {string[]} pathParts - Path segments
+ * @param {*} value - Value to set
+ */
+function setNestedValue(obj, pathParts, value) {
+  const parts = pathParts.filter(Boolean);
+  if (!parts.length) return;
+  let cur = obj;
+  for (let i = 0; i < parts.length - 1; i++) {
+    const p = parts[i];
+    if (!(p in cur) || typeof cur[p] !== 'object' || cur[p] === null || Array.isArray(cur[p])) {
+      cur[p] = {};
+    }
+    cur = cur[p];
+  }
+  const last = parts[parts.length - 1];
+  const existing = cur[last];
+  if (
+    existing &&
+    typeof existing === 'object' &&
+    !Array.isArray(existing) &&
+    value &&
+    typeof value === 'object' &&
+    !Array.isArray(value)
+  ) {
+    cur[last] = { ...existing, ...value };
+    return;
+  }
+  cur[last] = value;
+}
+
+/**
+ * @param {Object} parsed - Datasource config
+ * @param {Array<{ path: string, config: Object }>} pathEntries - Sorted deepest-first
+ * @param {Object} payloadTemplate - payloadTemplate (mutated)
+ */
+function enrichObjectLeavesFromNestedPaths(parsed, pathEntries, payloadTemplate) {
+  const propertiesNode = getSchemaNodeAtPath(parsed?.metadataSchema, ['properties']);
+  for (const { path } of pathEntries) {
+    if (!path.startsWith('properties.')) continue;
+    const leaf = path.slice('properties.'.length);
+    if (!leaf || leaf.includes('.')) continue;
+    if (!payloadTemplate.properties || typeof payloadTemplate.properties !== 'object') {
+      payloadTemplate.properties = placeholderFromSchemaNode(propertiesNode);
+    }
+    if (payloadTemplate.properties[leaf] === undefined || payloadTemplate.properties[leaf] === '') {
+      const leafNode = getSchemaNodeAtPath(parsed?.metadataSchema, ['properties', leaf]);
+      payloadTemplate.properties[leaf] = leafNode
+        ? placeholderFromSchemaNode(leafNode)
+        : 'sample';
+    }
+  }
+}
+
+/**
+ * @param {string} expressionPath - Path from {{ ... }} token
+ * @returns {string|null}
+ */
+function stripExpressionPathPrefix(expressionPath) {
+  const path = expressionPath.trim();
+  if (path.startsWith('raw.')) return path.slice(4);
+  if (path.startsWith('metadata.')) return path.slice(9);
+  return path;
+}
+
+/**
+ * @param {Object} parsed - Datasource config
+ * @param {Object} attrs - fieldMappings.attributes
+ * @returns {Array<{ attrKey: string, path: string|null, config: Object, expectedOnly: *|null }>}
+ */
+function buildPathEntriesFromAttributes(parsed, attrs) {
+  const pathEntries = [];
+  for (const [attrKey, config] of Object.entries(attrs)) {
+    const match = config?.expression?.match(/\{\{\s*([^}|]+)/);
+    if (!match) {
+      const metaNode = parsed.metadataSchema?.properties?.[attrKey];
+      const expected = metaNode
+        ? placeholderFromSchemaNode(metaNode)
+        : placeholderForType(config?.type || 'string');
+      pathEntries.push({ attrKey, path: null, config, expectedOnly: expected });
+      continue;
+    }
+    pathEntries.push({
+      attrKey,
+      path: stripExpressionPathPrefix(match[1]),
+      config,
+      expectedOnly: null
+    });
+  }
+  return pathEntries.sort((a, b) => {
+    const depthA = a.path ? a.path.split('.').length : 0;
+    const depthB = b.path ? b.path.split('.').length : 0;
+    return depthB - depthA;
+  });
+}
+
+/**
+ * @param {Object} parsed - Datasource config
+ * @param {Array} pathEntries - From buildPathEntriesFromAttributes
+ * @returns {{ payloadTemplate: Object, expectedResult: Object }}
+ */
+function buildTemplatesFromPathEntries(parsed, pathEntries) {
+  const payloadTemplate = {};
+  const expectedResult = {};
+  for (const entry of pathEntries) {
+    const { attrKey, path, config, expectedOnly } = entry;
+    const metaNode = parsed.metadataSchema?.properties?.[attrKey];
+    expectedResult[attrKey] = metaNode
+      ? placeholderFromSchemaNode(metaNode)
+      : (expectedOnly ?? placeholderForType(config?.type || 'string'));
+    if (!path) continue;
+    const parts = path.split('.');
+    setNestedValue(payloadTemplate, parts, resolvePlaceholderForPath(parsed, parts, config));
+  }
+  enrichObjectLeavesFromNestedPaths(parsed, pathEntries, payloadTemplate);
+  return { payloadTemplate, expectedResult };
+}
+
+/**
+ * Builds minimal payloadTemplate and expectedResult from attributes + metadataSchema.
+ *
+ * @param {Object} parsed - Parsed datasource (mutated)
+ * @param {string[]} changes - Change log
+ * @returns {boolean} True if testPayload was added or updated
+ */
+function repairTestPayload(parsed, changes) {
+  const attrs = parsed?.fieldMappings?.attributes;
+  if (!attrs || typeof attrs !== 'object') return false;
+
+  const pathEntries = buildPathEntriesFromAttributes(parsed, attrs);
+  const { payloadTemplate, expectedResult } = buildTemplatesFromPathEntries(parsed, pathEntries);
+
+  if (!parsed.testPayload) parsed.testPayload = {};
+  parsed.testPayload.payloadTemplate = payloadTemplate;
+  parsed.testPayload.expectedResult = expectedResult;
+  changes.push(
+    'Generated testPayload.payloadTemplate and testPayload.expectedResult from attributes and metadataSchema'
+  );
+  return true;
+}
+
+module.exports = {
+  primarySchemaType,
+  getSchemaNodeAtPath,
+  placeholderFromSchemaNode,
+  placeholderForType,
+  resolvePlaceholderForPath,
+  repairTestPayload
+};

package/lib/commands/repair-datasource.js

@@ -11,6 +11,13 @@
 
 const { repairOpenapiSection } = require('./repair-datasource-openapi');
 const { normalizeAutoRbacOperationKeys } = require('./repair-datasource-auto-rbac');
+const { repairTestPayload } = require('./repair-datasource-test-payload');
+const { repairComV2Manifest } = require('./repair-datasource-com-v2');
+const {
+  isDeprecatedNoneLlmManifest,
+  repairNoneLlmToServiceContract,
+  stripDeprecatedServiceCategory
+} = require('./repair-datasource-service-contract');
 
 const DEFAULT_SYNC = {
   mode: 'pull',
@@ -358,25 +365,6 @@ function repairSyncSection(parsed, changes) {
   return true;
 }
 
-function placeholderForType(type) {
-  if (type === 'number' || type === 'integer') return 0;
-  if (type === 'boolean') return false;
-  if (type === 'array') return [];
-  if (type === 'object') return {};
-  return '';
-}
-
-function setNested(obj, pathParts, value) {
-  let cur = obj;
-  for (let i = 0; i < pathParts.length - 1; i++) {
-    const p = pathParts[i];
-    if (!(p in cur) || typeof cur[p] !== 'object') cur[p] = {};
-    cur = cur[p];
-  }
-  const last = pathParts[pathParts.length - 1];
-  if (last) cur[last] = value;
-}
-
 /**
  * Removes unknown top-level keys from testPayload (v2.4.1 closed root).
  * @param {Object} parsed - Parsed datasource (mutated)
@@ -399,35 +387,38 @@ function sanitizeTestPayloadTopLevel(parsed, changes) {
 }
 
 /**
- * Builds minimal payloadTemplate and expectedResult from attribute expression paths.
- *
- * @param {Object} parsed - Parsed datasource (mutated)
- * @param {string[]} changes - Change log
- * @returns {boolean} True if testPayload was added or updated
+ * @param {Object} parsed
+ * @param {Object} options
+ * @returns {boolean}
  */
-function repairTestPayload(parsed, changes) {
-  const attrs = parsed?.fieldMappings?.attributes;
-  if (!attrs || typeof attrs !== 'object') return false;
-  const payloadTemplate = {};
-  const expectedResult = {};
-  for (const [key, config] of Object.entries(attrs)) {
-    const type = config?.type || 'string';
-    const placeholder = placeholderForType(type);
-    expectedResult[key] = placeholder;
-    const match = config?.expression?.match(/\{\{\s*([^}|]+)/);
-    if (match) {
-      let path = match[1].trim();
-      if (path.startsWith('raw.')) {
-        path = path.slice(4);
-      }
-      setNested(payloadTemplate, path.split('.'), placeholder);
-    }
+function shouldApplyServiceContractRepair(parsed, options) {
+  return (
+    isDeprecatedNoneLlmManifest(parsed) ||
+    (options.serviceContract && parsed?.resourceType === 'llm')
+  );
+}
+
+/**
+ * @param {Object} parsed
+ * @param {Object} options
+ * @param {boolean} none
+ * @param {string[]} changes
+ * @returns {boolean}
+ */
+function applyOptionalDatasourceRepairs(parsed, options, none, changes) {
+  let updated = false;
+  if (options.expose) {
+    updated = repairExposeFromAttributes(parsed, changes) || updated;
   }
-  if (!parsed.testPayload) parsed.testPayload = {};
-  parsed.testPayload.payloadTemplate = payloadTemplate;
-  parsed.testPayload.expectedResult = expectedResult;
-  changes.push('Generated testPayload.payloadTemplate and testPayload.expectedResult from attributes');
-  return true;
+  if (!none && options.sync) {
+    updated = repairSyncSection(parsed, changes) || updated;
+  }
+  if (options.test) {
+    sanitizeTestPayloadTopLevel(parsed, changes);
+    updated = repairTestPayload(parsed, changes) || updated;
+    updated = sanitizeTestPayloadTopLevel(parsed, changes) || updated;
+  }
+  return updated;
 }
 
 /**
@@ -440,32 +431,24 @@ function repairTestPayload(parsed, changes) {
  */
 function repairDatasourceFile(parsed, options = {}, changes = []) {
   const out = Array.isArray(changes) ? changes : [];
-  let updated = false;
-  const none = isNoneEntityType(parsed?.entityType);
-
-  updated = applyBaseDatasourceRepairs(parsed, none, out) || updated;
-  updated = applyOpenapiDatasourceRepairs(parsed, out) || updated;
+  let updated = stripDeprecatedServiceCategory(parsed, out);
 
-  if (options.expose) {
-    updated = repairExposeFromAttributes(parsed, out) || updated;
+  if (shouldApplyServiceContractRepair(parsed, options)) {
+    updated = repairNoneLlmToServiceContract(parsed, out) || updated;
   }
 
-  if (!none && options.sync) {
-    updated = repairSyncSection(parsed, out) || updated;
-  }
+  const none = isNoneEntityType(parsed?.entityType);
 
-  if (options.test) {
-    sanitizeTestPayloadTopLevel(parsed, out);
-    updated = repairTestPayload(parsed, out) || updated;
-    updated = sanitizeTestPayloadTopLevel(parsed, out) || updated;
-  }
+  updated = applyBaseDatasourceRepairs(parsed, none, out) || updated;
+  updated = applyOpenapiDatasourceRepairs(parsed, out) || updated;
+  updated = applyOptionalDatasourceRepairs(parsed, options, none, out) || updated;
 
   return { updated, changes: out };
 }
 
 function applyBaseDatasourceRepairs(parsed, none, changes) {
-  if (none) return false;
-  let updated = false;
+  let updated = repairComV2Manifest(parsed, changes);
+  if (none) return updated;
   updated = repairDimensionBindingShape(parsed, changes) || updated;
   updated = repairRootDimensionsFromAttributes(parsed, changes) || updated;
   updated = repairMetadataSchemaFromAttributes(parsed, changes) || updated;
@@ -494,5 +477,7 @@ module.exports = {
   DEFAULT_SYNC,
   MINIMAL_METADATA_SCHEMA,
   TEST_PAYLOAD_TOP_LEVEL_ALLOW,
-  isNoneEntityType
+  isNoneEntityType,
+  isDeprecatedNoneLlmManifest,
+  repairNoneLlmToServiceContract
 };

package/lib/commands/repair-env-template.js

@@ -10,9 +10,11 @@
 const path = require('path');
 const fs = require('fs');
 const { backupIntegrationFile } = require('../utils/integration-file-backup');
+const { trackRepairWrite } = require('./repair-changed-files');
 const { systemKeyToKvPrefix, kvEnvKeyToPath, securityKeyToVar } = require('../utils/credential-secrets-env');
 const { extractEnvTemplate } = require('../generator/split');
 const { generateExternalEnvTemplateContent } = require('../utils/external-env-template');
+const { commentOrphanKvEnvLines } = require('../utils/env-kv-orphan-lines');
 
 /**
  * Normalizes a keyvault config entry to canonical KV_* name and path-style value.
@@ -75,9 +77,16 @@ function addFromConfiguration(effective, config, prefix, seenNames, systemKey) {
  * @param {string} systemKey - System key for path namespace
  */
 function addFromAuthSecurity(effective, systemParsed, prefix, seenNames, systemKey) {
-  const security = systemParsed.authentication?.security;
+  const { getAuthCredentialTemplate } = require('../utils/authentication-shape');
+  const { isPlatformCatalogKvRef } = require('../utils/platform-kv-ref');
+  const credential = getAuthCredentialTemplate(systemParsed.authentication);
+  const security = credential?.security ?? systemParsed.authentication?.security;
   if (!security || typeof security !== 'object') return;
   for (const key of Object.keys(security)) {
+    const securityVal = security[key];
+    if (typeof securityVal === 'string' && isPlatformCatalogKvRef(securityVal)) {
+      continue;
+    }
     const envName = `KV_${prefix}_${securityKeyToVar(key)}`;
     if (seenNames.has(envName)) continue;
     const pathVal = kvEnvKeyToPath(envName, systemKey);
@@ -136,7 +145,7 @@ function buildExpectedByKey(effective) {
  * @param {Object} [systemParsed] - Parsed system config for template context (preferred)
  * @returns {boolean}
  */
-function createEnvTemplateIfMissing(envPath, expectedByKey, dryRun, changes, systemParsed) {
+function createEnvTemplateIfMissing(envPath, expectedByKey, dryRun, changes, systemParsed, backupCtx) {
   if (fs.existsSync(envPath)) return false;
   if (expectedByKey.size === 0) return false;
   const content = systemParsed
@@ -145,6 +154,7 @@ function createEnvTemplateIfMissing(envPath, expectedByKey, dryRun, changes, sys
   if (!content || !content.trim()) return false;
   const hasKeyValueLine = /^[A-Z_][A-Z0-9_]*=/m.test(content);
   if (!hasKeyValueLine) return false;
+  trackRepairWrite(envPath, backupCtx);
   if (!dryRun) {
     fs.writeFileSync(envPath, content + (content.endsWith('\n') ? '' : '\n'), { mode: 0o644, encoding: 'utf8' });
   }
@@ -229,6 +239,51 @@ function mergeEnvTemplateContent(content, expectedByKey) {
   return { output, changed: changedRef.value };
 }
 
+/**
+ * Comments active KV_* lines whose env key is not in the system-derived expected set.
+ * @param {string} content
+ * @param {Map<string, string>} expectedByKey
+ * @param {{ value: boolean }} changedRef
+ * @returns {string}
+ */
+function commentOrphanKvLines(content, expectedByKey, changedRef) {
+  const { content: next, changed } = commentOrphanKvEnvLines(content, expectedByKey);
+  if (changed) {
+    changedRef.value = true;
+  }
+  return next;
+}
+
+/**
+ * @param {string} envPath
+ * @param {string} output
+ * @param {boolean} dryRun
+ * @param {Object} [backupCtx]
+ */
+function persistEnvTemplate(envPath, output, dryRun, backupCtx) {
+  trackRepairWrite(envPath, backupCtx);
+  if (!dryRun) {
+    backupIntegrationFile(envPath, backupCtx);
+    fs.writeFileSync(envPath, output, { mode: 0o644, encoding: 'utf8' });
+  }
+}
+
+/**
+ * @param {string} content
+ * @param {Map<string, string>} expectedByKey
+ * @returns {{ output: string, mergeChanged: boolean, pruneChanged: boolean }}
+ */
+function mergeAndPruneEnvTemplate(content, expectedByKey) {
+  const mergeResult = mergeEnvTemplateContent(content, expectedByKey);
+  const pruneChanged = { value: false };
+  const output = commentOrphanKvLines(mergeResult.output, expectedByKey, pruneChanged);
+  return {
+    output,
+    mergeChanged: mergeResult.changed,
+    pruneChanged: pruneChanged.value
+  };
+}
+
 /**
  * Repairs env.template so KV_ names and path-style kv:// values match the system file.
  * @param {string} appPath - Application directory path
@@ -240,41 +295,35 @@ function mergeEnvTemplateContent(content, expectedByKey) {
  * @returns {boolean} True if env.template was repaired or created
  */
 function repairEnvTemplate(appPath, systemParsed, systemKey, dryRun, changes, backupCtx) {
-  const effective = buildEffectiveConfiguration(systemParsed, systemKey);
-  const expectedByKey = buildExpectedByKey(effective);
+  const expectedByKey = buildExpectedByKey(buildEffectiveConfiguration(systemParsed, systemKey));
   const envPath = path.join(appPath, 'env.template');
 
-  if (createEnvTemplateIfMissing(envPath, expectedByKey, dryRun, changes, systemParsed)) {
+  if (createEnvTemplateIfMissing(envPath, expectedByKey, dryRun, changes, systemParsed, backupCtx)) {
     return true;
   }
   if (!fs.existsSync(envPath)) {
     return false;
   }
 
-  const content = fs.readFileSync(envPath, 'utf8');
-  const { output, changed } = mergeEnvTemplateContent(content, expectedByKey);
-
-  if (changed && !dryRun) {
-    backupIntegrationFile(envPath, backupCtx);
-    fs.writeFileSync(envPath, output, { mode: 0o644, encoding: 'utf8' });
+  const { output, mergeChanged, pruneChanged } = mergeAndPruneEnvTemplate(
+    fs.readFileSync(envPath, 'utf8'),
+    expectedByKey
+  );
+  if (!mergeChanged && !pruneChanged) {
+    return false;
   }
-  if (changed) {
+
+  persistEnvTemplate(envPath, output, dryRun, backupCtx);
+  if (mergeChanged) {
     changes.push('Repaired env.template (KV_* names and path-style kv:// values)');
-    return true;
   }
-  return false;
+  if (pruneChanged) {
+    changes.push('Commented stale env.template KV_* lines (not in system authentication)');
+  }
+  return true;
 }
 
-/**
- * Returns true if a kv value looks like legacy format (KeyVault suffix or no path segments).
- * @param {string} val - Value from authentication.security or configuration
- * @returns {boolean}
- */
-function isLegacyKvValue(val) {
-  if (typeof val !== 'string' || !val.trim().toLowerCase().startsWith('kv://')) return false;
-  const after = val.trim().slice(5); // after 'kv://'
-  return after.includes('KeyVault') || !after.includes('/');
-}
+const { isLegacyKvValue } = require('../utils/platform-kv-ref');
 
 /**
  * Normalizes authentication.security values to path-style kv:// (kv://systemKey/variable).
@@ -341,7 +390,9 @@ function normalizeConfigurationSection(config, prefix, systemKey, changes) {
 function normalizeSystemFileAuthAndConfig(systemParsed, systemKey, changes) {
   const prefix = systemKeyToKvPrefix(systemKey);
   if (!prefix) return false;
-  const security = systemParsed.authentication?.security;
+  const { getAuthCredentialTemplate } = require('../utils/authentication-shape');
+  const credential = getAuthCredentialTemplate(systemParsed.authentication);
+  const security = credential?.security ?? systemParsed.authentication?.security;
   let updated = (security && typeof security === 'object' && normalizeSecuritySection(security, prefix, systemKey, changes));
   const config = systemParsed.configuration;
   if (Array.isArray(config)) {