@aifabrix/builder 2.41.0 → 2.42.1

package/lib/commands/repair-auth-config.js

@@ -0,0 +1,99 @@
+/**
+ * Normalizes system file authentication.security and configuration keyvault entries.
+ * @fileoverview Repair auth/config KV_* names and path-style kv:// values
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+'use strict';
+
+const { systemKeyToKvPrefix, securityKeyToVar, kvEnvKeyToPath } = require('../utils/credential-secrets-env');
+
+/**
+ * Returns true if a kv value looks like legacy format (KeyVault suffix or no path segments).
+ * @param {string} val - Value from authentication.security or configuration
+ * @returns {boolean}
+ */
+function isLegacyKvValue(val) {
+  if (typeof val !== 'string' || !val.trim().toLowerCase().startsWith('kv://')) return false;
+  const after = val.trim().slice(5); // after 'kv://'
+  return after.includes('KeyVault') || !after.includes('/');
+}
+
+/**
+ * Normalizes authentication.security keyvault entries to path-style kv:// values (kv://systemKey/variable).
+ * @param {Object} security - authentication.security object (mutated)
+ * @param {string} prefix - KV prefix (e.g. 'HUBSPOT')
+ * @param {string} systemKey - System key for path namespace
+ * @param {string[]} changes - Array to append change descriptions to
+ * @returns {boolean} True if any change was made
+ */
+function normalizeSecuritySection(security, prefix, systemKey, changes) {
+  let updated = false;
+  for (const key of Object.keys(security)) {
+    const val = security[key];
+    if (typeof val !== 'string' || !isLegacyKvValue(val)) continue;
+    const envName = `KV_${prefix}_${securityKeyToVar(key)}`;
+    const pathVal = kvEnvKeyToPath(envName, systemKey);
+    if (pathVal) {
+      security[key] = pathVal;
+      changes.push(`authentication.security.${key}: normalized to path-style ${pathVal}`);
+      updated = true;
+    }
+  }
+  return updated;
+}
+
+/**
+ * Normalizes configuration array keyvault entries to canonical KV_* names and path-style values.
+ * @param {Object[]} config - configuration array (mutated)
+ * @param {string} prefix - KV prefix (e.g. 'HUBSPOT')
+ * @param {string} systemKey - System key for path namespace
+ * @param {string[]} changes - Array to append change descriptions to
+ * @returns {boolean} True if any change was made
+ */
+function normalizeConfigurationSection(config, prefix, systemKey, changes) {
+  let updated = false;
+  for (let i = 0; i < config.length; i++) {
+    const entry = config[i];
+    if (!entry || !entry.name || (entry.location !== 'keyvault' && !String(entry.name).startsWith('KV_'))) continue;
+    const afterPrefix = entry.name.startsWith(`KV_${prefix}_`)
+      ? entry.name.slice(`KV_${prefix}_`.length)
+      : entry.name.replace(/^KV_[A-Z0-9]+_/, '');
+    const normalizedVar = afterPrefix.replace(/_/g, '').toUpperCase();
+    const canonicalName = `KV_${prefix}_${normalizedVar}`;
+    const pathVal = kvEnvKeyToPath(canonicalName, systemKey);
+    if (!pathVal) continue;
+    const pathValWithoutPrefix = pathVal.replace(/^kv:\/\//, '');
+    const valueLegacy = typeof entry.value === 'string' && (entry.value.includes('KeyVault') || !entry.value.includes('/'));
+    if (entry.name !== canonicalName || (valueLegacy && entry.value !== pathValWithoutPrefix)) {
+      config[i] = { ...entry, name: canonicalName, value: pathValWithoutPrefix, location: 'keyvault' };
+      changes.push(`configuration: normalized ${entry.name} → ${canonicalName}, value → path-style`);
+      updated = true;
+    }
+  }
+  return updated;
+}
+
+/**
+ * Normalizes system file authentication.security and configuration keyvault entries to canonical
+ * KV_* names and path-style kv:// values so upload validation and env.template align.
+ *
+ * @param {Object} systemParsed - Parsed system config (mutated)
+ * @param {string} systemKey - System key (e.g. 'hubspot')
+ * @param {string[]} changes - Array to append change descriptions to
+ * @returns {boolean} True if any change was made
+ */
+function normalizeSystemFileAuthAndConfig(systemParsed, systemKey, changes) {
+  const prefix = systemKeyToKvPrefix(systemKey);
+  if (!prefix) return false;
+  const security = systemParsed.authentication?.security;
+  let updated = (security && typeof security === 'object' && normalizeSecuritySection(security, prefix, systemKey, changes));
+  const config = systemParsed.configuration;
+  if (Array.isArray(config)) {
+    updated = normalizeConfigurationSection(config, prefix, systemKey, changes) || updated;
+  }
+  return updated;
+}
+
+module.exports = { normalizeSystemFileAuthAndConfig, isLegacyKvValue };

package/lib/commands/repair-datasource-keys.js

@@ -0,0 +1,208 @@
+/**
+ * Normalize datasource keys and filenames to canonical form during repair.
+ *
+ * Key: <systemKey>-<resourceType> or <systemKey>-<resourceType>-2, -3 for duplicates.
+ * Filename: <systemKey>-datasource-<suffix>.<ext> where suffix = key without leading systemKey-.
+ * Skips keys/filenames that already match the valid pattern (e.g. customer-extra, customer-1).
+ *
+ * @fileoverview Datasource key and filename normalization for repair
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+'use strict';
+
+const path = require('path');
+const fs = require('fs');
+const { loadConfigFile, writeConfigFile } = require('../utils/config-format');
+
+/**
+ * Returns suffix from a canonical-format filename: <systemKey>-datasource-<suffix>.<ext>.
+ *
+ * @param {string} fileName - Filename
+ * @param {string} systemKey - System key
+ * @returns {string|null} Suffix or null if not in canonical format
+ */
+function suffixFromCanonicalFilename(fileName, systemKey) {
+  const base = path.basename(fileName);
+  const ext = path.extname(fileName);
+  const withoutExt = base.slice(0, -ext.length);
+  const prefix = `${systemKey}-datasource-`;
+  if (!withoutExt.startsWith(prefix)) return null;
+  return withoutExt.slice(prefix.length) || null;
+}
+
+/**
+ * Returns true if the key already matches canonical form and should not be changed.
+ * Valid: <systemKey>-<resourceType> or <systemKey>-<resourceType>-<extra> (e.g. customer-extra, customer-1).
+ * When fileName is provided and is canonical, key may be just the suffix (e.g. record-storage).
+ * Invalid (will normalize): key ending with redundant -datasource (e.g. hubspot-demo-companies-datasource).
+ *
+ * @param {string} key - Datasource key
+ * @param {string} systemKey - System key
+ * @param {string} [fileName] - Optional filename; if canonical, key can be suffix-only
+ * @returns {boolean}
+ */
+function isKeyAlreadyCanonical(key, systemKey, fileName) {
+  if (!key || !systemKey) return false;
+  if (fileName && isFilenameAlreadyCanonical(fileName, systemKey)) {
+    const suffixFromFile = suffixFromCanonicalFilename(fileName, systemKey);
+    if (suffixFromFile && (key === suffixFromFile || key === `${systemKey}-${suffixFromFile}`)) {
+      return true;
+    }
+  }
+  if (!key.startsWith(systemKey + '-')) return false;
+  const suffix = key.slice(systemKey.length + 1);
+  if (!suffix) return false;
+  if (suffix.endsWith('-datasource')) return false;
+  return true;
+}
+
+/**
+ * Derives resourceType slug from key: strip systemKey prefix, then strip trailing -datasource if present.
+ *
+ * @param {string} key - Current datasource key
+ * @param {string} systemKey - System key
+ * @returns {string}
+ */
+function slugFromKey(key, systemKey) {
+  if (!key || !systemKey || !key.startsWith(systemKey + '-')) return key || '';
+  let suffix = key.slice(systemKey.length + 1);
+  if (suffix.endsWith('-datasource')) suffix = suffix.slice(0, -'-datasource'.length);
+  return suffix || key;
+}
+
+/**
+ * Returns canonical filename for a datasource: <systemKey>-datasource-<suffix>.<ext>.
+ *
+ * @param {string} canonicalKey - Canonical datasource key
+ * @param {string} systemKey - System key
+ * @param {string} ext - File extension including dot (e.g. .json)
+ * @returns {string}
+ */
+function canonicalDatasourceFilename(canonicalKey, systemKey, ext) {
+  const suffix = canonicalKey.startsWith(systemKey + '-')
+    ? canonicalKey.slice(systemKey.length + 1)
+    : canonicalKey;
+  return `${systemKey}-datasource-${suffix}${ext}`;
+}
+
+/**
+ * Returns true if filename already matches canonical pattern <systemKey>-datasource-<suffix>.<ext>.
+ *
+ * @param {string} fileName - Current filename
+ * @param {string} systemKey - System key
+ * @returns {boolean}
+ */
+function isFilenameAlreadyCanonical(fileName, systemKey) {
+  const base = path.basename(fileName);
+  const ext = path.extname(fileName);
+  const withoutExt = base.slice(0, -ext.length);
+  const prefix = `${systemKey}-datasource-`;
+  if (!withoutExt.startsWith(prefix)) return false;
+  const suffix = withoutExt.slice(prefix.length);
+  if (!suffix || suffix.endsWith('-datasource')) return false;
+  return true;
+}
+
+/**
+ * Normalizes datasource keys and filenames to canonical form. Runs early in repair.
+ * Updates file contents (key property), renames files when needed, and updates variables.externalIntegration.dataSources.
+ *
+ * @param {string} appPath - Application directory path
+ * @param {string[]} datasourceFiles - Current list of datasource filenames
+ * @param {string} systemKey - System key
+ * @param {Object} variables - Application variables (mutated: externalIntegration.dataSources updated)
+ * @param {boolean} dryRun - If true, do not write or rename
+ * @param {string[]} changes - Array to append change descriptions to
+ * @returns {{ updated: boolean, datasourceFiles: string[] }} Updated flag and new list of datasource filenames
+ */
+/* eslint-disable max-lines-per-function, max-statements, complexity -- Normalization loops and branching per file */
+function normalizeDatasourceKeysAndFilenames(appPath, datasourceFiles, systemKey, variables, dryRun, changes) {
+  if (!datasourceFiles || datasourceFiles.length === 0) {
+    return { updated: false, datasourceFiles: datasourceFiles || [] };
+  }
+
+  const slugCounts = new Map();
+  const fileInfos = [];
+
+  for (const fileName of datasourceFiles) {
+    const filePath = path.join(appPath, fileName);
+    if (!fs.existsSync(filePath)) continue;
+    let parsed;
+    try {
+      parsed = loadConfigFile(filePath);
+    } catch (err) {
+      fileInfos.push({ fileName, key: null, skip: true });
+      continue;
+    }
+    const key = (parsed && typeof parsed.key === 'string' && parsed.key.trim()) ? parsed.key.trim() : null;
+    if (isKeyAlreadyCanonical(key, systemKey, fileName) && isFilenameAlreadyCanonical(fileName, systemKey)) {
+      fileInfos.push({ fileName, key, skip: true });
+      continue;
+    }
+    const slug = slugFromKey(key || fileName, systemKey);
+    fileInfos.push({
+      fileName,
+      parsed,
+      key,
+      slug,
+      canonicalKey: null,
+      skip: false
+    });
+  }
+
+  for (const info of fileInfos) {
+    if (info.skip) continue;
+    const slug = info.slug;
+    const n = (slugCounts.get(slug) || 0) + 1;
+    slugCounts.set(slug, n);
+    info.canonicalKey = n === 1 ? `${systemKey}-${slug}` : `${systemKey}-${slug}-${n}`;
+  }
+
+  let updated = false;
+  const newDatasourceFiles = [];
+
+  for (const info of fileInfos) {
+    if (info.skip) {
+      newDatasourceFiles.push(info.fileName);
+      continue;
+    }
+    const { fileName, parsed, canonicalKey } = info;
+    const ext = path.extname(fileName);
+    const canonicalFileName = canonicalDatasourceFilename(canonicalKey, systemKey, ext);
+
+    if (parsed.key !== canonicalKey) {
+      parsed.key = canonicalKey;
+      if (!dryRun) writeConfigFile(path.join(appPath, fileName), parsed);
+      changes.push(`${fileName}: key → ${canonicalKey}`);
+      updated = true;
+    }
+    if (fileName !== canonicalFileName) {
+      const oldPath = path.join(appPath, fileName);
+      const newPath = path.join(appPath, canonicalFileName);
+      if (!dryRun && fs.existsSync(oldPath) && !fs.existsSync(newPath)) {
+        fs.renameSync(oldPath, newPath);
+      }
+      changes.push(`Renamed ${fileName} → ${canonicalFileName}`);
+      updated = true;
+      newDatasourceFiles.push(canonicalFileName);
+    } else {
+      newDatasourceFiles.push(fileName);
+    }
+  }
+
+  if (updated && variables.externalIntegration && Array.isArray(variables.externalIntegration.dataSources)) {
+    variables.externalIntegration.dataSources = [...newDatasourceFiles].sort();
+  }
+
+  return { updated, datasourceFiles: newDatasourceFiles };
+}
+
+module.exports = {
+  normalizeDatasourceKeysAndFilenames,
+  isKeyAlreadyCanonical,
+  slugFromKey,
+  canonicalDatasourceFilename,
+  isFilenameAlreadyCanonical
+};

package/lib/commands/repair-datasource.js

@@ -0,0 +1,235 @@
+/**
+ * Datasource repair helpers: align dimensions, metadataSchema, exposed, sync, testPayload
+ * with fieldMappings.attributes as source of truth.
+ *
+ * @fileoverview Repair datasource files for external integration
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+'use strict';
+
+const DEFAULT_SYNC = {
+  mode: 'pull',
+  batchSize: 500,
+  maxParallelRequests: 5
+};
+
+const MINIMAL_METADATA_SCHEMA = {
+  type: 'object',
+  additionalProperties: true
+};
+
+/**
+ * Returns the set of attribute keys from fieldMappings.attributes.
+ * @param {Object} parsed - Parsed datasource object
+ * @returns {Set<string>}
+ */
+function getAttributeKeys(parsed) {
+  const attrs = parsed?.fieldMappings?.attributes;
+  if (!attrs || typeof attrs !== 'object') return new Set();
+  return new Set(Object.keys(attrs));
+}
+
+/**
+ * Extracts paths from attribute expressions (e.g. {{ metadata.email }}).
+ * Skips record_ref: expressions.
+ * - topLevelKeys: first segment of each path (e.g. "metadata" from "metadata.id").
+ * - referencedSchemaPropertyNames: for paths like "metadata.xxx" or "metadata.xxx.yyy", the name "xxx"
+ *   (the first property under metadata). Used to prune metadataSchema.properties so we only keep
+ *   properties that are referenced; we do not compare against "metadata" or the schema would be wiped.
+ *
+ * @param {Object} attributes - fieldMappings.attributes object
+ * @returns {{ paths: string[], topLevelKeys: Set<string>, referencedSchemaPropertyNames: Set<string> }}
+ */
+function parsePathsFromExpressions(attributes) {
+  const paths = [];
+  const topLevelKeys = new Set();
+  const referencedSchemaPropertyNames = new Set();
+  if (!attributes || typeof attributes !== 'object') return { paths, topLevelKeys, referencedSchemaPropertyNames };
+  for (const attr of Object.values(attributes)) {
+    const expr = attr?.expression;
+    if (typeof expr !== 'string') continue;
+    if (/^\s*record_ref:/i.test(expr.trim())) continue;
+    const match = expr.match(/\{\{\s*([^}]+)\s*\}\}/);
+    if (!match) continue;
+    const path = match[1].trim().split('|')[0].trim();
+    if (path) {
+      paths.push(path);
+      const segments = path.split('.');
+      const first = segments[0];
+      if (first) topLevelKeys.add(first);
+      if (first === 'metadata' && segments.length >= 2 && segments[1]) {
+        referencedSchemaPropertyNames.add(segments[1]);
+      }
+    }
+  }
+  return { paths, topLevelKeys, referencedSchemaPropertyNames };
+}
+
+/**
+ * Removes dimension entries whose value is metadata.<attr> and attr is not in fieldMappings.attributes.
+ * Mutates parsed.fieldMappings.dimensions.
+ *
+ * @param {Object} parsed - Parsed datasource (mutated)
+ * @param {string[]} changes - Array to append change descriptions to
+ * @returns {boolean} True if any dimension was removed
+ */
+function repairDimensionsFromAttributes(parsed, changes) {
+  const dims = parsed?.fieldMappings?.dimensions;
+  if (!dims || typeof dims !== 'object') return false;
+  const attributeKeys = getAttributeKeys(parsed);
+  let updated = false;
+  for (const [dimKey, value] of Object.entries(dims)) {
+    if (typeof value !== 'string') continue;
+    if (!value.startsWith('metadata.')) continue;
+    const attr = value.slice('metadata.'.length).trim();
+    if (!attr || attributeKeys.has(attr)) continue;
+    delete dims[dimKey];
+    changes.push(`Removed dimension '${dimKey}': ${value} not in fieldMappings.attributes`);
+    updated = true;
+  }
+  return updated;
+}
+
+/**
+ * Ensures metadataSchema exists (minimal stub if missing). If present, prunes top-level
+ * properties not referenced by any attribute expression. Uses the first property name under
+ * "metadata" in paths (e.g. metadata.id → "id") so we do not remove schema properties that
+ * are referenced. If no metadata.xxx paths exist, we do not prune (keep all properties).
+ *
+ * @param {Object} parsed - Parsed datasource (mutated)
+ * @param {string[]} changes - Array to append change descriptions to
+ * @returns {boolean} True if schema was added or pruned
+ */
+function repairMetadataSchemaFromAttributes(parsed, changes) {
+  const { referencedSchemaPropertyNames } = parsePathsFromExpressions(parsed?.fieldMappings?.attributes ?? {});
+  if (!parsed.metadataSchema || typeof parsed.metadataSchema !== 'object') {
+    parsed.metadataSchema = { ...MINIMAL_METADATA_SCHEMA };
+    changes.push('Added minimal metadataSchema (was missing)');
+    return true;
+  }
+  const props = parsed.metadataSchema.properties;
+  if (!props || typeof props !== 'object') return false;
+  if (referencedSchemaPropertyNames.size === 0) return false;
+  const toRemove = Object.keys(props).filter(k => !referencedSchemaPropertyNames.has(k));
+  if (toRemove.length === 0) return false;
+  toRemove.forEach(k => delete props[k]);
+  changes.push(`Pruned metadataSchema.properties: removed [${toRemove.join(', ')}] (not referenced by attributes)`);
+  return true;
+}
+
+/**
+ * Sets exposed.attributes to the list of fieldMappings.attributes keys (sorted).
+ * Only when options.expose is true; caller should gate.
+ *
+ * @param {Object} parsed - Parsed datasource (mutated)
+ * @param {string[]} changes - Array to append change descriptions to
+ * @returns {boolean} True if exposed was updated
+ */
+function repairExposeFromAttributes(parsed, changes) {
+  const keys = Array.from(getAttributeKeys(parsed)).filter(Boolean).sort();
+  if (keys.length === 0) return false;
+  if (!parsed.exposed) parsed.exposed = {};
+  const prev = parsed.exposed.attributes;
+  const same = Array.isArray(prev) && prev.length === keys.length && prev.every((v, i) => v === keys[i]);
+  if (same) return false;
+  parsed.exposed.attributes = keys;
+  changes.push(`Set exposed.attributes to [${keys.join(', ')}]`);
+  return true;
+}
+
+/**
+ * Adds default sync section if missing or empty.
+ *
+ * @param {Object} parsed - Parsed datasource (mutated)
+ * @param {string[]} changes - Array to append change descriptions to
+ * @returns {boolean} True if sync was added
+ */
+function repairSyncSection(parsed, changes) {
+  const sync = parsed.sync;
+  if (sync && typeof sync === 'object' && Object.keys(sync).length > 0) return false;
+  parsed.sync = { ...DEFAULT_SYNC };
+  changes.push('Added default sync section (mode: pull, batchSize: 500, maxParallelRequests: 5)');
+  return true;
+}
+
+function placeholderForType(type) {
+  if (type === 'number' || type === 'integer') return 0;
+  if (type === 'boolean') return false;
+  if (type === 'array') return [];
+  if (type === 'object') return {};
+  return '';
+}
+
+function setNested(obj, pathParts, value) {
+  let cur = obj;
+  for (let i = 0; i < pathParts.length - 1; i++) {
+    const p = pathParts[i];
+    if (!(p in cur) || typeof cur[p] !== 'object') cur[p] = {};
+    cur = cur[p];
+  }
+  const last = pathParts[pathParts.length - 1];
+  if (last) cur[last] = value;
+}
+
+/**
+ * Builds minimal payloadTemplate and expectedResult from attribute expression paths.
+ *
+ * @param {Object} parsed - Parsed datasource (mutated)
+ * @param {string[]} changes - Array to append change descriptions to
+ * @returns {boolean} True if testPayload was added or updated
+ */
+function repairTestPayload(parsed, changes) {
+  const attrs = parsed?.fieldMappings?.attributes;
+  if (!attrs || typeof attrs !== 'object') return false;
+  const payloadTemplate = {};
+  const expectedResult = {};
+  for (const [key, config] of Object.entries(attrs)) {
+    const type = config?.type || 'string';
+    const placeholder = placeholderForType(type);
+    expectedResult[key] = placeholder;
+    const match = config?.expression?.match(/\{\{\s*([^}|]+)/);
+    if (match) {
+      const path = match[1].trim();
+      setNested(payloadTemplate, path.split('.'), placeholder);
+    }
+  }
+  if (!parsed.testPayload) parsed.testPayload = {};
+  parsed.testPayload.payloadTemplate = payloadTemplate;
+  parsed.testPayload.expectedResult = expectedResult;
+  changes.push('Generated testPayload.payloadTemplate and testPayload.expectedResult from attributes');
+  return true;
+}
+
+/**
+ * Runs all requested datasource repairs. Core: dimensions + metadataSchema. Optional: expose, sync, testPayload.
+ *
+ * @param {Object} parsed - Parsed datasource object (mutated)
+ * @param {Object} options - { expose?: boolean, sync?: boolean, test?: boolean }
+ * @param {string[]} [changes] - Optional array to append change descriptions to
+ * @returns {{ updated: boolean, changes: string[] }}
+ */
+function repairDatasourceFile(parsed, options = {}, changes = []) {
+  const out = Array.isArray(changes) ? changes : [];
+  let updated = false;
+  updated = repairDimensionsFromAttributes(parsed, out) || updated;
+  updated = repairMetadataSchemaFromAttributes(parsed, out) || updated;
+  if (options.expose) updated = repairExposeFromAttributes(parsed, out) || updated;
+  if (options.sync) updated = repairSyncSection(parsed, out) || updated;
+  if (options.test) updated = repairTestPayload(parsed, out) || updated;
+  return { updated, changes: out };
+}
+
+module.exports = {
+  getAttributeKeys,
+  parsePathsFromExpressions,
+  repairDimensionsFromAttributes,
+  repairMetadataSchemaFromAttributes,
+  repairExposeFromAttributes,
+  repairSyncSection,
+  repairTestPayload,
+  repairDatasourceFile,
+  DEFAULT_SYNC,
+  MINIMAL_METADATA_SCHEMA
+};