@aifabrix/builder 2.44.6 → 2.45.5

package/lib/datasource/log-viewer.js

@@ -13,7 +13,15 @@ const chalk = require('chalk');
 const logger = require('../utils/logger');
 const { resolveAppKeyForDatasource } = require('./resolve-app');
 const { getIntegrationPath } = require('../utils/paths');
-const { sectionTitle, headerKeyValue, formatBlockingError, successGlyph, failureGlyph } = require('../utils/cli-test-layout-chalk');
+const {
+  sectionTitle,
+  headerKeyValue,
+  formatBlockingError,
+  successGlyph,
+  failureGlyph
+} = require('../utils/cli-test-layout-chalk');
+const { statusGlyph } = require('../utils/datasource-test-run-display');
+const { trustDecisionToRowStatus } = require('./agent-trust-map');
 
 /**
  * Get the path to the latest log file in a directory matching a glob-like pattern
@@ -48,7 +56,13 @@ async function getLatestLogPath(logsDir, pattern) {
  */
 function isStructuralTestLogFileName(name) {
   if (!name || typeof name !== 'string' || !name.endsWith('.json')) return false;
-  if (name.startsWith('test-e2e-') || name.startsWith('test-integration-')) return false;
+  if (
+    name.startsWith('test-e2e-') ||
+    name.startsWith('test-integration-') ||
+    name.startsWith('test-trust-')
+  ) {
+    return false;
+  }
   return name.startsWith('test-');
 }
 
@@ -232,6 +246,85 @@ function structuralEnvelopeStatusLine(status) {
  * @param {Object} data - Parsed JSON { request, response?, error? }
  * @param {string} [fileName] - Log file name for header
  */
+function trustLogGlyph(trustDecision) {
+  return statusGlyph(trustDecisionToRowStatus(trustDecision));
+}
+
+function logTrustWarningsSection(run) {
+  const warnings = Array.isArray(run.highLevelWarnings) ? run.highLevelWarnings : [];
+  if (!warnings.length) return;
+  logger.log(sectionTitle('Warnings:'));
+  for (const w of warnings) {
+    logger.log(`  ${chalk.yellow('•')} ${chalk.white(String(w))}`);
+  }
+}
+
+function logTrustFindingsSection(run) {
+  const findings = Array.isArray(run.findings) ? run.findings : [];
+  if (!findings.length) return;
+  logger.log(sectionTitle('Findings:'));
+  const cap = 15;
+  for (let i = 0; i < Math.min(cap, findings.length); i += 1) {
+    const f = findings[i];
+    const code = f?.code || f?.stableCode || 'finding';
+    const msg = f?.message ? String(f.message) : '';
+    logger.log(`  ${chalk.gray(`• ${code}`)} ${msg}`.trim());
+  }
+  if (findings.length > cap) {
+    logger.log(chalk.gray(`  … and ${findings.length - cap} more (open JSON file for full list)`));
+  }
+}
+
+/**
+ * Format agent trust log (`datasource test-trust --debug` → `test-trust-*.json`).
+ * @param {Object} data - Parsed JSON { request, response?, error? }
+ * @param {string} [fileName] - Log file name for header
+ */
+function formatTrustLog(data, fileName) {
+  logger.log('');
+  logger.log(sectionTitle('Agent trust log'));
+  if (fileName) logger.log(chalk.gray(fileName));
+  const req = data.request || {};
+  logger.log('');
+  logger.log(sectionTitle('Request:'));
+  logger.log(headerKeyValue('datasourceKey:', String(req.datasourceKey ?? '—')));
+  if (req.revalidate !== undefined) {
+    logger.log(chalk.gray(`  revalidate: ${req.revalidate}`));
+  }
+  if (data.error) {
+    logger.log('');
+    logger.log(formatBlockingError(`Error: ${data.error}`));
+    return;
+  }
+  const run = data.response || {};
+  logger.log('');
+  logger.log(sectionTitle('Response:'));
+  logger.log(headerKeyValue('System:', String(run.systemKey ?? '—')));
+  const confPct = Math.round((Number(run.confidence) || 0) * 100);
+  logger.log(
+    headerKeyValue(
+      'Trust:',
+      `${trustLogGlyph(run.trustDecision)} ${run.trustDecision ?? '—'} · ${confPct}% · ${run.validationStatus ?? '—'}`
+    )
+  );
+  if (run.validatedAt) {
+    logger.log(chalk.gray(`  validatedAt: ${run.validatedAt}`));
+  }
+  if (run.inputHash) {
+    logger.log(chalk.gray(`  inputHash: ${truncate(String(run.inputHash), 40)}`));
+  }
+  if (run.cacheHit !== undefined && run.cacheHit !== null) {
+    logger.log(chalk.gray(`  cacheHit: ${run.cacheHit}`));
+  }
+  if (run.summary) {
+    logger.log('');
+    logger.log(chalk.white('  Summary'));
+    logger.log(`  ${run.summary}`);
+  }
+  logTrustWarningsSection(run);
+  logTrustFindingsSection(run);
+}
+
 function formatStructuralTestLog(data, fileName) {
   logger.log('');
   logger.log(sectionTitle('Structural validation log'));
@@ -263,7 +356,7 @@ function formatStructuralTestLog(data, fileName) {
 /**
  * Format log content by type
  * @param {Object} parsed - Parsed JSON log
- * @param {'test'|'test-e2e'|'test-integration'} logType - Log type
+ * @param {'test'|'test-e2e'|'test-integration'|'test-trust'} logType - Log type
  * @param {string} [fileName] - File name for header
  */
 function formatLogContent(parsed, logType, fileName) {
@@ -271,11 +364,15 @@ function formatLogContent(parsed, logType, fileName) {
     formatStructuralTestLog(parsed, fileName);
     return;
   }
+  if (logType === 'test-trust') {
+    formatTrustLog(parsed, fileName);
+    return;
+  }
   if (logType === 'test-e2e') {
     formatE2ELog(parsed, fileName);
-  } else {
-    formatIntegrationLog(parsed, fileName);
+    return;
   }
+  formatIntegrationLog(parsed, fileName);
 }
 
 /**
@@ -313,7 +410,7 @@ function parseDatasourceLogJson(content, logPath) {
  * @param {Object} options - Options
  * @param {string} [options.app] - App key (optional, resolved from key if omitted)
  * @param {string} [options.file] - Path to log file (overrides app resolution)
- * @param {'test'|'test-e2e'|'test-integration'} options.logType - Log type
+ * @param {'test'|'test-e2e'|'test-integration'|'test-trust'} options.logType - Log type
  * @throws {Error} When file not found or invalid JSON
  */
 /* eslint-disable-next-line max-statements -- Resolve path, read, parse, format */
@@ -336,6 +433,13 @@ async function runLogViewer(datasourceKey, options) {
           `No structural validation log found in ${logsDir}. Run: aifabrix datasource test <key> --debug`
         );
       }
+    } else if (logType === 'test-trust') {
+      logPath = await getLatestLogPath(logsDir, 'test-trust-');
+      if (!logPath) {
+        throw new Error(
+          `No test-trust log found in ${logsDir}. Run: aifabrix datasource test-trust <key> --debug`
+        );
+      }
     } else {
       const pattern = logType === 'test-e2e' ? 'test-e2e-' : 'test-integration-';
       logPath = await getLatestLogPath(logsDir, pattern);
@@ -360,6 +464,7 @@ module.exports = {
   formatE2ELog,
   formatIntegrationLog,
   formatStructuralTestLog,
+  formatTrustLog,
   parseDatasourceLogJson,
   runLogViewer
 };

package/lib/datasource/record-file-parser.js

@@ -0,0 +1,85 @@
+/**
+ * @fileoverview Parse JSON array and NDJSON record fixture files (plan 144)
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const fs = require('fs');
+
+/**
+ * @param {string} content
+ * @returns {Object[]}
+ */
+function parseNdjsonContent(content) {
+  const lines = String(content).split(/\r?\n/);
+  const records = [];
+  for (let i = 0; i < lines.length; i += 1) {
+    const line = lines[i].trim();
+    if (!line) continue;
+    let parsed;
+    try {
+      parsed = JSON.parse(line);
+    } catch (err) {
+      throw new Error(`NDJSON parse error at line ${i + 1}: ${err.message}`);
+    }
+    if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
+      throw new Error(`NDJSON line ${i + 1}: expected a JSON object`);
+    }
+    records.push(parsed);
+  }
+  return records;
+}
+
+/**
+ * @param {string} content
+ * @returns {Object[]}
+ */
+function parseJsonArrayContent(content) {
+  let parsed;
+  try {
+    parsed = JSON.parse(content);
+  } catch (err) {
+    throw new Error(`JSON parse error: ${err.message}`);
+  }
+  if (!Array.isArray(parsed)) {
+    throw new Error('JSON file must be a top-level array of objects');
+  }
+  parsed.forEach((row, index) => {
+    if (!row || typeof row !== 'object' || Array.isArray(row)) {
+      throw new Error(`JSON array index ${index}: expected an object`);
+    }
+  });
+  return parsed;
+}
+
+/**
+ * Parse a local record fixture file.
+ * @param {string} filePath
+ * @param {'json'|'ndjson'} format
+ * @returns {Object[]}
+ */
+function parseRecordFile(filePath, format) {
+  const content = fs.readFileSync(filePath, 'utf8');
+  if (format === 'ndjson') {
+    return parseNdjsonContent(content);
+  }
+  return parseJsonArrayContent(content);
+}
+
+/**
+ * Rough payload size estimate for dry-run (bytes).
+ * @param {Object[]} records
+ * @returns {number}
+ */
+function estimatePayloadBytes(records) {
+  return Buffer.byteLength(JSON.stringify(records), 'utf8');
+}
+
+module.exports = {
+  parseRecordFile,
+  parseNdjsonContent,
+  parseJsonArrayContent,
+  estimatePayloadBytes
+};

package/lib/datasource/record-mapper.js

@@ -0,0 +1,129 @@
+/**
+ * @fileoverview Map fixture payloads to canonical bulk sync records (plan 144)
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+/**
+ * @param {Object} obj
+ * @returns {boolean}
+ */
+function isCanonicalBulkRecord(obj) {
+  return (
+    obj &&
+    typeof obj === 'object' &&
+    typeof obj.key === 'string' &&
+    obj.key.trim().length > 0 &&
+    typeof obj.recordType === 'string' &&
+    obj.recordType.trim().length > 0 &&
+    obj.metadata !== undefined &&
+    obj.metadata !== null &&
+    typeof obj.metadata === 'object' &&
+    !Array.isArray(obj.metadata)
+  );
+}
+
+/**
+ * @param {string[]} primaryKeyFields
+ * @param {Object} payload
+ * @returns {string}
+ */
+function buildKeyFromPrimaryKey(primaryKeyFields, payload) {
+  if (!primaryKeyFields.length) {
+    throw new Error('Datasource primaryKey is empty; provide records with a key field');
+  }
+  if (primaryKeyFields.length === 1) {
+    const field = primaryKeyFields[0];
+    const value = payload[field];
+    if (value === undefined || value === null || String(value).trim() === '') {
+      throw new Error(`Missing primary key field '${field}' in record payload`);
+    }
+    return String(value);
+  }
+  const parts = [];
+  for (const field of primaryKeyFields) {
+    const value = payload[field];
+    if (value === undefined || value === null || String(value).trim() === '') {
+      throw new Error(`Missing composite primary key field '${field}' in record payload`);
+    }
+    parts.push(`${field}=${String(value)}`);
+  }
+  return parts.join(',');
+}
+
+/**
+ * @param {Object} datasource
+ * @param {Object} payload
+ * @returns {import('../api/types/records-bulk.types').ExternalBulkRecord}
+ */
+function mapPayloadToBulkRecord(datasource, payload) {
+  const primaryKey = Array.isArray(datasource.primaryKey) ? datasource.primaryKey : [];
+  const recordType =
+    typeof datasource.resourceType === 'string' && datasource.resourceType.trim()
+      ? datasource.resourceType.trim()
+      : 'record';
+  const key = buildKeyFromPrimaryKey(primaryKey, payload);
+  let displayName = '';
+  const labelKey = datasource.labelKey;
+  if (Array.isArray(labelKey) && labelKey.length > 0) {
+    displayName = labelKey
+      .map(field => payload[field])
+      .filter(v => v !== undefined && v !== null && String(v).trim() !== '')
+      .map(v => String(v))
+      .join(' ');
+  } else if (typeof labelKey === 'string' && labelKey.trim() && payload[labelKey] !== undefined && payload[labelKey] !== null) {
+    displayName = String(payload[labelKey]);
+  }
+  const metadata = { ...payload };
+  if (metadata.externalId === undefined) {
+    metadata.externalId = key;
+  }
+  return {
+    key,
+    displayName: displayName || '',
+    recordType,
+    metadata
+  };
+}
+
+/**
+ * @param {Object[]} rawRecords
+ * @param {Object} datasource
+ * @returns {import('../api/types/records-bulk.types').ExternalBulkRecord[]}
+ */
+function normalizeRecordsForBulk(rawRecords, datasource) {
+  if (!Array.isArray(rawRecords) || rawRecords.length === 0) {
+    throw new Error('Record file is empty after parse');
+  }
+  return rawRecords.map((row, index) => {
+    try {
+      if (isCanonicalBulkRecord(row)) {
+        const meta = { ...row.metadata };
+        if (meta.externalId === undefined) {
+          meta.externalId = row.key;
+        }
+        return {
+          key: String(row.key).trim(),
+          displayName:
+            row.displayName !== undefined && row.displayName !== null
+              ? String(row.displayName)
+              : '',
+          recordType: String(row.recordType).trim(),
+          metadata: meta
+        };
+      }
+      return mapPayloadToBulkRecord(datasource, row);
+    } catch (err) {
+      throw new Error(`Record ${index + 1}: ${err.message}`);
+    }
+  });
+}
+
+module.exports = {
+  isCanonicalBulkRecord,
+  buildKeyFromPrimaryKey,
+  mapPayloadToBulkRecord,
+  normalizeRecordsForBulk
+};

package/lib/datasource/resolve-app.js

@@ -5,49 +5,82 @@
  * @version 2.0.0
  */
 
-const path = require('path');
 const fs = require('fs');
 const {
   getIntegrationPath,
   listIntegrationAppNames,
   resolveIntegrationAppKeyFromCwd
 } = require('../utils/paths');
-const { resolveApplicationConfigPath } = require('../utils/app-config-resolver');
-const { loadConfigFile } = require('../utils/config-format');
+const {
+  appHasDatasourceIdentifier,
+  pickIntegrationAppForDatasourceIdentifier
+} = require('./datasource-identifier');
+
+/** @deprecated Use appHasDatasourceIdentifier — kept for tests and backward compatibility */
+function appHasDatasourceKey(appKey, datasourceKey) {
+  return appHasDatasourceIdentifier(appKey, datasourceKey);
+}
 
 /**
- * For one app, check if any of its datasource files has the given key
- * @param {string} appKey - Integration folder name (system key context)
- * @param {string} datasourceKey - Datasource key to match
- * @returns {boolean} True if this app has a datasource with that key
+ * @param {string} key
+ * @param {string} [explicitApp]
+ * @returns {{ appKey: string }|null}
  */
-function appHasDatasourceKey(appKey, datasourceKey) {
-  const appPath = getIntegrationPath(appKey);
-  let config;
-  try {
-    const configPath = resolveApplicationConfigPath(appPath);
-    config = loadConfigFile(configPath);
-  } catch {
-    return false;
+function resolveFromExplicitApp(key, explicitApp) {
+  if (!explicitApp || typeof explicitApp !== 'string' || !explicitApp.trim()) {
+    return null;
+  }
+  const appKey = explicitApp.trim();
+  if (fs.existsSync(getIntegrationPath(appKey))) {
+    return { appKey };
+  }
+  return null;
+}
+
+/**
+ * @param {string} key
+ * @param {string[]} appNames
+ * @returns {{ appKey: string }|null}
+ * @throws {Error}
+ */
+function resolveFromAppScan(key, appNames) {
+  const matches = appNames.filter(appName => appHasDatasourceIdentifier(appName, key));
+  if (matches.length === 1) {
+    return { appKey: matches[0] };
+  }
+  if (matches.length > 1) {
+    throw new Error(
+      `More than one app has this datasource; add --app <app>. Apps: ${matches.join(', ')}`
+    );
   }
-  const schemaBasePath = config.externalIntegration?.schemaBasePath || './';
-  const datasourceFiles = config.externalIntegration?.dataSources || [];
-  for (const f of datasourceFiles) {
-    if (!f || typeof f !== 'string') continue;
-    const fullPath = path.isAbsolute(schemaBasePath)
-      ? path.join(schemaBasePath, f)
-      : path.join(appPath, schemaBasePath, f);
-    if (!fs.existsSync(fullPath)) continue;
-    try {
-      const parsed = loadConfigFile(fullPath);
-      if (parsed && parsed.key === datasourceKey) return true;
-    } catch {
-      // skip unreadable or invalid files
-    }
+  return null;
+}
+
+/**
+ * @param {string} key
+ * @param {string[]} appNames
+ * @returns {{ appKey: string }|null}
+ */
+function resolveFromPrefixOrSegments(key, appNames) {
+  const prefixApp = pickIntegrationAppForDatasourceIdentifier(key, appNames);
+  if (prefixApp && appHasDatasourceIdentifier(prefixApp, key)) {
+    return { appKey: prefixApp };
   }
-  return false;
+  const segments = key.split('-');
+  if (segments.length < 2) {
+    return null;
+  }
+  const candidate = segments.slice(0, -1).join('-');
+  const candidatePath = getIntegrationPath(candidate);
+  if (fs.existsSync(candidatePath) && appHasDatasourceIdentifier(candidate, key)) {
+    return { appKey: candidate };
+  }
+  return null;
 }
 
+const RESOLVE_APP_HINT =
+  'Could not determine app context. Use --app <app>, a datasource key, a datasource JSON filename (e.g. myapp-datasource-contacts), or run from integration/<systemKey>/ directory.';
+
 /**
  * Resolve app key for a datasource: explicit --app, cwd, scan by key, or parse key convention.
  * @async
@@ -56,7 +89,6 @@ function appHasDatasourceKey(appKey, datasourceKey) {
  * @returns {Promise<{appKey: string}>} Resolved app key
  * @throws {Error} When app cannot be determined or multiple apps match
  */
-/* eslint-disable-next-line max-statements -- Resolution order: explicit, cwd, scan, parse */
 async function resolveAppKeyForDatasource(datasourceKey, explicitApp) {
   if (!datasourceKey || typeof datasourceKey !== 'string') {
     throw new Error('Datasource key is required');
@@ -66,41 +98,28 @@ async function resolveAppKeyForDatasource(datasourceKey, explicitApp) {
     throw new Error('Datasource key cannot be empty');
   }
 
-  if (explicitApp && typeof explicitApp === 'string' && explicitApp.trim()) {
-    const appPath = getIntegrationPath(explicitApp.trim());
-    if (fs.existsSync(appPath)) {
-      return { appKey: explicitApp.trim() };
-    }
+  const explicit = resolveFromExplicitApp(key, explicitApp);
+  if (explicit) {
+    return explicit;
   }
 
   const fromCwd = resolveIntegrationAppKeyFromCwd();
-  if (fromCwd && appHasDatasourceKey(fromCwd, key)) {
+  if (fromCwd && appHasDatasourceIdentifier(fromCwd, key)) {
     return { appKey: fromCwd };
   }
 
   const appNames = listIntegrationAppNames();
-  const matches = appNames.filter(appName => appHasDatasourceKey(appName, key));
-  if (matches.length === 1) {
-    return { appKey: matches[0] };
-  }
-  if (matches.length > 1) {
-    throw new Error(
-      `More than one app has this datasource; add --app <app>. Apps: ${matches.join(', ')}`
-    );
+  const scanned = resolveFromAppScan(key, appNames);
+  if (scanned) {
+    return scanned;
   }
 
-  const segments = key.split('-');
-  if (segments.length >= 2) {
-    const candidate = segments.slice(0, -1).join('-');
-    const candidatePath = getIntegrationPath(candidate);
-    if (fs.existsSync(candidatePath) && appHasDatasourceKey(candidate, key)) {
-      return { appKey: candidate };
-    }
+  const inferred = resolveFromPrefixOrSegments(key, appNames);
+  if (inferred) {
+    return inferred;
   }
 
-  throw new Error(
-    'Could not determine app context. Use --app <app> or run from integration/<systemKey>/ directory.'
-  );
+  throw new Error(RESOLVE_APP_HINT);
 }
 
 module.exports = {

package/lib/datasource/test-e2e.js

@@ -11,7 +11,7 @@ const chalk = require('chalk');
 const logger = require('../utils/logger');
 const { getIntegrationPath } = require('../utils/paths');
 const { resolveAppKeyForDatasource } = require('./resolve-app');
-const { infoLine } = require('../utils/cli-test-layout-chalk');
+const { formatProgress, formatSuccessLine, infoLine } = require('../utils/cli-test-layout-chalk');
 const { runUnifiedDatasourceValidation } = require('./unified-validation-run');
 const { includeDebugForRequest } = require('../utils/validation-run-request');
 const { e2eShapeFromEnvelope } = require('../utils/datasource-test-run-legacy-adapter');
@@ -41,7 +41,9 @@ function buildUnifiedE2eRunOptions(options, timeoutMs, pk) {
     minRecordCount: options.minRecordCount,
     capabilityKey: options.capabilityKey,
     timeout: timeoutMs,
-    sync: options.sync === true
+    sync: options.sync === true,
+    authConfig: options.authConfig,
+    dataplaneUrl: options.dataplaneUrl || options.dataplane
   };
 }
 
@@ -51,6 +53,25 @@ function logE2eDatasourceBanner(datasourceKey, verbose) {
   logger.log(infoLine(`🧪 Running E2E test for datasource: ${datasourceKey}`));
 }
 
+/**
+ * Compact per-datasource progress (default test-e2e / system rollup when not --verbose).
+ * @param {string} datasourceKey
+ * @param {boolean} verbose
+ */
+function logE2eDatasourceProgress(datasourceKey, verbose) {
+  if (verbose) return;
+  logger.log(formatProgress(`Running E2E validation for ${datasourceKey}…`));
+}
+
+/**
+ * @param {string} datasourceKey
+ * @param {boolean} verbose
+ */
+function logE2eDatasourceValidated(datasourceKey, verbose) {
+  if (verbose) return;
+  logger.log(formatSuccessLine(`${datasourceKey} validated`));
+}
+
 /**
  * Resolve primaryKeyValue for request body: string as-is, or read and parse JSON from @path
  * @param {string} [value]
@@ -174,6 +195,7 @@ async function runDatasourceTestE2E(datasourceKey, options = {}) {
   const cliWallStartedAt = Date.now();
   const { appKey } = await resolveAppKeyForDatasource(datasourceKey, options.app);
   logE2eDatasourceBanner(datasourceKey, options.verbose);
+  logE2eDatasourceProgress(datasourceKey, options.verbose);
   const pk = await resolvePrimaryKeyValue(options.primaryKeyValue);
   const timeoutMs = resolveE2ePollTimeoutMs(options);
   const requestMeta = buildE2eRequestMeta(datasourceKey, options, pk);
@@ -184,6 +206,8 @@ async function runDatasourceTestE2E(datasourceKey, options = {}) {
 
   await throwIfUnifiedE2EBlocked(unifiedResult, appKey, options, requestMeta);
 
+  logE2eDatasourceValidated(datasourceKey, options.verbose);
+
   const display = e2eShapeFromEnvelope(unifiedResult.envelope);
   Object.assign(display, { datasourceTestRun: unifiedResult.envelope });