@aifabrix/builder 2.45.0 → 2.46.0

package/lib/datasource/log-cleaner.js

@@ -0,0 +1,241 @@
+/**
+ * Remove saved datasource test debug JSON under integration/<app>/logs/.
+ * @fileoverview Local cleanup for test / test-integration / test-e2e / verify-trust logs
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+const path = require('path');
+const fsp = require('node:fs').promises;
+const chalk = require('chalk');
+const logger = require('../utils/logger');
+const { getIntegrationPath, listIntegrationAppNames } = require('../utils/paths');
+const { isStructuralTestLogFileName } = require('./log-viewer');
+const { formatSuccessLine } = require('../utils/cli-test-layout-chalk');
+
+/** @type {readonly string[]} */
+const LOG_CLEAN_TYPES = ['test', 'integration', 'e2e', 'trust', 'all'];
+
+/**
+ * @param {string} [type]
+ * @returns {'test'|'integration'|'e2e'|'trust'|'all'}
+ */
+function normalizeLogCleanType(type) {
+  const raw = (type === undefined || type === null ? 'all' : String(type)).trim().toLowerCase();
+  if (raw === '' || raw === 'all') return 'all';
+  if (LOG_CLEAN_TYPES.includes(raw)) return raw;
+  throw new Error(
+    `Invalid --type "${type}". Use: ${LOG_CLEAN_TYPES.join(', ')}`
+  );
+}
+
+/**
+ * @param {string} fileName
+ * @param {'test'|'integration'|'e2e'|'trust'|'all'} type
+ * @returns {boolean}
+ */
+function matchesLogCleanType(fileName, type) {
+  if (!fileName || !fileName.endsWith('.json')) return false;
+  if (type === 'all') return true;
+  if (type === 'test') return isStructuralTestLogFileName(fileName);
+  if (type === 'integration') {
+    return fileName.startsWith('test-integration-');
+  }
+  if (type === 'e2e') return fileName.startsWith('test-e2e-');
+  if (type === 'trust') return fileName.startsWith('verify-trust-');
+  return false;
+}
+
+/**
+ * @param {string} logsDir
+ * @param {'test'|'integration'|'e2e'|'trust'|'all'} type
+ * @returns {Promise<string[]>}
+ */
+async function listMatchingLogFiles(logsDir, type) {
+  let entries;
+  try {
+    entries = await fsp.readdir(logsDir, { withFileTypes: true });
+  } catch (err) {
+    if (err.code === 'ENOENT') return [];
+    throw err;
+  }
+  return entries
+    .filter(e => e.isFile() && matchesLogCleanType(e.name, type))
+    .map(e => path.join(logsDir, e.name))
+    .sort((a, b) => a.localeCompare(b));
+}
+
+/**
+ * @param {string} appKey
+ * @returns {Promise<void>}
+ */
+async function assertIntegrationAppExists(appKey) {
+  const appPath = getIntegrationPath(appKey);
+  try {
+    const st = await fsp.stat(appPath);
+    if (!st.isDirectory()) {
+      throw new Error(`Integration folder is not a directory: ${appPath}`);
+    }
+  } catch (err) {
+    if (err.code === 'ENOENT') {
+      throw new Error(`Integration folder not found: ${appPath}`);
+    }
+    throw err;
+  }
+}
+
+/**
+ * @param {Object} options
+ * @param {boolean} [options.all]
+ * @param {string} [options.app]
+ * @returns {Promise<string[]>}
+ */
+async function resolveTargetAppKeys(options) {
+  const useAll = Boolean(options.all);
+  const app = options.app && typeof options.app === 'string' ? options.app.trim() : '';
+  if (useAll && app) {
+    throw new Error('Use either --app <systemKey> or --all, not both');
+  }
+  if (!useAll && !app) {
+    throw new Error('Specify --app <systemKey> or --all');
+  }
+  if (useAll) {
+    return listIntegrationAppNames();
+  }
+  await assertIntegrationAppExists(app);
+  return [app];
+}
+
+/**
+ * @param {string[]} appKeys
+ * @param {'test'|'integration'|'e2e'|'trust'|'all'} type
+ * @returns {Promise<Array<{ appKey: string, logsDir: string, files: string[] }>>}
+ */
+async function collectLogFilesByApp(appKeys, type) {
+  const groups = [];
+  for (const appKey of appKeys) {
+    const logsDir = path.join(getIntegrationPath(appKey), 'logs');
+    const files = await listMatchingLogFiles(logsDir, type);
+    if (files.length > 0) {
+      groups.push({ appKey, logsDir, files });
+    }
+  }
+  return groups;
+}
+
+/**
+ * @param {Array<{ appKey: string, files: string[] }>} groups
+ * @param {boolean} dryRun
+ * @returns {Promise<{ removedCount: number, paths: string[] }>}
+ */
+async function removeCollectedFiles(groups, dryRun) {
+  const paths = groups.flatMap(g => g.files);
+  if (dryRun || paths.length === 0) {
+    return { removedCount: 0, paths };
+  }
+  await Promise.all(paths.map(filePath => fsp.unlink(filePath)));
+  return { removedCount: paths.length, paths };
+}
+
+/**
+ * @param {Object} options
+ * @param {boolean} options.dryRun
+ * @param {'test'|'integration'|'e2e'|'trust'|'all'} options.type
+ * @param {Array<{ appKey: string, files: string[] }>} groups
+ * @param {number} removedCount
+ */
+function printCleanLogsSummary(options, groups, removedCount) {
+  const totalFiles = groups.reduce((n, g) => n + g.files.length, 0);
+  const appCount = groups.length;
+  if (options.dryRun) {
+    if (totalFiles === 0) {
+      logger.log(chalk.gray('No matching log files found.'));
+      return;
+    }
+    logger.log('');
+    logger.log(
+      chalk.gray(
+        `Would remove ${totalFiles} file(s) from ${appCount} integration folder(s) (type: ${options.type}).`
+      )
+    );
+    return;
+  }
+  if (totalFiles === 0) {
+    logger.log(chalk.gray('No matching log files to remove.'));
+    return;
+  }
+  logger.log('');
+  logger.log(formatSuccessLine(`Removed ${removedCount} log file(s) from ${appCount} integration folder(s).`));
+}
+
+/**
+ * @param {boolean} dryRun
+ * @param {Array<{ appKey: string, logsDir: string, files: string[] }>} groups
+ */
+function printCleanLogsFileList(dryRun, groups) {
+  const prefix = dryRun ? 'Would remove' : 'Removing';
+  for (const { appKey, files } of groups) {
+    logger.log('');
+    logger.log(chalk.bold.white(`${appKey}/logs/`));
+    for (const filePath of files) {
+      logger.log(chalk.gray(`  ${prefix}: ${filePath}`));
+    }
+  }
+}
+
+/**
+ * @param {Object} result
+ * @param {boolean} result.dryRun
+ * @param {string} result.type
+ * @param {string[]} result.appKeys
+ * @param {string[]} result.paths
+ * @param {number} result.removedCount
+ */
+function printCleanLogsJson(result) {
+  const payload = {
+    dryRun: result.dryRun,
+    type: result.type,
+    appKeys: result.appKeys,
+    fileCount: result.paths.length,
+    paths: result.paths,
+    removedCount: result.removedCount
+  };
+  logger.log(JSON.stringify(payload, null, 2));
+}
+
+/**
+ * @async
+ * @param {Object} options
+ * @param {boolean} [options.all]
+ * @param {string} [options.app]
+ * @param {string} [options.type]
+ * @param {boolean} [options.dryRun]
+ * @param {boolean} [options.json]
+ * @returns {Promise<{ dryRun: boolean, type: string, appKeys: string[], paths: string[], removedCount: number }>}
+ */
+async function runCleanLogs(options = {}) {
+  const dryRun = Boolean(options.dryRun);
+  const json = Boolean(options.json);
+  const type = normalizeLogCleanType(options.type);
+  const appKeys = await resolveTargetAppKeys(options);
+  const groups = await collectLogFilesByApp(appKeys, type);
+  const { removedCount, paths } = await removeCollectedFiles(groups, dryRun);
+  const result = { dryRun, type, appKeys, paths, removedCount };
+  if (json) {
+    printCleanLogsJson(result);
+  } else {
+    printCleanLogsFileList(dryRun, groups);
+    printCleanLogsSummary({ dryRun, type }, groups, removedCount);
+  }
+  return result;
+}
+
+module.exports = {
+  LOG_CLEAN_TYPES,
+  normalizeLogCleanType,
+  matchesLogCleanType,
+  listMatchingLogFiles,
+  resolveTargetAppKeys,
+  collectLogFilesByApp,
+  runCleanLogs
+};

package/lib/datasource/log-viewer.js

@@ -13,7 +13,15 @@ const chalk = require('chalk');
 const logger = require('../utils/logger');
 const { resolveAppKeyForDatasource } = require('./resolve-app');
 const { getIntegrationPath } = require('../utils/paths');
-const { sectionTitle, headerKeyValue, formatBlockingError, successGlyph, failureGlyph } = require('../utils/cli-test-layout-chalk');
+const {
+  sectionTitle,
+  headerKeyValue,
+  formatBlockingError,
+  successGlyph,
+  failureGlyph
+} = require('../utils/cli-test-layout-chalk');
+const { statusGlyph } = require('../utils/datasource-test-run-display');
+const { trustDecisionToRowStatus } = require('./agent-trust-map');
 
 /**
  * Get the path to the latest log file in a directory matching a glob-like pattern
@@ -48,7 +56,13 @@ async function getLatestLogPath(logsDir, pattern) {
  */
 function isStructuralTestLogFileName(name) {
   if (!name || typeof name !== 'string' || !name.endsWith('.json')) return false;
-  if (name.startsWith('test-e2e-') || name.startsWith('test-integration-')) return false;
+  if (
+    name.startsWith('test-e2e-') ||
+    name.startsWith('test-integration-') ||
+    name.startsWith('verify-trust-')
+  ) {
+    return false;
+  }
   return name.startsWith('test-');
 }
 
@@ -232,6 +246,85 @@ function structuralEnvelopeStatusLine(status) {
  * @param {Object} data - Parsed JSON { request, response?, error? }
  * @param {string} [fileName] - Log file name for header
  */
+function trustLogGlyph(trustDecision) {
+  return statusGlyph(trustDecisionToRowStatus(trustDecision));
+}
+
+function logTrustWarningsSection(run) {
+  const warnings = Array.isArray(run.highLevelWarnings) ? run.highLevelWarnings : [];
+  if (!warnings.length) return;
+  logger.log(sectionTitle('Warnings:'));
+  for (const w of warnings) {
+    logger.log(`  ${chalk.yellow('•')} ${chalk.white(String(w))}`);
+  }
+}
+
+function logTrustFindingsSection(run) {
+  const findings = Array.isArray(run.findings) ? run.findings : [];
+  if (!findings.length) return;
+  logger.log(sectionTitle('Findings:'));
+  const cap = 15;
+  for (let i = 0; i < Math.min(cap, findings.length); i += 1) {
+    const f = findings[i];
+    const code = f?.code || f?.stableCode || 'finding';
+    const msg = f?.message ? String(f.message) : '';
+    logger.log(`  ${chalk.gray(`• ${code}`)} ${msg}`.trim());
+  }
+  if (findings.length > cap) {
+    logger.log(chalk.gray(`  … and ${findings.length - cap} more (open JSON file for full list)`));
+  }
+}
+
+/**
+ * Format agent trust log (`datasource verify-trust --debug` → `verify-trust-*.json`).
+ * @param {Object} data - Parsed JSON { request, response?, error? }
+ * @param {string} [fileName] - Log file name for header
+ */
+function formatTrustLog(data, fileName) {
+  logger.log('');
+  logger.log(sectionTitle('Agent trust log'));
+  if (fileName) logger.log(chalk.gray(fileName));
+  const req = data.request || {};
+  logger.log('');
+  logger.log(sectionTitle('Request:'));
+  logger.log(headerKeyValue('datasourceKey:', String(req.datasourceKey ?? '—')));
+  if (req.revalidate !== undefined) {
+    logger.log(chalk.gray(`  revalidate: ${req.revalidate}`));
+  }
+  if (data.error) {
+    logger.log('');
+    logger.log(formatBlockingError(`Error: ${data.error}`));
+    return;
+  }
+  const run = data.response || {};
+  logger.log('');
+  logger.log(sectionTitle('Response:'));
+  logger.log(headerKeyValue('System:', String(run.systemKey ?? '—')));
+  const confPct = Math.round((Number(run.confidence) || 0) * 100);
+  logger.log(
+    headerKeyValue(
+      'Trust:',
+      `${trustLogGlyph(run.trustDecision)} ${run.trustDecision ?? '—'} · ${confPct}% · ${run.validationStatus ?? '—'}`
+    )
+  );
+  if (run.validatedAt) {
+    logger.log(chalk.gray(`  validatedAt: ${run.validatedAt}`));
+  }
+  if (run.inputHash) {
+    logger.log(chalk.gray(`  inputHash: ${truncate(String(run.inputHash), 40)}`));
+  }
+  if (run.cacheHit !== undefined && run.cacheHit !== null) {
+    logger.log(chalk.gray(`  cacheHit: ${run.cacheHit}`));
+  }
+  if (run.summary) {
+    logger.log('');
+    logger.log(chalk.white('  Summary'));
+    logger.log(`  ${run.summary}`);
+  }
+  logTrustWarningsSection(run);
+  logTrustFindingsSection(run);
+}
+
 function formatStructuralTestLog(data, fileName) {
   logger.log('');
   logger.log(sectionTitle('Structural validation log'));
@@ -263,7 +356,7 @@ function formatStructuralTestLog(data, fileName) {
 /**
  * Format log content by type
  * @param {Object} parsed - Parsed JSON log
- * @param {'test'|'test-e2e'|'test-integration'} logType - Log type
+ * @param {'test'|'test-e2e'|'test-integration'|'verify-trust'} logType - Log type
  * @param {string} [fileName] - File name for header
  */
 function formatLogContent(parsed, logType, fileName) {
@@ -271,11 +364,15 @@ function formatLogContent(parsed, logType, fileName) {
     formatStructuralTestLog(parsed, fileName);
     return;
   }
+  if (logType === 'verify-trust') {
+    formatTrustLog(parsed, fileName);
+    return;
+  }
   if (logType === 'test-e2e') {
     formatE2ELog(parsed, fileName);
-  } else {
-    formatIntegrationLog(parsed, fileName);
+    return;
   }
+  formatIntegrationLog(parsed, fileName);
 }
 
 /**
@@ -313,7 +410,7 @@ function parseDatasourceLogJson(content, logPath) {
  * @param {Object} options - Options
  * @param {string} [options.app] - App key (optional, resolved from key if omitted)
  * @param {string} [options.file] - Path to log file (overrides app resolution)
- * @param {'test'|'test-e2e'|'test-integration'} options.logType - Log type
+ * @param {'test'|'test-e2e'|'test-integration'|'verify-trust'} options.logType - Log type
  * @throws {Error} When file not found or invalid JSON
  */
 /* eslint-disable-next-line max-statements -- Resolve path, read, parse, format */
@@ -336,6 +433,13 @@ async function runLogViewer(datasourceKey, options) {
           `No structural validation log found in ${logsDir}. Run: aifabrix datasource test <key> --debug`
         );
       }
+    } else if (logType === 'verify-trust') {
+      logPath = await getLatestLogPath(logsDir, 'verify-trust-');
+      if (!logPath) {
+        throw new Error(
+          `No verify-trust log found in ${logsDir}. Run: aifabrix datasource verify-trust <key> --debug`
+        );
+      }
     } else {
       const pattern = logType === 'test-e2e' ? 'test-e2e-' : 'test-integration-';
       logPath = await getLatestLogPath(logsDir, pattern);
@@ -360,6 +464,7 @@ module.exports = {
   formatE2ELog,
   formatIntegrationLog,
   formatStructuralTestLog,
+  formatTrustLog,
   parseDatasourceLogJson,
   runLogViewer
 };

package/lib/datasource/record-file-parser.js

@@ -0,0 +1,85 @@
+/**
+ * @fileoverview Parse JSON array and NDJSON record fixture files (plan 144)
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const fs = require('fs');
+
+/**
+ * @param {string} content
+ * @returns {Object[]}
+ */
+function parseNdjsonContent(content) {
+  const lines = String(content).split(/\r?\n/);
+  const records = [];
+  for (let i = 0; i < lines.length; i += 1) {
+    const line = lines[i].trim();
+    if (!line) continue;
+    let parsed;
+    try {
+      parsed = JSON.parse(line);
+    } catch (err) {
+      throw new Error(`NDJSON parse error at line ${i + 1}: ${err.message}`);
+    }
+    if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
+      throw new Error(`NDJSON line ${i + 1}: expected a JSON object`);
+    }
+    records.push(parsed);
+  }
+  return records;
+}
+
+/**
+ * @param {string} content
+ * @returns {Object[]}
+ */
+function parseJsonArrayContent(content) {
+  let parsed;
+  try {
+    parsed = JSON.parse(content);
+  } catch (err) {
+    throw new Error(`JSON parse error: ${err.message}`);
+  }
+  if (!Array.isArray(parsed)) {
+    throw new Error('JSON file must be a top-level array of objects');
+  }
+  parsed.forEach((row, index) => {
+    if (!row || typeof row !== 'object' || Array.isArray(row)) {
+      throw new Error(`JSON array index ${index}: expected an object`);
+    }
+  });
+  return parsed;
+}
+
+/**
+ * Parse a local record fixture file.
+ * @param {string} filePath
+ * @param {'json'|'ndjson'} format
+ * @returns {Object[]}
+ */
+function parseRecordFile(filePath, format) {
+  const content = fs.readFileSync(filePath, 'utf8');
+  if (format === 'ndjson') {
+    return parseNdjsonContent(content);
+  }
+  return parseJsonArrayContent(content);
+}
+
+/**
+ * Rough payload size estimate for dry-run (bytes).
+ * @param {Object[]} records
+ * @returns {number}
+ */
+function estimatePayloadBytes(records) {
+  return Buffer.byteLength(JSON.stringify(records), 'utf8');
+}
+
+module.exports = {
+  parseRecordFile,
+  parseNdjsonContent,
+  parseJsonArrayContent,
+  estimatePayloadBytes
+};

package/lib/datasource/record-mapper.js

@@ -0,0 +1,129 @@
+/**
+ * @fileoverview Map fixture payloads to canonical bulk sync records (plan 144)
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+/**
+ * @param {Object} obj
+ * @returns {boolean}
+ */
+function isCanonicalBulkRecord(obj) {
+  return (
+    obj &&
+    typeof obj === 'object' &&
+    typeof obj.key === 'string' &&
+    obj.key.trim().length > 0 &&
+    typeof obj.recordType === 'string' &&
+    obj.recordType.trim().length > 0 &&
+    obj.metadata !== undefined &&
+    obj.metadata !== null &&
+    typeof obj.metadata === 'object' &&
+    !Array.isArray(obj.metadata)
+  );
+}
+
+/**
+ * @param {string[]} primaryKeyFields
+ * @param {Object} payload
+ * @returns {string}
+ */
+function buildKeyFromPrimaryKey(primaryKeyFields, payload) {
+  if (!primaryKeyFields.length) {
+    throw new Error('Datasource primaryKey is empty; provide records with a key field');
+  }
+  if (primaryKeyFields.length === 1) {
+    const field = primaryKeyFields[0];
+    const value = payload[field];
+    if (value === undefined || value === null || String(value).trim() === '') {
+      throw new Error(`Missing primary key field '${field}' in record payload`);
+    }
+    return String(value);
+  }
+  const parts = [];
+  for (const field of primaryKeyFields) {
+    const value = payload[field];
+    if (value === undefined || value === null || String(value).trim() === '') {
+      throw new Error(`Missing composite primary key field '${field}' in record payload`);
+    }
+    parts.push(`${field}=${String(value)}`);
+  }
+  return parts.join(',');
+}
+
+/**
+ * @param {Object} datasource
+ * @param {Object} payload
+ * @returns {import('../api/types/records-bulk.types').ExternalBulkRecord}
+ */
+function mapPayloadToBulkRecord(datasource, payload) {
+  const primaryKey = Array.isArray(datasource.primaryKey) ? datasource.primaryKey : [];
+  const recordType =
+    typeof datasource.resourceType === 'string' && datasource.resourceType.trim()
+      ? datasource.resourceType.trim()
+      : 'record';
+  const key = buildKeyFromPrimaryKey(primaryKey, payload);
+  let displayName = '';
+  const labelKey = datasource.labelKey;
+  if (Array.isArray(labelKey) && labelKey.length > 0) {
+    displayName = labelKey
+      .map(field => payload[field])
+      .filter(v => v !== undefined && v !== null && String(v).trim() !== '')
+      .map(v => String(v))
+      .join(' ');
+  } else if (typeof labelKey === 'string' && labelKey.trim() && payload[labelKey] !== undefined && payload[labelKey] !== null) {
+    displayName = String(payload[labelKey]);
+  }
+  const metadata = { ...payload };
+  if (metadata.externalId === undefined) {
+    metadata.externalId = key;
+  }
+  return {
+    key,
+    displayName: displayName || '',
+    recordType,
+    metadata
+  };
+}
+
+/**
+ * @param {Object[]} rawRecords
+ * @param {Object} datasource
+ * @returns {import('../api/types/records-bulk.types').ExternalBulkRecord[]}
+ */
+function normalizeRecordsForBulk(rawRecords, datasource) {
+  if (!Array.isArray(rawRecords) || rawRecords.length === 0) {
+    throw new Error('Record file is empty after parse');
+  }
+  return rawRecords.map((row, index) => {
+    try {
+      if (isCanonicalBulkRecord(row)) {
+        const meta = { ...row.metadata };
+        if (meta.externalId === undefined) {
+          meta.externalId = row.key;
+        }
+        return {
+          key: String(row.key).trim(),
+          displayName:
+            row.displayName !== undefined && row.displayName !== null
+              ? String(row.displayName)
+              : '',
+          recordType: String(row.recordType).trim(),
+          metadata: meta
+        };
+      }
+      return mapPayloadToBulkRecord(datasource, row);
+    } catch (err) {
+      throw new Error(`Record ${index + 1}: ${err.message}`);
+    }
+  });
+}
+
+module.exports = {
+  isCanonicalBulkRecord,
+  buildKeyFromPrimaryKey,
+  mapPayloadToBulkRecord,
+  normalizeRecordsForBulk
+};