@aifabrix/builder 2.44.6 → 2.45.5

package/lib/datasource/datasource-identifier.js

@@ -0,0 +1,148 @@
+/**
+ * @fileoverview Match datasource CLI identifiers to integration files (key or filename stem).
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const path = require('path');
+const { getIntegrationPath } = require('../utils/paths');
+const { resolveApplicationConfigPath } = require('../utils/app-config-resolver');
+const { loadConfigFile } = require('../utils/config-format');
+
+/**
+ * @param {string} fileRef
+ * @returns {string}
+ */
+function datasourceFileStem(fileRef) {
+  return path.basename(String(fileRef || ''), path.extname(String(fileRef || '')));
+}
+
+/**
+ * True when identifier matches parsed `key` or the datasource JSON filename (with or without .json).
+ * @param {string} identifier
+ * @param {string} fileRef - Manifest-relative datasource path
+ * @param {Object|null} parsed
+ * @returns {boolean}
+ */
+function datasourceIdentifierMatchesFile(identifier, fileRef, parsed) {
+  const id = String(identifier || '').trim();
+  if (!id || !parsed || typeof parsed !== 'object') {
+    return false;
+  }
+  if (parsed.key === id) {
+    return true;
+  }
+  const stem = datasourceFileStem(fileRef);
+  if (stem === id) {
+    return true;
+  }
+  if (`${stem}.json` === id.toLowerCase()) {
+    return true;
+  }
+  return false;
+}
+
+/**
+ * Pick integration app folder for a datasource identifier (longest matching prefix wins).
+ * @param {string} identifier
+ * @param {string[]} appNames
+ * @returns {string|null}
+ */
+function pickIntegrationAppForDatasourceIdentifier(identifier, appNames) {
+  const id = String(identifier || '').trim();
+  if (!id) {
+    return null;
+  }
+  const matches = appNames.filter(app => id === app || id.startsWith(`${app}-`));
+  if (matches.length === 0) {
+    return null;
+  }
+  matches.sort((a, b) => b.length - a.length);
+  return matches[0];
+}
+
+/**
+ * List manifest datasource files for an integration app.
+ * @param {string} appKey
+ * @returns {{ appPath: string, schemaBasePath: string, datasourceFiles: string[] }|null}
+ */
+function listDatasourceFilesForApp(appKey) {
+  const appPath = getIntegrationPath(appKey);
+  let config;
+  try {
+    const configPath = resolveApplicationConfigPath(appPath);
+    config = loadConfigFile(configPath);
+  } catch {
+    return null;
+  }
+  const schemaBasePath = config.externalIntegration?.schemaBasePath || './';
+  const datasourceFiles = config.externalIntegration?.dataSources || [];
+  return { appPath, schemaBasePath, datasourceFiles };
+}
+
+/**
+ * @param {string} appPath
+ * @param {string} schemaBasePath
+ * @param {string} fileRef
+ * @returns {string}
+ */
+function resolveDatasourceFilePath(appPath, schemaBasePath, fileRef) {
+  if (path.isAbsolute(schemaBasePath)) {
+    return path.join(schemaBasePath, fileRef);
+  }
+  return path.join(appPath, schemaBasePath, fileRef);
+}
+
+/**
+ * Collect datasource file refs in one app that match the identifier.
+ * @param {string} appKey
+ * @param {string} identifier
+ * @returns {string[]}
+ */
+function collectDatasourceFileRefsMatchingIdentifier(appKey, identifier) {
+  const listed = listDatasourceFilesForApp(appKey);
+  if (!listed) {
+    return [];
+  }
+  const fsSync = require('fs');
+  const hits = [];
+  for (const fileRef of listed.datasourceFiles) {
+    if (!fileRef || typeof fileRef !== 'string') {
+      continue;
+    }
+    const fullPath = resolveDatasourceFilePath(listed.appPath, listed.schemaBasePath, fileRef);
+    if (!fsSync.existsSync(fullPath)) {
+      continue;
+    }
+    try {
+      const parsed = loadConfigFile(fullPath);
+      if (datasourceIdentifierMatchesFile(identifier, fileRef, parsed)) {
+        hits.push(fileRef);
+      }
+    } catch {
+      // skip unreadable or invalid files
+    }
+  }
+  return hits;
+}
+
+/**
+ * @param {string} appKey
+ * @param {string} identifier
+ * @returns {boolean}
+ */
+function appHasDatasourceIdentifier(appKey, identifier) {
+  return collectDatasourceFileRefsMatchingIdentifier(appKey, identifier).length > 0;
+}
+
+module.exports = {
+  datasourceFileStem,
+  datasourceIdentifierMatchesFile,
+  pickIntegrationAppForDatasourceIdentifier,
+  listDatasourceFilesForApp,
+  resolveDatasourceFilePath,
+  collectDatasourceFileRefsMatchingIdentifier,
+  appHasDatasourceIdentifier
+};

package/lib/datasource/datasource-load-export-context.js

@@ -0,0 +1,92 @@
+/**
+ * @fileoverview Shared context resolution for datasource load/export (plan 144)
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const { getConfig } = require('../core/config');
+const { setupIntegrationTestAuth } = require('../external-system/test-auth');
+const { resolveAppKeyForDatasource } = require('./resolve-app');
+const { getSystemKeyFromAppKey } = require('./integration-context');
+const { loadDatasourceForApp } = require('./unified-validation-run-resolve');
+const { findDatasourceFileByKey } = require('./integration-context');
+const { getIntegrationPath } = require('../utils/paths');
+const { resolveApplicationConfigPath } = require('../utils/app-config-resolver');
+const { computeEntitySuffix } = require('./local-data-paths');
+
+/**
+ * @param {string} appKey
+ * @throws {Error}
+ */
+async function assertExternalIntegrationApp(appKey) {
+  const pathsUtil = require('../utils/paths');
+  const appType = await pathsUtil.detectAppType(appKey).catch(() => null);
+  if (!appType || appType.baseDir !== 'integration') {
+    throw new Error(
+      'datasource load/export apply to external integration folders only (integration/<systemKey>/).'
+    );
+  }
+}
+
+/**
+ * @async
+ * @param {string} datasourceKey
+ * @param {Object} options
+ * @returns {Promise<Object>}
+ */
+async function resolveLoadExportContext(datasourceKey, options = {}) {
+  const { appKey } = await resolveAppKeyForDatasource(datasourceKey, options.app);
+  await assertExternalIntegrationApp(appKey);
+  const systemKey = await getSystemKeyFromAppKey(appKey);
+  const {
+    datasource,
+    datasourcePath,
+    datasourceKey: canonicalDatasourceKey
+  } = loadDatasourceForApp(appKey, datasourceKey, findDatasourceFileByKey);
+  const appPath = getIntegrationPath(appKey);
+  const manifestPath = resolveApplicationConfigPath(appPath);
+  const entitySuffix = computeEntitySuffix(systemKey, canonicalDatasourceKey);
+
+  let authConfig = options.authConfig;
+  let dataplaneUrl = options.dataplaneUrl || options.dataplane;
+  if (!authConfig || !dataplaneUrl) {
+    const configObj = await getConfig();
+    const auth = await setupIntegrationTestAuth(
+      appKey,
+      { environment: options.environment || options.env, dataplane: options.dataplane },
+      configObj
+    );
+    authConfig = auth.authConfig;
+    dataplaneUrl = auth.dataplaneUrl;
+  }
+
+  return {
+    appKey,
+    systemKey,
+    datasourceKey: canonicalDatasourceKey,
+    datasource,
+    datasourcePath,
+    manifestPath,
+    entitySuffix,
+    authConfig,
+    dataplaneUrl,
+    environment: options.environment || options.env
+  };
+}
+
+/**
+ * Gray manifest label segment for TTY (integration/<appKey>).
+ * @param {string} appKey
+ * @returns {string}
+ */
+function integrationManifestLabel(appKey) {
+  return `integration/${appKey}`;
+}
+
+module.exports = {
+  assertExternalIntegrationApp,
+  resolveLoadExportContext,
+  integrationManifestLabel
+};

package/lib/datasource/integration-context.js

@@ -9,6 +9,10 @@ const fs = require('fs').promises;
 const { getIntegrationPath } = require('../utils/paths');
 const { resolveApplicationConfigPath } = require('../utils/app-config-resolver');
 const { loadConfigFile } = require('../utils/config-format');
+const {
+  datasourceIdentifierMatchesFile,
+  resolveDatasourceFilePath
+} = require('./datasource-identifier');
 
 /**
  * @param {string} appKey - Integration app key
@@ -43,13 +47,11 @@ function findDatasourceFileByKey(appPath, schemaBasePath, datasourceFiles, datas
   const fsSync = require('fs');
   for (const f of datasourceFiles) {
     if (!f || typeof f !== 'string') continue;
-    const fullPath = path.isAbsolute(schemaBasePath)
-      ? path.join(schemaBasePath, f)
-      : path.join(appPath, schemaBasePath, f);
+    const fullPath = resolveDatasourceFilePath(appPath, schemaBasePath, f);
     if (!fsSync.existsSync(fullPath)) continue;
     try {
       const parsed = loadConfigFile(fullPath);
-      if (parsed && parsed.key === datasourceKey) return f;
+      if (datasourceIdentifierMatchesFile(datasourceKey, f, parsed)) return f;
     } catch {
       // skip unreadable or invalid files
     }

package/lib/datasource/local-data-paths.js

@@ -0,0 +1,145 @@
+/**
+ * @fileoverview Resolve integration/.data/ paths for datasource load/export (plan 144)
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const path = require('path');
+const fs = require('fs');
+const { getCwdIntegrationRoot, getIntegrationRoot } = require('../utils/paths');
+
+const DATA_DIR_NAME = '.data';
+
+/**
+ * Absolute path to integration/.data (cwd integration root preferred).
+ * @returns {string}
+ */
+function getIntegrationDataDir() {
+  const cwdRoot = getCwdIntegrationRoot();
+  const base = cwdRoot || getIntegrationRoot();
+  return path.join(base, DATA_DIR_NAME);
+}
+
+/**
+ * Entity suffix from datasource key and system key.
+ * @param {string} systemKey
+ * @param {string} datasourceKey
+ * @returns {string}
+ */
+function computeEntitySuffix(systemKey, datasourceKey) {
+  const sys = String(systemKey || '').trim();
+  const key = String(datasourceKey || '').trim();
+  const prefix = `${sys}-`;
+  if (sys && key.startsWith(prefix)) {
+    return key.slice(prefix.length) || key;
+  }
+  const lastHyphen = key.lastIndexOf('-');
+  if (lastHyphen > 0) {
+    return key.slice(lastHyphen + 1);
+  }
+  return key;
+}
+
+/**
+ * Default fixture basename (without extension).
+ * @param {string} systemKey
+ * @param {string} entitySuffix
+ * @returns {string}
+ */
+function defaultDataBasename(systemKey, entitySuffix) {
+  return `${systemKey}-data-${entitySuffix}`;
+}
+
+/**
+ * @param {string} dir
+ * @param {string} basename
+ * @param {'json'|'ndjson'} format
+ * @returns {string}
+ */
+function dataFilePath(dir, basename, format) {
+  const ext = format === 'ndjson' ? '.ndjson' : '.json';
+  return path.join(dir, `${basename}${ext}`);
+}
+
+/**
+ * Infer format from file extension.
+ * @param {string} filePath
+ * @returns {'json'|'ndjson'}
+ */
+function inferFormatFromPath(filePath) {
+  const ext = path.extname(filePath).toLowerCase();
+  if (ext === '.ndjson' || ext === '.jsonl') {
+    return 'ndjson';
+  }
+  return 'json';
+}
+
+/**
+ * Default integration/.data output path (file need not exist).
+ * @param {Object} opts
+ * @param {string} opts.systemKey
+ * @param {string} opts.entitySuffix
+ * @param {'json'|'ndjson'} [opts.format]
+ * @returns {{ filePath: string, format: 'json'|'ndjson' }}
+ */
+function resolveDefaultDataOutputPath(opts) {
+  const dir = getIntegrationDataDir();
+  const basename = defaultDataBasename(opts.systemKey, opts.entitySuffix);
+  const format = opts.format === 'ndjson' ? 'ndjson' : 'json';
+  return { filePath: dataFilePath(dir, basename, format), format };
+}
+
+/**
+ * Resolve data file: --file override or default integration/.data paths.
+ * @param {Object} opts
+ * @param {string} opts.systemKey
+ * @param {string} opts.entitySuffix
+ * @param {string} [opts.file] - CLI --file
+ * @param {'json'|'ndjson'} [opts.format] - explicit format
+ * @returns {{ filePath: string, format: 'json'|'ndjson' }}
+ */
+function resolveDataFilePath(opts) {
+  if (opts.file && String(opts.file).trim()) {
+    const filePath = path.resolve(String(opts.file).trim());
+    const format = opts.format || inferFormatFromPath(filePath);
+    return { filePath, format };
+  }
+
+  const dir = getIntegrationDataDir();
+  const basename = defaultDataBasename(opts.systemKey, opts.entitySuffix);
+  const jsonPath = dataFilePath(dir, basename, 'json');
+  const ndjsonPath = dataFilePath(dir, basename, 'ndjson');
+
+  if (opts.format === 'json' && fs.existsSync(jsonPath)) {
+    return { filePath: jsonPath, format: 'json' };
+  }
+  if (opts.format === 'ndjson' && fs.existsSync(ndjsonPath)) {
+    return { filePath: ndjsonPath, format: 'ndjson' };
+  }
+  if (!opts.format || opts.format === 'json') {
+    if (fs.existsSync(jsonPath)) {
+      return { filePath: jsonPath, format: 'json' };
+    }
+  }
+  if (!opts.format || opts.format === 'ndjson') {
+    if (fs.existsSync(ndjsonPath)) {
+      return { filePath: ndjsonPath, format: 'ndjson' };
+    }
+  }
+
+  throw new Error(
+    `No local data file found. Expected one of:\n  ${jsonPath}\n  ${ndjsonPath}\n` +
+      'Use --file <path> to override.'
+  );
+}
+
+module.exports = {
+  getIntegrationDataDir,
+  computeEntitySuffix,
+  defaultDataBasename,
+  inferFormatFromPath,
+  resolveDefaultDataOutputPath,
+  resolveDataFilePath
+};

package/lib/datasource/log-cleaner.js

@@ -0,0 +1,241 @@
+/**
+ * Remove saved datasource test debug JSON under integration/<app>/logs/.
+ * @fileoverview Local cleanup for test / test-integration / test-e2e / test-trust logs
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+const path = require('path');
+const fsp = require('node:fs').promises;
+const chalk = require('chalk');
+const logger = require('../utils/logger');
+const { getIntegrationPath, listIntegrationAppNames } = require('../utils/paths');
+const { isStructuralTestLogFileName } = require('./log-viewer');
+const { formatSuccessLine } = require('../utils/cli-test-layout-chalk');
+
+/** @type {readonly string[]} */
+const LOG_CLEAN_TYPES = ['test', 'integration', 'e2e', 'trust', 'all'];
+
+/**
+ * @param {string} [type]
+ * @returns {'test'|'integration'|'e2e'|'trust'|'all'}
+ */
+function normalizeLogCleanType(type) {
+  const raw = (type === undefined || type === null ? 'all' : String(type)).trim().toLowerCase();
+  if (raw === '' || raw === 'all') return 'all';
+  if (LOG_CLEAN_TYPES.includes(raw)) return raw;
+  throw new Error(
+    `Invalid --type "${type}". Use: ${LOG_CLEAN_TYPES.join(', ')}`
+  );
+}
+
+/**
+ * @param {string} fileName
+ * @param {'test'|'integration'|'e2e'|'trust'|'all'} type
+ * @returns {boolean}
+ */
+function matchesLogCleanType(fileName, type) {
+  if (!fileName || !fileName.endsWith('.json')) return false;
+  if (type === 'all') return true;
+  if (type === 'test') return isStructuralTestLogFileName(fileName);
+  if (type === 'integration') {
+    return fileName.startsWith('test-integration-');
+  }
+  if (type === 'e2e') return fileName.startsWith('test-e2e-');
+  if (type === 'trust') return fileName.startsWith('test-trust-');
+  return false;
+}
+
+/**
+ * @param {string} logsDir
+ * @param {'test'|'integration'|'e2e'|'trust'|'all'} type
+ * @returns {Promise<string[]>}
+ */
+async function listMatchingLogFiles(logsDir, type) {
+  let entries;
+  try {
+    entries = await fsp.readdir(logsDir, { withFileTypes: true });
+  } catch (err) {
+    if (err.code === 'ENOENT') return [];
+    throw err;
+  }
+  return entries
+    .filter(e => e.isFile() && matchesLogCleanType(e.name, type))
+    .map(e => path.join(logsDir, e.name))
+    .sort((a, b) => a.localeCompare(b));
+}
+
+/**
+ * @param {string} appKey
+ * @returns {Promise<void>}
+ */
+async function assertIntegrationAppExists(appKey) {
+  const appPath = getIntegrationPath(appKey);
+  try {
+    const st = await fsp.stat(appPath);
+    if (!st.isDirectory()) {
+      throw new Error(`Integration folder is not a directory: ${appPath}`);
+    }
+  } catch (err) {
+    if (err.code === 'ENOENT') {
+      throw new Error(`Integration folder not found: ${appPath}`);
+    }
+    throw err;
+  }
+}
+
+/**
+ * @param {Object} options
+ * @param {boolean} [options.all]
+ * @param {string} [options.app]
+ * @returns {Promise<string[]>}
+ */
+async function resolveTargetAppKeys(options) {
+  const useAll = Boolean(options.all);
+  const app = options.app && typeof options.app === 'string' ? options.app.trim() : '';
+  if (useAll && app) {
+    throw new Error('Use either --app <systemKey> or --all, not both');
+  }
+  if (!useAll && !app) {
+    throw new Error('Specify --app <systemKey> or --all');
+  }
+  if (useAll) {
+    return listIntegrationAppNames();
+  }
+  await assertIntegrationAppExists(app);
+  return [app];
+}
+
+/**
+ * @param {string[]} appKeys
+ * @param {'test'|'integration'|'e2e'|'trust'|'all'} type
+ * @returns {Promise<Array<{ appKey: string, logsDir: string, files: string[] }>>}
+ */
+async function collectLogFilesByApp(appKeys, type) {
+  const groups = [];
+  for (const appKey of appKeys) {
+    const logsDir = path.join(getIntegrationPath(appKey), 'logs');
+    const files = await listMatchingLogFiles(logsDir, type);
+    if (files.length > 0) {
+      groups.push({ appKey, logsDir, files });
+    }
+  }
+  return groups;
+}
+
+/**
+ * @param {Array<{ appKey: string, files: string[] }>} groups
+ * @param {boolean} dryRun
+ * @returns {Promise<{ removedCount: number, paths: string[] }>}
+ */
+async function removeCollectedFiles(groups, dryRun) {
+  const paths = groups.flatMap(g => g.files);
+  if (dryRun || paths.length === 0) {
+    return { removedCount: 0, paths };
+  }
+  await Promise.all(paths.map(filePath => fsp.unlink(filePath)));
+  return { removedCount: paths.length, paths };
+}
+
+/**
+ * @param {Object} options
+ * @param {boolean} options.dryRun
+ * @param {'test'|'integration'|'e2e'|'trust'|'all'} options.type
+ * @param {Array<{ appKey: string, files: string[] }>} groups
+ * @param {number} removedCount
+ */
+function printCleanLogsSummary(options, groups, removedCount) {
+  const totalFiles = groups.reduce((n, g) => n + g.files.length, 0);
+  const appCount = groups.length;
+  if (options.dryRun) {
+    if (totalFiles === 0) {
+      logger.log(chalk.gray('No matching log files found.'));
+      return;
+    }
+    logger.log('');
+    logger.log(
+      chalk.gray(
+        `Would remove ${totalFiles} file(s) from ${appCount} integration folder(s) (type: ${options.type}).`
+      )
+    );
+    return;
+  }
+  if (totalFiles === 0) {
+    logger.log(chalk.gray('No matching log files to remove.'));
+    return;
+  }
+  logger.log('');
+  logger.log(formatSuccessLine(`Removed ${removedCount} log file(s) from ${appCount} integration folder(s).`));
+}
+
+/**
+ * @param {boolean} dryRun
+ * @param {Array<{ appKey: string, logsDir: string, files: string[] }>} groups
+ */
+function printCleanLogsFileList(dryRun, groups) {
+  const prefix = dryRun ? 'Would remove' : 'Removing';
+  for (const { appKey, files } of groups) {
+    logger.log('');
+    logger.log(chalk.bold.white(`${appKey}/logs/`));
+    for (const filePath of files) {
+      logger.log(chalk.gray(`  ${prefix}: ${filePath}`));
+    }
+  }
+}
+
+/**
+ * @param {Object} result
+ * @param {boolean} result.dryRun
+ * @param {string} result.type
+ * @param {string[]} result.appKeys
+ * @param {string[]} result.paths
+ * @param {number} result.removedCount
+ */
+function printCleanLogsJson(result) {
+  const payload = {
+    dryRun: result.dryRun,
+    type: result.type,
+    appKeys: result.appKeys,
+    fileCount: result.paths.length,
+    paths: result.paths,
+    removedCount: result.removedCount
+  };
+  logger.log(JSON.stringify(payload, null, 2));
+}
+
+/**
+ * @async
+ * @param {Object} options
+ * @param {boolean} [options.all]
+ * @param {string} [options.app]
+ * @param {string} [options.type]
+ * @param {boolean} [options.dryRun]
+ * @param {boolean} [options.json]
+ * @returns {Promise<{ dryRun: boolean, type: string, appKeys: string[], paths: string[], removedCount: number }>}
+ */
+async function runCleanLogs(options = {}) {
+  const dryRun = Boolean(options.dryRun);
+  const json = Boolean(options.json);
+  const type = normalizeLogCleanType(options.type);
+  const appKeys = await resolveTargetAppKeys(options);
+  const groups = await collectLogFilesByApp(appKeys, type);
+  const { removedCount, paths } = await removeCollectedFiles(groups, dryRun);
+  const result = { dryRun, type, appKeys, paths, removedCount };
+  if (json) {
+    printCleanLogsJson(result);
+  } else {
+    printCleanLogsFileList(dryRun, groups);
+    printCleanLogsSummary({ dryRun, type }, groups, removedCount);
+  }
+  return result;
+}
+
+module.exports = {
+  LOG_CLEAN_TYPES,
+  normalizeLogCleanType,
+  matchesLogCleanType,
+  listMatchingLogFiles,
+  resolveTargetAppKeys,
+  collectLogFilesByApp,
+  runCleanLogs
+};