@aifabrix/builder 2.45.0 → 2.45.5

package/lib/datasource/bulk-loader-service.js

@@ -0,0 +1,127 @@
+/**
+ * @fileoverview Load local fixtures via dataplane bulk record sync (plan 144)
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const { resolveDataFilePath } = require('./local-data-paths');
+const { parseRecordFile, estimatePayloadBytes } = require('./record-file-parser');
+const { normalizeRecordsForBulk } = require('./record-mapper');
+const { resolveLoadExportContext } = require('./datasource-load-export-context');
+const { uploadRecordBatches } = require('./bulk-loader-run-batches');
+const { runWithDatasourceLoadExportSpinner } = require('../utils/datasource-load-export-ui');
+
+const DEFAULT_BATCH_SIZE = 100;
+
+/**
+ * @param {Object[]} records
+ * @param {number} batchSize
+ * @returns {Object[][]}
+ */
+function chunkRecords(records, batchSize) {
+  const size = Math.max(1, batchSize);
+  const chunks = [];
+  for (let i = 0; i < records.length; i += size) {
+    chunks.push(records.slice(i, i + size));
+  }
+  return chunks;
+}
+
+/**
+ * @param {Object} params
+ * @returns {Object}
+ */
+function buildLoadResultBase(params) {
+  const { ctx, filePath, format, records, batchSize, syncType, dryRun } = params;
+  return {
+    datasourceKey: ctx.datasourceKey,
+    systemKey: ctx.systemKey,
+    appKey: ctx.appKey,
+    dryRun,
+    file: filePath,
+    format,
+    recordCount: records.length,
+    batchSize,
+    syncType,
+    estimatedPayloadBytes: estimatePayloadBytes(records),
+    batches: [],
+    totals: { insertedCount: 0, updatedCount: 0, failedCount: 0 }
+  };
+}
+
+/**
+ * @async
+ * @param {Object} params
+ * @returns {Promise<Object>}
+ */
+async function executeDatasourceLoadWork(params) {
+  const { ctx, options, batchSize, syncType, dryRun } = params;
+  const { filePath, format } = resolveDataFilePath({
+    systemKey: ctx.systemKey,
+    entitySuffix: ctx.entitySuffix,
+    file: options.file,
+    format: options.format
+  });
+
+  const raw = parseRecordFile(filePath, format);
+  const records = normalizeRecordsForBulk(raw, ctx.datasource);
+  const result = buildLoadResultBase({
+    ctx,
+    filePath,
+    format,
+    records,
+    batchSize,
+    syncType,
+    dryRun
+  });
+
+  if (dryRun) {
+    return { ...result, context: ctx };
+  }
+
+  const { batches, totals } = await uploadRecordBatches({
+    chunks: chunkRecords(records, batchSize),
+    syncType,
+    dataplaneUrl: ctx.dataplaneUrl,
+    datasourceKey: ctx.datasourceKey,
+    authConfig: ctx.authConfig
+  });
+
+  return { ...result, batches, totals, context: ctx };
+}
+
+/**
+ * @async
+ * @param {string} datasourceKey
+ * @param {Object} options
+ * @returns {Promise<Object>}
+ */
+async function runDatasourceLoad(datasourceKey, options = {}) {
+  const ctx = await resolveLoadExportContext(datasourceKey, options);
+  const batchSize = options.batchSize
+    ? parseInt(String(options.batchSize), 10)
+    : DEFAULT_BATCH_SIZE;
+  const syncType = options.syncType || 'incremental';
+  const dryRun = options.dryRun === true;
+  const startedAt = Date.now();
+
+  const payload = await runWithDatasourceLoadExportSpinner(
+    () => executeDatasourceLoadWork({ ctx, options, batchSize, syncType, dryRun }),
+    {
+      operation: dryRun ? 'load-dry-run' : 'load',
+      datasourceKey: ctx.datasourceKey,
+      json: options.json === true
+    }
+  );
+
+  return { ...payload, durationMs: Date.now() - startedAt };
+}
+
+module.exports = {
+  runDatasourceLoad,
+  chunkRecords,
+  buildLoadResultBase,
+  DEFAULT_BATCH_SIZE
+};

package/lib/datasource/capability/run-capability-dimension.js

@@ -43,7 +43,8 @@ async function maybeLoadDimensionCatalogKeys() {
       page: 1,
       pageSize: 500
     });
-    const items = res?.data?.items || res?.data?.data?.items || res?.items || [];
+    const raw = res?.data?.items ?? res?.data?.data ?? res?.items;
+    const items = Array.isArray(raw) ? raw : raw?.items || [];
     const keys = new Set(
       Array.isArray(items) ? items.map((d) => String(d?.key || '').trim()).filter(Boolean) : []
     );

package/lib/datasource/datasource-exporter-resolve.js

@@ -0,0 +1,82 @@
+/**
+ * @fileoverview Export path and search body helpers (plan 144)
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const path = require('path');
+const { resolveDefaultDataOutputPath, inferFormatFromPath } = require('./local-data-paths');
+
+/**
+ * @param {string|undefined} raw
+ * @returns {Object|undefined}
+ */
+function parseFilterJson(raw) {
+  if (!raw || typeof raw !== 'string' || !raw.trim()) {
+    return undefined;
+  }
+  try {
+    const parsed = JSON.parse(raw);
+    if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
+      return parsed;
+    }
+    throw new Error('filter must be a JSON object');
+  } catch (err) {
+    throw new Error(`Invalid --filter JSON: ${err.message}`);
+  }
+}
+
+/**
+ * @param {string|undefined} raw
+ * @returns {string[]|undefined}
+ */
+function parseFieldsCsv(raw) {
+  if (!raw || typeof raw !== 'string' || !raw.trim()) {
+    return undefined;
+  }
+  return raw
+    .split(',')
+    .map(s => s.trim())
+    .filter(Boolean);
+}
+
+/**
+ * @param {Object} opts
+ * @returns {{ outputPath: string, format: 'json'|'ndjson', limit: number, intent: string, filters: Object|undefined, fields: string[]|undefined }}
+ */
+function resolveExportOptions(opts) {
+  let limit = opts.limit ? parseInt(String(opts.limit), 10) : 1000;
+  if (Number.isNaN(limit) || limit < 1) {
+    limit = 1000;
+  }
+  limit = Math.min(limit, 10000);
+
+  const intent = opts.intent || 'validation';
+  const filters = parseFilterJson(opts.filter);
+  const fields = parseFieldsCsv(opts.fields);
+
+  let outputPath;
+  let format = opts.format;
+  if (opts.file && String(opts.file).trim()) {
+    outputPath = path.resolve(String(opts.file).trim());
+    format = format || inferFormatFromPath(outputPath);
+  } else {
+    const resolved = resolveDefaultDataOutputPath({
+      systemKey: opts.systemKey,
+      entitySuffix: opts.entitySuffix,
+      format: format || 'json'
+    });
+    outputPath = resolved.filePath;
+    format = opts.format || resolved.format;
+  }
+
+  return { outputPath, format, limit, intent, filters, fields };
+}
+
+module.exports = {
+  parseFilterJson,
+  parseFieldsCsv,
+  resolveExportOptions
+};

package/lib/datasource/datasource-exporter-service.js

@@ -0,0 +1,144 @@
+/**
+ * @fileoverview Export governed records via Records Search (plan 144)
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const { searchRecords } = require('../api/records-search.api');
+const { resolveLoadExportContext } = require('./datasource-load-export-context');
+const { resolveExportOptions } = require('./datasource-exporter-resolve');
+const { runWithDatasourceLoadExportSpinner } = require('../utils/datasource-load-export-ui');
+
+const DEFAULT_LIMIT = 1000;
+const MAX_LIMIT = 10000;
+
+/**
+ * @param {Object[]} data
+ * @param {string[]|undefined} fields
+ * @returns {Object[]}
+ */
+function projectExportRows(data, fields) {
+  if (!fields || !fields.length) {
+    return data;
+  }
+  return data.map(row => {
+    if (!row || typeof row !== 'object') {
+      return row;
+    }
+    const meta =
+      row.metadata && typeof row.metadata === 'object' && !Array.isArray(row.metadata)
+        ? row.metadata
+        : row;
+    const out = {};
+    fields.forEach(field => {
+      if (Object.prototype.hasOwnProperty.call(meta, field)) {
+        out[field] = meta[field];
+      }
+    });
+    return out;
+  });
+}
+
+/**
+ * @param {string} filePath
+ * @param {'json'|'ndjson'} format
+ * @param {Object[]} rows
+ */
+function writeExportFile(filePath, format, rows) {
+  const dir = path.dirname(filePath);
+  fs.mkdirSync(dir, { recursive: true });
+  if (format === 'ndjson') {
+    const body = rows.map(row => JSON.stringify(row)).join('\n');
+    fs.writeFileSync(filePath, body.length ? `${body}\n` : '', 'utf8');
+    return;
+  }
+  fs.writeFileSync(filePath, `${JSON.stringify(rows, null, 2)}\n`, 'utf8');
+}
+
+/**
+ * @async
+ * @param {Object} ctx
+ * @param {Object} resolved
+ * @returns {Promise<Object>}
+ */
+async function executeDatasourceExportWork(ctx, resolved) {
+  const searchBody = {
+    intent: resolved.intent,
+    datasourceKeys: [ctx.datasourceKey],
+    searchMode: 'full',
+    limit: resolved.limit
+  };
+  if (resolved.filters) {
+    searchBody.filters = resolved.filters;
+  }
+
+  const response = await searchRecords(ctx.dataplaneUrl, ctx.authConfig, searchBody);
+  if (!response || response.success === false) {
+    const msg = response?.formattedError || response?.error || 'Records search request failed';
+    throw new Error(msg);
+  }
+
+  const data = Array.isArray(response.data) ? response.data : [];
+  const projected = projectExportRows(data, resolved.fields);
+  writeExportFile(resolved.outputPath, resolved.format, projected);
+
+  const meta = response.meta && typeof response.meta === 'object' ? response.meta : {};
+  const excluded = meta.excluded && typeof meta.excluded === 'object' ? meta.excluded : {};
+
+  return {
+    datasourceKey: ctx.datasourceKey,
+    systemKey: ctx.systemKey,
+    appKey: ctx.appKey,
+    outputFile: resolved.outputPath,
+    format: resolved.format,
+    recordCount: projected.length,
+    limit: resolved.limit,
+    intent: resolved.intent,
+    filters: resolved.filters || null,
+    fields: resolved.fields || null,
+    meta: { excluded, auditRef: meta.auditRef },
+    excludedAbac: Number(excluded.abac) || 0,
+    excludedFilter: Number(excluded.filter) || 0,
+    hitLimitCap: projected.length >= resolved.limit,
+    context: ctx
+  };
+}
+
+/**
+ * @async
+ * @param {string} datasourceKey
+ * @param {Object} options
+ * @returns {Promise<Object>}
+ */
+async function runDatasourceExport(datasourceKey, options = {}) {
+  const ctx = await resolveLoadExportContext(datasourceKey, options);
+  const resolved = resolveExportOptions({
+    ...options,
+    systemKey: ctx.systemKey,
+    entitySuffix: ctx.entitySuffix
+  });
+  const startedAt = Date.now();
+
+  const payload = await runWithDatasourceLoadExportSpinner(
+    () => executeDatasourceExportWork(ctx, resolved),
+    {
+      operation: 'export',
+      datasourceKey: ctx.datasourceKey,
+      json: options.json === true
+    }
+  );
+
+  return { ...payload, durationMs: Date.now() - startedAt };
+}
+
+module.exports = {
+  runDatasourceExport,
+  projectExportRows,
+  writeExportFile,
+  DEFAULT_LIMIT,
+  MAX_LIMIT
+};

package/lib/datasource/datasource-identifier.js

@@ -0,0 +1,148 @@
+/**
+ * @fileoverview Match datasource CLI identifiers to integration files (key or filename stem).
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const path = require('path');
+const { getIntegrationPath } = require('../utils/paths');
+const { resolveApplicationConfigPath } = require('../utils/app-config-resolver');
+const { loadConfigFile } = require('../utils/config-format');
+
+/**
+ * @param {string} fileRef
+ * @returns {string}
+ */
+function datasourceFileStem(fileRef) {
+  return path.basename(String(fileRef || ''), path.extname(String(fileRef || '')));
+}
+
+/**
+ * True when identifier matches parsed `key` or the datasource JSON filename (with or without .json).
+ * @param {string} identifier
+ * @param {string} fileRef - Manifest-relative datasource path
+ * @param {Object|null} parsed
+ * @returns {boolean}
+ */
+function datasourceIdentifierMatchesFile(identifier, fileRef, parsed) {
+  const id = String(identifier || '').trim();
+  if (!id || !parsed || typeof parsed !== 'object') {
+    return false;
+  }
+  if (parsed.key === id) {
+    return true;
+  }
+  const stem = datasourceFileStem(fileRef);
+  if (stem === id) {
+    return true;
+  }
+  if (`${stem}.json` === id.toLowerCase()) {
+    return true;
+  }
+  return false;
+}
+
+/**
+ * Pick integration app folder for a datasource identifier (longest matching prefix wins).
+ * @param {string} identifier
+ * @param {string[]} appNames
+ * @returns {string|null}
+ */
+function pickIntegrationAppForDatasourceIdentifier(identifier, appNames) {
+  const id = String(identifier || '').trim();
+  if (!id) {
+    return null;
+  }
+  const matches = appNames.filter(app => id === app || id.startsWith(`${app}-`));
+  if (matches.length === 0) {
+    return null;
+  }
+  matches.sort((a, b) => b.length - a.length);
+  return matches[0];
+}
+
+/**
+ * List manifest datasource files for an integration app.
+ * @param {string} appKey
+ * @returns {{ appPath: string, schemaBasePath: string, datasourceFiles: string[] }|null}
+ */
+function listDatasourceFilesForApp(appKey) {
+  const appPath = getIntegrationPath(appKey);
+  let config;
+  try {
+    const configPath = resolveApplicationConfigPath(appPath);
+    config = loadConfigFile(configPath);
+  } catch {
+    return null;
+  }
+  const schemaBasePath = config.externalIntegration?.schemaBasePath || './';
+  const datasourceFiles = config.externalIntegration?.dataSources || [];
+  return { appPath, schemaBasePath, datasourceFiles };
+}
+
+/**
+ * @param {string} appPath
+ * @param {string} schemaBasePath
+ * @param {string} fileRef
+ * @returns {string}
+ */
+function resolveDatasourceFilePath(appPath, schemaBasePath, fileRef) {
+  if (path.isAbsolute(schemaBasePath)) {
+    return path.join(schemaBasePath, fileRef);
+  }
+  return path.join(appPath, schemaBasePath, fileRef);
+}
+
+/**
+ * Collect datasource file refs in one app that match the identifier.
+ * @param {string} appKey
+ * @param {string} identifier
+ * @returns {string[]}
+ */
+function collectDatasourceFileRefsMatchingIdentifier(appKey, identifier) {
+  const listed = listDatasourceFilesForApp(appKey);
+  if (!listed) {
+    return [];
+  }
+  const fsSync = require('fs');
+  const hits = [];
+  for (const fileRef of listed.datasourceFiles) {
+    if (!fileRef || typeof fileRef !== 'string') {
+      continue;
+    }
+    const fullPath = resolveDatasourceFilePath(listed.appPath, listed.schemaBasePath, fileRef);
+    if (!fsSync.existsSync(fullPath)) {
+      continue;
+    }
+    try {
+      const parsed = loadConfigFile(fullPath);
+      if (datasourceIdentifierMatchesFile(identifier, fileRef, parsed)) {
+        hits.push(fileRef);
+      }
+    } catch {
+      // skip unreadable or invalid files
+    }
+  }
+  return hits;
+}
+
+/**
+ * @param {string} appKey
+ * @param {string} identifier
+ * @returns {boolean}
+ */
+function appHasDatasourceIdentifier(appKey, identifier) {
+  return collectDatasourceFileRefsMatchingIdentifier(appKey, identifier).length > 0;
+}
+
+module.exports = {
+  datasourceFileStem,
+  datasourceIdentifierMatchesFile,
+  pickIntegrationAppForDatasourceIdentifier,
+  listDatasourceFilesForApp,
+  resolveDatasourceFilePath,
+  collectDatasourceFileRefsMatchingIdentifier,
+  appHasDatasourceIdentifier
+};

package/lib/datasource/datasource-load-export-context.js

@@ -0,0 +1,92 @@
+/**
+ * @fileoverview Shared context resolution for datasource load/export (plan 144)
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const { getConfig } = require('../core/config');
+const { setupIntegrationTestAuth } = require('../external-system/test-auth');
+const { resolveAppKeyForDatasource } = require('./resolve-app');
+const { getSystemKeyFromAppKey } = require('./integration-context');
+const { loadDatasourceForApp } = require('./unified-validation-run-resolve');
+const { findDatasourceFileByKey } = require('./integration-context');
+const { getIntegrationPath } = require('../utils/paths');
+const { resolveApplicationConfigPath } = require('../utils/app-config-resolver');
+const { computeEntitySuffix } = require('./local-data-paths');
+
+/**
+ * @param {string} appKey
+ * @throws {Error}
+ */
+async function assertExternalIntegrationApp(appKey) {
+  const pathsUtil = require('../utils/paths');
+  const appType = await pathsUtil.detectAppType(appKey).catch(() => null);
+  if (!appType || appType.baseDir !== 'integration') {
+    throw new Error(
+      'datasource load/export apply to external integration folders only (integration/<systemKey>/).'
+    );
+  }
+}
+
+/**
+ * @async
+ * @param {string} datasourceKey
+ * @param {Object} options
+ * @returns {Promise<Object>}
+ */
+async function resolveLoadExportContext(datasourceKey, options = {}) {
+  const { appKey } = await resolveAppKeyForDatasource(datasourceKey, options.app);
+  await assertExternalIntegrationApp(appKey);
+  const systemKey = await getSystemKeyFromAppKey(appKey);
+  const {
+    datasource,
+    datasourcePath,
+    datasourceKey: canonicalDatasourceKey
+  } = loadDatasourceForApp(appKey, datasourceKey, findDatasourceFileByKey);
+  const appPath = getIntegrationPath(appKey);
+  const manifestPath = resolveApplicationConfigPath(appPath);
+  const entitySuffix = computeEntitySuffix(systemKey, canonicalDatasourceKey);
+
+  let authConfig = options.authConfig;
+  let dataplaneUrl = options.dataplaneUrl || options.dataplane;
+  if (!authConfig || !dataplaneUrl) {
+    const configObj = await getConfig();
+    const auth = await setupIntegrationTestAuth(
+      appKey,
+      { environment: options.environment || options.env, dataplane: options.dataplane },
+      configObj
+    );
+    authConfig = auth.authConfig;
+    dataplaneUrl = auth.dataplaneUrl;
+  }
+
+  return {
+    appKey,
+    systemKey,
+    datasourceKey: canonicalDatasourceKey,
+    datasource,
+    datasourcePath,
+    manifestPath,
+    entitySuffix,
+    authConfig,
+    dataplaneUrl,
+    environment: options.environment || options.env
+  };
+}
+
+/**
+ * Gray manifest label segment for TTY (integration/<appKey>).
+ * @param {string} appKey
+ * @returns {string}
+ */
+function integrationManifestLabel(appKey) {
+  return `integration/${appKey}`;
+}
+
+module.exports = {
+  assertExternalIntegrationApp,
+  resolveLoadExportContext,
+  integrationManifestLabel
+};

package/lib/datasource/integration-context.js

@@ -9,6 +9,10 @@ const fs = require('fs').promises;
 const { getIntegrationPath } = require('../utils/paths');
 const { resolveApplicationConfigPath } = require('../utils/app-config-resolver');
 const { loadConfigFile } = require('../utils/config-format');
+const {
+  datasourceIdentifierMatchesFile,
+  resolveDatasourceFilePath
+} = require('./datasource-identifier');
 
 /**
  * @param {string} appKey - Integration app key
@@ -43,13 +47,11 @@ function findDatasourceFileByKey(appPath, schemaBasePath, datasourceFiles, datas
   const fsSync = require('fs');
   for (const f of datasourceFiles) {
     if (!f || typeof f !== 'string') continue;
-    const fullPath = path.isAbsolute(schemaBasePath)
-      ? path.join(schemaBasePath, f)
-      : path.join(appPath, schemaBasePath, f);
+    const fullPath = resolveDatasourceFilePath(appPath, schemaBasePath, f);
     if (!fsSync.existsSync(fullPath)) continue;
     try {
       const parsed = loadConfigFile(fullPath);
-      if (parsed && parsed.key === datasourceKey) return f;
+      if (datasourceIdentifierMatchesFile(datasourceKey, f, parsed)) return f;
     } catch {
       // skip unreadable or invalid files
     }