@aifabrix/builder 2.45.0 → 2.46.0

package/lib/datasource/audit-evidence-verify-matrix.js

@@ -0,0 +1,205 @@
+/**
+ * @fileoverview Build matrix rows 1–9 for audit evidence verification
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const {
+  queryExecutions,
+  queryRbacDecisions,
+  queryAbacTraces,
+  getExecutionRbac,
+  getExecutionAbac,
+  getExecutionErrors,
+  getExecutionSteps,
+  getExecutionTrace
+} = require('../api/audit.api');
+const {
+  checkRow1ExecutionsByDatasource,
+  checkRow2RbacDecisionsList,
+  checkRow3AbacTracesList
+} = require('./audit-evidence-matrix-rows');
+const {
+  operationQueryVariants,
+  executionsCoverExpectedOperations
+} = require('./audit-evidence-operation-aliases');
+const {
+  checkRow8CorrelationGroup,
+  resolveRow8Executions,
+  checkRow9Traces,
+  isRbacSubResourceValid,
+  isAbacSubResourceValid,
+  isErrorsValid,
+  isStepsValid,
+  resolveExecutionIdsForChecks,
+  checkPerExecutionSub
+} = require('./audit-evidence-verify-helpers');
+
+/**
+ * @param {string} dataplaneUrl
+ * @param {Object} authConfig
+ * @param {string} correlationId
+ * @param {string[]} expectedOperations
+ * @returns {Promise<Array>}
+ */
+async function fetchCorrelationExecutionsByOperation(
+  dataplaneUrl,
+  authConfig,
+  correlationId,
+  expectedOperations
+) {
+  const deduped = new Map();
+  for (const rawOp of expectedOperations) {
+    const variants = operationQueryVariants(rawOp);
+    for (const op of variants) {
+      const res = await queryExecutions(dataplaneUrl, authConfig, {
+        correlationId,
+        operation: op,
+        pageSize: 20
+      });
+      for (const row of res.data || []) {
+        if (row && row.id) deduped.set(String(row.id), row);
+      }
+    }
+  }
+  const merged = [...deduped.values()];
+  return executionsCoverExpectedOperations(merged, expectedOperations) ? merged : [];
+}
+
+/**
+ * @param {Object} ctx
+ * @param {Object} polled
+ * @param {number} minExpected
+ * @returns {Promise<import('../api/types/audit.types').AuditEvidenceMatrixRow[]>}
+ */
+async function buildListMatrixRows(ctx, polled, minExpected) {
+  const executions = polled.data || [];
+  const { dataplaneUrl, authConfig } = ctx;
+  const totalItems =
+    polled.meta && polled.meta.totalItems !== undefined
+      ? polled.meta.totalItems
+      : executions.length;
+  const rows = [
+    checkRow1ExecutionsByDatasource({
+      executions,
+      totalItems,
+      minExpected
+    })
+  ];
+  const rbacList = await queryRbacDecisions(dataplaneUrl, authConfig, {
+    targetDatasourceKey: ctx.datasourceKey,
+    pageSize: 100
+  });
+  rows.push(
+    checkRow2RbacDecisionsList({
+      rbacItems: rbacList.data,
+      totalItems: rbacList.meta && rbacList.meta.totalItems
+    })
+  );
+  const abacList = await queryAbacTraces(dataplaneUrl, authConfig, {
+    datasourceKey: ctx.datasourceKey,
+    pageSize: 100
+  });
+  rows.push(
+    checkRow3AbacTracesList({
+      abacItems: abacList.data,
+      totalItems: abacList.meta && abacList.meta.totalItems
+    })
+  );
+  return rows;
+}
+
+/**
+ * @param {Object} ctx
+ * @param {string[]} execIds
+ * @returns {Promise<import('../api/types/audit.types').AuditEvidenceMatrixRow[]>}
+ */
+async function buildPerExecutionSubRows(ctx, execIds) {
+  const { dataplaneUrl, authConfig } = ctx;
+  const base = { dataplaneUrl, authConfig, executionId: execIds[0] };
+  return Promise.all([
+    checkPerExecutionSub({ ...base, rowNum: 4, label: 'rbac', fetcher: getExecutionRbac, isValid: isRbacSubResourceValid }),
+    checkPerExecutionSub({ ...base, rowNum: 5, label: 'abac', fetcher: getExecutionAbac, isValid: isAbacSubResourceValid }),
+    checkPerExecutionSub({ ...base, rowNum: 6, label: 'errors', fetcher: getExecutionErrors, isValid: isErrorsValid }),
+    checkPerExecutionSub({ ...base, rowNum: 7, label: 'steps', fetcher: getExecutionSteps, isValid: isStepsValid })
+  ]);
+}
+
+/**
+ * @param {Object} ctx
+ * @param {Array} executions
+ * @returns {Promise<Array>}
+ */
+async function resolveRow8PoolForMatrix(ctx, executions) {
+  const { dataplaneUrl, authConfig } = ctx;
+  let corrQueryRows = [];
+  if (ctx.correlationId) {
+    const corr = await queryExecutions(dataplaneUrl, authConfig, {
+      correlationId: ctx.correlationId,
+      pageSize: 100
+    });
+    corrQueryRows = corr.data || [];
+  }
+  const row8Pool = corrQueryRows.length ? corrQueryRows : executions;
+  const expected = ctx.expectedOperations || [];
+  if (!expected.length || !ctx.correlationId) {
+    return row8Pool;
+  }
+  const byOp = await fetchCorrelationExecutionsByOperation(
+    dataplaneUrl,
+    authConfig,
+    ctx.correlationId,
+    expected
+  );
+  return byOp.length ? byOp : row8Pool;
+}
+
+/**
+ * @param {Object} ctx
+ * @param {string[]} execIds
+ * @returns {Promise<Set<string>>}
+ */
+async function collectTraceIdsForRow9(ctx, execIds) {
+  const { dataplaneUrl, authConfig } = ctx;
+  const traceIds = new Set();
+  const sample = execIds.slice(0, Math.min(execIds.length, 12));
+  for (const id of sample) {
+    try {
+      const tr = await getExecutionTrace(dataplaneUrl, authConfig, id);
+      if (tr && tr.executionId) traceIds.add(String(tr.executionId));
+      else if (tr && tr.id) traceIds.add(String(tr.id));
+    } catch {
+      /* row 9 may fail */
+    }
+  }
+  return traceIds;
+}
+
+/**
+ * @param {Object} ctx
+ * @param {Array} executions
+ * @param {string[]} executionIds
+ * @returns {Promise<{ rows: import('../api/types/audit.types').AuditEvidenceMatrixRow[], execIds: string[] }>}
+ */
+async function buildPerExecutionAndTailRows(ctx, executions, executionIds) {
+  const execIds = resolveExecutionIdsForChecks(executions, executionIds);
+  const rows = await buildPerExecutionSubRows(ctx, execIds);
+  const row8Pool = await resolveRow8PoolForMatrix(ctx, executions);
+  const row8Executions = resolveRow8Executions(ctx, executions, row8Pool);
+  rows.push(
+    checkRow8CorrelationGroup({
+      correlationId: ctx.correlationId,
+      executions: row8Executions,
+      expectedOperations: ctx.expectedOperations
+    })
+  );
+  rows.push(checkRow9Traces(await collectTraceIdsForRow9(ctx, execIds), execIds));
+  return { rows, execIds };
+}
+
+module.exports = {
+  buildListMatrixRows,
+  buildPerExecutionAndTailRows
+};

package/lib/datasource/audit-evidence-verify.js

@@ -0,0 +1,108 @@
+/**
+ * @fileoverview Run Plan 407.1 nine-row audit evidence matrix against dataplane APIs
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const { queryExecutions } = require('../api/audit.api');
+const { pollExecutionsUntilReady } = require('./audit-evidence-poll');
+const {
+  buildListMatrixRows,
+  buildPerExecutionAndTailRows
+} = require('./audit-evidence-verify-matrix');
+const { checkRow8CorrelationGroup, checkRow9Traces } = require('./audit-evidence-verify-helpers');
+
+/**
+ * @param {Object} ctx
+ * @returns {number}
+ */
+/**
+ * E2E runs tag executions with testRunId as correlationId; prefer that for list poll.
+ * @param {Object} ctx
+ * @returns {Object}
+ */
+function buildExecutionsPollQuery(ctx) {
+  const base = { pageSize: 100, sort: '-startedAt' };
+  if (ctx.correlationId) {
+    return { ...base, correlationId: ctx.correlationId };
+  }
+  return { ...base, datasourceKey: ctx.datasourceKey };
+}
+
+function resolveMinExpected(ctx) {
+  const fromOps = ctx.expectedOperations && ctx.expectedOperations.length;
+  const fromIds = ctx.executionIds && ctx.executionIds.length;
+  if (fromIds && fromOps) {
+    return Math.max(1, Math.min(fromOps, fromIds));
+  }
+  return Math.max(1, fromOps || fromIds || 1);
+}
+
+/**
+ * @param {string} datasourceKey
+ * @param {Object} ctx
+ * @returns {import('../api/types/audit.types').AuditEvidenceVerification}
+ */
+function failedMissingIds(datasourceKey, ctx) {
+  return {
+    status: 'failed',
+    datasourceKey,
+    correlationId: ctx.correlationId || null,
+    executionIds: [],
+    matrix: [
+      {
+        row: 0,
+        status: 'failed',
+        detail: 'missing audit.executionIds in envelope',
+        code: 'auditIdsMissing'
+      }
+    ]
+  };
+}
+
+/**
+ * @param {Object} ctx
+ * @returns {Promise<import('../api/types/audit.types').AuditEvidenceVerification>}
+ */
+async function verifyAuditEvidenceMatrix(ctx) {
+  const datasourceKey = ctx.datasourceKey || '';
+  const executionIds = Array.isArray(ctx.executionIds) ? ctx.executionIds : [];
+  if (!executionIds.length && !ctx.correlationId) {
+    return failedMissingIds(datasourceKey, ctx);
+  }
+
+  const { dataplaneUrl, authConfig } = ctx;
+  const minExpected = resolveMinExpected(ctx);
+  const execQuery = buildExecutionsPollQuery(ctx);
+  const polled = await pollExecutionsUntilReady({
+    fetchExecutions: () => queryExecutions(dataplaneUrl, authConfig, execQuery),
+    minExpected,
+    executionIds,
+    maxWaitMs: ctx.maxWaitMs,
+    intervalMs: ctx.intervalMs
+  });
+
+  const executions = polled.data || [];
+  const matrix = await buildListMatrixRows(ctx, polled, minExpected);
+  const tail = await buildPerExecutionAndTailRows(ctx, executions, executionIds);
+  matrix.push(...tail.rows);
+
+  const passed = matrix.every(r => r.status === 'passed');
+  return {
+    status: passed ? 'passed' : 'failed',
+    datasourceKey,
+    correlationId: ctx.correlationId || null,
+    executionIds: tail.execIds,
+    matrix
+  };
+}
+
+module.exports = {
+  verifyAuditEvidenceMatrix,
+  buildExecutionsPollQuery,
+  resolveMinExpected,
+  checkRow8CorrelationGroup,
+  checkRow9Traces
+};

package/lib/datasource/bulk-loader-run-batches.js

@@ -0,0 +1,100 @@
+/**
+ * @fileoverview Batch upload loop for datasource load (plan 144)
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const { bulkSyncRecords } = require('../api/records-bulk.api');
+const { withTransientRetry, errorStatusCode } = require('./bulk-loader-service-retry');
+
+/**
+ * @param {Object} state
+ * @param {number} index
+ * @param {Error} err
+ * @param {Object[]} batchRecords
+ */
+function recordBatchFailure(state, index, err, batchRecords) {
+  state.batches.push({
+    index: index + 1,
+    total: state.chunkCount,
+    error: err.message || String(err),
+    statusCode: errorStatusCode(err),
+    recordCount: batchRecords.length,
+    failed: batchRecords.map(r => ({ key: r.key, message: err.message || String(err) }))
+  });
+  state.totals.failedCount += batchRecords.length;
+}
+
+/**
+ * @param {Object} state
+ * @param {number} index
+ * @param {Object} response
+ * @param {Object[]} batchRecords
+ */
+function recordBatchSuccess(state, index, response, batchRecords) {
+  if (!response || typeof response !== 'object') {
+    throw new Error('Bulk sync returned empty response');
+  }
+  if (response.success === false) {
+    throw new Error(
+      response.formattedError ||
+        response.error ||
+        `Bulk sync failed (HTTP ${response.status || 0})`
+    );
+  }
+  const payload =
+    response.data !== undefined && response.data !== null ? response.data : response;
+  const inserted = payload.insertedCount || 0;
+  const updated = payload.updatedCount || 0;
+  const processed = payload.totalProcessed || batchRecords.length;
+  const failedInBatch = Math.max(0, batchRecords.length - processed);
+
+  state.batches.push({
+    index: index + 1,
+    total: state.chunkCount,
+    insertedCount: inserted,
+    updatedCount: updated,
+    totalProcessed: processed,
+    failed: failedInBatch > 0 ? [{ message: 'Some records were not processed' }] : []
+  });
+  state.totals.insertedCount += inserted;
+  state.totals.updatedCount += updated;
+  state.totals.failedCount += failedInBatch;
+}
+
+/**
+ * @async
+ * @param {Object} params
+ * @returns {Promise<{ batches: Object[], totals: Object }>}
+ */
+async function uploadRecordBatches(params) {
+  const { chunks, syncType, dataplaneUrl, datasourceKey, authConfig } = params;
+  const state = {
+    batches: [],
+    totals: { insertedCount: 0, updatedCount: 0, failedCount: 0 },
+    chunkCount: chunks.length
+  };
+
+  for (let i = 0; i < chunks.length; i += 1) {
+    const batchRecords = chunks[i];
+    const body = { syncType, sync: false, records: batchRecords };
+    try {
+      const response = await withTransientRetry(() =>
+        bulkSyncRecords(dataplaneUrl, datasourceKey, authConfig, body)
+      );
+      recordBatchSuccess(state, i, response, batchRecords);
+    } catch (err) {
+      recordBatchFailure(state, i, err, batchRecords);
+    }
+  }
+
+  return { batches: state.batches, totals: state.totals };
+}
+
+module.exports = {
+  uploadRecordBatches,
+  recordBatchFailure,
+  recordBatchSuccess
+};

package/lib/datasource/bulk-loader-service-retry.js

@@ -0,0 +1,52 @@
+/**
+ * @fileoverview Retry helpers for bulk loader (plan 144)
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const MAX_RETRIES = 3;
+const RETRY_STATUS = new Set([429, 500, 502, 503, 504]);
+
+/**
+ * @param {Error} err
+ * @returns {number|null}
+ */
+function errorStatusCode(err) {
+  if (!err || typeof err !== 'object') return null;
+  if (typeof err.statusCode === 'number') return err.statusCode;
+  if (err.response && typeof err.response.status === 'number') return err.response.status;
+  return null;
+}
+
+function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+/**
+ * @async
+ * @param {Function} fn
+ * @returns {Promise<*>}
+ */
+async function withTransientRetry(fn) {
+  let lastErr;
+  for (let attempt = 0; attempt < MAX_RETRIES; attempt += 1) {
+    try {
+      return await fn();
+    } catch (err) {
+      lastErr = err;
+      const code = errorStatusCode(err);
+      if (!code || !RETRY_STATUS.has(code) || attempt >= MAX_RETRIES - 1) {
+        throw err;
+      }
+      await sleep(500 * (attempt + 1));
+    }
+  }
+  throw lastErr;
+}
+
+module.exports = {
+  withTransientRetry,
+  errorStatusCode
+};

package/lib/datasource/bulk-loader-service.js

@@ -0,0 +1,127 @@
+/**
+ * @fileoverview Load local fixtures via dataplane bulk record sync (plan 144)
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const { resolveDataFilePath } = require('./local-data-paths');
+const { parseRecordFile, estimatePayloadBytes } = require('./record-file-parser');
+const { normalizeRecordsForBulk } = require('./record-mapper');
+const { resolveLoadExportContext } = require('./datasource-load-export-context');
+const { uploadRecordBatches } = require('./bulk-loader-run-batches');
+const { runWithDatasourceLoadExportSpinner } = require('../utils/datasource-load-export-ui');
+
+const DEFAULT_BATCH_SIZE = 100;
+
+/**
+ * @param {Object[]} records
+ * @param {number} batchSize
+ * @returns {Object[][]}
+ */
+function chunkRecords(records, batchSize) {
+  const size = Math.max(1, batchSize);
+  const chunks = [];
+  for (let i = 0; i < records.length; i += size) {
+    chunks.push(records.slice(i, i + size));
+  }
+  return chunks;
+}
+
+/**
+ * @param {Object} params
+ * @returns {Object}
+ */
+function buildLoadResultBase(params) {
+  const { ctx, filePath, format, records, batchSize, syncType, dryRun } = params;
+  return {
+    datasourceKey: ctx.datasourceKey,
+    systemKey: ctx.systemKey,
+    appKey: ctx.appKey,
+    dryRun,
+    file: filePath,
+    format,
+    recordCount: records.length,
+    batchSize,
+    syncType,
+    estimatedPayloadBytes: estimatePayloadBytes(records),
+    batches: [],
+    totals: { insertedCount: 0, updatedCount: 0, failedCount: 0 }
+  };
+}
+
+/**
+ * @async
+ * @param {Object} params
+ * @returns {Promise<Object>}
+ */
+async function executeDatasourceLoadWork(params) {
+  const { ctx, options, batchSize, syncType, dryRun } = params;
+  const { filePath, format } = resolveDataFilePath({
+    systemKey: ctx.systemKey,
+    entitySuffix: ctx.entitySuffix,
+    file: options.file,
+    format: options.format
+  });
+
+  const raw = parseRecordFile(filePath, format);
+  const records = normalizeRecordsForBulk(raw, ctx.datasource);
+  const result = buildLoadResultBase({
+    ctx,
+    filePath,
+    format,
+    records,
+    batchSize,
+    syncType,
+    dryRun
+  });
+
+  if (dryRun) {
+    return { ...result, context: ctx };
+  }
+
+  const { batches, totals } = await uploadRecordBatches({
+    chunks: chunkRecords(records, batchSize),
+    syncType,
+    dataplaneUrl: ctx.dataplaneUrl,
+    datasourceKey: ctx.datasourceKey,
+    authConfig: ctx.authConfig
+  });
+
+  return { ...result, batches, totals, context: ctx };
+}
+
+/**
+ * @async
+ * @param {string} datasourceKey
+ * @param {Object} options
+ * @returns {Promise<Object>}
+ */
+async function runDatasourceLoad(datasourceKey, options = {}) {
+  const ctx = await resolveLoadExportContext(datasourceKey, options);
+  const batchSize = options.batchSize
+    ? parseInt(String(options.batchSize), 10)
+    : DEFAULT_BATCH_SIZE;
+  const syncType = options.syncType || 'incremental';
+  const dryRun = options.dryRun === true;
+  const startedAt = Date.now();
+
+  const payload = await runWithDatasourceLoadExportSpinner(
+    () => executeDatasourceLoadWork({ ctx, options, batchSize, syncType, dryRun }),
+    {
+      operation: dryRun ? 'load-dry-run' : 'load',
+      datasourceKey: ctx.datasourceKey,
+      json: options.json === true
+    }
+  );
+
+  return { ...payload, durationMs: Date.now() - startedAt };
+}
+
+module.exports = {
+  runDatasourceLoad,
+  chunkRecords,
+  buildLoadResultBase,
+  DEFAULT_BATCH_SIZE
+};

package/lib/datasource/capability/run-capability-dimension.js

@@ -43,7 +43,8 @@ async function maybeLoadDimensionCatalogKeys() {
       page: 1,
       pageSize: 500
     });
-    const items = res?.data?.items || res?.data?.data?.items || res?.items || [];
+    const raw = res?.data?.items ?? res?.data?.data ?? res?.items;
+    const items = Array.isArray(raw) ? raw : raw?.items || [];
     const keys = new Set(
       Array.isArray(items) ? items.map((d) => String(d?.key || '').trim()).filter(Boolean) : []
     );

package/lib/datasource/datasource-exporter-resolve.js

@@ -0,0 +1,82 @@
+/**
+ * @fileoverview Export path and search body helpers (plan 144)
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const path = require('path');
+const { resolveDefaultDataOutputPath, inferFormatFromPath } = require('./local-data-paths');
+
+/**
+ * @param {string|undefined} raw
+ * @returns {Object|undefined}
+ */
+function parseFilterJson(raw) {
+  if (!raw || typeof raw !== 'string' || !raw.trim()) {
+    return undefined;
+  }
+  try {
+    const parsed = JSON.parse(raw);
+    if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
+      return parsed;
+    }
+    throw new Error('filter must be a JSON object');
+  } catch (err) {
+    throw new Error(`Invalid --filter JSON: ${err.message}`);
+  }
+}
+
+/**
+ * @param {string|undefined} raw
+ * @returns {string[]|undefined}
+ */
+function parseFieldsCsv(raw) {
+  if (!raw || typeof raw !== 'string' || !raw.trim()) {
+    return undefined;
+  }
+  return raw
+    .split(',')
+    .map(s => s.trim())
+    .filter(Boolean);
+}
+
+/**
+ * @param {Object} opts
+ * @returns {{ outputPath: string, format: 'json'|'ndjson', limit: number, intent: string, filters: Object|undefined, fields: string[]|undefined }}
+ */
+function resolveExportOptions(opts) {
+  let limit = opts.limit ? parseInt(String(opts.limit), 10) : 1000;
+  if (Number.isNaN(limit) || limit < 1) {
+    limit = 1000;
+  }
+  limit = Math.min(limit, 10000);
+
+  const intent = opts.intent || 'validation';
+  const filters = parseFilterJson(opts.filter);
+  const fields = parseFieldsCsv(opts.fields);
+
+  let outputPath;
+  let format = opts.format;
+  if (opts.file && String(opts.file).trim()) {
+    outputPath = path.resolve(String(opts.file).trim());
+    format = format || inferFormatFromPath(outputPath);
+  } else {
+    const resolved = resolveDefaultDataOutputPath({
+      systemKey: opts.systemKey,
+      entitySuffix: opts.entitySuffix,
+      format: format || 'json'
+    });
+    outputPath = resolved.filePath;
+    format = opts.format || resolved.format;
+  }
+
+  return { outputPath, format, limit, intent, filters, fields };
+}
+
+module.exports = {
+  parseFilterJson,
+  parseFieldsCsv,
+  resolveExportOptions
+};