@aifabrix/builder 2.45.0 → 2.45.5

package/lib/commands/credential-push.js

@@ -16,6 +16,7 @@ const { getDeploymentAuth, requireBearerForDataplanePipeline } = require('../uti
 const { resolveDataplaneUrl } = require('../utils/dataplane-resolver');
 const { getIntegrationPath } = require('../utils/paths');
 const { pushCredentialSecrets } = require('../utils/credential-secrets-env');
+const { runWithCredentialPushSpinner } = require('../utils/credential-push-ui');
 const { generateControllerManifest } = require('../generator/external-controller-manifest');
 
 /**
@@ -84,11 +85,15 @@ async function runCredentialPush(systemKey) {
   const dataplaneUrl = await resolveDataplaneUrl(controllerUrl, environment, authConfig);
 
   const payload = await buildPayload(systemKey);
-  const pushResult = await pushCredentialSecrets(dataplaneUrl, authConfig, {
-    envFilePath,
-    appName: systemKey,
-    payload
-  });
+  const pushResult = await runWithCredentialPushSpinner(
+    () =>
+      pushCredentialSecrets(dataplaneUrl, authConfig, {
+        envFilePath,
+        appName: systemKey,
+        payload
+      }),
+    { systemKey }
+  );
 
   logPushResult(pushResult);
   return { pushed: pushResult.pushed || 0 };

package/lib/commands/datasource-load-export-cli.js

@@ -0,0 +1,178 @@
+/**
+ * @fileoverview CLI for `datasource load` and `datasource export` (plan 144)
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const logger = require('../utils/logger');
+const { formatBlockingError } = require('../utils/cli-test-layout-chalk');
+const { runDatasourceLoad } = require('../datasource/bulk-loader-service');
+const { runDatasourceExport } = require('../datasource/datasource-exporter-service');
+const { displayDatasourceLoadTTY } = require('../utils/datasource-load-display');
+const { displayDatasourceExportTTY } = require('../utils/datasource-export-display');
+
+function loadHelpAfter() {
+  return `
+Examples:
+  $ aifabrix datasource load hubspot-test-datasource-company
+  $ aifabrix datasource load hubspot-test-company --file ./fixtures/rows.ndjson -v
+  $ aifabrix datasource load hubspot-test-company --dry-run
+  $ af ds load hubspot-test-company --batch-size 50 --sync-type incremental
+
+Notes:
+  - Imports local JSON/NDJSON via dataplane bulk record sync (not direct DB).
+  - Default file: integration/.data/<systemKey>-data-<entity>.json or .ndjson
+  - External integration folders only (integration/<systemKey>/).
+`;
+}
+
+function exportHelpAfter() {
+  return `
+Examples:
+  $ aifabrix datasource export hubspot-test-datasource-company
+  $ aifabrix datasource export hubspot-test-company --format ndjson --limit 500
+  $ aifabrix datasource export hubspot-test-company --filter '{"status":{"eq":"active"}}' -v
+  $ af ds export hubspot-test-company --fields email,name --strict
+
+Notes:
+  - Exports governed records search results (ABAC applied), not a raw DB dump.
+  - Default output: integration/.data/<systemKey>-data-<entity>.json or .ndjson
+  - Maximum 10000 rows per run (dataplane search limit).
+`;
+}
+
+/**
+ * @param {Object} result
+ * @returns {number}
+ */
+function computeLoadExitCode(result) {
+  if (result.dryRun) {
+    return 0;
+  }
+  return (result.totals.failedCount || 0) > 0 ? 1 : 0;
+}
+
+/**
+ * @param {Object} result
+ * @param {Object} options
+ * @returns {number}
+ */
+function computeExportExitCode(result, options) {
+  if (options.strict === true && result.recordCount === 0) {
+    return 1;
+  }
+  return 0;
+}
+
+async function datasourceLoadAction(datasourceKey, options) {
+  try {
+    const result = await runDatasourceLoad(datasourceKey, {
+      app: options.app,
+      env: options.env,
+      file: options.file,
+      verbose: options.verbose,
+      dryRun: options.dryRun,
+      batchSize: options.batchSize,
+      syncType: options.syncType,
+      format: options.format
+    });
+
+    if (options.json) {
+      const { context: _ctx, ...payload } = result;
+      logger.log(JSON.stringify(payload, null, 2));
+    } else {
+      displayDatasourceLoadTTY(result, { environment: options.env, verbose: options.verbose });
+    }
+
+    process.exit(computeLoadExitCode(result));
+  } catch (err) {
+    logger.error(formatBlockingError(`Load failed: ${err.message}`));
+    const code = err.message && /auth|login|token|version|unreachable/i.test(err.message) ? 3 : 1;
+    process.exit(code);
+  }
+}
+
+async function datasourceExportAction(datasourceKey, options) {
+  try {
+    const result = await runDatasourceExport(datasourceKey, {
+      app: options.app,
+      env: options.env,
+      file: options.file,
+      verbose: options.verbose,
+      format: options.format,
+      filter: options.filter,
+      fields: options.fields,
+      limit: options.limit,
+      intent: options.intent,
+      strict: options.strict
+    });
+
+    if (options.json) {
+      const { context: _ctx, ...payload } = result;
+      logger.log(JSON.stringify(payload, null, 2));
+    } else {
+      displayDatasourceExportTTY(result, { environment: options.env, verbose: options.verbose });
+    }
+
+    process.exit(computeExportExitCode(result, options));
+  } catch (err) {
+    logger.error(formatBlockingError(`Export failed: ${err.message}`));
+    const code = err.message && /auth|login|token|version|unreachable/i.test(err.message) ? 3 : 1;
+    process.exit(code);
+  }
+}
+
+/**
+ * @param {import('commander').Command} datasource
+ */
+function setupDatasourceLoadExportCommands(datasource) {
+  datasource
+    .command('load <datasourceKey>')
+    .description('Import local JSON/NDJSON records via dataplane bulk sync')
+    .option('-a, --app <app>', 'Integration folder name')
+    .option('-e, --env <env>', 'Environment: dev, tst, or pro')
+    .option('--file <path>', 'Input file (default: integration/.data/...)')
+    .option('--format <json|ndjson>', 'File format when extension is ambiguous')
+    .option('--batch-size <n>', 'Records per bulk request', '100')
+    .option(
+      '--sync-type <incremental|bulk|validate>',
+      'Dataplane syncType',
+      'incremental'
+    )
+    .option('--dry-run', 'Parse and validate only; no upload')
+    .option('-v, --verbose', 'Per-batch progress')
+    .option('--json', 'Machine-readable result JSON')
+    .addHelpText('after', loadHelpAfter())
+    .action(datasourceLoadAction);
+
+  datasource
+    .command('export <datasourceKey>')
+    .description('Export governed records to local JSON/NDJSON via records search')
+    .option('-a, --app <app>', 'Integration folder name')
+    .option('-e, --env <env>', 'Environment: dev, tst, or pro')
+    .option('--file <path>', 'Output file (default: integration/.data/...)')
+    .option('--format <json|ndjson>', 'Output format')
+    .option('--filter <json>', 'Records search filters (JSON object)')
+    .option('--fields <csv>', 'Project metadata keys in export rows')
+    .option('--limit <n>', 'Max records (1–10000)', '1000')
+    .option(
+      '--intent <retrieval|grounding|analytics|validation>',
+      'Search intent',
+      'validation'
+    )
+    .option('--strict', 'Exit 1 when zero rows are exported')
+    .option('-v, --verbose', 'Search stats and file size')
+    .option('--json', 'Machine-readable result JSON')
+    .addHelpText('after', exportHelpAfter())
+    .action(datasourceExportAction);
+}
+
+module.exports = {
+  setupDatasourceLoadExportCommands,
+  datasourceLoadAction,
+  datasourceExportAction,
+  computeLoadExitCode,
+  computeExportExitCode
+};

package/lib/commands/datasource-test-trust-cli.js

@@ -0,0 +1,114 @@
+/**
+ * @fileoverview CLI wiring for `datasource test-trust` (404.5 / plan 143).
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const logger = require('../utils/logger');
+const { formatBlockingError } = require('../utils/cli-test-layout-chalk');
+const { runDatasourceAgentTrust } = require('../datasource/agent-trust-run');
+const { writeTrustDebugLogAndPrint } = require('../datasource/agent-trust-debug-log');
+const { displayAgentTrustRunTTY } = require('../utils/agent-trust-run-display');
+const { computeExitCodeFromTrustRun } = require('../utils/agent-trust-run-exit');
+const { attachDatasourceTestCommonOptions } = require('./datasource-unified-test-cli.options');
+const { cliOptsSkipSync } = require('../utils/cli-sync-options');
+
+function datasourceTestTrustHelpAfter() {
+  return `
+Examples:
+  $ aifabrix datasource test-trust hubspot-companies
+  $ aifabrix datasource test-trust hubspot-companies --app test-e2e-hubspot -v
+  $ aifabrix datasource test-trust hubspot-companies --revalidate --strict
+  $ aifabrix test-trust test-e2e-hubspot
+
+Notes:
+  - Semantic trust only (agent metadata validation). For live API proof use test-e2e.
+  - Local files are published before run unless --no-sync.
+  - With --debug, JSON is saved under integration/<app>/logs/ (test-trust-<key>-*.json). View with: aifabrix datasource log-trust <key>
+`;
+}
+
+async function runDatasourceTestTrustOnce(datasourceKey, options) {
+  const { trustRun, apiError } = await runDatasourceAgentTrust(datasourceKey, {
+    app: options.app,
+    environment: options.env,
+    noSync: options.noSync === true,
+    revalidate: options.revalidate === true,
+    summary: options.summary === true,
+    timeout: options.timeout
+  });
+  return { trustRun, apiError };
+}
+
+async function finalizeDatasourceTestTrust(datasourceKey, options, result) {
+  const displayOpts = {
+    json: options.json,
+    verbose: options.verbose,
+    environment: options.env,
+    noSync: options.noSync === true,
+    strict: options.strict === true
+  };
+  if (options.json) {
+    logger.log(JSON.stringify(result.trustRun, null, 2));
+  } else if (options.summary) {
+    const tr = result.trustRun;
+    logger.log(
+      `${tr.datasourceKey}: ${tr.trustDecision} (${tr.validationStatus}, ${Math.round((tr.confidence || 0) * 100)}%)`
+    );
+  } else {
+    displayAgentTrustRunTTY(result.trustRun, displayOpts);
+  }
+  const exitOpts = {
+    warningsAsErrors: options.warningsAsErrors === true,
+    strict: options.strict === true
+  };
+  return computeExitCodeFromTrustRun(result.trustRun, exitOpts);
+}
+
+async function datasourceTestTrustAction(datasourceKey, options, cmd) {
+  try {
+    const merged = { ...options, noSync: cliOptsSkipSync(options, cmd) };
+    const result = await runDatasourceTestTrustOnce(datasourceKey, merged);
+    const { resolveAppKeyForDatasource } = require('../datasource/resolve-app');
+    const { appKey } = await resolveAppKeyForDatasource(datasourceKey, options.app);
+    if (options.debug) {
+      await writeTrustDebugLogAndPrint(appKey, datasourceKey, {
+        request: { datasourceKey, revalidate: options.revalidate === true },
+        response: result.trustRun,
+        error: result.apiError ? result.apiError.message : null
+      });
+    }
+    const exitCode = await finalizeDatasourceTestTrust(datasourceKey, merged, result);
+    process.exit(exitCode);
+  } catch (err) {
+    logger.error(formatBlockingError('Agent trust validation failed:'), err.message);
+    process.exit(3);
+  }
+}
+
+function setupDatasourceTestTrustCommand(datasource) {
+  const cmd = datasource
+    .command('test-trust <datasourceKey>')
+    .description('Semantic agent metadata validation for one datasource (dataplane 404.5)');
+  attachDatasourceTestCommonOptions(cmd, {
+    includeNoAsync: false,
+    includePayload: false,
+    timeoutHelp: 'HTTP timeout for validate request (ms)',
+    timeoutDefault: '120000'
+  });
+  cmd
+    .option('--no-sync', 'Skip publishing local integration files before validate')
+    .option('--revalidate', 'Force new validation (ignore inputHash cache)')
+    .option('--strict', 'Exit 1 unless trustDecision is trusted')
+    .addHelpText('after', datasourceTestTrustHelpAfter())
+    .action(datasourceTestTrustAction);
+}
+
+module.exports = {
+  setupDatasourceTestTrustCommand,
+  runDatasourceTestTrustOnce,
+  finalizeDatasourceTestTrust,
+  datasourceTestTrustAction
+};

package/lib/commands/datasource-unified-test-cli.js

@@ -28,6 +28,7 @@ const {
   finalizeDatasourceTestE2ELegacyPath,
   displayDatasourceTestE2EEnvelopeResults
 } = require('./datasource-unified-test-e2e-cli-helpers');
+const { maybeVerifyAuditAfterE2E } = require('../datasource/audit-evidence-run');
 const {
   datasourceTestHelpAfter,
   datasourceTestIntegrationHelpAfter,
@@ -333,7 +334,7 @@ async function runDatasourceTestE2ECliOnce(datasourceKey, options) {
     minProcessed: options.minProcessed,
     minRecordCount: options.minRecordCount,
     timeout: options.timeout,
-    capabilityKey: options.capability,
+    capabilityKey: options.capabilityKey,
     sync: options.sync === true
   });
   const unifiedModes =
@@ -352,7 +353,7 @@ async function runDatasourceTestE2ECliOnce(datasourceKey, options) {
         warningsAsErrors: options.warningsAsErrors,
         requireCert: options.requireCert,
         debug: options.debug,
-        requestedCapabilityKey: options.capability,
+        requestedCapabilityKey: options.capabilityKey,
         strictCapabilityScope: options.strictCapabilityScope === true
       }
     );
@@ -370,24 +371,24 @@ async function runDatasourceTestE2ECliOnce(datasourceKey, options) {
 
 async function runDatasourceTestE2ECliAction(datasourceKey, options) {
   const { exitCode, envelope } = await runDatasourceTestE2ECliOnce(datasourceKey, options);
-  await afterUnifiedValidationCertSync(exitCode, datasourceKey, options, 'datasource test-e2e', envelope);
-  process.exit(exitCode);
+  const finalExit = await maybeVerifyAuditAfterE2E(exitCode, datasourceKey, envelope, options);
+  await afterUnifiedValidationCertSync(
+    finalExit,
+    datasourceKey,
+    options,
+    'datasource test-e2e',
+    envelope
+  );
+  process.exit(finalExit);
 }
 
 async function e2eTestCommandAction(datasourceKey, capabilityKey, options) {
   const optObj = options && typeof options === 'object' ? options : {};
   const capFromArg = typeof capabilityKey === 'string' ? capabilityKey.trim() : '';
-  const capFromFlag = optObj.capability !== undefined && optObj.capability !== null
-    ? String(optObj.capability).trim()
-    : '';
-  const requestedCapability = capFromArg || capFromFlag;
-  if (capFromArg && capFromFlag && capFromArg !== capFromFlag) {
-    logger.warn(
-      chalk.yellow('⚠ Capability mismatch:'),
-      `using positional '${capFromArg}' instead of --capability '${capFromFlag}'.`
-    );
-  }
-  const mergedOptions = { ...optObj, capability: requestedCapability || undefined };
+  const mergedOptions = {
+    ...optObj,
+    capabilityKey: capFromArg || undefined
+  };
   try {
     if (mergedOptions.watch) {
       const { appKey } = await resolveAppKeyForDatasource(datasourceKey, mergedOptions.app);
@@ -400,14 +401,20 @@ async function e2eTestCommandAction(datasourceKey, capabilityKey, options) {
         runOnce: async() => {
           try {
             const r = await runDatasourceTestE2ECliOnce(datasourceKey, mergedOptions);
-            await afterUnifiedValidationCertSync(
+            const finalExit = await maybeVerifyAuditAfterE2E(
               r.exitCode,
               datasourceKey,
+              r.envelope,
+              mergedOptions
+            );
+            await afterUnifiedValidationCertSync(
+              finalExit,
+              datasourceKey,
               mergedOptions,
               'datasource test-e2e (watch)',
               r.envelope
             );
-            return r;
+            return { exitCode: finalExit, envelope: r.envelope };
           } catch (err) {
             logger.error(formatBlockingError('E2E test failed:'), err.message);
             return { exitCode: 3, envelope: null };
@@ -459,5 +466,6 @@ module.exports = {
   runIntegrationOnceForWatch,
   integrationTestCommandAction,
   runDatasourceTestE2ECliOnce,
+  maybeVerifyAuditAfterE2E,
   e2eTestCommandAction
 };

package/lib/commands/datasource-unified-test-cli.options.js

@@ -44,6 +44,7 @@ Examples:
   $ aifabrix datasource test-e2e hubspot-users --app test-e2e-hubspot -v --debug
   $ aifabrix datasource test-e2e my-documents-ds --min-vector-hits 7 -v --debug
   $ aifabrix datasource test-e2e hubspot-users -a test-e2e-hubspot --no-async
+  $ aifabrix datasource test-e2e hubspot-users --sync --verify-audit -v
   $ aifabrix datasource test-e2e hubspot-users read
 
 Notes:
@@ -53,6 +54,7 @@ Notes:
       aifabrix datasource test-integration <datasourceKey>
   - For system-level E2E rollup across datasources, use:
       aifabrix test-e2e <systemKey>
+  - To prove governance audit evidence after a green E2E, add --verify-audit (see verify-audit command).
 `;
 }
 
@@ -175,11 +177,24 @@ function attachDatasourceTestE2eExclusiveOptions(cmd) {
       '--primary-key-value <value|@path>',
       'Primary key value or path to JSON file (e.g. @pk.json) for e2eOptions.primaryKeyValue'
     )
-    .option('--capability <key>', 'Capability drill-down (deprecated; use positional [capabilityKey])')
     .option(
       '--strict-capability-scope',
-      'Exit 1 if a capability drill-down is requested but the report lists more than one capabilities[] row (plan §2.3)'
-    );
+      'Exit 1 when positional [capabilityKey] is set but the report lists more than one capabilities[] row'
+    )
+    .option(
+      '--verify-audit',
+      'After successful E2E, verify the nine-row audit evidence matrix on the dataplane (requires audit:read)'
+    )
+    .option(
+      '--no-verify-audit',
+      'Skip audit evidence verification even if enabled by default in a future release'
+    )
+    .option(
+      '--audit-poll-ms <ms>',
+      'Max wait for audit rows after E2E when using --verify-audit (default 15000)',
+      '15000'
+    )
+    .option('--audit-poll-interval-ms <ms>', 'Audit poll interval when using --verify-audit (default 2000)', '2000');
 }
 
 module.exports = {

package/lib/commands/datasource-unified-test-e2e-cli-helpers.js

@@ -46,7 +46,7 @@ function exitCodeFromDatasourceTestRunEnvelope(env, options) {
     warningsAsErrors: options.warningsAsErrors === true,
     requireCert: options.requireCert === true
   });
-  const scope = analyzeCapabilityScope(env, options.capability);
+  const scope = analyzeCapabilityScope(env, options.capabilityKey);
   if (options.strictCapabilityScope === true && scope.violated) {
     code = Math.max(code, 1);
   }
@@ -62,10 +62,10 @@ function exitCodeFromDatasourceTestRunEnvelope(env, options) {
 function finalizeDatasourceTestE2ELegacyPath(data, options) {
   displayE2EResults(data, options.verbose);
   logDatasourceTestRunDebugAppendix(data.datasourceTestRun, options.debug);
-  logE2eCapabilityFocusFromEnvelope(data.datasourceTestRun, options.capability);
+  logE2eCapabilityFocusFromEnvelope(data.datasourceTestRun, options.capabilityKey);
   const env = data.datasourceTestRun;
   if (env) {
-    emitCapabilityScopeDiagnostics(env, { requestedCapabilityKey: options.capability });
+    emitCapabilityScopeDiagnostics(env, { requestedCapabilityKey: options.capabilityKey });
     const code = exitCodeFromDatasourceTestRunEnvelope(env, options);
     return code === null ? 1 : code;
   }
@@ -91,10 +91,10 @@ function displayDatasourceTestE2EEnvelopeResults(datasourceKey, env, options) {
     {
       debug: options.debug,
       runType: 'e2e',
-      requestedCapabilityKey: options.capability
+      requestedCapabilityKey: options.capabilityKey
     }
   );
-  logE2eCapabilityFocusFromEnvelope(env, options.capability);
+  logE2eCapabilityFocusFromEnvelope(env, options.capabilityKey);
 }
 
 module.exports = {

package/lib/commands/datasource-verify-audit-cli.js

@@ -0,0 +1,96 @@
+/**
+ * @fileoverview `aifabrix datasource verify-audit` — re-run 407.1 matrix from last E2E log
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const logger = require('../utils/logger');
+const { formatBlockingError } = require('../utils/cli-test-layout-chalk');
+const { runAuditEvidenceVerification } = require('../datasource/audit-evidence-run');
+const { loadEnvelopeFromLatestE2eLog } = require('../datasource/audit-evidence-extract');
+
+const VERIFY_AUDIT_HELP_AFTER = `
+Examples:
+  $ aifabrix datasource verify-audit test-e2e-hubspot-companies
+  $ aifabrix datasource verify-audit test-e2e-hubspot-companies --app test-e2e-hubspot -v
+  $ aifabrix datasource verify-audit test-e2e-hubspot-companies --correlation-id <testRunId>
+  $ aifabrix datasource verify-audit test-e2e-hubspot-companies --json
+
+Notes:
+  - Re-checks audit evidence from the latest test-e2e debug log (or --file) without re-running E2E.
+  - Requires audit:read on the dataplane (same login as test-e2e).
+  - Pair with: aifabrix datasource test-e2e <datasourceKey> --verify-audit
+`;
+
+/**
+ * @param {string} datasourceKey
+ * @param {Object} options
+ */
+async function verifyAuditCommandAction(datasourceKey, options) {
+  const envelope = await loadEnvelopeFromLatestE2eLog(datasourceKey, options);
+  if (!envelope) {
+    logger.error(
+      formatBlockingError('verify-audit failed:'),
+      'No test-e2e debug log found (run test-e2e with --debug first)'
+    );
+    process.exit(3);
+    return;
+  }
+  const pollMs = parseInt(String(options.auditPollMs || '15000'), 10);
+  const pollInterval = parseInt(String(options.auditPollIntervalMs || '2000'), 10);
+  const { exitCode } = await runAuditEvidenceVerification(datasourceKey, envelope, {
+    app: options.app,
+    env: options.env,
+    verbose: options.verbose,
+    json: options.json,
+    correlationId: options.correlationId,
+    executionId: options.executionId,
+    quiet: false,
+    auditPollMaxWaitMs: Number.isFinite(pollMs) ? pollMs : 15000,
+    auditPollIntervalMs: Number.isFinite(pollInterval) ? pollInterval : 2000
+  });
+  process.exit(exitCode);
+}
+
+/**
+ * @param {import('commander').Command} datasource
+ */
+function setupDatasourceVerifyAuditCommand(datasource) {
+  datasource
+    .command('verify-audit <datasourceKey>')
+    .description(
+      'Verify audit evidence matrix from latest E2E log (or --file) without re-running E2E'
+    )
+    .option(
+      '-a, --app <app>',
+      'Integration folder name (optional: resolve from cwd or datasource key if single match)'
+    )
+    .option('-e, --env <env>', 'Environment: dev, tst, or pro')
+    .option('-v, --verbose', 'Show matrix row codes and execution id count')
+    .option('--json', 'Print auditEvidenceVerification JSON block')
+    .option('-f, --file <path>', 'Path to test-e2e debug log JSON')
+    .option('--correlation-id <id>', 'Override correlation/testRunId filter')
+    .option('--execution-id <id>', 'Include execution id for sub-resource checks')
+    .option(
+      '--audit-poll-ms <ms>',
+      'Max wait for audit rows to appear (default 15000)',
+      '15000'
+    )
+    .option('--audit-poll-interval-ms <ms>', 'Poll interval (default 2000)', '2000')
+    .addHelpText('after', VERIFY_AUDIT_HELP_AFTER)
+    .action(async(datasourceKey, options) => {
+      try {
+        await verifyAuditCommandAction(datasourceKey, options);
+      } catch (error) {
+        logger.error(formatBlockingError('verify-audit failed:'), error.message);
+        process.exit(3);
+      }
+    });
+}
+
+module.exports = {
+  setupDatasourceVerifyAuditCommand,
+  verifyAuditCommandAction
+};