@aifabrix/builder 2.45.0 → 2.45.5

package/lib/external-system/test-system-level.js

@@ -16,7 +16,7 @@ const { getIntegrationPath } = require('../utils/paths');
 const { runUnifiedDatasourceValidation } = require('../datasource/unified-validation-run');
 
 async function runOneDatasourceKey(appName, datasourceKey, opts) {
-  const { environment, runType, debug, timeoutMs } = opts;
+  const { environment, runType, debug, timeoutMs, authConfig, dataplaneUrl } = opts;
   try {
     const uni = await runUnifiedDatasourceValidation(datasourceKey, {
       app: appName,
@@ -26,7 +26,9 @@ async function runOneDatasourceKey(appName, datasourceKey, opts) {
       verbose: false,
       timeout: timeoutMs,
       async: true,
-      noAsync: false
+      noAsync: false,
+      authConfig,
+      dataplaneUrl
     });
 
     if (uni.apiError) {
@@ -89,7 +91,17 @@ async function maybeWriteSystemRunLog(appName, systemKey, datasourceKeys, runTyp
  * @param {number} [params.timeout] - Request timeout
  * @returns {Promise<{success: boolean, datasourceResults: Object[]}>}
  */
-async function runSystemLevelTest({ appName, systemKey, datasourceKeys, environment, runType, debug, timeout }) {
+async function runSystemLevelTest({
+  appName,
+  systemKey,
+  datasourceKeys,
+  environment,
+  runType,
+  debug,
+  timeout,
+  authConfig,
+  dataplaneUrl
+}) {
   if (!Array.isArray(datasourceKeys) || datasourceKeys.length === 0) {
     return { success: true, datasourceResults: [] };
   }
@@ -100,7 +112,16 @@ async function runSystemLevelTest({ appName, systemKey, datasourceKeys, environm
     // Keep sequential to avoid spiking external APIs / dataplane concurrency in CLI runs.
     // (Plan allows parallel later, but UX needs stable logs first.)
     // eslint-disable-next-line no-await-in-loop -- intentional sequential execution
-    results.push(await runOneDatasourceKey(appName, key, { environment, runType, debug, timeoutMs }));
+    results.push(
+      await runOneDatasourceKey(appName, key, {
+        environment,
+        runType,
+        debug,
+        timeoutMs,
+        authConfig,
+        dataplaneUrl
+      })
+    );
   }
 
   const success = results.every(r => r.success);

package/lib/external-system/test.js

@@ -486,7 +486,9 @@ async function testExternalSystemIntegration(appName, options = {}) {
           environment: options.environment,
           runType: 'integration',
           debug: options.debug,
-          timeout
+          timeout,
+          authConfig,
+          dataplaneUrl
         });
         results.success = systemResult.success;
         results.datasourceResults = systemResult.datasourceResults;

package/lib/governance/governance-pack-loader.js

@@ -0,0 +1,70 @@
+/**
+ * @fileoverview Load governance scenario pack YAML from integration folder
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const yaml = require('js-yaml');
+const { getIntegrationPath } = require('../utils/paths');
+
+/**
+ * Resolve pack file path for test-governance.
+ * @param {string} systemKey
+ * @param {Object} options
+ * @param {string} [options.pack] - Explicit path (absolute or relative to integration root)
+ * @param {string} [options.app] - Integration folder override
+ * @returns {string}
+ */
+function resolveGovernancePackPath(systemKey, options = {}) {
+  const integrationPath = options.app
+    ? path.isAbsolute(options.app)
+      ? options.app
+      : path.resolve(process.cwd(), options.app)
+    : getIntegrationPath(systemKey);
+
+  if (options.pack) {
+    const packPath = path.isAbsolute(options.pack)
+      ? options.pack
+      : path.join(integrationPath, options.pack);
+    if (!fs.existsSync(packPath)) {
+      throw new Error(`Scenario pack not found: ${packPath}`);
+    }
+    return packPath;
+  }
+
+  const candidates = [
+    path.join(integrationPath, 'scenarios', 'default.yaml'),
+    path.join(integrationPath, 'scenarios', `${systemKey}-v1.yaml`)
+  ];
+  for (const candidate of candidates) {
+    if (fs.existsSync(candidate)) {
+      return candidate;
+    }
+  }
+  throw new Error(
+    `No scenario pack found under ${path.join(integrationPath, 'scenarios')}. ` +
+      'Use --pack <path> (e.g. scenarios/test-protection-v1.yaml).'
+  );
+}
+
+/**
+ * @param {string} packPath
+ * @returns {Object}
+ */
+function loadGovernancePackYaml(packPath) {
+  const raw = fs.readFileSync(packPath, 'utf8');
+  const doc = yaml.load(raw);
+  if (!doc || typeof doc !== 'object') {
+    throw new Error(`Invalid scenario pack YAML: ${packPath}`);
+  }
+  return doc;
+}
+
+module.exports = {
+  resolveGovernancePackPath,
+  loadGovernancePackYaml
+};

package/lib/governance/governance-run-display.js

@@ -0,0 +1,119 @@
+/**
+ * @fileoverview TTY display for governance scenario acceptance runs
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const chalk = require('chalk');
+const logger = require('../utils/logger');
+const {
+  sectionTitle,
+  headerKeyValue,
+  metadata: metaGray,
+  formatSuccessParagraph,
+  formatBlockingError
+} = require('../utils/cli-test-layout-chalk');
+
+const SEP = '────────────────────────────────';
+
+/**
+ * @param {'pass'|'fail'} status
+ * @returns {string}
+ */
+function scenarioGlyph(status) {
+  return status === 'pass' ? chalk.green('✔') : chalk.red('✖');
+}
+
+/**
+ * @param {import('../api/types/governance-scenarios.types').GovernanceScenarioResult} scenario
+ * @param {boolean} verbose
+ */
+function logScenarioVerdict(scenario, verbose) {
+  logger.log(`  ${scenarioGlyph(scenario.status)} ${scenario.verdict}`);
+  if (!verbose) return;
+  logger.log(
+    metaGray(
+      `      Subject: ${scenario.subjectDisplayName} (${scenario.subjectUserId})`
+    )
+  );
+  if (scenario.subjectGroups && scenario.subjectGroups.length > 0) {
+    logger.log(metaGray(`      Groups: ${scenario.subjectGroups.join(', ')}`));
+  }
+  logger.log(
+    metaGray(
+      `      Visible keys: ${scenario.visibleKeyCount} · excluded ABAC: ${scenario.excludedAbac}` +
+        (scenario.auditRef ? ` · audit: ${scenario.auditRef}` : '')
+    )
+  );
+  if (scenario.unexpectedVisibleKeys && scenario.unexpectedVisibleKeys.length > 0) {
+    logger.log(
+      metaGray(`      Unexpected: ${scenario.unexpectedVisibleKeys.join(', ')}`)
+    );
+  }
+  if (scenario.missingRequiredKeys && scenario.missingRequiredKeys.length > 0) {
+    logger.log(metaGray(`      Missing: ${scenario.missingRequiredKeys.join(', ')}`));
+  }
+  if (scenario.fixHint) {
+    logger.log(metaGray(`      Fix: ${scenario.fixHint}`));
+  }
+}
+
+/**
+ * @param {string} systemKey
+ * @param {import('../api/types/governance-scenarios.types').GovernanceScenariosRunResponse} result
+ * @param {{ verbose?: boolean, packPath?: string }} opts
+ */
+function displayGovernanceRunTTY(systemKey, result, opts = {}) {
+  const agg = result.summary.failed > 0 ? 'fail' : 'ok';
+  logger.log(SEP);
+  logger.log(sectionTitle('Governed search acceptance (key sets only — no record payload)'));
+  logger.log(headerKeyValue('System:', systemKey));
+  logger.log(headerKeyValue('Pack:', result.packKey));
+  if (opts.packPath) {
+    logger.log(metaGray(`  File: ${opts.packPath}`));
+  }
+  logger.log(
+    headerKeyValue(
+      'Summary:',
+      `${result.summary.passed}/${result.summary.total} passed`
+    )
+  );
+  logger.log(metaGray('  Does not call vendor APIs (use test-e2e for connectivity).'));
+  logger.log('');
+  for (const scenario of result.scenarios || []) {
+    logScenarioVerdict(scenario, opts.verbose === true);
+  }
+  logger.log('');
+  if (agg === 'ok') {
+    logger.log(formatSuccessParagraph('All governance scenarios passed'));
+  } else {
+    logger.log(formatBlockingError(`${result.summary.failed} scenario(s) failed`));
+  }
+  logger.log(SEP);
+}
+
+/**
+ * @param {import('../api/types/governance-scenarios.types').GovernanceScenariosRunResponse} result
+ */
+function printGovernanceRunJson(result) {
+  logger.log(JSON.stringify(result, null, 2));
+}
+
+/**
+ * @param {{ total: number, passed: number, failed: number }} summary
+ * @returns {number}
+ */
+function exitCodeFromGovernanceSummary(summary) {
+  if (!summary || summary.failed > 0) {
+    return 1;
+  }
+  return 0;
+}
+
+module.exports = {
+  displayGovernanceRunTTY,
+  printGovernanceRunJson,
+  exitCodeFromGovernanceSummary
+};

package/lib/identity/identity-apply-core.js

@@ -0,0 +1,108 @@
+/**
+ * @fileoverview Shared helpers for identity apply
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const { getUser, listUsers } = require('../api/users.api');
+
+/**
+ * @param {Object} response
+ * @returns {any}
+ */
+function unwrapControllerData(response) {
+  if (!response || response.success === false) {
+    return null;
+  }
+  return response?.data?.data ?? response?.data ?? response;
+}
+
+/**
+ * @param {Object} response
+ * @returns {string}
+ */
+function apiErrorMessage(response) {
+  if (!response) {
+    return 'Request failed';
+  }
+  const err = response.error;
+  if (typeof err === 'string') {
+    return err;
+  }
+  if (err && typeof err === 'object') {
+    return err.formattedError || err.message || err.error || 'Request failed';
+  }
+  return response.formattedError || response.message || 'Request failed';
+}
+
+/**
+ * @param {Object} response
+ * @throws {Error}
+ */
+function throwIfApiFailed(response) {
+  if (!response || response.success === false) {
+    throw new Error(apiErrorMessage(response));
+  }
+}
+
+/**
+ * @param {Object|null|undefined} stats
+ * @param {{ allowEmptySync?: boolean }} [opts]
+ * @throws {Error}
+ */
+function assertSyncStatsOk(stats, opts = {}) {
+  const syncErrors = Number(stats?.errors ?? 0);
+  if (syncErrors > 0) {
+    throw new Error(
+      `Dataplane full sync reported ${syncErrors} error(s). ` +
+        `usersProcessed=${stats?.usersProcessed ?? 0} groupsProcessed=${stats?.groupsProcessed ?? 0}. ` +
+        'Check miso-controller logs for the sync request.'
+    );
+  }
+  const processed = Number(stats?.usersProcessed ?? 0);
+  if (processed === 0 && !opts.allowEmptySync) {
+    throw new Error(
+      'Dataplane full sync completed with usersProcessed=0. Use --allow-empty-sync to accept, or fix identity on the controller.'
+    );
+  }
+}
+
+/**
+ * @async
+ * @param {string} controllerUrl
+ * @param {Object} authConfig
+ * @param {{ csvId?: string, email: string }} userSpec
+ * @returns {Promise<Object|null>}
+ */
+async function findExistingUser(controllerUrl, authConfig, userSpec) {
+  const csvId = String(userSpec.csvId || '').trim();
+  if (csvId) {
+    const byId = await getUser(controllerUrl, authConfig, csvId);
+    const row = unwrapControllerData(byId);
+    if (row && row.id) {
+      return row;
+    }
+  }
+  const email = String(userSpec.email || '').trim();
+  const listed = await listUsers(controllerUrl, authConfig, {
+    search: email,
+    pageSize: 10,
+    page: 1
+  });
+  const items = unwrapControllerData(listed);
+  const list = Array.isArray(items) ? items : items?.data;
+  if (!Array.isArray(list)) {
+    return null;
+  }
+  return list.find((u) => String(u.email || '').toLowerCase() === email.toLowerCase()) || null;
+}
+
+module.exports = {
+  unwrapControllerData,
+  apiErrorMessage,
+  throwIfApiFailed,
+  assertSyncStatsOk,
+  findExistingUser
+};

package/lib/identity/identity-apply-phases.js

@@ -0,0 +1,244 @@
+/**
+ * @fileoverview Phased execution for identity apply
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const { getGroup, createGroup } = require('../api/groups.api');
+const { createUser } = require('../api/users.api');
+const { addUserToGroup } = require('../api/user-groups.api');
+const { clearAuthCache } = require('../api/auth-cache.api');
+const { fullSyncToDataplane } = require('../api/dataplane-sync.api');
+const {
+  unwrapControllerData,
+  throwIfApiFailed,
+  assertSyncStatsOk,
+  apiErrorMessage,
+  findExistingUser
+} = require('./identity-apply-core');
+
+/**
+ * @param {string} msg
+ * @returns {boolean}
+ */
+function isAlreadyMemberError(msg) {
+  return /already a member|409|Conflict/i.test(String(msg || ''));
+}
+
+/**
+ * @async
+ * @param {string} controllerUrl
+ * @param {Object} authConfig
+ * @param {{ name: string, displayName: string }} spec
+ * @param {boolean} dryRun
+ * @returns {Promise<{ id: string, created: boolean }>}
+ */
+async function ensureGroup(controllerUrl, authConfig, spec, dryRun) {
+  const name = spec.name;
+  const existingRes = await getGroup(controllerUrl, authConfig, name);
+  const existing = unwrapControllerData(existingRes);
+  if (existing && existing.id) {
+    return { id: existing.id, created: false };
+  }
+  if (dryRun) {
+    return { id: `(dry-run:${name})`, created: true };
+  }
+  const createdRes = await createGroup(controllerUrl, authConfig, {
+    name,
+    displayName: spec.displayName || name
+  });
+  throwIfApiFailed(createdRes);
+  const created = unwrapControllerData(createdRes);
+  return { id: created.id, created: true };
+}
+
+/**
+ * @async
+ * @param {string} controllerUrl
+ * @param {Object} authConfig
+ * @param {Object} userSpec
+ * @param {boolean} dryRun
+ * @returns {Promise<{ id: string, created: boolean }>}
+ */
+async function ensureUser(controllerUrl, authConfig, userSpec, dryRun) {
+  const existing = await findExistingUser(controllerUrl, authConfig, userSpec);
+  if (existing && existing.id) {
+    return { id: existing.id, created: false };
+  }
+  if (dryRun) {
+    return { id: `(dry-run:${userSpec.email})`, created: true };
+  }
+  const body = {
+    email: userSpec.email,
+    firstName: userSpec.firstName,
+    lastName: userSpec.lastName,
+    displayName: userSpec.displayName,
+    username: userSpec.username,
+    status: 'active'
+  };
+  Object.keys(body).forEach((k) => {
+    if (body[k] === undefined || body[k] === '') {
+      delete body[k];
+    }
+  });
+  const createdRes = await createUser(controllerUrl, authConfig, body);
+  throwIfApiFailed(createdRes);
+  const created = unwrapControllerData(createdRes);
+  return { id: created.id, created: true };
+}
+
+/**
+ * @async
+ * @param {string} controllerUrl
+ * @param {Object} authConfig
+ * @param {Map} groups
+ * @param {boolean} dryRun
+ * @returns {Promise<{ map: Map<string, string>, created: number, processed: number }>}
+ */
+async function applyGroupsPhase(controllerUrl, authConfig, groups, dryRun) {
+  const map = new Map();
+  let created = 0;
+  let processed = 0;
+  for (const spec of groups.values()) {
+    const out = await ensureGroup(controllerUrl, authConfig, spec, dryRun);
+    map.set(spec.name, out.id);
+    processed += 1;
+    if (out.created) {
+      created += 1;
+    }
+  }
+  return { map, created, processed };
+}
+
+/**
+ * @async
+ * @param {string} controllerUrl
+ * @param {Object} authConfig
+ * @param {Map} users
+ * @param {boolean} dryRun
+ * @returns {Promise<{ map: Map<string, string>, created: number, processed: number }>}
+ */
+async function applyUsersPhase(controllerUrl, authConfig, users, dryRun) {
+  const map = new Map();
+  let created = 0;
+  let processed = 0;
+  for (const [userKey, userSpec] of users.entries()) {
+    const out = await ensureUser(controllerUrl, authConfig, userSpec, dryRun);
+    map.set(userKey, out.id);
+    processed += 1;
+    if (out.created) {
+      created += 1;
+    }
+  }
+  return { map, created, processed };
+}
+
+/**
+ * @async
+ * @param {string} controllerUrl
+ * @param {Object} authConfig
+ * @param {Object[]} memberships
+ * @param {Map<string, string>} userIdByKey
+ * @param {Map<string, string>} groupIdByName
+ * @param {boolean} dryRun
+ * @returns {Promise<{ created: number, processed: number, skipped: number }>}
+ */
+/**
+ * @async
+ * @param {string} controllerUrl
+ * @param {Object} authConfig
+ * @param {Object} m
+ * @param {Map<string, string>} userIdByKey
+ * @param {Map<string, string>} groupIdByName
+ * @param {boolean} dryRun
+ * @returns {Promise<'created'|'skipped'|'dry'>}
+ */
+async function applyOneMembership(controllerUrl, authConfig, m, userIdByKey, groupIdByName, dryRun) {
+  const userId = userIdByKey.get(m.userKey);
+  const groupId = groupIdByName.get(m.groupName);
+  if (!userId || !groupId) {
+    return 'skipped';
+  }
+  if (dryRun) {
+    return 'dry';
+  }
+  const res = await addUserToGroup(controllerUrl, authConfig, userId, groupId, {});
+  if (res && res.success !== false) {
+    return 'created';
+  }
+  if (isAlreadyMemberError(apiErrorMessage(res))) {
+    return 'skipped';
+  }
+  throw new Error(apiErrorMessage(res));
+}
+
+async function applyMembershipsPhase(
+  controllerUrl,
+  authConfig,
+  memberships,
+  userIdByKey,
+  groupIdByName,
+  dryRun
+) {
+  let created = 0;
+  let processed = 0;
+  let skipped = 0;
+  for (const m of memberships) {
+    const userId = userIdByKey.get(m.userKey);
+    const groupId = groupIdByName.get(m.groupName);
+    if (!userId || !groupId) {
+      continue;
+    }
+    processed += 1;
+    const outcome = await applyOneMembership(
+      controllerUrl,
+      authConfig,
+      m,
+      userIdByKey,
+      groupIdByName,
+      dryRun
+    );
+    if (outcome === 'created' || outcome === 'dry') {
+      created += 1;
+    } else if (outcome === 'skipped') {
+      skipped += 1;
+    }
+  }
+  return { created, processed, skipped };
+}
+
+/**
+ * @async
+ * @param {string} controllerUrl
+ * @param {Object} authConfig
+ * @param {Object} opts
+ * @returns {Promise<Object|null>}
+ */
+async function maybePurgeAndSync(controllerUrl, authConfig, opts) {
+  if (opts.dryRun) {
+    return null;
+  }
+  if (opts.purgeCache) {
+    const cacheRes = await clearAuthCache(controllerUrl, authConfig);
+    throwIfApiFailed(cacheRes);
+  }
+  if (!opts.sync || !opts.envKey) {
+    return null;
+  }
+  const syncRes = await fullSyncToDataplane(controllerUrl, authConfig, opts.envKey);
+  throwIfApiFailed(syncRes);
+  const stats = unwrapControllerData(syncRes);
+  assertSyncStatsOk(stats, { allowEmptySync: opts.allowEmptySync });
+  return stats;
+}
+
+module.exports = {
+  ensureGroup,
+  ensureUser,
+  applyGroupsPhase,
+  applyUsersPhase,
+  applyMembershipsPhase,
+  maybePurgeAndSync
+};

package/lib/identity/identity-apply-service.js

@@ -0,0 +1,77 @@
+/**
+ * @fileoverview Bulk identity apply (CSV) against miso-controller
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const { parseUsersCsvFile, buildApplyPlanFromRows } = require('./identity-csv-parser');
+const {
+  applyGroupsPhase,
+  applyUsersPhase,
+  applyMembershipsPhase,
+  maybePurgeAndSync
+} = require('./identity-apply-phases');
+
+const {
+  unwrapControllerData,
+  apiErrorMessage,
+  throwIfApiFailed,
+  findExistingUser
+} = require('./identity-apply-core');
+
+/**
+ * @async
+ * @param {string} controllerUrl
+ * @param {Object} authConfig
+ * @param {Object} options
+ * @returns {Promise<Object>}
+ */
+async function runIdentityApply(controllerUrl, authConfig, options) {
+  const { rows } = parseUsersCsvFile(options.filePath, options.filterPrefix);
+  if (!rows.length) {
+    throw new Error('No data rows to apply after parsing CSV');
+  }
+  const plan = buildApplyPlanFromRows(rows);
+  const dryRun = Boolean(options.dryRun);
+
+  const groupPhase = await applyGroupsPhase(controllerUrl, authConfig, plan.groups, dryRun);
+  const userPhase = await applyUsersPhase(controllerUrl, authConfig, plan.users, dryRun);
+  const memPhase = await applyMembershipsPhase(
+    controllerUrl,
+    authConfig,
+    plan.memberships,
+    userPhase.map,
+    groupPhase.map,
+    dryRun
+  );
+
+  const syncStats = await maybePurgeAndSync(controllerUrl, authConfig, {
+    dryRun,
+    purgeCache: Boolean(options.purgeCache),
+    sync: Boolean(options.sync),
+    envKey: options.envKey,
+    allowEmptySync: Boolean(options.allowEmptySync)
+  });
+
+  return {
+    groupsProcessed: groupPhase.processed,
+    groupsCreated: groupPhase.created,
+    usersProcessed: userPhase.processed,
+    usersCreated: userPhase.created,
+    membershipsProcessed: memPhase.processed,
+    membershipsCreated: memPhase.created,
+    membershipsSkipped: memPhase.skipped,
+    dryRun,
+    syncStats
+  };
+}
+
+module.exports = {
+  unwrapControllerData,
+  apiErrorMessage,
+  throwIfApiFailed,
+  runIdentityApply,
+  findExistingUser
+};