@aifabrix/builder 2.44.5 → 2.44.6

package/lib/datasource/capability/validate-capability-slice.js

@@ -0,0 +1,35 @@
+/**
+ * Optional capability-key checks after schema validation.
+ *
+ * @fileoverview capability validate command helpers
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+const { normalizeCapabilityKey } = require('./capability-key');
+
+/**
+ * Lists structural locations where a capability key should appear.
+ *
+ * @param {object} parsed - Datasource JSON
+ * @param {string} rawKey - Capability key
+ * @returns {{ key: string, missing: string[] }}
+ */
+function checkCapabilitySlices(parsed, rawKey) {
+  const key = normalizeCapabilityKey(rawKey, 'capability');
+  const missing = [];
+  if (!Array.isArray(parsed.capabilities) || !parsed.capabilities.includes(key)) {
+    missing.push(`capabilities[] (missing "${key}")`);
+  }
+  if (!parsed.openapi?.operations?.[key]) {
+    missing.push(`openapi.operations.${key}`);
+  }
+  if (!parsed.execution?.cip?.operations?.[key]) {
+    missing.push(`execution.cip.operations.${key}`);
+  }
+  return { key, missing };
+}
+
+module.exports = {
+  checkCapabilitySlices
+};

package/lib/datasource/list.js

@@ -3,7 +3,10 @@ const { formatBlockingError } = require('../utils/cli-test-layout-chalk');
  * Datasource List Command
  *
  * Lists datasources from the dataplane (GET /api/v1/external).
- * Resolves dataplane URL from the controller, then calls the dataplane list API.
+ * Resolves dataplane URL from the controller, then calls the dataplane list API with
+ * pagination (pageSize up to API max) so results are not capped at the first page.
+ * Optional key prefix uses API filter `key:like:<prefix>%` (SQL LIKE prefix) plus a
+ * client-side safety pass on the merged rows.
  *
  * @fileoverview Datasource listing for AI Fabrix Builder
  * @author AI Fabrix Team
@@ -18,6 +21,90 @@ const { resolveDataplaneUrl } = require('../utils/dataplane-resolver');
 const { formatApiError } = require('../utils/api-error-handler');
 const logger = require('../utils/logger');
 
+/** Dataplane GET /api/v1/external allows pageSize up to 100 */
+const LIST_DATASOURCE_PAGE_SIZE = 100;
+
+/**
+ * Filter query fragment for datasource keys that start with a prefix (dataplane list API).
+ * @param {string} prefix - Trimmed non-empty prefix
+ * @returns {string}
+ */
+function buildKeyPrefixFilterParam(prefix) {
+  return `key:like:${prefix}%`;
+}
+
+/**
+ * @param {Object|null|undefined} meta - Response `meta` object when present
+ * @returns {number|null}
+ */
+function readTotalPagesFromMeta(meta) {
+  if (meta && typeof meta.totalPages === 'number' && meta.totalPages > 0) {
+    return meta.totalPages;
+  }
+  return null;
+}
+
+/**
+ * @param {number} batchLen
+ * @param {number} pageSize
+ * @param {number} page - 1-based page index
+ * @param {number|null} totalPages
+ * @returns {boolean}
+ */
+function shouldStopDatasourcePaging(batchLen, pageSize, page, totalPages) {
+  if (batchLen < pageSize) return true;
+  if (typeof totalPages === 'number' && page >= totalPages) return true;
+  return false;
+}
+
+/**
+ * Loads every page of datasources from the dataplane for the current query.
+ * @async
+ * @param {string} dataplaneUrl
+ * @param {Object} authConfig
+ * @param {string|null} keyPrefix - Trimmed prefix or null for full list
+ * @returns {Promise<Array<Object>>}
+ */
+async function fetchAllDatasourcePages(dataplaneUrl, authConfig, keyPrefix) {
+  const merged = [];
+  let page = 1;
+  for (;;) {
+    if (page > 10000) break;
+    const params = { page, pageSize: LIST_DATASOURCE_PAGE_SIZE };
+    if (keyPrefix) params.filter = buildKeyPrefixFilterParam(keyPrefix);
+    const response = await listDatasourcesFromDataplane(dataplaneUrl, authConfig, params);
+    if (!response.success || !response.data) {
+      const err = new Error('LIST_DATASOURCES_FAILED');
+      err.response = response;
+      throw err;
+    }
+    const batch = extractDatasources(response);
+    if (!batch.length) break;
+    merged.push(...batch);
+    const totalPages = readTotalPagesFromMeta(response.data.meta);
+    if (shouldStopDatasourcePaging(batch.length, LIST_DATASOURCE_PAGE_SIZE, page, totalPages)) {
+      break;
+    }
+    page += 1;
+  }
+  return merged;
+}
+
+/**
+ * @returns {Promise<Array<Object>|null>} Rows or null if API failure was handled (exit)
+ */
+async function fetchDatasourcesForList(dataplaneUrl, authConfig, keyPrefix) {
+  try {
+    return await fetchAllDatasourcePages(dataplaneUrl, authConfig, keyPrefix);
+  } catch (err) {
+    if (err && err.response) {
+      handleDatasourceApiError(err.response, dataplaneUrl);
+      return null;
+    }
+    throw err;
+  }
+}
+
 /**
  * Extracts datasources array from API response
  * Handles multiple response formats similar to applications list
@@ -100,21 +187,47 @@ function extractDatasources(response) {
   return datasources;
 }
 
+/**
+ * Keeps datasources whose `key` starts with the given prefix (trimmed). No-op if prefix empty.
+ *
+ * @param {Array<Object>} datasources - Datasource objects from API
+ * @param {string} [prefix] - Match against datasource key (prefix match)
+ * @returns {Array<Object>}
+ */
+function filterDatasourcesByKeyPrefix(datasources, prefix) {
+  if (!prefix || typeof prefix !== 'string') return datasources;
+  const p = prefix.trim();
+  if (!p) return datasources;
+  return datasources.filter((ds) => {
+    const k = String(ds?.key ?? '');
+    return k.startsWith(p);
+  });
+}
+
 /**
  * Displays datasources in a formatted table
  *
  * @function displayDatasources
  * @param {Array} datasources - Array of datasource objects
  * @param {string} environment - Environment key
- * @param {string} dataplaneUrl - Dataplane URL for header display
+ * @param {string} [dataplaneUrl] - Dataplane URL for header display
+ * @param {string|null} [keyPrefix] - When set, header notes filter on datasource key
  */
-function displayDatasources(datasources, environment, dataplaneUrl) {
+function displayDatasources(datasources, environment, dataplaneUrl, keyPrefix = null) {
   const environmentName = environment || 'dev';
-  const header = `Datasources in ${environmentName} environment${dataplaneUrl ? ` (${dataplaneUrl})` : ''}`;
+  const prefixNote =
+    keyPrefix && String(keyPrefix).trim()
+      ? ` — datasource keys starting with "${String(keyPrefix).trim()}"`
+      : '';
+  const header = `Datasources in ${environmentName} environment${dataplaneUrl ? ` (${dataplaneUrl})` : ''}${prefixNote}`;
 
   if (datasources.length === 0) {
     logger.log(chalk.bold(`\n📋 ${header}:\n`));
-    logger.log(chalk.gray('  No datasources found in this environment.\n'));
+    const emptyDetail =
+      keyPrefix && String(keyPrefix).trim()
+        ? `  No datasources with keys starting with "${String(keyPrefix).trim()}".\n`
+        : '  No datasources found in this environment.\n';
+    logger.log(chalk.gray(emptyDetail));
     return;
   }
 
@@ -133,14 +246,6 @@ function displayDatasources(datasources, environment, dataplaneUrl) {
   logger.log('');
 }
 
-/**
- * Lists datasources from an environment
- *
- * @async
- * @function listDatasources
- * @param {Object} _options - Command options (unused, kept for compatibility)
- * @throws {Error} If listing fails
- */
 /**
  * Gets device token from config
  * @async
@@ -275,7 +380,11 @@ function setupAuthConfig(token, controllerUrl) {
   };
 }
 
-async function listDatasources(_options) {
+/**
+ * @param {Object} [options]
+ * @param {string} [options.keyPrefix] - If set, only datasources whose `key` starts with this string (after trim)
+ */
+async function listDatasources(options = {}) {
   const config = await getConfig();
 
   // Resolve environment from config.yaml (no flags)
@@ -296,21 +405,25 @@ async function listDatasources(_options) {
   // Resolve dataplane URL first (required for list call)
   const dataplaneUrl = await resolveAndValidateDataplaneUrl(controllerUrl, environment, authConfig);
 
-  // List datasources from dataplane (GET /api/v1/external)
-  const response = await listDatasourcesFromDataplane(dataplaneUrl, authConfig);
+  const rawPrefix = options.keyPrefix;
+  const keyPrefix =
+    typeof rawPrefix === 'string' && rawPrefix.trim() ? rawPrefix.trim() : null;
 
-  if (!response.success || !response.data) {
-    handleDatasourceApiError(response, dataplaneUrl);
-    return; // Ensure we don't continue after exit
-  }
+  let datasources = await fetchDatasourcesForList(dataplaneUrl, authConfig, keyPrefix);
+  if (datasources === null) return;
 
-  const datasources = extractDatasources(response);
-  displayDatasources(datasources, environment, dataplaneUrl);
+  if (keyPrefix) {
+    datasources = filterDatasourcesByKeyPrefix(datasources, keyPrefix);
+  }
+  displayDatasources(datasources, environment, dataplaneUrl, keyPrefix);
 }
 
 module.exports = {
   listDatasources,
   displayDatasources,
-  extractDatasources
+  extractDatasources,
+  fetchAllDatasourcePages,
+  buildKeyPrefixFilterParam,
+  filterDatasourcesByKeyPrefix
 };
 

package/lib/datasource/log-viewer.js

@@ -320,10 +320,8 @@ function parseDatasourceLogJson(content, logPath) {
 async function runLogViewer(datasourceKey, options) {
   const { app, file, logType } = options;
   let logPath;
-  let fileName;
   if (file && typeof file === 'string' && file.trim()) {
     logPath = path.isAbsolute(file) ? file : path.resolve(process.cwd(), file.trim());
-    fileName = path.basename(logPath);
   } else {
     if (!datasourceKey || typeof datasourceKey !== 'string') {
       throw new Error('Datasource key is required when --file is not provided');
@@ -347,11 +345,11 @@ async function runLogViewer(datasourceKey, options) {
         );
       }
     }
-    fileName = path.basename(logPath);
   }
+  const resolvedLogPath = path.resolve(logPath);
   const content = await fsp.readFile(logPath, 'utf8');
   const parsed = parseDatasourceLogJson(content, logPath);
-  formatLogContent(parsed, logType, fileName);
+  formatLogContent(parsed, logType, resolvedLogPath);
 }
 
 module.exports = {

package/lib/datasource/unified-validation-run.js

@@ -16,11 +16,53 @@ const { getSystemKeyFromAppKey, findDatasourceFileByKey } = require('./integrati
 const { loadDatasourceForApp } = require('./unified-validation-run-resolve');
 const { buildUnifiedValidationBody } = require('./unified-validation-run-body');
 const { postValidationRunAndOptionalPoll } = require('./unified-validation-run-post');
-const { publishDatasourceViaPipeline } = require('../api/pipeline.api');
 const { requireBearerForDataplanePipeline } = require('../utils/token-manager');
-const { formatApiError } = require('../utils/api-error-handler');
 const logger = require('../utils/logger');
 
+function _resourceTypeFromDatasource(datasource) {
+  return datasource && typeof datasource === 'object' && typeof datasource.resourceType === 'string'
+    ? datasource.resourceType
+    : null;
+}
+
+function _extractOpKeyFromDpSec013Message(msg, datasourceKey) {
+  if (typeof msg !== 'string' || !msg.trim()) return '';
+  const m = msg.match(new RegExp(`^${datasourceKey}:([^\\s-]+)\\s*-`));
+  return m && m[1] ? String(m[1]).trim() : '';
+}
+
+function annotateMissingAutoRbacPermissions(envelope, datasourceKey, datasource) {
+  const issues =
+    envelope && envelope.validation && Array.isArray(envelope.validation.issues)
+      ? envelope.validation.issues
+      : [];
+  if (!issues.length) return;
+
+  const resourceType = _resourceTypeFromDatasource(datasource);
+  if (!resourceType) return;
+
+  for (const iss of issues) {
+    if (!iss || iss.code !== 'DP-SEC-013' || typeof iss.message !== 'string') continue;
+    const operationKey = _extractOpKeyFromDpSec013Message(iss.message, datasourceKey);
+    if (!operationKey) continue;
+
+    iss.details = iss.details && typeof iss.details === 'object' ? iss.details : {};
+    iss.details.resolvedPermission = `${resourceType}:${operationKey}`;
+  }
+}
+
+async function maybeSyncDatasourceToDataplane({ options, authConfig, systemKey }) {
+  if (options.sync !== true) return;
+  requireBearerForDataplanePipeline(authConfig);
+  logger.log(chalk.cyan('Syncing local config to dataplane…'));
+  // Publish full external system (system + all datasources) to ensure system-level RBAC/permissions
+  // are updated before validation runs. Datasource-only publish can still fail DP-SEC-013 when the
+  // dataplane system permissions[] are stale.
+  const { uploadExternalSystem } = require('../commands/upload');
+  await uploadExternalSystem(systemKey, { minimal: true });
+  logger.log(formatSuccessLine('Sync complete'));
+}
+
 /**
  * Resolve datasource JSON path and loaded object (same rules as test-integration).
  * Re-export for tests.
@@ -49,19 +91,7 @@ async function runUnifiedDatasourceValidation(datasourceKey, options) {
   const configObj = await getConfig();
   const { authConfig, dataplaneUrl } = await setupIntegrationTestAuth(appKey, options, configObj);
 
-  if (options.sync === true) {
-    requireBearerForDataplanePipeline(authConfig);
-    logger.log(chalk.cyan('Syncing local config to dataplane…'));
-    const publishResponse = await publishDatasourceViaPipeline(dataplaneUrl, systemKey, authConfig, datasource);
-    if (!publishResponse || publishResponse.success === false) {
-      const msg =
-        (publishResponse && (publishResponse.formattedError || publishResponse.error)) ||
-        formatApiError(publishResponse, dataplaneUrl) ||
-        'Publish failed';
-      throw new Error(`Sync failed: ${msg}`);
-    }
-    logger.log(formatSuccessLine('Sync complete'));
-  }
+  await maybeSyncDatasourceToDataplane({ options, authConfig, systemKey });
 
   const useAsync = options.noAsync ? false : options.async !== false;
   const timeoutMs = parseInt(String(options.timeout || '30000'), 10) || 30000;
@@ -76,7 +106,7 @@ async function runUnifiedDatasourceValidation(datasourceKey, options) {
     options
   });
 
-  return postValidationRunAndOptionalPoll({
+  const result = await postValidationRunAndOptionalPoll({
     dataplaneUrl,
     authConfig,
     body,
@@ -85,6 +115,11 @@ async function runUnifiedDatasourceValidation(datasourceKey, options) {
     noAsync: options.noAsync === true || options.async === false,
     verbosePoll: options.verbose === true
   });
+
+  if (result && result.envelope) {
+    annotateMissingAutoRbacPermissions(result.envelope, datasourceKey, datasource);
+  }
+  return result;
 }
 
 module.exports = {

package/lib/datasource/validate.js

@@ -229,8 +229,60 @@ async function validateDatasourceFile(filePathOrKey) {
   };
 }
 
+/**
+ * Validates parsed datasource JSON (same rules as {@link validateDatasourceFile}, without reading a file).
+ *
+ * @param {Object} parsed - Parsed datasource object
+ * @returns {{ valid: boolean, errors: string[], warnings: string[], summary: Object|null }}
+ */
+function validateDatasourceParsed(parsed) {
+  if (!parsed || typeof parsed !== 'object') {
+    return {
+      valid: false,
+      errors: ['Datasource JSON must be an object'],
+      warnings: [],
+      summary: null
+    };
+  }
+
+  const summary = buildDatasourceValidateSummary(parsed);
+
+  const validate = loadExternalDataSourceSchema();
+  const schemaValid = validate(parsed);
+
+  if (!schemaValid) {
+    return {
+      valid: false,
+      errors: formatValidationErrors(validate.errors, { rootData: parsed, dedupe: true }),
+      warnings: [],
+      summary
+    };
+  }
+
+  const fieldRefErrors = validateFieldReferences(parsed);
+  const abacErrors = validateAbac(parsed);
+  const postSchemaErrors = [...fieldRefErrors, ...abacErrors];
+  if (postSchemaErrors.length > 0) {
+    return {
+      valid: false,
+      errors: postSchemaErrors,
+      warnings: [],
+      summary
+    };
+  }
+
+  return {
+    valid: true,
+    errors: [],
+    warnings: [],
+    summary
+  };
+}
+
 module.exports = {
   validateDatasourceFile,
-  resolveValidateInputPath
+  validateDatasourceParsed,
+  resolveValidateInputPath,
+  resolvePathFromIntegrationDatasourceKey
 };
 

package/lib/deployment/deploy-poll-ui.js

@@ -0,0 +1,60 @@
+/**
+ * TTY ora spinner + non-TTY lines for deploy pipeline polling (aligned with guided infra commands).
+ *
+ * @fileoverview Deploy poll presentation helpers
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const chalk = require('chalk');
+const ora = require('ora');
+const logger = require('../utils/logger');
+const { buildDeployPollSpinnerText } = require('./deployer-status');
+
+function shouldUseDeployPollSpinner() {
+  return Boolean(process && process.stdout && process.stdout.isTTY);
+}
+
+/**
+ * @param {number} maxAttempts - Max polling attempts (shown in UI)
+ * @param {{ silent?: boolean }} [options]
+ * @returns {{ onPollProgress: Function, finish: () => void }}
+ */
+function createDeployPollHandlers(maxAttempts, options = {}) {
+  const silent = options && options.silent === true;
+  if (silent) {
+    return {
+      onPollProgress: () => {},
+      finish: () => {}
+    };
+  }
+  if (shouldUseDeployPollSpinner()) {
+    const spinner = ora({
+      text: buildDeployPollSpinnerText(null, 0, maxAttempts),
+      spinner: 'dots'
+    }).start();
+    return {
+      onPollProgress: (deploymentData, attempt, maxA) => {
+        spinner.text = buildDeployPollSpinnerText(deploymentData, attempt, maxA);
+      },
+      finish: () => {
+        spinner.stop();
+      }
+    };
+  }
+  return {
+    onPollProgress: (deploymentData, attempt, maxA) => {
+      const status = deploymentData.status ?? 'pending';
+      const progress = deploymentData.progress ?? 0;
+      logger.log(chalk.gray(`Status: ${status} (${progress}%) (attempt ${attempt + 1}/${maxA})`));
+    },
+    finish: () => {}
+  };
+}
+
+module.exports = {
+  createDeployPollHandlers,
+  shouldUseDeployPollSpinner
+};

package/lib/deployment/deployer-status.js

@@ -53,7 +53,7 @@ function extractDeploymentData(response) {
 }
 
 /**
- * Logs deployment progress
+ * Logs deployment progress (non-spinner / programmatic polling)
  * @param {Object} deploymentData - Deployment data
  * @param {number} attempt - Current attempt
  * @param {number} maxAttempts - Maximum attempts
@@ -64,6 +64,19 @@ function logDeploymentProgress(deploymentData, attempt, maxAttempts) {
   logger.log(chalk.blue(`   Status: ${status} (${progress}%) (attempt ${attempt + 1}/${maxAttempts})`));
 }
 
+/**
+ * Single-line ora text for deploy polling (updates in place; same style as guided infra spinners).
+ * @param {Object|null|undefined} deploymentData - Latest deployment payload (optional before first response)
+ * @param {number} attempt - Zero-based attempt index
+ * @param {number} maxAttempts - Max polling attempts
+ * @returns {string}
+ */
+function buildDeployPollSpinnerText(deploymentData, attempt, maxAttempts) {
+  const status = deploymentData?.status ?? 'pending';
+  const progress = deploymentData && Number.isFinite(Number(deploymentData.progress)) ? Number(deploymentData.progress) : 0;
+  return `Deploying application... Status: ${status} (${progress}%) (attempt ${attempt + 1}/${maxAttempts})`;
+}
+
 /**
  * Process deployment status response
  * @param {Object} response - API response
@@ -71,9 +84,17 @@ function logDeploymentProgress(deploymentData, attempt, maxAttempts) {
  * @param {number} maxAttempts - Maximum attempts
  * @param {number} interval - Polling interval
  * @param {string} deploymentId - Deployment ID for error messages
+ * @param {Function|null} [onProgress] - If set, called instead of logDeploymentProgress with (deploymentData, attempt, maxAttempts)
  * @returns {Promise<Object|null>} Deployment data if terminal, null if needs to continue polling
  */
-async function processDeploymentStatusResponse(response, attempt, maxAttempts, interval, deploymentId) {
+async function processDeploymentStatusResponse(
+  response,
+  attempt,
+  maxAttempts,
+  interval,
+  deploymentId,
+  onProgress = null
+) {
   if (!response.success || !response.data) {
     handleDeploymentStatusError(response, deploymentId);
   }
@@ -83,7 +104,11 @@ async function processDeploymentStatusResponse(response, attempt, maxAttempts, i
     return deploymentData;
   }
 
-  logDeploymentProgress(deploymentData, attempt, maxAttempts);
+  if (typeof onProgress === 'function') {
+    onProgress(deploymentData, attempt, maxAttempts);
+  } else {
+    logDeploymentProgress(deploymentData, attempt, maxAttempts);
+  }
   if (attempt < maxAttempts - 1) {
     await new Promise(resolve => setTimeout(resolve, interval));
   }
@@ -97,5 +122,6 @@ module.exports = {
   handleDeploymentStatusError,
   extractDeploymentData,
   logDeploymentProgress,
+  buildDeployPollSpinnerText,
   processDeploymentStatusResponse
 };