@aifabrix/builder 2.45.5 → 2.47.0

package/lib/validation/validate-display.js

@@ -1,389 +1,35 @@
-const {
-  headerKeyValue,
-  metadata,
-  formatDatasourceListRow,
-  formatBlockingError,
-  formatSuccessLine,
-  formatSuccessParagraph,
-  successGlyph,
-  failureGlyph
-} = require('../utils/cli-test-layout-chalk');
 /**
  * @fileoverview Validation display for `aifabrix validate` (tty-summary: cli-layout.mdc, layout.md).
  * @author AI Fabrix Team
  */
 
+'use strict';
+
 const path = require('path');
 const chalk = require('chalk');
 const logger = require('../utils/logger');
-const { loadConfigFile } = require('../utils/config-format');
-const { flattenRootDimensionsForDisplay } = require('./dimension-display-helpers');
 const {
-  logSectionTitle,
-  logOkRowRest,
-  logWarnRow,
-  logDimLine,
-  logErrorDetail
-} = require('./validate-display-log-helpers');
-
-/**
- * Displays application validation results
- * @function displayApplicationValidation
- * @param {Object} application - Application validation result
- */
-function displayApplicationValidation(application) {
-  if (!application) {
-    return;
-  }
-
-  logSectionTitle('Application');
-  if (application.valid) {
-    logOkRowRest('Application configuration is valid');
-  } else {
-    logger.log(`  ${failureGlyph()} ${chalk.red('Application configuration has errors:')}`);
-    if (application.errors && application.errors.length > 0) {
-      application.errors.forEach(error => {
-        logErrorDetail(error);
-      });
-    }
-  }
-  if (application.warnings && application.warnings.length > 0) {
-    application.warnings.forEach(warning => {
-      logger.log(`    ${chalk.yellow('⚠')} ${chalk.white(warning)}`);
-    });
-  }
-}
-
-/**
- * Extracts dimensions from a datasource file for CLI display.
- * Local bindings become `metadata.<field>`; FK bindings become `fk:<fk>→<dimension>...` plus optional `(actor: ...)`.
- * @function extractDimensionsFromDatasource
- * @param {string} filePath - Path to datasource file
- * @returns {Object} Dimensions info { dimensions: Object, dimensionKeys: string[], hasDimensions: boolean }
- */
-function extractDimensionsFromDatasource(filePath) {
-  try {
-    const parsed = loadConfigFile(filePath);
-
-    const rootFlat = flattenRootDimensionsForDisplay(parsed.dimensions);
-    const abacDimensions = parsed.abac?.dimensions || {};
-    const allDimensions = { ...rootFlat, ...abacDimensions };
-    const dimensionKeys = Object.keys(allDimensions);
-
-    return {
-      dimensions: allDimensions,
-      dimensionKeys,
-      hasDimensions: dimensionKeys.length > 0
-    };
-  } catch {
-    return { dimensions: {}, dimensionKeys: [], hasDimensions: false };
-  }
-}
-
-/**
- * Displays external files validation results
- * @function displayExternalFilesValidation
- * @param {Array} externalFiles - External files validation results
- */
-function displayExternalFilesValidation(externalFiles) {
-  if (!externalFiles || externalFiles.length === 0) {
-    return;
-  }
-
-  logSectionTitle('External integration files');
-  externalFiles.forEach(file => {
-    if (file.valid) {
-      logger.log(formatDatasourceListRow('ok', file.file, file.type));
-    } else {
-      logger.log(formatDatasourceListRow('fail', file.file, file.type));
-      file.errors.forEach(error => {
-        logErrorDetail(error);
-      });
-    }
-    if (file.warnings && file.warnings.length > 0) {
-      file.warnings.forEach(warning => {
-        logger.log(`    ${chalk.yellow('⚠')} ${chalk.white(warning)}`);
-      });
-    }
-  });
-}
-
-/**
- * Displays Dimensions (ABAC) validation results for datasources
- * @function displayDimensionsValidation
- * @param {Array} externalFiles - External files validation results
- * @returns {Array} Warnings to add to aggregated warnings
- */
-function displayDimensionsValidation(externalFiles) {
-  if (!externalFiles || externalFiles.length === 0) {
-    return [];
-  }
-
-  const datasourceFiles = externalFiles.filter(f =>
-    f.type === 'datasource' || f.type === 'external-datasource'
-  );
-
-  if (datasourceFiles.length === 0) {
-    return [];
-  }
-
-  logSectionTitle('Dimensions (ABAC)');
-
-  const warnings = [];
-  let anyDatasourceHasDimensions = false;
-
-  datasourceFiles.forEach(file => {
-    const dimensionsInfo = extractDimensionsFromDatasource(file.path);
-
-    if (dimensionsInfo.hasDimensions) {
-      anyDatasourceHasDimensions = true;
-      logger.log(formatDatasourceListRow('ok', file.file, file.type));
-      dimensionsInfo.dimensionKeys.forEach(key => {
-        const mapping = dimensionsInfo.dimensions[key];
-        logDimLine(`${key} → ${mapping}`);
-      });
-    } else {
-      logWarnRow(`${file.file} — no dimensions configured`);
-      warnings.push(`${file.file} - no dimensions configured, ABAC filtering disabled`);
-    }
-  });
-
-  if (!anyDatasourceHasDimensions) {
-    logWarnRow('No dimensions configured — ABAC filtering disabled for all datasources');
-  }
-
-  return warnings;
-}
-
-/**
- * Displays RBAC validation results
- * @function displayRbacValidation
- * @param {Object} rbac - RBAC validation result
- */
-function displayRbacValidation(rbac) {
-  if (!rbac) {
-    return;
-  }
-
-  logSectionTitle('RBAC configuration');
-  if (rbac.valid) {
-    logOkRowRest('RBAC configuration is valid');
-  } else {
-    logger.log(`  ${failureGlyph()} ${chalk.red('RBAC configuration has errors:')}`);
-    rbac.errors.forEach(error => {
-      logErrorDetail(error);
-    });
-  }
-  if (rbac.warnings && rbac.warnings.length > 0) {
-    rbac.warnings.forEach(warning => {
-      logger.log(`    ${chalk.yellow('⚠')} ${chalk.white(warning)}`);
-    });
-  }
-}
-
-/**
- * Displays file validation results (for direct file validation)
- * @function displayFileValidation
- * @param {Object} result - Validation result
- */
-function displayFileValidation(result) {
-  if (!result.file) {
-    return;
-  }
-
-  logSectionTitle('File');
-  logger.log(`  ${headerKeyValue('Path:', result.file)}`);
-  logger.log(`  ${headerKeyValue('Type:', String(result.type))}`);
-  if (result.valid) {
-    logOkRowRest('File is valid');
-  } else {
-    logger.log(`  ${failureGlyph()} ${chalk.red('File has errors:')}`);
-    result.errors.forEach(error => {
-      logErrorDetail(error);
-    });
-  }
-  if (result.warnings && result.warnings.length > 0) {
-    result.warnings.forEach(warning => {
-      logger.log(`    ${chalk.yellow('⚠')} ${chalk.white(warning)}`);
-    });
-  }
-}
-
-/**
- * Displays aggregated warnings
- * @function displayAggregatedWarnings
- * @param {Array} warnings - Array of warning messages
- */
-function displayAggregatedWarnings(warnings) {
-  if (!warnings || warnings.length === 0) {
-    return;
-  }
-
-  logSectionTitle('Warnings');
-  warnings.forEach(warning => {
-    logger.log(`  ${chalk.yellow('⚠')} ${chalk.white(warning)}`);
-  });
-}
-
-/**
- * Displays application validation step
- * @function displayApplicationStep
- * @param {Object} application - Application validation result
- * @returns {void}
- */
-function displayApplicationStep(application) {
-  logSectionTitle('Application');
-  if (application.valid) {
-    logOkRowRest('Application configuration is valid');
-  } else {
-    logger.log(`  ${failureGlyph()} ${chalk.red('Application configuration has errors:')}`);
-    application.errors.forEach(error => {
-      logErrorDetail(error);
-    });
-  }
-  if (application.warnings && application.warnings.length > 0) {
-    application.warnings.forEach(warning => {
-      logger.log(`    ${chalk.yellow('⚠')} ${chalk.white(warning)}`);
-    });
-  }
-}
-
-/**
- * Displays external integration files step
- * @function displayComponentsStep
- * @param {Object} components - Components validation result
- * @returns {void}
- */
-function displayComponentsStep(components) {
-  const hasFiles = components.files && components.files.length > 0;
-  const hasErrors = components.errors && components.errors.length > 0;
-  if (!hasFiles && !hasErrors) {
-    return;
-  }
-
-  logSectionTitle('External integration files');
-  if (hasFiles) {
-    components.files.forEach(file => {
-      if (file.valid) {
-        logger.log(formatDatasourceListRow('ok', file.file, file.type));
-      } else {
-        logger.log(formatDatasourceListRow('fail', file.file, file.type));
-      }
-    });
-  }
-  if (hasErrors) {
-    components.errors.forEach(error => {
-      logger.log(formatBlockingError(error));
-    });
-  }
-}
-
-/**
- * Displays dimensions validation for datasources
- * @function displayDimensionsStep
- * @param {Array} datasourceFiles - Datasource files
- * @returns {void}
- */
-function displayDimensionsStep(datasourceFiles) {
-  if (datasourceFiles.length === 0) {
-    return;
-  }
-
-  logSectionTitle('Dimensions (ABAC)');
-  datasourceFiles.forEach(file => {
-    try {
-      const dimensionsInfo = extractDimensionsFromDatasource(file.path || file.file);
-      if (dimensionsInfo.hasDimensions) {
-        logger.log(formatDatasourceListRow('ok', file.file, file.type));
-        dimensionsInfo.dimensionKeys.forEach(key => {
-          const mapping = dimensionsInfo.dimensions[key];
-          logDimLine(`${key} → ${mapping}`);
-        });
-      } else {
-        logWarnRow(`${file.file} — no dimensions configured`);
-      }
-    } catch {
-      // Skip if file path not available
-    }
-  });
-}
-
-/**
- * Displays deployment manifest step
- * @function displayManifestStep
- * @param {Object} manifest - Manifest validation result
- * @param {Array} componentFiles - Component files
- * @returns {void}
- */
-function displayManifestStep(manifest, componentFiles) {
-  logSectionTitle('Deployment manifest');
-  if (manifest.skipped) {
-    logger.log(`  ${chalk.gray('⏭')} ${chalk.white('Skipped (fix errors above first)')}`);
-  } else if (manifest.valid) {
-    logOkRowRest('Full deployment manifest is valid');
-    if (componentFiles) {
-      const datasourceFiles = componentFiles.filter(f => f.type === 'datasource' || f.type === 'external-datasource');
-      logger.log(`    ${successGlyph()} ${chalk.white('System configuration valid')}`);
-      logger.log(`    ${successGlyph()} ${chalk.white(`${datasourceFiles.length} datasource(s) valid`)}`);
-      logger.log(`    ${successGlyph()} ${chalk.white('Schema validation passed')}`);
-    }
-  } else {
-    logger.log(`  ${failureGlyph()} ${chalk.red('Full deployment manifest validation failed:')}`);
-    const errs = manifest.errors && manifest.errors.length > 0 ? manifest.errors : [];
-    if (errs.length > 0) {
-      errs.forEach(error => {
-        const msg = typeof error === 'string' ? error : String(error);
-        logErrorDetail(msg);
-      });
-    } else {
-      logErrorDetail('No error details available (check schema and manifest structure).');
-    }
-  }
-  if (manifest.warnings && manifest.warnings.length > 0) {
-    manifest.warnings.forEach(warning => {
-      logger.log(`    ${chalk.yellow('⚠')} ${chalk.white(warning)}`);
-    });
-  }
-}
-
-/**
- * Displays step-by-step validation results for external systems
- * @function displayStepByStepValidation
- * @param {Object} result - Validation result with steps
- */
-function displayStepByStepValidation(result) {
-  if (result.valid) {
-    logger.log(formatSuccessParagraph('Validation passed!'));
-  } else {
-    logger.log(`\n${formatBlockingError('Validation failed!')}`);
-  }
-
-  displayApplicationStep(result.steps.application);
-
-  if (!result.steps.components.valid || (result.steps.components.files && result.steps.components.files.length > 0)) {
-    displayComponentsStep(result.steps.components);
-  }
-
-  const datasourceFiles = result.steps.components.files?.filter(f =>
-    f.type === 'datasource' || f.type === 'external-datasource'
-  ) || [];
-  displayDimensionsStep(datasourceFiles);
-
-  if (result.rbac) {
-    displayRbacValidation(result.rbac);
-  }
-
-  displayManifestStep(result.steps.manifest, result.steps.components.files);
-
-  if (result.warnings && result.warnings.length > 0) {
-    displayAggregatedWarnings(result.warnings);
-  }
-
-  displayOverallStatus(result);
-  if (result.appPath) {
-    logger.log(`\n${metadata(path.resolve(result.appPath))}`);
-  }
-}
+  headerKeyValue,
+  metadata,
+  formatBlockingError,
+  formatSuccessLine,
+  formatSuccessParagraph,
+  failureGlyph
+} = require('../utils/cli-test-layout-chalk');
+const { logSectionTitle } = require('./validate-display-log-helpers');
+const {
+  displayApplicationValidation,
+  displayExternalFilesValidation,
+  displayRbacValidation,
+  displayFileValidation,
+  displayAggregatedWarnings,
+  displayOverallStatus
+} = require('./validate-display-sections');
+const {
+  displayDimensionsValidation,
+  extractDimensionsFromDatasource
+} = require('./validate-display-dimensions');
+const { displayStepByStepValidation } = require('./validate-display-steps');
 
 /**
  * Displays batch validation results (per-app blocks then summary).
@@ -418,31 +64,6 @@ function displayBatchValidationResults(batchResult) {
   }
 }
 
-/**
- * Displays overall validation status
- * @param {Object} result - Validation result
- */
-function displayOverallStatus(result) {
-  let hasErrors = result.errors && result.errors.length > 0;
-  if (!hasErrors && result.steps) {
-    const stepErrors = [result.steps.application, result.steps.components, result.steps.manifest]
-      .filter(Boolean)
-      .some(s => s.errors && s.errors.length > 0);
-    if (stepErrors) {
-      hasErrors = true;
-    }
-  }
-  const hasWarnings = result.warnings && result.warnings.length > 0;
-  logger.log('');
-  if (hasErrors) {
-    logger.log(headerKeyValue('Overall:', chalk.red('Failed')));
-  } else if (hasWarnings) {
-    logger.log(headerKeyValue('Overall:', chalk.yellow('Passed with warnings')));
-  } else {
-    logger.log(headerKeyValue('Overall:', chalk.green('Passed')));
-  }
-}
-
 /**
  * Displays validation results in a user-friendly format
  *
@@ -481,6 +102,7 @@ function displayValidationResults(result) {
     logger.log(`\n${metadata(path.resolve(result.appPath))}`);
   }
 }
+
 module.exports = {
   displayValidationResults,
   displayBatchValidationResults,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aifabrix/builder",
-  "version": "2.45.5",
+  "version": "2.47.0",
   "description": "AI Fabrix Local Fabric & Deployment SDK",
   "main": "lib/index.js",
   "bin": {

package/templates/applications/dataplane/application.yaml

@@ -64,9 +64,15 @@ build:
   dockerfile: builder/dataplane/Dockerfile  # Dockerfile path (relative to project root)
   envOutputPath: ../../.env       # Copy to repo root for local dev
   language: python                # Runtime language for template selection (typescript or python)
-  reloadStart: uvicorn app.main:app --host 0.0.0.0 --port ${PORT:-3001} --reload  # PORT set from port above at run time; default 3001 must match port
+  # Dev reload: polling + watch /app/app only (avoids EMFILE on full repo bind-mount). See run_local.py, make af-reload.
+  reloadStart: WATCHFILES_FORCE_POLLING=1 uvicorn app.main:app --host 0.0.0.0 --port ${PORT:-3001} --reload --reload-dir /app/app
   # Pulled-image local compose (no bind-mount): drives `command` in generated docker-compose. Override for non-standard images (e.g. stub ACR tags).
   imageRun: exec python -m uvicorn app.main:app --host 0.0.0.0 --port ${PORT:-3001}
+  # Image policy (442.0): explicit Dockerfile COPY; worker/beat via python -m app.core.*
+  # integration/ in image: <systemKey>-deploy.json + application.json|yaml only (no test-*, wizard-configs).
+  worker: true
+  imageRunWorker: exec python -m app.core.celery_worker_main
+  imageRunBeat: exec python -m app.core.celery_beat_main
 
 # =============================================================================
 # Portal Input Configuration (Deployment Wizard)

package/templates/applications/dataplane/env.template

@@ -20,22 +20,33 @@ LOG_FORMAT=json
 # Docker default (no volume): use LOG_FILE_PATH=/app/logs/app.log (directory created in image).
 # Production with persistent logs: use /mnt/data/logs/app.log and mount a volume at /mnt/data.
 LOG_FILE_PATH=/mnt/data/logs/app.log
-# If true, run without Redis/Celery (single process)
+# If true, run without Redis/Celery (single process). Set false on API/worker/beat in production.
 LOCAL_MODE=true
 
+# Celery worker process tuning (dataplane-worker container; optional overrides)
+# CELERY_WORKER_QUEUES=external_data,document_sync,document_processing,audit
+# CELERY_WORKER_CONCURRENCY=2
+# CELERY_LOG_LEVEL=info
+
 # When API_KEY is set, a matching Bearer token bypasses OAuth2 validation
 # Same kv:// as miso-controller API_KEY so pipeline Bearer bypass uses one secrets.local entry
 API_KEY=kv://miso-controller-secrets-apiKeyVault
+# Comma-separated realm roles attached to API_KEY bypass (JWT realm_access.roles). Default: operator.
+# Add integration RBAC role keys from published manifests when manual tests need selector/MCP access, e.g.:
+# API_KEY_BYPASS_ROLES=operator,<role-from-manifest-rbac>
+# API_KEY_BYPASS_ROLES=operator
 
 # API Configuration
 API_V1_STR=/api/v1
 VERSION=1.9.5
 # Minimum supported aifabrix-builder CLI semver (advertised on /health and /api/v1/health; empty = no gate)
 # MIN_BUILDER_CLI_VERSION=2.45.0
-MIN_BUILDER_CLI_VERSION=2.45.0
+MIN_BUILDER_CLI_VERSION=2.45.5
 # Base URL for the dataplane web server (used for default OAuth2 callback URL when redirectUri is omitted)
 # url:// resolves after kv://; includes front-door path from application.yaml (e.g. /data)
 DATAPLANE_WEB_SERVER_URL=url://public
+# Relative URL path where dataplane is served by reverse proxy (for prefixed deployments, e.g. /data).
+DATAPLANE_RELATIVE_PATH=url://vdir-public
 DATAPLANE_INTERNAL_URL=url://internal
 
 # CORS Configuration
@@ -106,6 +117,7 @@ ABAC_POLICY_PARSE_CACHE_MAX_ENTRIES=2048
 # =============================================================================
 
 # MISO Application Client Credentials (per application)
+# Backend-only: never expose MISO_CLIENTSECRET to the Vite frontend (use VITE_* in app-ui).
 MISO_CLIENTID=kv://dataplane-client-idKeyVault
 MISO_CLIENTSECRET=kv://dataplane-client-secretKeyVault
 
@@ -123,24 +135,17 @@ KEYCLOAK_REALM=aifabrix
 MISO_WEB_SERVER_URL=url://miso-controller-public
 # Internal: server-to-controller API calls (Docker: service:port; local: host:port — no front-door path).
 MISO_CONTROLLER_URL=url://miso-controller-internal
+# Browser logout page on miso-controller (path relative to MISO_WEB_SERVER_URL; not dataplane /login).
+MISO_LOGOUT_URL=/logout
 
 # Environment key sent to miso-controller (e.g. environmentKey on RBAC/register) when not supplied in the payload.
 # Set MISO_PIPELINE_ENV_KEY=dev when the controller installation uses dev (e.g. MISO_CLIENTID=miso-controller-dev-dataplane).
 # If unset, derived from MISO_CLIENTID (e.g. dev from miso-controller-dev-dataplane).
 MISO_PIPELINE_ENV_KEY=
 
-# =============================================================================
-# AI/LLM CONFIGURATION
-# =============================================================================
-
-# OpenAI Configuration
-OPENAI_API_KEY=kv://secrets-openaiApiKeyVault
-
-# Azure OpenAI Configuration
-AZURE_OPENAI_ENDPOINT=kv://azure-openaiapi-urlKeyVault
-AZURE_OPENAI_API_KEY=kv://secrets-azureOpenaiApiKeyVault
-AZURE_OPENAI_API_VERSION=2024-02-15-preview
-AZURE_OPENAI_DEPLOYMENT_NAME=gpt-4o
+# LLM: chat/embed route through governed enterprise-llm only. Provider credentials
+# live on LLM provider external systems (integration/openai, azure-openai, ollama).
+AI_VALIDATION_LLM_MODEL=gpt-4o-mini
 
 # =============================================================================
 # AUDIT CONFIGURATION
@@ -177,6 +182,19 @@ AUDIT_ASYNC_SHADOW_ENQUEUE=false
 # Set true for local dev policy CRUD; pair with VITE_POLICY_MANAGEMENT_ENABLED=true in app-ui.
 POLICY_MANAGEMENT_API_ENABLED=false
 
+# Developer Access tab help links (optional; empty hides banner links in UI).
+# VITE_DEVELOPER_GUIDE_URL=
+# VITE_API_EXAMPLES_URL=
+# VITE_MCP_INTEGRATION_GUIDE_URL=
+
+# First-party HTTP internal SpecFile (398.2 / plan 397). When true, lifespan upserts
+# internal:dataplane-first-party-http from the live COM-merged OpenAPI schema.
+FIRST_PARTY_HTTP_INTERNAL_SPEC_SYNC_ON_STARTUP=false
+# Upsert internal:dataplane-first-party-http after pipeline external-system publish.
+FIRST_PARTY_HTTP_INTERNAL_SPEC_SYNC_ON_PUBLISH=false
+# Optional external_systems.id for spec_files FK; empty uses first row when startup sync is on.
+FIRST_PARTY_HTTP_INTERNAL_SPEC_EXTERNAL_SYSTEM_ID=
+
 # =============================================================================
 # OBSERVABILITY CONFIGURATION
 # =============================================================================
@@ -189,6 +207,11 @@ OPENTELEMETRY_ENDPOINT=
 # INTEGRATION CERTIFICATE SIGNING (RS256 PEM)
 # =============================================================================
 # Read by PemRsaCertificateSigner.from_environment in app/validation/certificates/signer.py.
+# Workforce execute signing (437.4). Falls back to CERTIFICATE_* keys, then ephemeral dev RSA.
+DATAPLANE_PRIVATE_KEY=
+DATAPLANE_PUBLIC_KEY=
+DATAPLANE_PUBLIC_KEY_ID=
+
 # When CERTIFICATE_PRIVATE_KEY and CERTIFICATE_PUBLIC_KEY are both set (non-empty PEM), the
 # engine uses RS256. When unset and LOCAL_MODE=true, a process-local ephemeral RSA keypair is used
 # (still RS256, with publicKey + publicKeyFingerprint on issued certs). Non-local requires PEM keys.
@@ -198,6 +221,19 @@ CERTIFICATE_PUBLIC_KEY=
 # Optional public key identifier for issued certificates; default if unset: dataplane-signing-key
 CERTIFICATE_PUBLIC_KEY_ID=
 
+# =============================================================================
+# VALIDATION ENGINE — YAML CONTRACTS
+# =============================================================================
+# Load rules from repo contracts/ and schedule hybrid YAML tasks (see docs/adr/0001-validation-engine-yaml-contracts.md).
+# Code defaults when unset: VALIDATION_CONTRACTS_ENABLED=false, VALIDATION_RULESET=legacy.
+# hybrid = legacy Python categories + YAML rules; yaml = YAML/hybrid only (requires contracts enabled).
+# Optional override for contract bundle root (default: <dataplane-repo>/contracts):
+# VALIDATION_CONTRACTS_ROOT=
+VALIDATION_CONTRACTS_ENABLED=true
+VALIDATION_RULESET=hybrid
+# Attach advisory yamlSuggestedLevel + contract metadata on certification details (does not change ICC/PDS level selection).
+VALIDATION_CERTIFICATION_YAML_ENABLED=true
+
 # After a successful POST /api/v1/validation/run (validation engine with certification passed, or E2E success),
 # persist an active integration certificate without a separate issue call. Set false to require explicit issue only.
 VALIDATION_AUTO_ISSUE_INTEGRATION_CERTIFICATE=true