@aifabrix/builder 2.45.5 → 2.47.0

package/lib/utils/developer-volume-id.js

@@ -0,0 +1,33 @@
+/**
+ * Developer ID suffix variants for Docker volume names (`dev6` vs `dev06`).
+ *
+ * @fileoverview Volume name suffix normalization for multi-dev Docker
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+/**
+ * @param {string|number} developerId
+ * @returns {string[]}
+ */
+function collectDeveloperVolumeIdSuffixes(developerId) {
+  const raw = String(developerId).trim();
+  const idNum = parseInt(raw, 10);
+  if (Number.isNaN(idNum) || idNum === 0) {
+    return ['0'];
+  }
+  const suffixes = new Set([String(idNum), raw]);
+  if (raw.length === 1) {
+    suffixes.add(raw.padStart(2, '0'));
+  }
+  if (String(idNum).length === 1) {
+    suffixes.add(String(idNum).padStart(2, '0'));
+  }
+  return [...suffixes];
+}
+
+module.exports = {
+  collectDeveloperVolumeIdSuffixes
+};

package/lib/utils/docker-build.js

@@ -264,10 +264,7 @@ async function executeDockerBuildWithTag(effectiveImageName, imageName, dockerfi
 
   await executeBuild(effectiveImageName, dockerfilePath, contextPath, tag, options);
 
-  const wantCompatTag =
-    effectiveImageName !== imageName &&
-    options &&
-    options.base === true;
+  const wantCompatTag = effectiveImageName !== imageName && imageName;
   if (!wantCompatTag) {
     return;
   }

package/lib/utils/env-copy.js

@@ -272,14 +272,16 @@ async function patchEnvContentForLocal(envContent, variables) {
  * @param {string} outputPath - Resolved `build.envOutputPath`
  * @param {string} envOutputPathLabel - Label for log (e.g. raw `build.envOutputPath` value)
  */
-async function syncWrittenBuilderEnvToOutputPath(envPath, outputPath, envOutputPathLabel) {
+async function syncWrittenBuilderEnvToOutputPath(envPath, outputPath, envOutputPathLabel, copyOptions = {}) {
   const { parseEnvContentToMap, mergeEnvMapIntoContent } = require('../core/secrets');
   const raw = fs.readFileSync(envPath, 'utf8');
   const synced = substituteMntDataForLocal(raw, outputPath);
   let toWrite = synced;
-  if (fs.existsSync(outputPath)) {
+  if (!copyOptions.freshEnv && fs.existsSync(outputPath)) {
     const existingContent = fs.readFileSync(outputPath, 'utf8');
-    toWrite = mergeEnvMapIntoContent(existingContent, parseEnvContentToMap(synced));
+    toWrite = mergeEnvMapIntoContent(existingContent, parseEnvContentToMap(synced), {
+      appendMissingFromNewMap: true
+    });
   }
   fs.writeFileSync(outputPath, toWrite, { mode: 0o600 });
   logger.log(formatSuccessLine(`Copied resolved .env to envOutputPath (${envOutputPathLabel})`));
@@ -300,10 +302,10 @@ async function writeLocalEnvToOutputPath(outputPath, appName, secretsPath, envOu
   let localEnvContent = await generateEnvContent(appName, secretsPath, 'local', false, genOpts);
   localEnvContent = substituteMntDataForLocal(localEnvContent, outputPath);
   let toWrite = localEnvContent;
-  if (fs.existsSync(outputPath)) {
+  if (!extraOpts.freshEnv && fs.existsSync(outputPath)) {
     const existingContent = fs.readFileSync(outputPath, 'utf8');
     const localMap = parseEnvContentToMap(localEnvContent);
-    toWrite = mergeEnvMapIntoContent(existingContent, localMap);
+    toWrite = mergeEnvMapIntoContent(existingContent, localMap, { appendMissingFromNewMap: true });
   }
   fs.writeFileSync(outputPath, toWrite, { mode: 0o600 });
   logger.log(formatSuccessLine(`Generated local .env at: ${envOutputPathLabel}`));
@@ -350,15 +352,16 @@ async function copyOrRegenerateEnvForNamedApp(opts) {
     secretsPath,
     label,
     preferLocal,
-    appPathForLocal
+    appPathForLocal,
+    freshEnv
   } = opts;
-  const localOpts = { appPath: appPathForLocal || undefined };
+  const localOpts = { appPath: appPathForLocal || undefined, freshEnv: freshEnv === true };
   if (preferLocal) {
     await writeLocalEnvToOutputPath(outputPath, appName, secretsPath, label, localOpts);
     return;
   }
   if (fs.existsSync(envPath)) {
-    await syncWrittenBuilderEnvToOutputPath(envPath, outputPath, label);
+    await syncWrittenBuilderEnvToOutputPath(envPath, outputPath, label, { freshEnv: freshEnv === true });
     return;
   }
   await writeLocalEnvToOutputPath(outputPath, appName, secretsPath, label, localOpts);
@@ -386,6 +389,7 @@ async function processEnvVariables(envPath, variablesPath, appName, secretsPath,
     (variablesPath ? path.dirname(variablesPath) : null);
   const preferLocal =
     copyOptions && typeof copyOptions === 'object' && copyOptions.preferLocalEnvOutputPath === true;
+  const freshEnv = copyOptions && typeof copyOptions === 'object' && copyOptions.freshEnv === true;
   if (appName) {
     await copyOrRegenerateEnvForNamedApp({
       envPath,
@@ -394,7 +398,8 @@ async function processEnvVariables(envPath, variablesPath, appName, secretsPath,
       secretsPath,
       label,
       preferLocal,
-      appPathForLocal
+      appPathForLocal,
+      freshEnv
     });
   } else {
     await writePatchedEnvToOutputPath(envPath, outputPath, variables, label);

package/lib/utils/env-kv-orphan-lines.js

@@ -0,0 +1,43 @@
+/**
+ * Comment active KV_* env lines that are not in the expected key set (stale auth vars).
+ *
+ * @fileoverview Shared orphan KV line handling for env.template repair and resolve merge
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+/**
+ * Comments active KV_* lines whose env key is not in the expected set.
+ *
+ * @param {string} content - .env or env.template content
+ * @param {Set<string>|Map<string, unknown>} expectedKeys - Keys that should stay active
+ * @returns {{ content: string, changed: boolean }}
+ */
+function commentOrphanKvEnvLines(content, expectedKeys) {
+  const expected =
+    expectedKeys instanceof Set ? expectedKeys : new Set(expectedKeys.keys());
+  const changedRef = { value: false };
+  const out = (content || '').split(/\r?\n/).map((line) => {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith('#')) {
+      return line;
+    }
+    const eq = line.indexOf('=');
+    if (eq <= 0 || !/^KV_/i.test(trimmed)) {
+      return line;
+    }
+    const key = line.substring(0, eq).trim();
+    if (expected.has(key)) {
+      return line;
+    }
+    changedRef.value = true;
+    return `${line.match(/^\s*/)[0]}# ${trimmed}`;
+  });
+  const joined = out.join('\n');
+  const normalized = joined.length > 0 && !joined.endsWith('\n') ? `${joined}\n` : joined;
+  return { content: normalized, changed: changedRef.value };
+}
+
+module.exports = { commentOrphanKvEnvLines };

package/lib/utils/error-formatters/network-errors.js

@@ -10,6 +10,7 @@ const { formatBlockingError } = require('../cli-test-layout-chalk');
  */
 
 const chalk = require('chalk');
+const { serviceLabelDisplayName } = require('../service-url-label');
 
 /**
  * Normalizes error message to string
@@ -56,9 +57,16 @@ function addControllerUrlToMessage(lines, errorData) {
  * @param {Object} errorData - Error response data
  */
 function formatConnectionRefusedError(lines, errorData) {
-  lines.push(chalk.yellow('Cannot connect to controller.'));
+  const label = serviceLabelDisplayName(errorData?.serviceLabel || 'service');
+  lines.push(chalk.yellow(`Cannot connect to ${label}.`));
   addControllerUrlToMessage(lines, errorData);
-  lines.push(chalk.gray('Check if the controller is running.'));
+  if (label === 'dataplane') {
+    lines.push(chalk.gray('Check if the dataplane is running (e.g. aifabrix run dataplane --reload).'));
+  } else if (label === 'controller') {
+    lines.push(chalk.gray('Check if the controller is running.'));
+  } else {
+    lines.push(chalk.gray('Check the service URL and network.'));
+  }
 }
 
 /**
@@ -68,9 +76,10 @@ function formatConnectionRefusedError(lines, errorData) {
  * @param {Object} errorData - Error response data
  */
 function formatHostnameNotFoundError(lines, errorData) {
-  lines.push(chalk.yellow('Controller hostname not found.'));
+  const label = serviceLabelDisplayName(errorData?.serviceLabel || 'service');
+  lines.push(chalk.yellow(`${label.charAt(0).toUpperCase()}${label.slice(1)} hostname not found.`));
   addControllerUrlToMessage(lines, errorData);
-  lines.push(chalk.gray('Check your controller URL.'));
+  lines.push(chalk.gray(`Check your ${label} URL (see aifabrix auth status).`));
 }
 
 /**

package/lib/utils/external-env-template.js

@@ -13,6 +13,7 @@ const path = require('path');
 const fs = require('fs');
 const Handlebars = require('handlebars');
 const { systemKeyToKvPrefix, kvEnvKeyToPath, securityKeyToVar } = require('./credential-secrets-env');
+const { getAuthCredentialTemplate } = require('./authentication-shape');
 
 /**
  * Builds hint string from portalInput (options → enum, validation → min-max or pattern).
@@ -43,6 +44,7 @@ const FALLBACK_SECURITY_BY_AUTH = {
   aad: ['clientId', 'clientSecret'],
   apikey: ['apiKey'],
   apiKey: ['apiKey'],
+  bearertoken: ['token'],
   basic: ['username', 'password'],
   queryParam: ['paramValue'],
   oidc: [],
@@ -62,9 +64,15 @@ function buildAuthSecureVarsFromSystem(system) {
   const systemKey = system?.key || 'external-system';
   const prefix = systemKeyToKvPrefix(systemKey);
   if (!prefix) return authSecureVars;
-  const security = system?.authentication?.security || system?.auth?.security;
-  const authMethod = (system?.authentication?.method || system?.authentication?.type ||
-    system?.auth?.method || system?.auth?.type || 'apikey').toLowerCase();
+  const authBlock = system?.authentication || system?.auth;
+  const credential = getAuthCredentialTemplate(authBlock);
+  const security = credential?.security || authBlock?.security;
+  const authMethod = (
+    credential?.method ||
+    authBlock?.method ||
+    authBlock?.type ||
+    'apikey'
+  ).toLowerCase();
   if (security && typeof security === 'object' && Object.keys(security).length > 0) {
     for (const key of Object.keys(security)) {
       const envName = `KV_${prefix}_${securityKeyToVar(key)}`;

package/lib/utils/external-system-readiness-core.js

@@ -28,6 +28,15 @@ function isPublicationResultShape(obj) {
   return !!(obj && typeof obj === 'object' && (obj.uploadId || obj.system) && Array.isArray(obj.datasources));
 }
 
+/**
+ * True when dataplane skipped publish because manifest matches running config.
+ * @param {*} publication - Parsed PublicationResult
+ * @returns {boolean}
+ */
+function isPublicationUnchanged(publication) {
+  return !!(publication && publication.uploadStatus === 'unchanged');
+}
+
 /**
  * Returns publication payload from upload API response.
  * @param {*} res - Response from uploadApplicationViaPipeline
@@ -432,6 +441,7 @@ module.exports = {
   unwrapApiData,
   unwrapPublicationResult,
   isPublicationResultShape,
+  isPublicationUnchanged,
   classifyDatasourceTierADetail,
   classifyDatasourceTierA,
   formatTierAPartialHint,

package/lib/utils/external-system-readiness-display.js

@@ -128,25 +128,94 @@ function logServerValidationWarnings(validationBody) {
   }
 }
 
+/**
+ * @param {Object[]} results - Probe result rows
+ * @returns {boolean}
+ */
+function connectivityCheckFailed(results) {
+  return results.some(
+    row =>
+      row?.endpointTestResults?.success === false
+      || row?.endpointTestResults?.endpointReachable === false
+  );
+}
+
 /**
  * @param {Object} results - Probe result rows
  * @param {Object} summary - summarizeProbeResults output
  */
 function logProbeCredentialLine(results, summary) {
   logSeparator();
-  logSectionTitle('Credential Test:');
-  const anyEndpointFail = results.some(row => row?.endpointTestResults?.success === false);
-  if (anyEndpointFail) {
+  logSectionTitle('Connectivity check:');
+  if (connectivityCheckFailed(results)) {
     logger.log(formatBlockingError('Failed (see Key Issues / endpoint test)'));
-  } else if (summary.failed > 0) {
-    logger.log(formatBlockingError('Some datasource checks failed'));
-  } else if (summary.partial > 0) {
-    logger.log(chalk.yellow('⚠ Completed with warnings'));
   } else {
-    logger.log(formatSuccessLine('Passed'));
+    logger.log(formatSuccessLine('Passed (system credential / testEndpoint)'));
+  }
+  if (summary.partial > 0 || summary.failed > 0) {
+    logSeparator();
+    logSectionTitle('Datasource readiness:');
+    if (summary.failed > 0) {
+      logger.log(formatBlockingError('Some datasource checks failed'));
+    } else {
+      logger.log(
+        chalk.yellow(
+          '⚠ Partial — validation or certification warnings (not a failed credential test)'
+        )
+      );
+    }
+    logger.log(
+      chalk.gray(
+        '  Upload --probe runs structural validation + certification tiers; live .env connectivity is separate.'
+      )
+    );
+  }
+}
+
+/**
+ * @param {Object[]} results
+ */
+function logProbeEmptyResultsWarningIfNeeded(results) {
+  if (results.length === 0) {
+    logger.log(chalk.yellow('⚠ No runtime datasource results were returned by the dataplane probe.'));
+    logger.log(chalk.gray('   Endpoint credential test may still pass even if datasource checks were not reported.'));
   }
 }
 
+/**
+ * @param {Array<{ key: string, lines: string[] }>} issues
+ */
+function logProbeKeyIssues(issues) {
+  if (issues.length === 0) return;
+  logSeparator();
+  logSectionTitle('Key Issues:');
+  for (const { key, lines } of issues) {
+    logger.log(chalk.white(key));
+    for (const line of lines) {
+      logger.log(chalk.red(`- ${line}`));
+    }
+  }
+}
+
+/**
+ * @param {Object[]} results
+ * @param {Object} summary
+ * @returns {string[]}
+ */
+function buildProbeNextActionHints(results, summary) {
+  const hints = [];
+  if (connectivityCheckFailed(results)) {
+    hints.push('Fix API credentials or permissions (connectivity check failed)');
+  }
+  if (summary.failed > 0) {
+    hints.push('Fix datasource errors listed under Key Issues');
+  }
+  if (summary.partial > 0 && summary.failed === 0) {
+    hints.push('Review Key Issues for certification/validation warnings (ICC, MCP, dimensions)');
+  }
+  return hints;
+}
+
 /**
  * @param {Object} probeRaw - Unwrapped POST /validation/run body
  * @param {string} systemKey
@@ -156,25 +225,13 @@ function logProbeRuntimeBlock(probeRaw, systemKey) {
   const summary = summarizeProbeResults(results);
   logSeparator();
   logger.log(chalk.bold('Runtime Readiness:'));
-  if (results.length === 0) {
-    logger.log(chalk.yellow('⚠ No runtime datasource results were returned by the dataplane probe.'));
-    logger.log(chalk.gray('   Endpoint credential test may still pass even if datasource checks were not reported.'));
-  }
+  logProbeEmptyResultsWarningIfNeeded(results);
   logDatasourceTable(summary.rows, summary);
-  if (summary.issues.length > 0) {
-    logSeparator();
-    logSectionTitle('Key Issues:');
-    for (const { key, lines } of summary.issues) {
-      logger.log(chalk.white(key));
-      for (const line of lines) {
-        logger.log(chalk.red(`- ${line}`));
-      }
-    }
-  }
+  logProbeKeyIssues(summary.issues);
   logProbeCredentialLine(results, summary);
   logSeparator();
   logNextActions(
-    ['Fix API credentials or permissions if endpoint tests failed'],
+    buildProbeNextActionHints(results, summary),
     `Run: aifabrix datasource test-e2e <datasourceKey> --app ${systemKey}`
   );
 }

package/lib/utils/external-system-validators.js

@@ -392,8 +392,7 @@ function validateAgainstSchema(data, schema) {
     validateSchema: false
   });
   addFormats(ajv);
-  // Some schemas (e.g. external-datasource.schema.json) reference these by $id (aifabrix://schema/type/*).
-  // Adding them up-front keeps validation offline-safe and avoids unresolved $ref failures.
+  // Type schemas are referenced by $id from external-system and external-datasource manifests.
   ajv.addSchema(require('../schema/type/document-storage.json'));
   ajv.addSchema(require('../schema/type/message-service.json'));
   ajv.addSchema(require('../schema/type/vector-store.json'));
@@ -404,6 +403,13 @@ function validateAgainstSchema(data, schema) {
   }
   // Normalize schema to handle nullable without type
   const normalizedSchema = normalizeSchema(schemaCopy);
+  const schemaId = normalizedSchema.$id || '';
+  if (schemaId.includes('external-datasource')) {
+    const externalSystemSchema = require('../schema/external-system.schema.json');
+    if (!ajv.getSchema(externalSystemSchema.$id)) {
+      ajv.addSchema(externalSystemSchema, externalSystemSchema.$id);
+    }
+  }
   const validate = ajv.compile(normalizedSchema);
   const valid = validate(data);
 

package/lib/utils/health-check.js

@@ -357,7 +357,10 @@ async function performHealthCheckAttempt(healthCheckUrl, attempt, maxAttempts, d
     }
     const healthCheckPassed = await checkHealthEndpoint(healthCheckUrl, debug);
     if (healthCheckPassed) {
-      logger.log(formatSuccessLine('Application is healthy'));
+      const { isSetupQuietOutput } = require('./setup-quiet-output');
+      if (!isSetupQuietOutput()) {
+        logger.log(formatSuccessLine('Application is healthy'));
+      }
       if (debug) {
         logger.log(chalk.gray(`[DEBUG] Health check passed after ${attempt + 1} attempt(s)`));
       }

package/lib/utils/help-builder.js

@@ -73,7 +73,12 @@ const CATEGORIES = [
       { name: 'app' },
       { name: 'credential' },
       { name: 'deployment' },
-      { name: 'integration-client' }
+      { name: 'integration-client' },
+      { name: 'dimension' },
+      { name: 'dimension value', term: 'dimension value <command>' },
+      { name: 'resource-type' },
+      { name: 'identity' },
+      { name: 'protection' }
     ]
   },
   {
@@ -97,18 +102,18 @@ const CATEGORIES = [
       { name: 'upload', term: 'upload <systemKey>' },
       { name: 'delete', term: 'delete <systemKey>' },
       { name: 'repair', term: 'repair <systemKey>' },
+      { name: 'channel', term: 'channel scaffold <channelTarget>' },
       { name: 'datasource' },
       { name: 'datasource load', term: 'datasource load <datasourceKey>' },
       { name: 'datasource export', term: 'datasource export <datasourceKey>' },
-      { name: 'dimension' },
-      { name: 'dimension value', term: 'dimension value <command>' },
-      { name: 'identity' },
-      { name: 'protection' },
       { name: 'test', term: 'test <appKey|systemKey>' },
+      { name: 'test-integration', term: 'test-integration <appKey|systemKey>' },
       { name: 'test-e2e', term: 'test-e2e <appKey|systemKey>' },
-      { name: 'test-governance', term: 'test-governance <appKey|systemKey>' },
-      { name: 'test-trust', term: 'test-trust <appKey|systemKey>' },
-      { name: 'test-integration', term: 'test-integration <appKey|systemKey>' }
+      { name: 'verify-operations', term: 'verify-operations <systemKey>' },
+      { name: 'verify-trust', term: 'verify-trust <systemKey>' },
+      { name: 'verify-governance', term: 'verify-governance <systemKey>' },
+      { name: 'governance', term: 'governance <generate|verify|download|upload> <systemKey>' },
+      { name: 'lifecycle', term: 'lifecycle <systemKey>' }
     ]
   },
   {

package/lib/utils/keycloak-realm-ready.js

@@ -0,0 +1,101 @@
+/**
+ * Poll Keycloak until the `aifabrix` realm is available (post-wipe / fresh install).
+ *
+ * @fileoverview Keycloak realm readiness for setup auth
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const { computeAppBaseUrl } = require('./platform-controller-url');
+const config = require('../core/config');
+const { readUrlsLocalRegistrySync } = require('./urls-local-registry');
+const { normalizeFrontDoorPatternForHealth } = require('./health-check-url');
+
+/**
+ * @param {string} keycloakBase
+ * @returns {string}
+ */
+function buildRealmWellKnownUrl(keycloakBase) {
+  const base = String(keycloakBase || '').replace(/\/+$/, '');
+  return `${base}/realms/aifabrix/.well-known/openid-configuration`;
+}
+
+/**
+ * @async
+ * @param {string} url
+ * @returns {Promise<boolean>}
+ */
+async function probeRealmWellKnown(url) {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), 8000);
+  try {
+    const res = await fetch(url, { method: 'GET', signal: controller.signal });
+    return res.ok;
+  } catch {
+    return false;
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+function computeKeycloakDirectBaseUrlFromRegistry(registry, developerIdNum) {
+  const portRaw = registry && registry['keycloak-port'];
+  const port =
+    typeof portRaw === 'number'
+      ? portRaw
+      : typeof portRaw === 'string' && /^\d+$/.test(portRaw.trim())
+        ? parseInt(portRaw.trim(), 10)
+        : null;
+  if (!port || port <= 0) {
+    return null;
+  }
+  const patternRaw = registry && registry['keycloak-pattern'];
+  const pattern = typeof patternRaw === 'string' && patternRaw.trim() ? patternRaw.trim() : null;
+  const mount = pattern ? normalizeFrontDoorPatternForHealth(pattern) : null;
+  const adjusted = port + (Number.isFinite(developerIdNum) ? developerIdNum : 0) * 100;
+  return `http://localhost:${adjusted}${mount || ''}`;
+}
+
+/**
+ * Wait until Keycloak serves the aifabrix realm OIDC metadata.
+ *
+ * @async
+ * @param {{ timeoutSeconds?: number, intervalMs?: number }} [opts]
+ * @returns {Promise<void>}
+ * @throws {Error} On timeout
+ */
+async function waitForKeycloakRealmReady(opts = {}) {
+  const timeoutSeconds = Number.isFinite(opts.timeoutSeconds) ? opts.timeoutSeconds : 180;
+  const intervalMs = Number.isFinite(opts.intervalMs) ? opts.intervalMs : 3000;
+  const keycloakBase = await computeAppBaseUrl('keycloak');
+  const url = buildRealmWellKnownUrl(keycloakBase);
+  const registry = readUrlsLocalRegistrySync();
+  const developerIdRaw = await config.getDeveloperId();
+  const developerIdNum = typeof developerIdRaw === 'string' ? parseInt(developerIdRaw, 10) : developerIdRaw;
+  const directBase = computeKeycloakDirectBaseUrlFromRegistry(registry, developerIdNum);
+  const directUrl = directBase ? buildRealmWellKnownUrl(directBase) : null;
+  const deadline = Date.now() + timeoutSeconds * 1000;
+
+  while (Date.now() < deadline) {
+    if (await probeRealmWellKnown(url)) {
+      return;
+    }
+    if (directUrl && (await probeRealmWellKnown(directUrl))) {
+      return;
+    }
+    await new Promise((resolve) => setTimeout(resolve, intervalMs));
+  }
+
+  throw new Error(
+    `Keycloak realm "aifabrix" was not ready within ${timeoutSeconds}s (checked ${url}${directUrl ? ` and ${directUrl}` : ''}). ` +
+      'Ensure Miso Controller finished onboarding after the database wipe.'
+  );
+}
+
+module.exports = {
+  buildRealmWellKnownUrl,
+  waitForKeycloakRealmReady,
+  computeKeycloakDirectBaseUrlFromRegistry
+};