@aifabrix/builder 2.45.5 → 2.46.0

package/lib/utils/credential-secrets-env.js

@@ -280,21 +280,146 @@ function storedCountFromResponse(res, fallback) {
  * @param {Array<{ key: string, value: string }>} items - Items to send
  * @returns {Promise<{ pushed: number, warning?: string }>}
  */
+/**
+ * For bearerToken auth, credential template resolves security.token only.
+ * Legacy .env keys KV_*_APIKEY with a plain token wrongly map to kv://.../apiKey.
+ * Prefer token path; drop duplicate apiKey when both would be pushed.
+ *
+ * @param {Map<string, string>} itemsByKey
+ * @param {string} [systemKey]
+ * @param {Object} [authConfig]
+ */
+function reconcileBearerTokenSecretPaths(itemsByKey, systemKey, authConfig) {
+  const method = (
+    authConfig?.method ||
+    authConfig?.type ||
+    ''
+  )
+    .toString()
+    .toLowerCase()
+    .replace(/_/g, '');
+  if (method !== 'bearertoken' || !systemKey) return;
+  const tokenPath = `kv://${systemKey}/token`;
+  const apiKeyPath = `kv://${systemKey}/apiKey`;
+  if (!itemsByKey.has(apiKeyPath)) return;
+  if (!itemsByKey.has(tokenPath)) {
+    itemsByKey.set(tokenPath, itemsByKey.get(apiKeyPath));
+  }
+  itemsByKey.delete(apiKeyPath);
+}
+
+/**
+ * Optional dev/lab dataplane API key (Bearer bypass) for credential secret push only.
+ * Does not alter pipeline publish auth when AIFABRIX_DEPLOYMENT_AUTH=client-credentials.
+ * @returns {string|null}
+ */
+function resolveLabDataplaneApiKey() {
+  const candidates = [
+    process.env.AIFABRIX_DATAPLANE_API_KEY,
+    process.env.DATAPLANE_API_KEY,
+    process.env.API_KEY
+  ];
+  for (const raw of candidates) {
+    const trimmed = typeof raw === 'string' ? raw.trim() : '';
+    if (trimmed) return trimmed;
+  }
+  return null;
+}
+
+/**
+ * @param {string} token
+ * @returns {{ type: string, token: string }}
+ */
+function apiKeyAuthConfig(token) {
+  return { type: 'bearer', token };
+}
+
+const PERMISSION_DENIED_WARNING =
+  'Could not push credential secrets (permission denied or unauthenticated). Ensure dataplane role has credential:create if you use KV_* in .env, or set AIFABRIX_DATAPLANE_API_KEY for local lab bypass.';
+
+/**
+ * @param {Object} res
+ * @returns {boolean}
+ */
+function isStoreResponseFailed(res) {
+  return Boolean(res && (res.success === false || res.data?.success === false));
+}
+
+/**
+ * @param {Object} res
+ * @returns {number|undefined}
+ */
+function storeResponseStatus(res) {
+  return res?.status ?? res?.statusCode;
+}
+
+/**
+ * @param {Object} res
+ * @returns {{ pushed: number, warning: string }}
+ */
+function storeErrorWarning(res) {
+  const errMsg =
+    res.formattedError ||
+    res.data?.formattedError ||
+    res.error ||
+    res.data?.error ||
+    'Failed to push credential secrets to dataplane.';
+  return { pushed: 0, warning: errMsg };
+}
+
+/**
+ * @param {string} dataplaneUrl
+ * @param {Object} authConfig
+ * @param {Array<{ key: string, value: string }>} items
+ * @returns {Promise<{ pushed: number }|null>}
+ */
+async function tryLabApiKeyRetry(dataplaneUrl, authConfig, items) {
+  const labKey = resolveLabDataplaneApiKey();
+  if (!labKey || authConfig?.token === labKey) {
+    return null;
+  }
+  const retry = await storeCredentialSecrets(
+    dataplaneUrl,
+    apiKeyAuthConfig(labKey),
+    items
+  );
+  if (isStoreResponseFailed(retry)) {
+    return null;
+  }
+  return { pushed: storedCountFromResponse(retry, items.length) };
+}
+
+/**
+ * @param {string} dataplaneUrl
+ * @param {Object} authConfig
+ * @param {Array<{ key: string, value: string }>} items
+ * @param {Object} res
+ * @returns {Promise<{ pushed: number, warning?: string }>}
+ */
+async function handleFailedStoreResponse(dataplaneUrl, authConfig, items, res) {
+  const status = storeResponseStatus(res);
+  if (status === 403 || status === 401) {
+    const retryResult = await tryLabApiKeyRetry(dataplaneUrl, authConfig, items);
+    if (retryResult) {
+      return retryResult;
+    }
+    return { pushed: 0, warning: PERMISSION_DENIED_WARNING };
+  }
+  return storeErrorWarning(res);
+}
+
 async function sendCredentialSecrets(dataplaneUrl, authConfig, items) {
   try {
     const res = await storeCredentialSecrets(dataplaneUrl, authConfig, items);
-    const failed = res && (res.success === false || res.data?.success === false);
-    if (failed) {
-      const status = res.status ?? res.statusCode;
-      if (status === 403 || status === 401) {
-        return { pushed: 0, warning: 'Could not push credential secrets (permission denied or unauthenticated). Ensure dataplane role has credential:create if you use KV_* in .env.' };
-      }
-      const errMsg = res.formattedError || res.data?.formattedError || res.error || res.data?.error || 'Failed to push credential secrets to dataplane.';
-      return { pushed: 0, warning: errMsg };
+    if (isStoreResponseFailed(res)) {
+      return handleFailedStoreResponse(dataplaneUrl, authConfig, items, res);
     }
     return { pushed: storedCountFromResponse(res, items.length) };
   } catch (err) {
-    return { pushed: 0, warning: err.message || 'Failed to push credential secrets to dataplane.' };
+    return {
+      pushed: 0,
+      warning: err.message || 'Failed to push credential secrets to dataplane.'
+    };
   }
 }
 
@@ -322,6 +447,7 @@ async function pushCredentialSecrets(dataplaneUrl, authConfig, options = {}) {
   const itemsByKey = new Map();
   buildItemsFromEnv(envFilePath, secrets, itemsByKey, appName);
   buildItemsFromPayload(payload, secrets, itemsByKey);
+  reconcileBearerTokenSecretPaths(itemsByKey, appName, authConfig);
 
   const items = Array.from(itemsByKey.entries())
     .filter(([k]) => isValidKvPath(k))
@@ -362,6 +488,8 @@ module.exports = {
   collectKvEnvVarsAsSecretItems,
   collectKvRefsFromPayload,
   pushCredentialSecrets,
+  reconcileBearerTokenSecretPaths,
+  resolveLabDataplaneApiKey,
   kvEnvKeyToPath,
   getKvPathSegmentForSecurityKey,
   systemKeyToKvPrefix,

package/lib/utils/dataplane-resolver.js

@@ -8,7 +8,127 @@
  * @version 2.0.0
  */
 
+const fs = require('fs');
+const path = require('path');
+
+const config = require('../core/config');
 const { discoverDataplaneUrl } = require('../commands/wizard-dataplane');
+const pathsUtil = require('./paths');
+const { loadConfigFile } = require('./config-format');
+
+/** Matches `templates/applications/dataplane/application.yaml` and `calculateComposePort`. */
+const DEFAULT_DATAPLANE_MANIFEST_PORT = 3001;
+
+/**
+ * @param {string} url
+ * @returns {number|null}
+ */
+function localhostPortFromUrl(url) {
+  if (!url || typeof url !== 'string') return null;
+  const normalized = url.replace(/127\.0\.0\.1/g, 'localhost');
+  const match = normalized.match(/localhost:(\d+)/);
+  if (!match) return null;
+  const port = parseInt(match[1], 10);
+  return Number.isFinite(port) ? port : null;
+}
+
+/**
+ * Read dataplane `port` from materialized builder config or bundled template.
+ *
+ * @returns {number}
+ */
+function loadDataplaneManifestListenPort() {
+  const tryLoad = (configPath) => {
+    if (!fs.existsSync(configPath)) {
+      return null;
+    }
+    const variables = loadConfigFile(configPath);
+    const port = variables && variables.port !== undefined ? Number(variables.port) : NaN;
+    return Number.isFinite(port) && port > 0 ? port : null;
+  };
+
+  try {
+    const builderPath = pathsUtil.getBuilderPath('dataplane');
+    if (fs.existsSync(builderPath)) {
+      try {
+        const fromBuilder = tryLoad(pathsUtil.resolveApplicationConfigPath(builderPath));
+        if (fromBuilder) {
+          return fromBuilder;
+        }
+      } catch {
+        // Fall through to bundled template when builder dir exists but config is missing.
+      }
+    }
+  } catch {
+    // ignore
+  }
+
+  const templatePath = path.join(
+    pathsUtil.getProjectRoot(),
+    'templates',
+    'applications',
+    'dataplane',
+    'application.yaml'
+  );
+  const fromTemplate = tryLoad(templatePath);
+  return fromTemplate || DEFAULT_DATAPLANE_MANIFEST_PORT;
+}
+
+/**
+ * Local Docker host port: `application.yaml` `port` (default 3001) + developerId×100.
+ * Same rule as {@link calculateComposePort} / `aifabrix up-platform` — not
+ * `APP_SERVICE_ENV_DEFAULTS_LOCAL.DATAPLANE_PUBLIC_PORT` (3011), which is for other local profiles.
+ *
+ * @returns {Promise<string>}
+ */
+async function expectedLocalDataplaneBaseUrl() {
+  const developerIdRaw = await config.getDeveloperId();
+  const idNum =
+    typeof developerIdRaw === 'string'
+      ? parseInt(developerIdRaw, 10)
+      : Number(developerIdRaw);
+  const devId = Number.isFinite(idNum) && idNum >= 0 ? idNum : 0;
+  const basePort = loadDataplaneManifestListenPort();
+  const hostPort = devId === 0 ? basePort : basePort + devId * 100;
+  return `http://localhost:${hostPort}`;
+}
+
+/**
+ * Controller application.url can lag after dev-id changes; prefer builder host port formula for localhost.
+ * @param {string} discoveredUrl
+ * @returns {Promise<string>}
+ */
+async function alignDiscoveredDataplaneUrl(discoveredUrl) {
+  if (!discoveredUrl || !/localhost|127\.0\.0\.1/i.test(discoveredUrl)) {
+    return discoveredUrl;
+  }
+  try {
+    const expectedUrl = await expectedLocalDataplaneBaseUrl();
+    const discoveredPort = localhostPortFromUrl(discoveredUrl);
+    const expectedPort = localhostPortFromUrl(expectedUrl);
+    if (
+      discoveredPort !== null &&
+      expectedPort !== null &&
+      discoveredPort !== expectedPort
+    ) {
+      return expectedUrl;
+    }
+  } catch {
+    // Keep controller-discovered URL when developer config is unavailable.
+  }
+  return discoveredUrl;
+}
+
+/**
+ * @returns {string|null}
+ */
+function dataplaneUrlFromProcessEnv() {
+  const raw = process.env.DP || process.env.DATAPLANE_URL;
+  if (!raw || typeof raw !== 'string') {
+    return null;
+  }
+  return raw.trim().replace(/\/$/, '');
+}
 
 /**
  * Resolve dataplane URL by discovering from controller
@@ -22,9 +142,16 @@ const { discoverDataplaneUrl } = require('../commands/wizard-dataplane');
  * @throws {Error} If dataplane URL cannot be resolved
  */
 async function resolveDataplaneUrl(controllerUrl, environment, authConfig, opts = {}) {
-  return await discoverDataplaneUrl(controllerUrl, environment, authConfig, opts);
+  const envUrl = dataplaneUrlFromProcessEnv();
+  if (envUrl) {
+    return envUrl;
+  }
+  const discovered = await discoverDataplaneUrl(controllerUrl, environment, authConfig, opts);
+  return alignDiscoveredDataplaneUrl(discovered);
 }
 
 module.exports = {
-  resolveDataplaneUrl
+  resolveDataplaneUrl,
+  expectedLocalDataplaneBaseUrl,
+  alignDiscoveredDataplaneUrl
 };

package/lib/utils/declarative-url-ports.js

@@ -21,7 +21,7 @@ function parseDeveloperIdNum(raw) {
 }
 
 /**
- * Docker published host port: port + devId*100 (dev 0 → base port).
+ * Host port for compose publish, localhost URLs, and health checks: port + devId×100 (dev 0 → base port).
  * @param {number} appPort - Manifest listen port
  * @param {number} developerIdNum
  * @returns {number}
@@ -31,13 +31,13 @@ function publishedHostPort(appPort, developerIdNum) {
 }
 
 /**
- * Local workstation host port: port + 10 + devId*100 (dev 0 → port+10).
+ * Same as {@link publishedHostPort} (legacy name kept for call sites).
  * @param {number} appPort
  * @param {number} developerIdNum
  * @returns {number}
  */
 function localHostPort(appPort, developerIdNum) {
-  return appPort + 10 + developerIdNum * 100;
+  return publishedHostPort(appPort, developerIdNum);
 }
 
 module.exports = {

package/lib/utils/deployment-auth-mode.js

@@ -0,0 +1,41 @@
+/**
+ * Deployment auth mode for pipeline publish / external integration CLI.
+ * @fileoverview AIFABRIX_DEPLOYMENT_AUTH and options.deploymentAuth parsing
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+/**
+ * @typedef {'auto'|'client-credentials'|'device'} DeploymentAuthMode
+ */
+
+/**
+ * @param {Object} [options] - CLI or caller options
+ * @returns {DeploymentAuthMode}
+ */
+function getDeploymentAuthMode(options = {}) {
+  const raw = options.deploymentAuth || process.env.AIFABRIX_DEPLOYMENT_AUTH || 'auto';
+  const mode = String(raw).trim().toLowerCase().replace(/_/g, '-');
+  if (mode === 'client-credentials' || mode === 'client') {
+    return 'client-credentials';
+  }
+  if (mode === 'device' || mode === 'device-token' || mode === 'bearer') {
+    return 'device';
+  }
+  return 'auto';
+}
+
+/**
+ * @param {DeploymentAuthMode} mode
+ * @returns {boolean}
+ */
+function shouldUseDeviceTokenForDeployment(mode) {
+  return mode !== 'client-credentials';
+}
+
+module.exports = {
+  getDeploymentAuthMode,
+  shouldUseDeviceTokenForDeployment
+};

package/lib/utils/developer-volume-id.js

@@ -0,0 +1,33 @@
+/**
+ * Developer ID suffix variants for Docker volume names (`dev6` vs `dev06`).
+ *
+ * @fileoverview Volume name suffix normalization for multi-dev Docker
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+/**
+ * @param {string|number} developerId
+ * @returns {string[]}
+ */
+function collectDeveloperVolumeIdSuffixes(developerId) {
+  const raw = String(developerId).trim();
+  const idNum = parseInt(raw, 10);
+  if (Number.isNaN(idNum) || idNum === 0) {
+    return ['0'];
+  }
+  const suffixes = new Set([String(idNum), raw]);
+  if (raw.length === 1) {
+    suffixes.add(raw.padStart(2, '0'));
+  }
+  if (String(idNum).length === 1) {
+    suffixes.add(String(idNum).padStart(2, '0'));
+  }
+  return [...suffixes];
+}
+
+module.exports = {
+  collectDeveloperVolumeIdSuffixes
+};

package/lib/utils/env-copy.js

@@ -272,14 +272,16 @@ async function patchEnvContentForLocal(envContent, variables) {
  * @param {string} outputPath - Resolved `build.envOutputPath`
  * @param {string} envOutputPathLabel - Label for log (e.g. raw `build.envOutputPath` value)
  */
-async function syncWrittenBuilderEnvToOutputPath(envPath, outputPath, envOutputPathLabel) {
+async function syncWrittenBuilderEnvToOutputPath(envPath, outputPath, envOutputPathLabel, copyOptions = {}) {
   const { parseEnvContentToMap, mergeEnvMapIntoContent } = require('../core/secrets');
   const raw = fs.readFileSync(envPath, 'utf8');
   const synced = substituteMntDataForLocal(raw, outputPath);
   let toWrite = synced;
-  if (fs.existsSync(outputPath)) {
+  if (!copyOptions.freshEnv && fs.existsSync(outputPath)) {
     const existingContent = fs.readFileSync(outputPath, 'utf8');
-    toWrite = mergeEnvMapIntoContent(existingContent, parseEnvContentToMap(synced));
+    toWrite = mergeEnvMapIntoContent(existingContent, parseEnvContentToMap(synced), {
+      appendMissingFromNewMap: true
+    });
   }
   fs.writeFileSync(outputPath, toWrite, { mode: 0o600 });
   logger.log(formatSuccessLine(`Copied resolved .env to envOutputPath (${envOutputPathLabel})`));
@@ -300,10 +302,10 @@ async function writeLocalEnvToOutputPath(outputPath, appName, secretsPath, envOu
   let localEnvContent = await generateEnvContent(appName, secretsPath, 'local', false, genOpts);
   localEnvContent = substituteMntDataForLocal(localEnvContent, outputPath);
   let toWrite = localEnvContent;
-  if (fs.existsSync(outputPath)) {
+  if (!extraOpts.freshEnv && fs.existsSync(outputPath)) {
     const existingContent = fs.readFileSync(outputPath, 'utf8');
     const localMap = parseEnvContentToMap(localEnvContent);
-    toWrite = mergeEnvMapIntoContent(existingContent, localMap);
+    toWrite = mergeEnvMapIntoContent(existingContent, localMap, { appendMissingFromNewMap: true });
   }
   fs.writeFileSync(outputPath, toWrite, { mode: 0o600 });
   logger.log(formatSuccessLine(`Generated local .env at: ${envOutputPathLabel}`));
@@ -350,15 +352,16 @@ async function copyOrRegenerateEnvForNamedApp(opts) {
     secretsPath,
     label,
     preferLocal,
-    appPathForLocal
+    appPathForLocal,
+    freshEnv
   } = opts;
-  const localOpts = { appPath: appPathForLocal || undefined };
+  const localOpts = { appPath: appPathForLocal || undefined, freshEnv: freshEnv === true };
   if (preferLocal) {
     await writeLocalEnvToOutputPath(outputPath, appName, secretsPath, label, localOpts);
     return;
   }
   if (fs.existsSync(envPath)) {
-    await syncWrittenBuilderEnvToOutputPath(envPath, outputPath, label);
+    await syncWrittenBuilderEnvToOutputPath(envPath, outputPath, label, { freshEnv: freshEnv === true });
     return;
   }
   await writeLocalEnvToOutputPath(outputPath, appName, secretsPath, label, localOpts);
@@ -386,6 +389,7 @@ async function processEnvVariables(envPath, variablesPath, appName, secretsPath,
     (variablesPath ? path.dirname(variablesPath) : null);
   const preferLocal =
     copyOptions && typeof copyOptions === 'object' && copyOptions.preferLocalEnvOutputPath === true;
+  const freshEnv = copyOptions && typeof copyOptions === 'object' && copyOptions.freshEnv === true;
   if (appName) {
     await copyOrRegenerateEnvForNamedApp({
       envPath,
@@ -394,7 +398,8 @@ async function processEnvVariables(envPath, variablesPath, appName, secretsPath,
       secretsPath,
       label,
       preferLocal,
-      appPathForLocal
+      appPathForLocal,
+      freshEnv
     });
   } else {
     await writePatchedEnvToOutputPath(envPath, outputPath, variables, label);

package/lib/utils/env-kv-orphan-lines.js

@@ -0,0 +1,43 @@
+/**
+ * Comment active KV_* env lines that are not in the expected key set (stale auth vars).
+ *
+ * @fileoverview Shared orphan KV line handling for env.template repair and resolve merge
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+/**
+ * Comments active KV_* lines whose env key is not in the expected set.
+ *
+ * @param {string} content - .env or env.template content
+ * @param {Set<string>|Map<string, unknown>} expectedKeys - Keys that should stay active
+ * @returns {{ content: string, changed: boolean }}
+ */
+function commentOrphanKvEnvLines(content, expectedKeys) {
+  const expected =
+    expectedKeys instanceof Set ? expectedKeys : new Set(expectedKeys.keys());
+  const changedRef = { value: false };
+  const out = (content || '').split(/\r?\n/).map((line) => {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith('#')) {
+      return line;
+    }
+    const eq = line.indexOf('=');
+    if (eq <= 0 || !/^KV_/i.test(trimmed)) {
+      return line;
+    }
+    const key = line.substring(0, eq).trim();
+    if (expected.has(key)) {
+      return line;
+    }
+    changedRef.value = true;
+    return `${line.match(/^\s*/)[0]}# ${trimmed}`;
+  });
+  const joined = out.join('\n');
+  const normalized = joined.length > 0 && !joined.endsWith('\n') ? `${joined}\n` : joined;
+  return { content: normalized, changed: changedRef.value };
+}
+
+module.exports = { commentOrphanKvEnvLines };

package/lib/utils/error-formatters/network-errors.js

@@ -10,6 +10,7 @@ const { formatBlockingError } = require('../cli-test-layout-chalk');
  */
 
 const chalk = require('chalk');
+const { serviceLabelDisplayName } = require('../service-url-label');
 
 /**
  * Normalizes error message to string
@@ -56,9 +57,16 @@ function addControllerUrlToMessage(lines, errorData) {
  * @param {Object} errorData - Error response data
  */
 function formatConnectionRefusedError(lines, errorData) {
-  lines.push(chalk.yellow('Cannot connect to controller.'));
+  const label = serviceLabelDisplayName(errorData?.serviceLabel || 'service');
+  lines.push(chalk.yellow(`Cannot connect to ${label}.`));
   addControllerUrlToMessage(lines, errorData);
-  lines.push(chalk.gray('Check if the controller is running.'));
+  if (label === 'dataplane') {
+    lines.push(chalk.gray('Check if the dataplane is running (e.g. aifabrix run dataplane --reload).'));
+  } else if (label === 'controller') {
+    lines.push(chalk.gray('Check if the controller is running.'));
+  } else {
+    lines.push(chalk.gray('Check the service URL and network.'));
+  }
 }
 
 /**
@@ -68,9 +76,10 @@ function formatConnectionRefusedError(lines, errorData) {
  * @param {Object} errorData - Error response data
  */
 function formatHostnameNotFoundError(lines, errorData) {
-  lines.push(chalk.yellow('Controller hostname not found.'));
+  const label = serviceLabelDisplayName(errorData?.serviceLabel || 'service');
+  lines.push(chalk.yellow(`${label.charAt(0).toUpperCase()}${label.slice(1)} hostname not found.`));
   addControllerUrlToMessage(lines, errorData);
-  lines.push(chalk.gray('Check your controller URL.'));
+  lines.push(chalk.gray(`Check your ${label} URL (see aifabrix auth status).`));
 }
 
 /**

package/lib/utils/external-env-template.js

@@ -43,6 +43,7 @@ const FALLBACK_SECURITY_BY_AUTH = {
   aad: ['clientId', 'clientSecret'],
   apikey: ['apiKey'],
   apiKey: ['apiKey'],
+  bearertoken: ['token'],
   basic: ['username', 'password'],
   queryParam: ['paramValue'],
   oidc: [],

package/lib/utils/external-system-readiness-core.js

@@ -28,6 +28,15 @@ function isPublicationResultShape(obj) {
   return !!(obj && typeof obj === 'object' && (obj.uploadId || obj.system) && Array.isArray(obj.datasources));
 }
 
+/**
+ * True when dataplane skipped publish because manifest matches running config.
+ * @param {*} publication - Parsed PublicationResult
+ * @returns {boolean}
+ */
+function isPublicationUnchanged(publication) {
+  return !!(publication && publication.uploadStatus === 'unchanged');
+}
+
 /**
  * Returns publication payload from upload API response.
  * @param {*} res - Response from uploadApplicationViaPipeline
@@ -432,6 +441,7 @@ module.exports = {
   unwrapApiData,
   unwrapPublicationResult,
   isPublicationResultShape,
+  isPublicationUnchanged,
   classifyDatasourceTierADetail,
   classifyDatasourceTierA,
   formatTierAPartialHint,