@aifabrix/builder 2.46.0 → 2.47.1

package/lib/utils/credential-secrets-env.js

@@ -1,7 +1,5 @@
 /**
  * Collect KV_* env vars as secret store items and push to dataplane.
- * Reads integration .env, resolves kv:// in values, scans payload for kv:// refs,
- * and pushes plain values to POST /api/v1/credential/secret.
  *
  * @fileoverview Credential secrets push from .env and payload (Dataplane)
  * @author AI Fabrix Team
@@ -11,217 +9,45 @@
 const fs = require('fs');
 const { loadSecrets } = require('../core/secrets');
 const { storeCredentialSecrets } = require('../api/credential.api');
-
-const KV_PREFIX = 'KV_';
-const KV_REF_PATTERN = /kv:\/\/([a-zA-Z0-9_\-/]+)/g;
-
-/**
- * Converts systemKey to KV_* prefix (e.g. hubspot -> HUBSPOT, my-hubspot -> MY_HUBSPOT).
- * @param {string} systemKey - System key
- * @returns {string}
- */
-function systemKeyToKvPrefix(systemKey) {
-  if (!systemKey || typeof systemKey !== 'string') return '';
-  return systemKey.replace(/-/g, '_').toUpperCase();
-}
-
-/**
- * Maps authentication security key (camelCase) to env VAR (UPPERCASE, no underscores).
- * Used for canonical KV_<APPKEY>_<VAR> names (e.g. clientId → CLIENTID).
- * @param {string} securityKey - Security key (e.g. 'clientId', 'clientSecret')
- * @returns {string}
- */
-function securityKeyToVar(securityKey) {
-  if (!securityKey || typeof securityKey !== 'string') return '';
-  return securityKey.replace(/_/g, '').toUpperCase();
-}
-
-/** Known single-segment variable suffixes (uppercase) for inferring var vs namespace in env keys */
-const VAR_SUFFIXES = new Set(['ID', 'SECRET', 'KEY', 'TOKEN', 'URL', 'USERNAME', 'PASSWORD']);
-
-/**
- * Converts var segment(s) from env key to path-style camelCase (e.g. CLIENT_ID → clientId, CLIENTID → clientId).
- * @param {string[]} varSegments - One or two segments (e.g. ['CLIENT', 'ID'], ['CLIENTID'])
- * @returns {string}
- */
-function varSegmentsToCamelCase(varSegments) {
-  if (!varSegments || varSegments.length === 0) return '';
-  if (varSegments.length === 1) {
-    const s = varSegments[0].toLowerCase();
-    if (s.endsWith('id') && s.length > 2) return s.slice(0, -2) + 'Id';
-    if (s.endsWith('secret') && s.length > 6) return s.slice(0, -6) + 'Secret';
-    if (s.endsWith('key') && s.length > 3) return s.slice(0, -3) + 'Key';
-    if (s.endsWith('token') && s.length > 5) return s.slice(0, -5) + 'Token';
-    if (s.endsWith('url') && s.length > 3) return s.slice(0, -3) + 'Url';
-    return s;
-  }
-  return varSegments.map((seg, i) => {
-    const lower = seg.toLowerCase();
-    return i === 0 ? lower : lower.charAt(0).toUpperCase() + lower.slice(1);
-  }).join('');
-}
-
-/**
- * Builds kv path from segments when systemKey is provided (path = kv://systemKey/variable).
- * @param {string[]} segments - Parsed segments after KV_ prefix
- * @param {string} systemKey - System key (e.g. 'microsoft-teams')
- * @returns {string|null}
- */
-function kvPathWithSystemKey(segments, systemKey) {
-  const prefixInKey = systemKey.replace(/-/g, '_').toUpperCase();
-  const prefixSegs = prefixInKey.split('_').filter(Boolean);
-  if (segments.length <= prefixSegs.length) return null;
-  const prefixMatch = prefixSegs.every((p, i) => segments[i] === p);
-  if (!prefixMatch) return null;
-  const varSegments = segments.slice(prefixSegs.length);
-  const pathVar = varSegmentsToCamelCase(varSegments);
-  return pathVar ? `kv://${systemKey}/${pathVar}` : null;
-}
-
-/**
- * Builds kv path from segments when systemKey is not provided (infers namespace and variable).
- * @param {string[]} segments - Parsed segments after KV_ prefix
- * @returns {string|null}
- */
-function kvPathInferred(segments) {
-  if (segments.length === 1) return `kv://${segments[0].toLowerCase()}`;
-  const varSegmentCount = (segments.length >= 2 && VAR_SUFFIXES.has(segments[segments.length - 1])) ? 2 : 1;
-  const namespace = segments.slice(0, -varSegmentCount).map(s => s.toLowerCase()).join('-');
-  const varSegments = segments.slice(-varSegmentCount);
-  const pathVar = varSegmentsToCamelCase(varSegments);
-  return (namespace && pathVar) ? `kv://${namespace}/${pathVar}` : null;
-}
-
-/**
- * Returns the path segment used in kv://&lt;systemKey&gt;/&lt;segment&gt; for a given security key.
- * Uses the same derivation as env key → path (securityKeyToVar + varSegmentsToCamelCase).
- * @param {string} securityKey - Security key (e.g. 'apiKey', 'clientId', 'clientSecret')
- * @returns {string} Canonical path segment (e.g. 'apiKey', 'clientId')
- */
-function getKvPathSegmentForSecurityKey(securityKey) {
-  if (!securityKey || typeof securityKey !== 'string') return '';
-  return varSegmentsToCamelCase([securityKeyToVar(securityKey)]);
-}
-
-/**
- * Converts KV_* env key to kv:// path in format kv://&lt;system-key&gt;/&lt;variable&gt;.
- * System-key uses hyphens (e.g. microsoft-teams); variable is camelCase (e.g. clientId).
- * When systemKey is provided, uses it as the path namespace; otherwise infers from segments.
- *
- * @param {string} envKey - Env var name (e.g. KV_MICROSOFT_TEAMS_CLIENT_ID)
- * @param {string} [systemKey] - Optional system key (e.g. 'microsoft-teams'); when provided, path is kv://systemKey/variable
- * @returns {string|null} kv:// path or null if invalid
- */
-function kvEnvKeyToPath(envKey, systemKey) {
-  if (!envKey || typeof envKey !== 'string' || !envKey.toUpperCase().startsWith(KV_PREFIX)) return null;
-  const rest = envKey.slice(KV_PREFIX.length).trim();
-  if (!rest) return null;
-  const segments = rest.split('_').filter(Boolean);
-  if (segments.length === 0) return null;
-
-  const hasSystemKey = typeof systemKey === 'string' && systemKey.length > 0;
-  if (hasSystemKey) return kvPathWithSystemKey(segments, systemKey);
-  return kvPathInferred(segments);
-}
-
-/**
- * Collects KV_* entries from env map as secret items (key = kv path, value = raw).
- * When value is a kv:// path, uses it as the key so it matches payload paths (e.g. kv://microsoft-teams/clientId).
- * Otherwise derives path from env key via kvEnvKeyToPath(envKey).
- *
- * @param {Object.<string, string>} envMap - Key-value map from .env
- * @returns {Array<{ key: string, value: string }>} Items (key = kv://..., value = raw)
- */
-function collectKvEnvVarsAsSecretItems(envMap, systemKey) {
-  if (!envMap || typeof envMap !== 'object') {
-    return [];
-  }
-  const items = [];
-  for (const [envKey, rawValue] of Object.entries(envMap)) {
-    const value = typeof rawValue === 'string' ? rawValue.trim() : '';
-    if (value === '') continue;
-    let kvPath = null;
-    if (value.startsWith('kv://') && isValidKvPath(value)) {
-      kvPath = value;
-    }
-    if (!kvPath) kvPath = kvEnvKeyToPath(envKey, systemKey) || kvEnvKeyToPath(envKey);
-    if (!kvPath) continue;
-    items.push({ key: kvPath, value });
-  }
-  return items;
-}
-
-/**
- * Resolves a single value if it is a kv:// reference using secrets map.
- * Supports path-style keys (e.g. secrets/foo) and hyphen-style (secrets-foo).
- *
- * @param {Object} secrets - Loaded secrets object
- * @param {string} value - Value (may be "kv://..." or plain)
- * @returns {string|null} Resolved plain value or null if unresolved
- */
-function resolveKvValue(secrets, value) {
-  if (typeof value !== 'string') return null;
-  const trimmed = value.trim();
-  if (!trimmed.startsWith('kv://')) {
-    return trimmed;
-  }
-  const pathMatch = trimmed.match(/^kv:\/\/([a-zA-Z0-9_\-/]+)$/);
-  if (!pathMatch) return null;
-  const pathKey = pathMatch[1];
-  const resolved = secrets[pathKey];
-  if (resolved === undefined) return null;
-  return typeof resolved === 'string' ? resolved : String(resolved);
-}
+const {
+  collectKvEnvVarsAsSecretItems,
+  collectKvRefsFromPayload,
+  getKvPathSegmentForSecurityKey,
+  isNonEmptySecretValue,
+  isValidKvPath,
+  kvEnvKeyToPath,
+  resolveKvValue,
+  securityKeyToVar,
+  systemKeyToKvPrefix
+} = require('./credential-secrets-env-kv');
 
 /**
- * Recursively collects all kv:// references from a JSON-serializable payload.
- *
- * @param {Object|Array|string|number|boolean|null} obj - Upload payload (application + dataSources)
- * @param {Set<string>} [acc] - Accumulator set (internal)
- * @returns {string[]} Unique kv:// refs (e.g. ["kv://secrets/foo"])
+ * Parses .env-style content into key-value map (first = separates key and value).
+ * @param {string} content - Raw .env content
+ * @returns {Object.<string, string>}
  */
-function collectKvRefsFromPayload(obj, acc = new Set()) {
-  if (obj === null || obj === undefined) {
-    return Array.from(acc);
-  }
-  if (typeof obj === 'string') {
-    let m;
-    KV_REF_PATTERN.lastIndex = 0;
-    while ((m = KV_REF_PATTERN.exec(obj)) !== null) {
-      acc.add(`kv://${m[1]}`);
-    }
-    return Array.from(acc);
-  }
-  if (Array.isArray(obj)) {
-    for (const item of obj) {
-      collectKvRefsFromPayload(item, acc);
-    }
-    return Array.from(acc);
-  }
-  if (typeof obj === 'object') {
-    for (const v of Object.values(obj)) {
-      collectKvRefsFromPayload(v, acc);
+function parseEnvToMap(content) {
+  if (!content || typeof content !== 'string') return {};
+  const map = {};
+  const lines = content.split(/\r?\n/);
+  for (const line of lines) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith('#')) continue;
+    const eq = trimmed.indexOf('=');
+    if (eq > 0) {
+      const key = trimmed.substring(0, eq).trim();
+      const value = trimmed.substring(eq + 1);
+      map[key] = value;
     }
-    return Array.from(acc);
   }
-  return Array.from(acc);
-}
-
-/**
- * Validates that key is a well-formed kv path (kv:// with alphanumeric/hyphen/slash segments).
- *
- * @param {string} key - kv:// path
- * @returns {boolean}
- */
-function isValidKvPath(key) {
-  return typeof key === 'string' && /^kv:\/\/[a-z0-9][a-z0-9\-/]*$/i.test(key.trim());
+  return map;
 }
 
 /**
- * Builds secret items from .env file (KV_* vars, values resolved).
- * @param {string} envFilePath - Path to .env
- * @param {Object} secrets - Loaded secrets
- * @param {Map<string, string>} itemsByKey - Mutable map to add items to
+ * @param {string} envFilePath
+ * @param {Object} secrets
+ * @param {Map<string, string>} itemsByKey
+ * @param {string} [systemKey]
  */
 function buildItemsFromEnv(envFilePath, secrets, itemsByKey, systemKey) {
   if (!envFilePath || typeof envFilePath !== 'string' || !fs.existsSync(envFilePath)) return;
@@ -231,8 +57,11 @@ function buildItemsFromEnv(envFilePath, secrets, itemsByKey, systemKey) {
     const fromEnv = collectKvEnvVarsAsSecretItems(envMap, systemKey);
     for (const { key, value } of fromEnv) {
       const resolved = resolveKvValue(secrets, value);
-      // Skip placeholder: value that equals the kv path (e.g. from env.template) must not be pushed as the secret
-      if (resolved !== null && resolved !== undefined && isValidKvPath(key) && resolved.trim() !== key.trim()) {
+      if (
+        isNonEmptySecretValue(resolved) &&
+        isValidKvPath(key) &&
+        String(resolved).trim() !== key.trim()
+      ) {
         itemsByKey.set(key, resolved);
       }
     }
@@ -242,10 +71,9 @@ function buildItemsFromEnv(envFilePath, secrets, itemsByKey, systemKey) {
 }
 
 /**
- * Builds secret items from payload kv:// refs not already in itemsByKey.
- * @param {Object} payload - Upload payload
- * @param {Object} secrets - Loaded secrets
- * @param {Map<string, string>} itemsByKey - Mutable map to add items to
+ * @param {Object} payload
+ * @param {Object} secrets
+ * @param {Map<string, string>} itemsByKey
  */
 function buildItemsFromPayload(payload, secrets, itemsByKey) {
   if (!payload || typeof payload !== 'object') return;
@@ -255,16 +83,15 @@ function buildItemsFromPayload(payload, secrets, itemsByKey) {
     if (existingKeys.has(ref)) continue;
     const pathKey = ref.replace(/^kv:\/\//, '');
     const resolved = secrets[pathKey];
-    if (resolved !== null && resolved !== undefined && isValidKvPath(ref)) {
+    if (isNonEmptySecretValue(resolved) && isValidKvPath(ref)) {
       itemsByKey.set(ref, typeof resolved === 'string' ? resolved : String(resolved));
     }
   }
 }
 
 /**
- * Derives stored count from API response.
- * @param {Object} res - Response from storeCredentialSecrets
- * @param {number} fallback - Fallback when stored not in response
+ * @param {Object} res
+ * @param {number} fallback
  * @returns {number}
  */
 function storedCountFromResponse(res, fallback) {
@@ -274,17 +101,6 @@ function storedCountFromResponse(res, fallback) {
 }
 
 /**
- * Sends items to dataplane credential API; returns pushed count or warning.
- * @param {string} dataplaneUrl - Dataplane URL
- * @param {Object} authConfig - Auth config
- * @param {Array<{ key: string, value: string }>} items - Items to send
- * @returns {Promise<{ pushed: number, warning?: string }>}
- */
-/**
- * For bearerToken auth, credential template resolves security.token only.
- * Legacy .env keys KV_*_APIKEY with a plain token wrongly map to kv://.../apiKey.
- * Prefer token path; drop duplicate apiKey when both would be pushed.
- *
  * @param {Map<string, string>} itemsByKey
  * @param {string} [systemKey]
  * @param {Object} [authConfig]
@@ -309,8 +125,6 @@ function reconcileBearerTokenSecretPaths(itemsByKey, systemKey, authConfig) {
 }
 
 /**
- * Optional dev/lab dataplane API key (Bearer bypass) for credential secret push only.
- * Does not alter pipeline publish auth when AIFABRIX_DEPLOYMENT_AUTH=client-credentials.
  * @returns {string|null}
  */
 function resolveLabDataplaneApiKey() {
@@ -408,6 +222,12 @@ async function handleFailedStoreResponse(dataplaneUrl, authConfig, items, res) {
   return storeErrorWarning(res);
 }
 
+/**
+ * @param {string} dataplaneUrl
+ * @param {Object} authConfig
+ * @param {Array<{ key: string, value: string }>} items
+ * @returns {Promise<{ pushed: number, warning?: string }>}
+ */
 async function sendCredentialSecrets(dataplaneUrl, authConfig, items) {
   try {
     const res = await storeCredentialSecrets(dataplaneUrl, authConfig, items);
@@ -424,17 +244,10 @@ async function sendCredentialSecrets(dataplaneUrl, authConfig, items) {
 }
 
 /**
- * Pushes credential secrets to dataplane: from .env (KV_*) and from payload kv:// refs.
- * Resolves all kv:// in values via loadSecrets; sends only plain values. Best-effort:
- * on 403/401 logs warning and returns; never logs secret values.
- *
- * @param {string} dataplaneUrl - Dataplane base URL
- * @param {Object} authConfig - Auth config (Bearer)
- * @param {Object} options - Options
- * @param {string} [options.envFilePath] - Path to .env (integration/<systemKey>/.env)
- * @param {string} [options.appName] - App/system name for loadSecrets context
- * @param {Object} [options.payload] - Upload payload { application, dataSources } for kv scan
- * @returns {Promise<{ pushed: number, keys?: string[], skipped?: boolean, warning?: string }>} Count pushed, keys (on success), skipped (when nothing to push), optional warning
+ * @param {string} dataplaneUrl
+ * @param {Object} authConfig
+ * @param {Object} options
+ * @returns {Promise<{ pushed: number, keys?: string[], skipped?: boolean, warning?: string }>}
  */
 async function pushCredentialSecrets(dataplaneUrl, authConfig, options = {}) {
   const { envFilePath, appName, payload } = options;
@@ -461,29 +274,6 @@ async function pushCredentialSecrets(dataplaneUrl, authConfig, options = {}) {
   return sendResult;
 }
 
-/**
- * Parses .env-style content into key-value map (first = separates key and value).
- *
- * @param {string} content - Raw .env content
- * @returns {Object.<string, string>}
- */
-function parseEnvToMap(content) {
-  if (!content || typeof content !== 'string') return {};
-  const map = {};
-  const lines = content.split(/\r?\n/);
-  for (const line of lines) {
-    const trimmed = line.trim();
-    if (!trimmed || trimmed.startsWith('#')) continue;
-    const eq = trimmed.indexOf('=');
-    if (eq > 0) {
-      const key = trimmed.substring(0, eq).trim();
-      const value = trimmed.substring(eq + 1);
-      map[key] = value;
-    }
-  }
-  return map;
-}
-
 module.exports = {
   collectKvEnvVarsAsSecretItems,
   collectKvRefsFromPayload,

package/lib/utils/dataplane-resolver.js

@@ -1,123 +1,15 @@
 /**
  * Dataplane URL Resolver
  *
- * Resolves dataplane URL by discovering from controller
+ * Resolves dataplane URL by discovering from controller (application.url in Miso).
+ * The controller registration is the source of truth — the CLI does not rewrite ports.
  *
  * @fileoverview Dataplane URL resolution utilities
  * @author AI Fabrix Team
  * @version 2.0.0
  */
 
-const fs = require('fs');
-const path = require('path');
-
-const config = require('../core/config');
 const { discoverDataplaneUrl } = require('../commands/wizard-dataplane');
-const pathsUtil = require('./paths');
-const { loadConfigFile } = require('./config-format');
-
-/** Matches `templates/applications/dataplane/application.yaml` and `calculateComposePort`. */
-const DEFAULT_DATAPLANE_MANIFEST_PORT = 3001;
-
-/**
- * @param {string} url
- * @returns {number|null}
- */
-function localhostPortFromUrl(url) {
-  if (!url || typeof url !== 'string') return null;
-  const normalized = url.replace(/127\.0\.0\.1/g, 'localhost');
-  const match = normalized.match(/localhost:(\d+)/);
-  if (!match) return null;
-  const port = parseInt(match[1], 10);
-  return Number.isFinite(port) ? port : null;
-}
-
-/**
- * Read dataplane `port` from materialized builder config or bundled template.
- *
- * @returns {number}
- */
-function loadDataplaneManifestListenPort() {
-  const tryLoad = (configPath) => {
-    if (!fs.existsSync(configPath)) {
-      return null;
-    }
-    const variables = loadConfigFile(configPath);
-    const port = variables && variables.port !== undefined ? Number(variables.port) : NaN;
-    return Number.isFinite(port) && port > 0 ? port : null;
-  };
-
-  try {
-    const builderPath = pathsUtil.getBuilderPath('dataplane');
-    if (fs.existsSync(builderPath)) {
-      try {
-        const fromBuilder = tryLoad(pathsUtil.resolveApplicationConfigPath(builderPath));
-        if (fromBuilder) {
-          return fromBuilder;
-        }
-      } catch {
-        // Fall through to bundled template when builder dir exists but config is missing.
-      }
-    }
-  } catch {
-    // ignore
-  }
-
-  const templatePath = path.join(
-    pathsUtil.getProjectRoot(),
-    'templates',
-    'applications',
-    'dataplane',
-    'application.yaml'
-  );
-  const fromTemplate = tryLoad(templatePath);
-  return fromTemplate || DEFAULT_DATAPLANE_MANIFEST_PORT;
-}
-
-/**
- * Local Docker host port: `application.yaml` `port` (default 3001) + developerId×100.
- * Same rule as {@link calculateComposePort} / `aifabrix up-platform` — not
- * `APP_SERVICE_ENV_DEFAULTS_LOCAL.DATAPLANE_PUBLIC_PORT` (3011), which is for other local profiles.
- *
- * @returns {Promise<string>}
- */
-async function expectedLocalDataplaneBaseUrl() {
-  const developerIdRaw = await config.getDeveloperId();
-  const idNum =
-    typeof developerIdRaw === 'string'
-      ? parseInt(developerIdRaw, 10)
-      : Number(developerIdRaw);
-  const devId = Number.isFinite(idNum) && idNum >= 0 ? idNum : 0;
-  const basePort = loadDataplaneManifestListenPort();
-  const hostPort = devId === 0 ? basePort : basePort + devId * 100;
-  return `http://localhost:${hostPort}`;
-}
-
-/**
- * Controller application.url can lag after dev-id changes; prefer builder host port formula for localhost.
- * @param {string} discoveredUrl
- * @returns {Promise<string>}
- */
-async function alignDiscoveredDataplaneUrl(discoveredUrl) {
-  if (!discoveredUrl || !/localhost|127\.0\.0\.1/i.test(discoveredUrl)) {
-    return discoveredUrl;
-  }
-  try {
-    const expectedUrl = await expectedLocalDataplaneBaseUrl();
-    const discoveredPort = localhostPortFromUrl(discoveredUrl);
-    const expectedPort = localhostPortFromUrl(expectedUrl);
-    if (
-      discoveredPort !== null &&
-      expectedPort !== null &&
-      discoveredPort !== expectedPort
-    ) {
-      return expectedUrl;
-    }
-  } catch {
-    // Keep controller-discovered URL when developer config is unavailable.
-  }
-  return discoveredUrl;
-}
 
 /**
  * @returns {string|null}
@@ -146,12 +38,9 @@ async function resolveDataplaneUrl(controllerUrl, environment, authConfig, opts
   if (envUrl) {
     return envUrl;
   }
-  const discovered = await discoverDataplaneUrl(controllerUrl, environment, authConfig, opts);
-  return alignDiscoveredDataplaneUrl(discovered);
+  return discoverDataplaneUrl(controllerUrl, environment, authConfig, opts);
 }
 
 module.exports = {
-  resolveDataplaneUrl,
-  expectedLocalDataplaneBaseUrl,
-  alignDiscoveredDataplaneUrl
+  resolveDataplaneUrl
 };

package/lib/utils/declarative-url-ports.js

@@ -40,8 +40,31 @@ function localHostPort(appPort, developerIdNum) {
   return publishedHostPort(appPort, developerIdNum);
 }
 
+/**
+ * Host port for local dataplane Docker (`aifabrix up-dataplane` / `run --env dev|tst`).
+ * Matches {@link computeDataplanePortOverride} in up-dataplane.js (+10 dev, +20 tst).
+ *
+ * @param {number} appPort - Manifest listen port (dataplane default 3001)
+ * @param {number} developerIdNum
+ * @param {string} [environmentKey] - dev | tst | other
+ * @returns {number}
+ */
+function dataplaneDockerHostPort(appPort, developerIdNum, environmentKey = 'dev') {
+  const id = Number.isFinite(developerIdNum) && developerIdNum >= 0 ? developerIdNum : 0;
+  const baseHostPort = appPort + id * 100;
+  const env = String(environmentKey || '').trim().toLowerCase();
+  if (env === 'dev') {
+    return baseHostPort + 10;
+  }
+  if (env === 'tst') {
+    return baseHostPort + 20;
+  }
+  return id === 0 ? appPort : baseHostPort;
+}
+
 module.exports = {
   parseDeveloperIdNum,
   publishedHostPort,
-  localHostPort
+  localHostPort,
+  dataplaneDockerHostPort
 };