@aifabrix/builder 2.45.0 → 2.46.0

package/lib/utils/pipeline-upload-auth-retry.js

@@ -0,0 +1,82 @@
+/**
+ * Retry dataplane pipeline upload with application client token when device Bearer fails.
+ *
+ * @fileoverview Pipeline publish auth fallback when device Bearer is rejected
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const chalk = require('chalk');
+const logger = require('./logger');
+const { uploadApplicationViaPipeline } = require('../api/pipeline.api');
+const { getDeploymentAuth } = require('./token-manager');
+const { resolveControllerUrl } = require('./controller-url');
+const config = require('../core/config');
+
+/**
+ * @param {Object} apiResponse
+ * @returns {boolean}
+ */
+function isPipelineAuthFailure(apiResponse) {
+  if (!apiResponse || apiResponse.success !== false) {
+    return false;
+  }
+  const status = Number(apiResponse.status) || 0;
+  if (status === 401 || status === 403) {
+    return true;
+  }
+  const text = String(
+    apiResponse.formattedError || apiResponse.error || apiResponse.message || ''
+  ).toLowerCase();
+  return text.includes('invalid token') || text.includes('unauthorized') || text.includes('authentication');
+}
+
+/**
+ * @param {string} systemKey
+ * @returns {Promise<Object|null>}
+ */
+async function resolveClientTokenAuthForPipelineRetry(systemKey) {
+  try {
+    const controllerUrl = await resolveControllerUrl();
+    const environment = await config.resolveEnvironment();
+    return await getDeploymentAuth(controllerUrl, environment, systemKey, {
+      deploymentAuth: 'client-credentials'
+    });
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * POST /api/v1/pipeline/upload with optional client-token retry after device Bearer 401.
+ *
+ * @async
+ * @param {string} dataplaneUrl
+ * @param {Object} authConfig
+ * @param {Object} payload
+ * @param {string} systemKey - External system key for credential lookup on retry
+ * @returns {Promise<Object>}
+ */
+async function uploadApplicationViaPipelineWithAuthRetry(dataplaneUrl, authConfig, payload, systemKey) {
+  const first = await uploadApplicationViaPipeline(dataplaneUrl, authConfig, payload);
+  if (!isPipelineAuthFailure(first) || authConfig.type !== 'bearer') {
+    return first;
+  }
+  const clientAuth = await resolveClientTokenAuthForPipelineRetry(systemKey);
+  if (!clientAuth || clientAuth.type === 'bearer') {
+    return first;
+  }
+  logger.log(
+    chalk.yellow(
+      'Device login token was rejected for pipeline publish; retrying with application client token…'
+    )
+  );
+  return uploadApplicationViaPipeline(dataplaneUrl, clientAuth, payload);
+}
+
+module.exports = {
+  isPipelineAuthFailure,
+  uploadApplicationViaPipelineWithAuthRetry
+};

package/lib/utils/platform-controller-url.js

@@ -0,0 +1,156 @@
+/**
+ * Absolute public base URLs for platform apps (Keycloak, Miso Controller, Dataplane).
+ * Used by setup (`platform-controller`), guided footers, and declarative routing.
+ *
+ * @fileoverview Platform app public URL resolution (Traefik / localhost + dev ports)
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const config = require('../core/config');
+const pathsUtil = require('./paths');
+const { loadConfigFile } = require('./config-format');
+const { computePublicUrlBaseString } = require('./url-declarative-public-base');
+const { joinUrlPath, normalizeFrontDoorPatternForHealth } = require('./health-check-url');
+const { getDevPorts } = require('./dev-config');
+
+const MISO_CONTROLLER_APP = 'miso-controller';
+
+/**
+ * Bundled `templates/applications/<app>/application.yaml` (fresh install has no builder copy yet).
+ *
+ * @param {string} appName
+ * @returns {Object|null}
+ */
+function loadBundledApplicationVariables(appName) {
+  const templateConfigPath = path.join(
+    pathsUtil.getProjectRoot(),
+    'templates',
+    'applications',
+    appName,
+    'application.yaml'
+  );
+  if (!fs.existsSync(templateConfigPath)) {
+    return null;
+  }
+  const variables = loadConfigFile(templateConfigPath);
+  return variables && typeof variables === 'object' ? variables : null;
+}
+
+/**
+ * Load platform app variables from materialized builder config, or from bundled template
+ * when `aifabrix setup` runs before `up-platform` has copied templates.
+ *
+ * @param {string} appName
+ * @returns {Object}
+ */
+function loadAppVariables(appName) {
+  const builderPath = pathsUtil.getBuilderPath(appName);
+  if (fs.existsSync(builderPath)) {
+    try {
+      const configPath = pathsUtil.resolveApplicationConfigPath(builderPath);
+      return loadConfigFile(configPath) || {};
+    } catch (err) {
+      const msg = err && err.message ? String(err.message) : '';
+      if (!msg.includes('Application config not found')) {
+        throw err;
+      }
+    }
+  }
+  const fromTemplate = loadBundledApplicationVariables(appName);
+  if (fromTemplate) {
+    return fromTemplate;
+  }
+  throw new Error(
+    `Application config not found for platform app "${appName}". ` +
+      'Run `aifabrix setup` or `aifabrix up-platform` to materialize builder templates.'
+  );
+}
+
+function computeLocalFullPlatformFrontDoorUrl(params) {
+  const { setupPlatformMode, remoteServer, developerIdNum, frontDoorRouting } = params;
+  if (setupPlatformMode !== 'full') {
+    return null;
+  }
+  if (remoteServer && String(remoteServer).trim()) {
+    return null;
+  }
+  const ports = getDevPorts(Number.isFinite(developerIdNum) ? developerIdNum : 0);
+  const mount = normalizeFrontDoorPatternForHealth(frontDoorRouting.pattern);
+  // Full local platform uses a single front-door port (3000 + devId×100), not the infra Traefik port.
+  return joinUrlPath(`http://localhost:${ports.app}`, mount);
+}
+
+/**
+ * Compute the absolute base URL for a platform app (same rules as guided `up-platform` footer).
+ *
+ * @async
+ * @param {string} appName - e.g. `miso-controller`, `keycloak`, `dataplane`
+ * @returns {Promise<string>}
+ */
+async function computeAppBaseUrl(appName) {
+  const variables = loadAppVariables(appName);
+  const basePort = Number(variables.port || 3000);
+  const developerIdRaw = await config.getDeveloperId();
+  const developerIdNum =
+    typeof developerIdRaw === 'string' ? parseInt(developerIdRaw, 10) : developerIdRaw;
+  const userCfg = await config.getConfig();
+  const remoteServer = await config.getRemoteServer();
+  const infraTlsEnabled = Boolean(userCfg && userCfg.tlsEnabled);
+
+  const fd = variables.frontDoorRouting || null;
+  const frontDoorEnabled = Boolean(fd && fd.enabled === true);
+  const traefikOn = Boolean(userCfg && userCfg.traefik);
+  const pathActive = Boolean(traefikOn && frontDoorEnabled);
+  const setupPlatformMode = await config.getSetupPlatformMode();
+
+  const publicBase = computePublicUrlBaseString({
+    traefik: traefikOn,
+    pathActive,
+    hostTemplate: fd ? fd.host : null,
+    tls: fd ? fd.tls : true,
+    developerIdRaw,
+    remoteServer,
+    profile: 'docker',
+    listenPort: basePort,
+    developerIdNum,
+    infraTlsEnabled
+  });
+
+  if (pathActive) {
+    const localFullUrl = computeLocalFullPlatformFrontDoorUrl({
+      setupPlatformMode,
+      remoteServer,
+      developerIdNum,
+      frontDoorRouting: fd
+    });
+    if (localFullUrl) return localFullUrl;
+    const mount = normalizeFrontDoorPatternForHealth(fd.pattern);
+    return joinUrlPath(publicBase, mount);
+  }
+
+  return String(publicBase).replace(/\/+$/, '');
+}
+
+/**
+ * Absolute Miso Controller URL for local platform install / setup auth.
+ *
+ * @async
+ * @returns {Promise<string>}
+ */
+async function resolvePlatformControllerUrl() {
+  return computeAppBaseUrl(MISO_CONTROLLER_APP);
+}
+
+module.exports = {
+  MISO_CONTROLLER_APP,
+  loadAppVariables,
+  loadBundledApplicationVariables,
+  computeAppBaseUrl,
+  resolvePlatformControllerUrl
+};

package/lib/utils/postgres-platform-bootstrap.js

@@ -0,0 +1,199 @@
+/**
+ * Recreate platform Postgres databases and roles after setup "wipe data".
+ *
+ * Infra init scripts run only on first volume creation; dropping DBs in a
+ * running Postgres container requires explicit bootstrap before Keycloak/Miso start.
+ *
+ * @fileoverview Platform database bootstrap after postgres wipe
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const config = require('../core/config');
+const adminSecrets = require('../core/admin-secrets');
+const logger = require('./logger');
+const chalk = require('chalk');
+const { successGlyph } = require('./cli-test-layout-chalk');
+const { getPostgresContainerName } = require('./postgres-wipe');
+const dockerExec = require('./docker-exec');
+const { isSetupQuietOutput } = require('./setup-quiet-output');
+
+const SUPERUSER_ROLE = 'pgadmin';
+
+/** @type {{ dbName: string, secretKey: string }[]} */
+const PLATFORM_DATABASE_SLOTS = [
+  { dbName: 'keycloak', secretKey: 'databases-keycloak-0-passwordKeyVault' },
+  { dbName: 'miso', secretKey: 'databases-miso-controller-0-passwordKeyVault' },
+  { dbName: 'miso-logs', secretKey: 'databases-miso-controller-1-passwordKeyVault' },
+  { dbName: 'dataplane', secretKey: 'databases-dataplane-0-passwordKeyVault' },
+  { dbName: 'dataplane-vector', secretKey: 'databases-dataplane-1-passwordKeyVault' },
+  { dbName: 'dataplane-logs', secretKey: 'databases-dataplane-2-passwordKeyVault' },
+  { dbName: 'dataplane-records', secretKey: 'databases-dataplane-3-passwordKeyVault' }
+];
+
+/**
+ * @param {string} dbName
+ * @returns {string}
+ */
+function pgRoleName(dbName) {
+  return `${String(dbName).replace(/-/g, '_')}_user`;
+}
+
+/**
+ * @param {string} s
+ * @returns {string}
+ */
+function escapePgString(s) {
+  if (s === null || s === undefined || typeof s !== 'string') return '';
+  return s.replace(/'/g, '\'\'');
+}
+
+/**
+ * @param {string} urlOrPassword
+ * @returns {string|null}
+ */
+function extractPasswordFromUrlOrValue(urlOrPassword) {
+  if (typeof urlOrPassword !== 'string' || urlOrPassword.length === 0) return null;
+  if (!urlOrPassword.includes('://')) return urlOrPassword;
+  try {
+    const u = new URL(
+      urlOrPassword.replace(/\$\{DB_HOST\}/g, 'postgres').replace(/\$\{DB_PORT\}/g, '5432')
+    );
+    return u.password || null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Escape SQL for embedding in a double-quoted shell argument (-c "...").
+ * Bash expands `$` and `$$` (PID) inside double quotes; backslash-escape every `$`.
+ *
+ * @param {string} s
+ * @returns {string}
+ */
+function escapeShellDoubleQuoted(s) {
+  return s.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\$/g, '\\$');
+}
+
+/**
+ * @async
+ * @param {string} container
+ * @param {string} sql
+ * @param {string} adminPassword
+ * @returns {Promise<void>}
+ */
+async function runPsqlCommand(container, sql, adminPassword, database = 'postgres') {
+  const escapedSql = escapeShellDoubleQuoted(sql);
+  const cmd =
+    `docker exec -e PGPASSWORD -i ${container} psql -U ${SUPERUSER_ROLE} -d ${database} ` +
+    `-v ON_ERROR_STOP=1 -c "${escapedSql}"`;
+  await dockerExec.execWithDockerEnv(cmd, { env: { PGPASSWORD: adminPassword } });
+}
+
+/**
+ * @async
+ * @param {string} container
+ * @param {string} dbName
+ * @param {string} role
+ * @param {string} password
+ * @param {string} adminPassword
+ * @returns {Promise<void>}
+ */
+async function ensureRoleAndDatabase(container, dbName, role, password, adminPassword) {
+  if (!/^[A-Za-z_][A-Za-z0-9_-]*$/.test(dbName) || !/^[A-Za-z_][A-Za-z0-9_-]*$/.test(role)) {
+    throw new Error(`Unsafe database or role name: ${dbName} / ${role}`);
+  }
+  const pwd = escapePgString(password);
+  await runPsqlCommand(
+    container,
+    `DO $$ BEGIN IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = '${role}') THEN CREATE USER "${role}" WITH PASSWORD '${pwd}'; ELSE ALTER USER "${role}" WITH PASSWORD '${pwd}'; END IF; END $$;`,
+    adminPassword
+  );
+  const exists = await dockerExec.execWithDockerEnv(
+    `docker exec -e PGPASSWORD -i ${container} psql -U ${SUPERUSER_ROLE} -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname = '${dbName}'"`,
+    { env: { PGPASSWORD: adminPassword } }
+  );
+  const hasDb = String((exists && exists.stdout) || '').trim() === '1';
+  if (!hasDb) {
+    await runPsqlCommand(
+      container,
+      `CREATE DATABASE "${dbName}" OWNER "${role}";`,
+      adminPassword
+    );
+  }
+  await runPsqlCommand(
+    container,
+    `GRANT ALL ON SCHEMA public TO "${role}"; ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO "${role}";`,
+    adminPassword,
+    dbName
+  );
+}
+
+/**
+ * @async
+ * @param {string} container
+ * @param {{ dbName: string, secretKey: string }} slot
+ * @param {Object} secrets
+ * @param {string} adminPassword
+ * @returns {Promise<string>}
+ */
+async function ensurePlatformDatabaseSlot(container, slot, secrets, adminPassword) {
+  const raw = secrets[slot.secretKey];
+  const password = extractPasswordFromUrlOrValue(raw);
+  if (!password || !password.trim()) {
+    throw new Error(
+      `Secret ${slot.secretKey} is missing or empty. Run "aifabrix up-infra" to ensure infra secrets.`
+    );
+  }
+  const role = pgRoleName(slot.dbName);
+  await ensureRoleAndDatabase(container, slot.dbName, role, password.trim(), adminPassword);
+  if (!isSetupQuietOutput()) {
+    logger.log(chalk.gray(` ${successGlyph()} Ensured database ${slot.dbName} (${role})`));
+  }
+  return slot.dbName;
+}
+
+/**
+ * @async
+ * @returns {Promise<{ container: string, adminPassword: string, secrets: Object }>}
+ */
+async function resolveBootstrapContext() {
+  const devId = await config.getDeveloperId();
+  const container = getPostgresContainerName(devId);
+  const admin = await adminSecrets.readAndDecryptAdminSecrets();
+  const adminPassword = admin && admin.POSTGRES_PASSWORD;
+  if (!adminPassword) {
+    throw new Error('POSTGRES_PASSWORD not found in admin-secrets.env. Run "aifabrix up-infra" first.');
+  }
+  const secretsEnsure = require('../core/secrets-ensure');
+  await secretsEnsure.ensureInfraSecrets();
+  const { loadSecrets } = require('../core/secrets');
+  const secrets = await loadSecrets(undefined);
+  return { container, adminPassword, secrets };
+}
+
+/**
+ * Recreate platform DB users and databases from resolved kv secrets.
+ *
+ * @async
+ * @returns {Promise<string[]>} database names ensured
+ */
+async function bootstrapPlatformPostgresDatabases() {
+  const { container, adminPassword, secrets } = await resolveBootstrapContext();
+  const ensured = [];
+  for (const slot of PLATFORM_DATABASE_SLOTS) {
+    ensured.push(await ensurePlatformDatabaseSlot(container, slot, secrets, adminPassword));
+  }
+  return ensured;
+}
+
+module.exports = {
+  PLATFORM_DATABASE_SLOTS,
+  bootstrapPlatformPostgresDatabases,
+  pgRoleName,
+  extractPasswordFromUrlOrValue,
+  escapeShellDoubleQuoted
+};

package/lib/utils/postgres-wipe.js

@@ -22,6 +22,7 @@ const dockerExec = require('./docker-exec');
 const logger = require('./logger');
 const chalk = require('chalk');
 const { successGlyph } = require('./cli-test-layout-chalk');
+const { isSetupQuietOutput } = require('./setup-quiet-output');
 
 /** Roles that must never be dropped (Postgres-managed predefined roles). */
 const PROTECTED_ROLES = new Set([
@@ -190,15 +191,21 @@ async function wipePostgresData() {
   }
 
   const databases = await listUserDatabases(container, password);
+  const quiet = isSetupQuietOutput();
+
   for (const dbName of databases) {
     await dropDatabase(container, dbName, password);
-    logger.log(chalk.gray(` ${successGlyph()} Dropped database ${dbName}`));
+    if (!quiet) {
+      logger.log(chalk.gray(` ${successGlyph()} Dropped database ${dbName}`));
+    }
   }
 
   const roles = await listDropableRoles(container, password);
   for (const role of roles) {
     await dropRole(container, role, password);
-    logger.log(chalk.gray(` ${successGlyph()} Dropped role ${role}`));
+    if (!quiet) {
+      logger.log(chalk.gray(` ${successGlyph()} Dropped role ${role}`));
+    }
   }
 
   return { databases, roles };

package/lib/utils/resolve-env-mode.js

@@ -0,0 +1,47 @@
+/**
+ * Chooses `local` vs `docker` for `generateEnvFile` / `resolve` (integration → local, no Docker rewrite).
+ *
+ * @fileoverview Resolve environment mode for env generation
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const path = require('path');
+
+/** Platform builder apps that run in Docker and need docker host/port transforms on resolve. */
+const PLATFORM_DOCKER_APP_KEYS = new Set(['keycloak', 'miso-controller', 'dataplane']);
+
+/**
+ * @param {string} appPath - Resolved app directory from getResolveAppPath
+ * @returns {boolean}
+ */
+function isIntegrationAppPath(appPath) {
+  if (!appPath || typeof appPath !== 'string') return false;
+  const normalized = path.resolve(appPath);
+  const marker = `${path.sep}integration${path.sep}`;
+  return normalized.includes(marker);
+}
+
+/**
+ * @param {string} appPath - Resolved application directory
+ * @param {string} [appName] - App or system key
+ * @returns {'local'|'docker'}
+ */
+function resolveGenerateEnvEnvironment(appPath, appName) {
+  if (isIntegrationAppPath(appPath)) {
+    return 'local';
+  }
+  const key = String(appName || '').trim().toLowerCase();
+  if (PLATFORM_DOCKER_APP_KEYS.has(key)) {
+    return 'docker';
+  }
+  return 'docker';
+}
+
+module.exports = {
+  PLATFORM_DOCKER_APP_KEYS,
+  isIntegrationAppPath,
+  resolveGenerateEnvEnvironment
+};

package/lib/utils/secrets-generator.js

@@ -77,42 +77,44 @@ function loadYamlTolerantOfDuplicateKeys(content) {
 }
 
 /**
- * Skips commented or empty lines when scanning env.template
- * @param {string} line - Single line
+ * True when secrets map already has a non-empty value for a kv path (flat or nested).
+ * @param {Object} secrets
+ * @param {string} pathStr - Path after kv:// (e.g. hubspot-demo/token)
  * @returns {boolean}
  */
-function isCommentOrEmptyLine(line) {
-  const t = line.trim();
-  return t === '' || t.startsWith('#');
+function secretPathExistsInStore(secrets, pathStr) {
+  if (!secrets || typeof secrets !== 'object' || !pathStr) {
+    return false;
+  }
+  const direct = secrets[pathStr];
+  if (direct !== undefined && direct !== null && String(direct).trim() !== '') {
+    return true;
+  }
+  if (!pathStr.includes('/')) {
+    return false;
+  }
+  let current = secrets;
+  for (const segment of pathStr.split('/')) {
+    if (current === null || current === undefined || typeof current !== 'object' || !(segment in current)) {
+      return false;
+    }
+    current = current[segment];
+  }
+  return current !== undefined && current !== null && String(current).trim() !== '';
 }
 
 /**
- * Finds missing secret keys from template (skips commented and empty lines)
+ * Finds missing secret keys from template (skips commented and empty lines).
+ * Uses full kv:// paths (e.g. hubspot-demo/token), aligned with collectMissingSecrets.
  * @function findMissingSecretKeys
  * @param {string} envTemplate - Environment template content
  * @param {Object} existingSecrets - Existing secrets object
  * @returns {string[]} Array of missing secret keys
  */
 function findMissingSecretKeys(envTemplate, existingSecrets) {
-  const kvPattern = /kv:\/\/([a-zA-Z0-9-_]+)/g;
-  const missingKeys = [];
-  const seenKeys = new Set();
-  const lines = envTemplate.split('\n');
-
-  for (const line of lines) {
-    if (isCommentOrEmptyLine(line)) continue;
-    let match;
-    kvPattern.lastIndex = 0;
-    while ((match = kvPattern.exec(line)) !== null) {
-      const secretKey = match[1];
-      if (!seenKeys.has(secretKey) && !(secretKey in existingSecrets)) {
-        missingKeys.push(secretKey);
-        seenKeys.add(secretKey);
-      }
-    }
-  }
-
-  return missingKeys;
+  const { collectUniqueKvPathStrings } = require('./secrets-kv-refs');
+  const paths = collectUniqueKvPathStrings(envTemplate);
+  return paths.filter(pathStr => !secretPathExistsInStore(existingSecrets, pathStr));
 }
 
 /**
@@ -275,6 +277,8 @@ function loadExistingSecrets(resolvedPath) {
  * @throws {Error} If save fails
  */
 function saveSecretsFile(resolvedPath, secrets) {
+  const { assertWritableSecretsPathForTests } = require('./aifabrix-test-runtime-guard');
+  assertWritableSecretsPathForTests(resolvedPath);
   const dir = path.dirname(resolvedPath);
   if (!fs.existsSync(dir)) {
     fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
@@ -356,6 +360,8 @@ function appendSecretsToFile(resolvedPath, secrets) {
   if (!secrets || typeof secrets !== 'object' || Object.keys(secrets).length === 0) {
     return;
   }
+  const { assertWritableSecretsPathForTests } = require('./aifabrix-test-runtime-guard');
+  assertWritableSecretsPathForTests(resolvedPath);
   const dir = path.dirname(resolvedPath);
   if (!fs.existsSync(dir)) {
     fs.mkdirSync(dir, { recursive: true, mode: 0o700 });