@aifabrix/builder 2.46.0 → 2.47.1

package/lib/utils/build-copy.js

@@ -14,6 +14,11 @@ const fs = require('fs').promises;
 const fsSync = require('fs');
 const path = require('path');
 const paths = require('./paths');
+const { formatWarningLine } = require('./cli-test-layout-chalk');
+const logger = require('./logger');
+
+/** Builder `.env` is not copied into the dev workspace (plan 139: resolve in-memory; run uses `.env.run`). */
+const BUILD_WORKSPACE_SKIP_FILES = new Set(['.env']);
 
 /**
  * Copies all files from builder directory to developer-specific directory
@@ -81,8 +86,11 @@ async function copyDirectory(sourceDir, targetDir) {
   const entries = await fs.readdir(sourceDir);
 
   for (const entry of entries) {
-    // Skip hidden files and directories (but allow .env, .gitignore, etc.)
-    if (entry.startsWith('.') && entry !== '.env' && entry !== '.gitignore') {
+    if (BUILD_WORKSPACE_SKIP_FILES.has(entry)) {
+      continue;
+    }
+    // Skip hidden files and directories (allow .gitignore)
+    if (entry.startsWith('.') && entry !== '.gitignore') {
       continue;
     }
 
@@ -92,12 +100,35 @@ async function copyDirectory(sourceDir, targetDir) {
     const stats = await fs.stat(sourcePath);
 
     if (stats.isDirectory()) {
-      // Recursively copy subdirectories
       await copyDirectory(sourcePath, targetPath);
     } else if (stats.isFile()) {
-      // Copy file
-      await fs.copyFile(sourcePath, targetPath);
+      await copyFileToWorkspace(sourcePath, targetPath, entry);
+    }
+  }
+}
+
+/**
+ * Copy a single file into the dev workspace; skip unreadable files instead of failing the build.
+ * @async
+ * @param {string} sourcePath
+ * @param {string} targetPath
+ * @param {string} entryName
+ */
+async function copyFileToWorkspace(sourcePath, targetPath, entryName) {
+  try {
+    await fs.copyFile(sourcePath, targetPath);
+  } catch (err) {
+    const code = err && err.code ? String(err.code) : '';
+    if (code === 'EACCES' || code === 'EPERM') {
+      logger.log(
+        formatWarningLine(
+          `Skipped ${entryName} during workspace copy (permission denied). ` +
+            'Use "aifabrix resolve <app>" for a persistent .env when needed.'
+        )
+      );
+      return;
     }
+    throw err;
   }
 }
 

package/lib/utils/build-resolve-image.js

@@ -11,9 +11,10 @@
 const chalk = require('chalk');
 const logger = require('./logger');
 const { execWithDockerEnv } = require('./docker-exec');
+const { buildDevImageName } = require('./image-name');
 const {
-  resolveDockerImageRef,
-  getRepositoryPathFromConfig
+  getRepositoryPathFromConfig,
+  resolveDockerImageRef
 } = require('./resolve-docker-image-ref');
 
 /**
@@ -47,20 +48,44 @@ function repositoryTail(repoPath) {
   return parts[parts.length - 1] || repoPath;
 }
 
+/**
+ * Remove registry host prefix from a Docker repository path (local build only).
+ * @param {string} repository
+ * @returns {string}
+ */
+function stripRegistryHostFromRepository(repository) {
+  if (!repository || typeof repository !== 'string') {
+    return repository;
+  }
+  const parts = repository.split('/');
+  if (parts.length <= 1) {
+    return repository;
+  }
+  const host = parts[0];
+  if (host.includes('.') || host.includes(':') || host === 'localhost') {
+    return parts.slice(1).join('/');
+  }
+  return repository;
+}
+
 /**
  * Strip dev-scoped or -extra suffix from a Docker repository path.
  * @param {string} repo
  * @returns {string}
  */
 function baseRepositoryFromDevScoped(repo) {
-  const devMatch = repo.match(/^(.*)-dev\d+$/);
+  const localRepo = stripRegistryHostFromRepository(repo);
+  const tail = repositoryTail(localRepo);
+  const devMatch = tail.match(/^(.*)-dev\d+$/);
   if (devMatch) {
-    return devMatch[1];
+    const prefix = localRepo.slice(0, localRepo.length - tail.length);
+    return `${prefix}${devMatch[1]}`;
   }
-  if (repo.endsWith('-extra')) {
-    return repo.slice(0, -'-extra'.length);
+  if (tail.endsWith('-extra')) {
+    const prefix = localRepo.slice(0, localRepo.length - tail.length);
+    return `${prefix}${tail.slice(0, -'-extra'.length)}`;
   }
-  return repo;
+  return localRepo;
 }
 
 /**
@@ -140,7 +165,33 @@ async function discoverLocalBaseImageRepository(appName, developerId) {
 }
 
 /**
- * Repository path (may include registry prefix) used for `docker build -t`.
+ * Resolve local `docker build -t` repository path(s) from the manifest or discovery.
+ * When `application.yaml` declares `image`, the manifest ref is used exactly (registry + name).
+ * Developer-scoped `-devN` suffixes apply only when the manifest omits `image` and Docker is scanned.
+ *
+ * @param {string} appName
+ * @param {Object} variables - application manifest
+ * @param {string|number} developerId
+ * @param {Object} [options] - CLI build options (`base`, `tag`, `registry`)
+ * @returns {Promise<{ buildImageName: string, compatBaseName: string|null }>}
+ */
+async function resolveBuildDockerTagNames(appName, variables, developerId, options = {}) {
+  if (!wantsLocalDockerImageDiscovery(variables)) {
+    const { imageName } = resolveDockerImageRef(appName, variables, options);
+    return { buildImageName: imageName, compatBaseName: null };
+  }
+  const baseRepo = await resolveBuildImageRepositoryName(appName, variables, developerId);
+  const buildImageName = buildDevImageName(baseRepo, developerId);
+  const wantCompat =
+    buildImageName !== baseRepo && options && options.base === true;
+  return {
+    buildImageName,
+    compatBaseName: wantCompat ? baseRepo : null
+  };
+}
+
+/**
+ * Repository path for local `docker build -t` when manifest omits `image` (discovery / app key).
  * @param {string} appName
  * @param {Object} variables - application manifest
  * @param {string|number} developerId
@@ -148,18 +199,21 @@ async function discoverLocalBaseImageRepository(appName, developerId) {
  */
 async function resolveBuildImageRepositoryName(appName, variables, developerId) {
   if (!wantsLocalDockerImageDiscovery(variables)) {
-    return resolveDockerImageRef(appName, variables, {}).imageName;
+    return getRepositoryPathFromConfig(variables, appName);
   }
   const discovered = await discoverLocalBaseImageRepository(appName, developerId);
   if (discovered) {
-    logger.log(chalk.blue(`Using image repository from local Docker: ${discovered}`));
-    return discovered;
+    const localRepo = stripRegistryHostFromRepository(discovered);
+    logger.log(chalk.blue(`Using image repository from local Docker: ${localRepo}`));
+    return localRepo;
   }
   return getRepositoryPathFromConfig(variables, appName);
 }
 
 module.exports = {
+  resolveBuildDockerTagNames,
   resolveBuildImageRepositoryName,
   wantsLocalDockerImageDiscovery,
-  pickBaseFromDockerLines
+  pickBaseFromDockerLines,
+  stripRegistryHostFromRepository
 };

package/lib/utils/celery-companion-apps.js

@@ -0,0 +1,53 @@
+/**
+ * Worker companion helpers for run/logs/down lifecycle.
+ *
+ * @fileoverview Loads application.yaml and resolves worker sidecar app keys
+ */
+
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const yaml = require('js-yaml');
+const paths = require('./paths');
+const {
+  getWorkerCompanionAppKeys,
+  resolveWorkerSidecars
+} = require('./worker-sidecars');
+
+/**
+ * @param {string} appName
+ * @param {object|null|undefined} appConfig
+ * @returns {boolean}
+ */
+function isCeleryWorkerStackEnabled(appName, appConfig) {
+  return resolveWorkerSidecars(appName, appConfig).enabled;
+}
+
+/**
+ * @param {string} appName
+ * @param {object|null|undefined} appConfig
+ * @returns {string[]}
+ */
+function getCeleryCompanionAppKeys(appName, appConfig) {
+  return getWorkerCompanionAppKeys(appName, appConfig);
+}
+
+/**
+ * @param {string} appName
+ * @returns {object|null}
+ */
+function loadBuilderApplicationConfig(appName) {
+  const configPath = path.join(paths.getBuilderPath(appName), 'application.yaml');
+  if (!fs.existsSync(configPath)) {
+    return null;
+  }
+  return yaml.load(fs.readFileSync(configPath, 'utf8'));
+}
+
+module.exports = {
+  isCeleryWorkerStackEnabled,
+  getCeleryCompanionAppKeys,
+  loadBuilderApplicationConfig,
+  resolveWorkerSidecars
+};

package/lib/utils/cli-utils.js

@@ -215,6 +215,10 @@ function formatAzureError(errorMsg) {
       '   Use format: *.azurecr.io (e.g., myacr.azurecr.io)'
     ];
   }
+  // Docker build failures may mention azurecr.io in the local tag; show the real build error instead.
+  if (errorMsg.includes('Docker build failed') || errorMsg.includes('Build failed: Docker')) {
+    return null;
+  }
   // Only match ACR-specific authentication errors, not general authentication failures
   if (errorMsg.includes('ACR') || errorMsg.includes('azurecr.io') ||
       (errorMsg.includes('authenticate') && (errorMsg.includes('registry') || errorMsg.includes('container')))) {

package/lib/utils/compose-celery-worker.js

@@ -0,0 +1,23 @@
+/**
+ * Worker/beat compose fragment for local runs.
+ *
+ * @fileoverview Compose context for optional worker and beat services from application.yaml
+ */
+
+'use strict';
+
+const { buildWorkerSidecarComposeContext } = require('./worker-sidecars');
+
+/**
+ * @param {string} appKey
+ * @param {object} appConfig
+ * @param {{ devId: string|number, idNum: number }} ids
+ * @returns {ReturnType<typeof buildWorkerSidecarComposeContext>}
+ */
+function buildCeleryWorkerComposeContext(appKey, appConfig, ids) {
+  return buildWorkerSidecarComposeContext(appKey, appConfig, ids);
+}
+
+module.exports = {
+  buildCeleryWorkerComposeContext
+};

package/lib/utils/compose-generate-docker-compose.js

@@ -14,6 +14,7 @@ const buildCopy = require('./build-copy');
 const paths = require('./paths');
 const { getInfraDirName } = require('../infrastructure/helpers');
 const { resolveMisoEnvironment } = require('./compose-miso-env');
+const { buildCeleryWorkerComposeContext } = require('./compose-celery-worker');
 
 /**
  * When running a pulled Python image (no bind-mounted source), the image CMD may invoke the
@@ -149,7 +150,11 @@ function buildComposeTemplateContext(ctx) {
     devMountPath: ctx.devMountPath,
     reloadStartCommand: ctx.reloadStartCommand,
     infraPgpassPath: ctx.useInfraPgpass ? ctx.infraPgpassPath : null,
-    useInfraPgpass: !!ctx.useInfraPgpass
+    useInfraPgpass: !!ctx.useInfraPgpass,
+    workerSidecars: buildCeleryWorkerComposeContext(ctx.appName, ctx.appConfig, {
+      devId: ctx.devId,
+      idNum: ctx.idNum
+    })
   };
 }
 
@@ -303,6 +308,9 @@ async function generateDockerComposeImpl(deps, appName, appConfig, options) {
       ...layouts,
       ...side,
       idNum: head.idNum,
+      appName,
+      appConfig,
+      devId: head.devId,
       options
     })
   );

package/lib/utils/configuration-env-resolver.js

@@ -190,6 +190,40 @@ function retemplateConfigurationFromEnvTemplate(configArray, envTemplateVariable
   }
 }
 
+/**
+ * Re-templates every system variable definition to {{NAME}} (download/export safe).
+ *
+ * @param {Array<{ name?: string, location?: string }>} configArray - Configuration array (mutated)
+ */
+function retemplateAllVariableDefinitions(configArray) {
+  if (!Array.isArray(configArray)) return;
+  for (const item of configArray) {
+    if (!item || String(item.location || 'variable').toLowerCase() !== 'variable') continue;
+    const name = item.name && String(item.name).trim();
+    if (name) {
+      item.value = `{{${name}}}`;
+    }
+  }
+}
+
+/**
+ * Re-templates datasource override literals to {{NAME}} placeholders.
+ *
+ * @param {Array<{ name?: string, value?: string }>} configArray - Datasource configuration (mutated)
+ */
+function retemplateDatasourceConfigurationOverrides(configArray) {
+  if (!Array.isArray(configArray)) return;
+  for (const item of configArray) {
+    if (!item || !item.name) continue;
+    const name = String(item.name).trim();
+    const raw = item.value;
+    if (typeof raw !== 'string') continue;
+    const value = raw.trim();
+    if (!value || value.startsWith('kv://') || value.includes('{{')) continue;
+    item.value = `{{${name}}}`;
+  }
+}
+
 /**
  * Reads env.template at integration path, re-templates the given configuration array,
  * and returns the updated array (mutates in place). If env.template is missing, does nothing.
@@ -215,6 +249,8 @@ module.exports = {
   collectResolvedVariableConfigurationNames,
   getEnvTemplateVariableNames,
   retemplateConfigurationFromEnvTemplate,
+  retemplateAllVariableDefinitions,
+  retemplateDatasourceConfigurationOverrides,
   retemplateConfigurationForDownload,
   substituteVarPlaceholders
 };

package/lib/utils/controller-url.js

@@ -145,7 +145,10 @@ async function isControllerHealthReachable(controllerUrl) {
     return false;
   }
   try {
-    const res = await fetch(`${base}/health`, { signal: AbortSignal.timeout(2500) });
+    const { fetchWithOptionalDevPlatformTls } = require('./dev-platform-tls');
+    const res = await fetchWithOptionalDevPlatformTls(`${base}/health`, {
+      signal: AbortSignal.timeout(2500)
+    });
     return res.ok;
   } catch {
     return false;

package/lib/utils/credential-secrets-env-kv.js

@@ -0,0 +1,216 @@
+/**
+ * KV_* env key ↔ kv:// path mapping and payload reference scanning.
+ *
+ * @fileoverview KV path helpers for credential-secrets-env
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+const KV_PREFIX = 'KV_';
+const KV_REF_PATTERN = /kv:\/\/([a-zA-Z0-9_\-/]+)/g;
+
+/**
+ * Converts systemKey to KV_* prefix (e.g. hubspot -> HUBSPOT, my-hubspot -> MY_HUBSPOT).
+ * @param {string} systemKey - System key
+ * @returns {string}
+ */
+function systemKeyToKvPrefix(systemKey) {
+  if (!systemKey || typeof systemKey !== 'string') return '';
+  return systemKey.replace(/-/g, '_').toUpperCase();
+}
+
+/**
+ * Maps authentication security key (camelCase) to env VAR (UPPERCASE, no underscores).
+ * @param {string} securityKey - Security key (e.g. 'clientId', 'clientSecret')
+ * @returns {string}
+ */
+function securityKeyToVar(securityKey) {
+  if (!securityKey || typeof securityKey !== 'string') return '';
+  return securityKey.replace(/_/g, '').toUpperCase();
+}
+
+/** Known single-segment variable suffixes (uppercase) for inferring var vs namespace in env keys */
+const VAR_SUFFIXES = new Set(['ID', 'SECRET', 'KEY', 'TOKEN', 'URL', 'USERNAME', 'PASSWORD']);
+
+/**
+ * Converts var segment(s) from env key to path-style camelCase.
+ * @param {string[]} varSegments - One or two segments (e.g. ['CLIENT', 'ID'], ['CLIENTID'])
+ * @returns {string}
+ */
+function varSegmentsToCamelCase(varSegments) {
+  if (!varSegments || varSegments.length === 0) return '';
+  if (varSegments.length === 1) {
+    const s = varSegments[0].toLowerCase();
+    if (s.endsWith('id') && s.length > 2) return s.slice(0, -2) + 'Id';
+    if (s.endsWith('secret') && s.length > 6) return s.slice(0, -6) + 'Secret';
+    if (s.endsWith('key') && s.length > 3) return s.slice(0, -3) + 'Key';
+    if (s.endsWith('token') && s.length > 5) return s.slice(0, -5) + 'Token';
+    if (s.endsWith('url') && s.length > 3) return s.slice(0, -3) + 'Url';
+    return s;
+  }
+  return varSegments.map((seg, i) => {
+    const lower = seg.toLowerCase();
+    return i === 0 ? lower : lower.charAt(0).toUpperCase() + lower.slice(1);
+  }).join('');
+}
+
+/**
+ * @param {string[]} segments - Parsed segments after KV_ prefix
+ * @param {string} systemKey - System key (e.g. 'microsoft-teams')
+ * @returns {string|null}
+ */
+function kvPathWithSystemKey(segments, systemKey) {
+  const prefixInKey = systemKey.replace(/-/g, '_').toUpperCase();
+  const prefixSegs = prefixInKey.split('_').filter(Boolean);
+  if (segments.length <= prefixSegs.length) return null;
+  const prefixMatch = prefixSegs.every((p, i) => segments[i] === p);
+  if (!prefixMatch) return null;
+  const varSegments = segments.slice(prefixSegs.length);
+  const pathVar = varSegmentsToCamelCase(varSegments);
+  return pathVar ? `kv://${systemKey}/${pathVar}` : null;
+}
+
+/**
+ * @param {string[]} segments - Parsed segments after KV_ prefix
+ * @returns {string|null}
+ */
+function kvPathInferred(segments) {
+  if (segments.length === 1) return `kv://${segments[0].toLowerCase()}`;
+  const varSegmentCount = (segments.length >= 2 && VAR_SUFFIXES.has(segments[segments.length - 1])) ? 2 : 1;
+  const namespace = segments.slice(0, -varSegmentCount).map(s => s.toLowerCase()).join('-');
+  const varSegments = segments.slice(-varSegmentCount);
+  const pathVar = varSegmentsToCamelCase(varSegments);
+  return (namespace && pathVar) ? `kv://${namespace}/${pathVar}` : null;
+}
+
+/**
+ * @param {string} securityKey - Security key (e.g. 'apiKey', 'clientId')
+ * @returns {string}
+ */
+function getKvPathSegmentForSecurityKey(securityKey) {
+  if (!securityKey || typeof securityKey !== 'string') return '';
+  return varSegmentsToCamelCase([securityKeyToVar(securityKey)]);
+}
+
+/**
+ * @param {string} envKey - Env var name (e.g. KV_MICROSOFT_TEAMS_CLIENT_ID)
+ * @param {string} [systemKey] - Optional system key
+ * @returns {string|null}
+ */
+function kvEnvKeyToPath(envKey, systemKey) {
+  if (!envKey || typeof envKey !== 'string' || !envKey.toUpperCase().startsWith(KV_PREFIX)) return null;
+  const rest = envKey.slice(KV_PREFIX.length).trim();
+  if (!rest) return null;
+  const segments = rest.split('_').filter(Boolean);
+  if (segments.length === 0) return null;
+
+  const hasSystemKey = typeof systemKey === 'string' && systemKey.length > 0;
+  if (hasSystemKey) return kvPathWithSystemKey(segments, systemKey);
+  return kvPathInferred(segments);
+}
+
+/**
+ * @param {Object.<string, string>} envMap - Key-value map from .env
+ * @param {string} [systemKey]
+ * @returns {Array<{ key: string, value: string }>}
+ */
+function collectKvEnvVarsAsSecretItems(envMap, systemKey) {
+  if (!envMap || typeof envMap !== 'object') {
+    return [];
+  }
+  const items = [];
+  for (const [envKey, rawValue] of Object.entries(envMap)) {
+    const value = typeof rawValue === 'string' ? rawValue.trim() : '';
+    if (value === '') continue;
+    let kvPath = null;
+    if (value.startsWith('kv://') && isValidKvPath(value)) {
+      kvPath = value;
+    }
+    if (!kvPath) kvPath = kvEnvKeyToPath(envKey, systemKey) || kvEnvKeyToPath(envKey);
+    if (!kvPath) continue;
+    items.push({ key: kvPath, value });
+  }
+  return items;
+}
+
+/**
+ * @param {Object} secrets - Loaded secrets object
+ * @param {string} value - Value (may be "kv://..." or plain)
+ * @returns {string|null}
+ */
+function resolveKvValue(secrets, value) {
+  if (typeof value !== 'string') return null;
+  const trimmed = value.trim();
+  if (!trimmed.startsWith('kv://')) {
+    return trimmed;
+  }
+  const pathMatch = trimmed.match(/^kv:\/\/([a-zA-Z0-9_\-/]+)$/);
+  if (!pathMatch) return null;
+  const pathKey = pathMatch[1];
+  const resolved = secrets[pathKey];
+  if (resolved === undefined) return null;
+  return typeof resolved === 'string' ? resolved : String(resolved);
+}
+
+/**
+ * @param {Object|Array|string|number|boolean|null} obj
+ * @param {Set<string>} [acc]
+ * @returns {string[]}
+ */
+function collectKvRefsFromPayload(obj, acc = new Set()) {
+  if (obj === null || obj === undefined) {
+    return Array.from(acc);
+  }
+  if (typeof obj === 'string') {
+    let m;
+    KV_REF_PATTERN.lastIndex = 0;
+    while ((m = KV_REF_PATTERN.exec(obj)) !== null) {
+      acc.add(`kv://${m[1]}`);
+    }
+    return Array.from(acc);
+  }
+  if (Array.isArray(obj)) {
+    for (const item of obj) {
+      collectKvRefsFromPayload(item, acc);
+    }
+    return Array.from(acc);
+  }
+  if (typeof obj === 'object') {
+    for (const v of Object.values(obj)) {
+      collectKvRefsFromPayload(v, acc);
+    }
+    return Array.from(acc);
+  }
+  return Array.from(acc);
+}
+
+/**
+ * @param {string} key - kv:// path
+ * @returns {boolean}
+ */
+function isValidKvPath(key) {
+  return typeof key === 'string' && /^kv:\/\/[a-z0-9][a-z0-9\-/]*$/i.test(key.trim());
+}
+
+/**
+ * @param {unknown} resolved
+ * @returns {boolean}
+ */
+function isNonEmptySecretValue(resolved) {
+  if (resolved === null || resolved === undefined) {
+    return false;
+  }
+  return String(resolved).trim() !== '';
+}
+
+module.exports = {
+  collectKvEnvVarsAsSecretItems,
+  collectKvRefsFromPayload,
+  getKvPathSegmentForSecurityKey,
+  isNonEmptySecretValue,
+  isValidKvPath,
+  kvEnvKeyToPath,
+  resolveKvValue,
+  securityKeyToVar,
+  systemKeyToKvPrefix
+};