@aifabrix/builder 2.46.0 → 2.47.0

package/lib/utils/paths.js

@@ -315,6 +315,30 @@ function getDevDirectory(appName, developerId) {
   return baseDir;
 }
 
+/**
+ * Target directory for `aifabrix create` — always under {@link process.cwd} (`builder/` or
+ * `integration/`) even when those folders do not exist yet. Platform system apps still
+ * materialize under {@link getAppsMaterializationParent}.
+ *
+ * Post-create lookup uses {@link getAppPath} / {@link getBuilderPath} / {@link getIntegrationPath}.
+ *
+ * @param {string} appName - Application name
+ * @param {string} [appType] - Application type (`external` or other)
+ * @returns {string} Absolute path to the new application directory
+ */
+function getAppCreatePath(appName, appType) {
+  if (!appName || typeof appName !== 'string') {
+    throw new Error('App name is required and must be a string');
+  }
+  if (appType === 'external') {
+    return path.join(path.resolve(process.cwd()), 'integration', appName);
+  }
+  if (isSystemBuilderAppName(appName)) {
+    return path.join(getAppsMaterializationParent(), 'builder', appName);
+  }
+  return path.join(path.resolve(process.cwd()), 'builder', appName);
+}
+
 /**
  * Gets the application path (builder or integration folder).
  * Matches {@link getBuilderPath} / {@link getIntegrationPath} (cwd `integration/` / `builder/`, then
@@ -881,6 +905,7 @@ module.exports = {
   getApplicationsBaseDir,
   getDevDirectory,
   getAppPath,
+  getAppCreatePath,
   getAppsMaterializationParent,
   getCwdIntegrationRoot,
   getCwdBuilderRoot,

package/lib/utils/platform-controller-url.js

@@ -18,6 +18,7 @@ const { loadConfigFile } = require('./config-format');
 const { computePublicUrlBaseString } = require('./url-declarative-public-base');
 const { joinUrlPath, normalizeFrontDoorPatternForHealth } = require('./health-check-url');
 const { getDevPorts } = require('./dev-config');
+const { flagsForSetupPlatformMode } = require('../core/setup-platform-mode');
 
 const MISO_CONTROLLER_APP = 'miso-controller';
 
@@ -74,7 +75,7 @@ function loadAppVariables(appName) {
 
 function computeLocalFullPlatformFrontDoorUrl(params) {
   const { setupPlatformMode, remoteServer, developerIdNum, frontDoorRouting } = params;
-  if (setupPlatformMode !== 'full') {
+  if (!flagsForSetupPlatformMode(setupPlatformMode).traefik) {
     return null;
   }
   if (remoteServer && String(remoteServer).trim()) {

package/lib/utils/platform-kv-ref.js

@@ -0,0 +1,57 @@
+/**
+ * Platform installation kv:// refs from infra.parameter.yaml (flat keys, no path segments).
+ *
+ * @fileoverview Distinguish platform catalog secrets from per-integration path-style kv refs
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const { getInfraParameterCatalog } = require('../parameters/infra-parameter-catalog');
+
+/**
+ * Returns the single-segment path after kv:// when the ref has no slashes.
+ *
+ * @param {string} val - kv:// reference
+ * @returns {string|null}
+ */
+function kvRefSegment(val) {
+  if (typeof val !== 'string' || !val.trim().toLowerCase().startsWith('kv://')) return null;
+  const after = val.trim().slice(5);
+  if (!after || after.includes('/')) return null;
+  return after;
+}
+
+/**
+ * True when val is kv://<key> and key exists in shipped infra.parameter.yaml.
+ *
+ * @param {string} val - kv:// reference
+ * @returns {boolean}
+ */
+function isPlatformCatalogKvRef(val) {
+  const segment = kvRefSegment(val);
+  if (!segment) return false;
+  try {
+    const catalog = getInfraParameterCatalog();
+    return Boolean(catalog.findEntryForKey(segment));
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * True when repair should normalize a kv value to path-style kv://systemKey/variable.
+ * Platform catalog refs (e.g. kv://secrets-openaiApiKeyVault) are never legacy.
+ *
+ * @param {string} val - Value from authentication.security or configuration
+ * @returns {boolean}
+ */
+function isLegacyKvValue(val) {
+  if (typeof val !== 'string' || !val.trim().toLowerCase().startsWith('kv://')) return false;
+  if (isPlatformCatalogKvRef(val)) return false;
+  const after = val.trim().slice(5);
+  return after.includes('KeyVault') || !after.includes('/');
+}
+
+module.exports = { kvRefSegment, isPlatformCatalogKvRef, isLegacyKvValue };

package/lib/utils/runtime-configuration-download.js

@@ -0,0 +1,189 @@
+/**
+ * Sync runtime configuration_values from dataplane on download (values → env.template; manifest → placeholders).
+ *
+ * @fileoverview Download-time runtime configuration sync
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+const fs = require('fs');
+const fsPromises = require('fs').promises;
+const path = require('path');
+const yaml = require('js-yaml');
+
+const {
+  getExternalSystemConfigurationResolved,
+  getExternalDataSourceConfigurationResolved
+} = require('../api/runtime-configuration.api');
+const {
+  retemplateAllVariableDefinitions,
+  retemplateDatasourceConfigurationOverrides
+} = require('./configuration-env-resolver');
+const { mergeEnvTemplateWithExisting } = require('../generator/split');
+
+/**
+ * @param {Object} response - Dataplane API client response
+ * @returns {Record<string, string>}
+ */
+function extractResolvedVariableValues(response) {
+  if (!response || !response.success) {
+    return {};
+  }
+  const body = response.data && typeof response.data === 'object' ? response.data : {};
+  const variables = body.variables && typeof body.variables === 'object' ? body.variables : {};
+  const out = {};
+  for (const [name, entry] of Object.entries(variables)) {
+    if (!entry || typeof entry.value !== 'string') {
+      continue;
+    }
+    const value = entry.value.trim();
+    if (!value || value.startsWith('kv://')) {
+      continue;
+    }
+    out[name] = value;
+  }
+  return out;
+}
+
+/**
+ * @param {string} dataplaneUrl
+ * @param {Object} authConfig
+ * @param {string} systemKey
+ * @param {Array<{ key?: string }>} dataSources
+ * @returns {Promise<Record<string, string>>}
+ */
+async function collectResolvedConfigurationEnvUpdates(
+  dataplaneUrl,
+  authConfig,
+  systemKey,
+  dataSources
+) {
+  const updates = {};
+  const systemResponse = await getExternalSystemConfigurationResolved(
+    dataplaneUrl,
+    systemKey,
+    authConfig
+  );
+  Object.assign(updates, extractResolvedVariableValues(systemResponse));
+
+  for (const ds of dataSources || []) {
+    if (!ds?.key) {
+      continue;
+    }
+    const dsResponse = await getExternalDataSourceConfigurationResolved(
+      dataplaneUrl,
+      ds.key,
+      authConfig
+    );
+    Object.assign(updates, extractResolvedVariableValues(dsResponse));
+  }
+  return updates;
+}
+
+/**
+ * @param {string} integrationPath
+ * @param {Record<string, string>} updates
+ * @returns {Promise<boolean>}
+ */
+async function mergeConfigurationUpdatesIntoEnvTemplate(integrationPath, updates) {
+  const keys = Object.keys(updates || {});
+  if (!keys.length) {
+    return false;
+  }
+  const envTemplatePath = path.join(integrationPath, 'env.template');
+  const expectedByKey = new Map();
+  for (const key of keys) {
+    expectedByKey.set(key, `${key}=${updates[key]}`);
+  }
+  const existingContent = fs.existsSync(envTemplatePath)
+    ? await fsPromises.readFile(envTemplatePath, 'utf8')
+    : '';
+  const merged = mergeEnvTemplateWithExisting(existingContent, expectedByKey);
+  await fsPromises.writeFile(envTemplatePath, merged, { mode: 0o644, encoding: 'utf8' });
+  return true;
+}
+
+/**
+ * @param {string} filePath
+ * @returns {Promise<boolean>}
+ */
+async function retemplateConfigurationFile(filePath, { system = false } = {}) {
+  if (!filePath || !fs.existsSync(filePath)) {
+    return false;
+  }
+  const raw = await fsPromises.readFile(filePath, 'utf8');
+  const isJson = filePath.endsWith('.json');
+  const parsed = isJson ? JSON.parse(raw) : yaml.load(raw);
+  if (!parsed || typeof parsed !== 'object') {
+    return false;
+  }
+  if (system) {
+    retemplateAllVariableDefinitions(parsed.configuration);
+  } else {
+    retemplateDatasourceConfigurationOverrides(parsed.configuration);
+  }
+  const next = isJson ? `${JSON.stringify(parsed, null, 2)}\n` : yaml.dump(parsed, { indent: 2, lineWidth: -1 });
+  await fsPromises.writeFile(filePath, next, { mode: 0o644, encoding: 'utf8' });
+  return true;
+}
+
+/**
+ * @param {string} integrationPath
+ * @param {string} systemKey
+ * @returns {Promise<void>}
+ */
+async function retemplateDownloadedConfigurationFiles(integrationPath, systemKey) {
+  for (const ext of ['yaml', 'json']) {
+    const systemPath = path.join(integrationPath, `${systemKey}-system.${ext}`);
+    if (fs.existsSync(systemPath)) {
+      await retemplateConfigurationFile(systemPath, { system: true });
+    }
+  }
+  const entries = await fsPromises.readdir(integrationPath);
+  for (const name of entries) {
+    if (!name.includes('-datasource-') || (!name.endsWith('.yaml') && !name.endsWith('.json'))) {
+      continue;
+    }
+    await retemplateConfigurationFile(path.join(integrationPath, name), { system: false });
+  }
+}
+
+/**
+ * Pulls configuration_values from dataplane, merges env.template, and re-templates manifests.
+ *
+ * @param {Object} params
+ * @param {string} params.dataplaneUrl
+ * @param {Object} params.authConfig
+ * @param {string} params.systemKey
+ * @param {string} params.integrationPath
+ * @param {Array<{ key?: string }>} params.dataSources
+ * @returns {Promise<{ merged: number, names: string[] }>}
+ */
+async function syncRuntimeConfigurationFromDataplane({
+  dataplaneUrl,
+  authConfig,
+  systemKey,
+  integrationPath,
+  dataSources
+}) {
+  const updates = await collectResolvedConfigurationEnvUpdates(
+    dataplaneUrl,
+    authConfig,
+    systemKey,
+    dataSources
+  );
+  const names = Object.keys(updates);
+  if (names.length) {
+    await mergeConfigurationUpdatesIntoEnvTemplate(integrationPath, updates);
+  }
+  await retemplateDownloadedConfigurationFiles(integrationPath, systemKey);
+  return { merged: names.length, names };
+}
+
+module.exports = {
+  extractResolvedVariableValues,
+  collectResolvedConfigurationEnvUpdates,
+  mergeConfigurationUpdatesIntoEnvTemplate,
+  retemplateDownloadedConfigurationFiles,
+  syncRuntimeConfigurationFromDataplane
+};

package/lib/utils/runtime-configuration-values.js

@@ -0,0 +1,119 @@
+/**
+ * Push runtime configuration_values to dataplane after publish (never logs values).
+ *
+ * @fileoverview Post-publish configuration PUT helpers
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+const {
+  putExternalSystemConfiguration,
+  putExternalDataSourceConfiguration
+} = require('../api/runtime-configuration.api');
+const { buildResolvedEnvMapForIntegration } = require('./configuration-env-resolver');
+
+/**
+ * @param {Array<{ name?: string, location?: string }>|undefined} systemConfiguration
+ * @param {Object.<string, string>} envMap
+ * @returns {Array<{ name: string, value: string }>}
+ */
+function collectSystemVariablePutRequests(systemConfiguration, envMap) {
+  if (!Array.isArray(systemConfiguration)) {
+    return [];
+  }
+  const requests = [];
+  for (const item of systemConfiguration) {
+    if (!item || !item.name) {
+      continue;
+    }
+    const location = String(item.location || 'variable').toLowerCase();
+    if (location !== 'variable') {
+      continue;
+    }
+    const name = String(item.name).trim();
+    const raw = envMap[name];
+    if (raw === undefined || raw === null) {
+      continue;
+    }
+    const value = String(raw).trim();
+    if (!value || value.startsWith('kv://') || value.includes('{{')) {
+      continue;
+    }
+    requests.push({ name, value });
+  }
+  return requests;
+}
+
+/**
+ * Datasource overrides are name+value only; values may already be resolved in upload payload.
+ *
+ * @param {Array<{ name?: string, value?: string }>|undefined} datasourceConfiguration
+ * @returns {Array<{ name: string, value: string }>}
+ */
+function collectDatasourceOverridePutRequests(datasourceConfiguration) {
+  if (!Array.isArray(datasourceConfiguration)) {
+    return [];
+  }
+  const requests = [];
+  for (const item of datasourceConfiguration) {
+    if (!item || !item.name) {
+      continue;
+    }
+    const name = String(item.name).trim();
+    const raw = item.value;
+    if (typeof raw !== 'string') {
+      continue;
+    }
+    const value = raw.trim();
+    if (!value || value.startsWith('kv://') || value.includes('{{')) {
+      continue;
+    }
+    requests.push({ name, value });
+  }
+  return requests;
+}
+
+/**
+ * Upserts system-scoped and datasource-scoped configuration values after publish.
+ *
+ * @param {string} dataplaneUrl
+ * @param {Object} authConfig
+ * @param {string} systemKey
+ * @param {{ application?: { configuration?: unknown[] }, dataSources?: Array<{ key?: string, configuration?: unknown[] }> }} payload
+ * @returns {Promise<{ pushed: number, names: string[] }>}
+ */
+async function pushRuntimeConfigurationValues(dataplaneUrl, authConfig, systemKey, payload) {
+  const { envMap } = await buildResolvedEnvMapForIntegration(systemKey);
+  const names = [];
+  let pushed = 0;
+
+  const systemRequests = collectSystemVariablePutRequests(
+    payload?.application?.configuration,
+    envMap
+  );
+  for (const body of systemRequests) {
+    await putExternalSystemConfiguration(dataplaneUrl, systemKey, authConfig, body);
+    names.push(body.name);
+    pushed += 1;
+  }
+
+  for (const ds of payload?.dataSources || []) {
+    if (!ds?.key) {
+      continue;
+    }
+    const dsRequests = collectDatasourceOverridePutRequests(ds.configuration);
+    for (const body of dsRequests) {
+      await putExternalDataSourceConfiguration(dataplaneUrl, ds.key, authConfig, body);
+      names.push(`${ds.key}:${body.name}`);
+      pushed += 1;
+    }
+  }
+
+  return { pushed, names };
+}
+
+module.exports = {
+  collectSystemVariablePutRequests,
+  collectDatasourceOverridePutRequests,
+  pushRuntimeConfigurationValues
+};

package/lib/utils/schema-loader.js

@@ -28,6 +28,23 @@ function resetValidators() {
   externalDataSourceValidator = null;
 }
 
+/**
+ * Register AJV schemas required to compile or validate external-datasource manifests.
+ * Includes type schemas and external-system (for $defs/subscriptionFilter cross-ref).
+ *
+ * @function registerExternalDatasourceAjvDependencies
+ * @param {import('ajv').default} ajv - AJV instance
+ */
+function registerExternalDatasourceAjvDependencies(ajv) {
+  ajv.addSchema(require('../schema/type/document-storage.json'));
+  ajv.addSchema(require('../schema/type/message-service.json'));
+  ajv.addSchema(require('../schema/type/vector-store.json'));
+  const externalSystemSchema = require('../schema/external-system.schema.json');
+  if (!ajv.getSchema(externalSystemSchema.$id)) {
+    ajv.addSchema(externalSystemSchema, externalSystemSchema.$id);
+  }
+}
+
 /**
  * Loads and compiles external-system schema
  * Caches the compiled validator for performance
@@ -115,10 +132,7 @@ function loadExternalDataSourceSchema() {
 
   const ajv = new Ajv({ allErrors: true, strict: false, strictSchema: false });
   addFormats(ajv);
-  // external-datasource.schema.json references these by $id (aifabrix://schema/type/*)
-  ajv.addSchema(require('../schema/type/document-storage.json'));
-  ajv.addSchema(require('../schema/type/message-service.json'));
-  ajv.addSchema(require('../schema/type/vector-store.json'));
+  registerExternalDatasourceAjvDependencies(ajv);
   externalDataSourceValidator = ajv.compile(schemaToCompile);
 
   return externalDataSourceValidator;
@@ -333,6 +347,7 @@ function detectSchemaType(filePath, content) {
 module.exports = {
   loadExternalSystemSchema,
   loadExternalDataSourceSchema,
+  registerExternalDatasourceAjvDependencies,
   detectSchemaType,
   detectSchemaTypeFromParsed,
   resetValidators

package/lib/utils/worker-sidecars.js

@@ -0,0 +1,142 @@
+/**
+ * Worker sidecar resolution from application.yaml build settings.
+ *
+ * @fileoverview Generic worker/beat companions driven by build.worker and imageRun* fields
+ */
+
+'use strict';
+
+const { buildDefaultLocalContainerName } = require('./environment-scoped-resources');
+
+/**
+ * @param {object|null|undefined} build
+ * @param {string} field
+ * @returns {string|null}
+ */
+function readImageRunCommand(build, field) {
+  if (!build || typeof build !== 'object') {
+    return null;
+  }
+  const raw = build[field];
+  if (typeof raw !== 'string') {
+    return null;
+  }
+  const trimmed = raw.trim();
+  return trimmed ? trimmed : null;
+}
+
+/**
+ * @param {object|null|undefined} appConfig
+ * @returns {boolean}
+ */
+function isWorkerStackEnabled(appConfig) {
+  if (!appConfig || typeof appConfig !== 'object') {
+    return false;
+  }
+  return appConfig.build?.worker === true;
+}
+
+/**
+ * @param {string} appKey
+ * @param {object|null|undefined} appConfig
+ * @returns {{
+ *   enabled: boolean,
+ *   worker: { key: string, command: string }|null,
+ *   beat: { key: string, command: string }|null,
+ *   companions: Array<{ role: string, key: string, command: string }>
+ * }}
+ */
+function resolveWorkerSidecars(appKey, appConfig) {
+  const empty = { enabled: false, worker: null, beat: null, companions: [] };
+  if (!isWorkerStackEnabled(appConfig)) {
+    return empty;
+  }
+
+  const build = appConfig.build && typeof appConfig.build === 'object' ? appConfig.build : {};
+  const workerCommand = readImageRunCommand(build, 'imageRunWorker');
+  const beatCommand = readImageRunCommand(build, 'imageRunBeat');
+  const companions = [];
+
+  if (workerCommand) {
+    companions.push({ role: 'worker', key: `${appKey}-worker`, command: workerCommand });
+  }
+  if (beatCommand) {
+    companions.push({ role: 'beat', key: `${appKey}-beat`, command: beatCommand });
+  }
+
+  if (companions.length === 0) {
+    return empty;
+  }
+
+  const worker = companions.find((entry) => entry.role === 'worker') || null;
+  const beat = companions.find((entry) => entry.role === 'beat') || null;
+
+  return {
+    enabled: true,
+    worker,
+    beat,
+    companions
+  };
+}
+
+/**
+ * @param {string} appKey
+ * @param {object|null|undefined} appConfig
+ * @returns {string[]}
+ */
+function getWorkerCompanionAppKeys(appKey, appConfig) {
+  return resolveWorkerSidecars(appKey, appConfig).companions.map((entry) => entry.key);
+}
+
+/**
+ * @param {string} appKey
+ * @param {object} appConfig
+ * @param {{ devId: string|number, idNum: number }} ids
+ * @returns {{
+ *   enabled: boolean,
+ *   worker?: { key: string, containerName: string, command: string },
+ *   beat?: { key: string, containerName: string, command: string }
+ * }}
+ */
+function buildWorkerSidecarComposeContext(appKey, appConfig, ids) {
+  const sidecars = resolveWorkerSidecars(appKey, appConfig);
+  if (!sidecars.enabled) {
+    return { enabled: false };
+  }
+
+  const context = { enabled: true };
+  if (sidecars.worker) {
+    context.worker = {
+      key: sidecars.worker.key,
+      containerName: buildDefaultLocalContainerName(sidecars.worker.key, ids.devId, ids.idNum),
+      command: sidecars.worker.command
+    };
+  }
+  if (sidecars.beat) {
+    context.beat = {
+      key: sidecars.beat.key,
+      containerName: buildDefaultLocalContainerName(sidecars.beat.key, ids.devId, ids.idNum),
+      command: sidecars.beat.command
+    };
+  }
+  return context;
+}
+
+/**
+ * Azure startupCommand from compose imageRun (drops leading exec).
+ *
+ * @param {string} imageRun
+ * @returns {string}
+ */
+function imageRunToStartupCommand(imageRun) {
+  return imageRun.replace(/^exec\s+/i, '').trim();
+}
+
+module.exports = {
+  readImageRunCommand,
+  isWorkerStackEnabled,
+  resolveWorkerSidecars,
+  getWorkerCompanionAppKeys,
+  buildWorkerSidecarComposeContext,
+  imageRunToStartupCommand
+};

package/lib/validation/external-manifest-validator.js

@@ -16,6 +16,7 @@ const path = require('path');
 const { formatValidationErrors } = require('../utils/error-formatter');
 const { validateFieldReferences } = require('../datasource/field-reference-validator');
 const { validateAbac } = require('../datasource/abac-validator');
+const { validateManifestRuntimeConfiguration } = require('./runtime-configuration-rules');
 
 /**
  * Load and normalize external-system + external-datasource schema objects for manifest AJV.
@@ -247,6 +248,7 @@ async function validateControllerManifest(manifest) {
   validateInlineSystem(manifest, ajv, externalSystemSchema, errors);
   validateDatasources(manifest, ajv, externalDatasourceSchema, errors, warnings);
   validateDatasourceSystemKeyAlignment(manifest, errors);
+  errors.push(...validateManifestRuntimeConfiguration(manifest));
   validateConditionalRequirements(manifest, errors, warnings);
   validateRequiredFields(manifest, errors);