@aifabrix/builder 2.37.0 → 2.37.5

package/.cursor/rules/project-rules.mdc

@@ -70,6 +70,23 @@ lib/
 └── schema/             # JSON schemas
 ```
 
+### Generated Output (integration/ and builder/)
+
+Files under **integration/** and **builder/** are **auto-generated** by the CLI. When fixing bugs or changing behavior, validate **where** each file is produced so fixes go into the generator, not only into the generated artifact.
+
+- **integration/** – External system / wizard output:
+  - Path: `integration/<appName>/` (see `lib/utils/paths.js` → `getIntegrationPath`).
+  - Generated by: `lib/generator/wizard.js` (`generateWizardFiles`, `generateConfigFilesForWizard`), `lib/external-system/download.js`, wizard commands in `lib/commands/wizard-core.js` and `lib/commands/wizard.js`.
+  - Typical outputs: `variables.yaml`, `env.template`, `README.md`, `*-system.json`, `*-datasource*.json`, `*-deploy.json`, deploy script (`deploy.js`), and optionally `wizard.yaml`, `error.log`.
+- **builder/** – Application (non-external) output:
+  - Path: `builder/<appName>/` or custom root via `AIFABRIX_BUILDER_DIR` (see `lib/utils/paths.js` → `getBuilderPath`).
+  - Generated by: app create/register flow, `lib/generator/index.js`, `lib/commands/up-common.js`, `lib/core/secrets.js`, and related app/deploy logic.
+  - Typical outputs: `variables.yaml`, `env.template`, deploy JSON, `.env` (from secrets), and app-specific config.
+
+**Editable vs generated:**
+- Some generated files are **intended to be edited** (e.g. `variables.yaml`, `env.template`, `README.md`, `wizard.yaml`). Improvements to defaults or structure still belong in the generator/templates.
+- **When debugging:** First identify the **source of generation** (which module and function write the file). Fix bugs in that generator or template; avoid treating a one-off edit in integration/ or builder/ as the permanent fix unless it’s a deliberate local override.
+
 ### CLI Command Pattern
 Commands are defined in `lib/cli.js` using Commander.js:
 ```javascript
@@ -850,6 +867,7 @@ Define request/response types using JSDoc `@typedef`:
 
 ### Must Do (✅)
 - ✅ Validate all inputs (app names, file paths, URLs)
+- ✅ When fixing bugs in integration/ or builder/ output: identify the generator that produces the file and fix the source (lib/generator, lib/commands, templates), not only the generated artifact
 - ✅ Use try-catch for all async operations
 - ✅ Provide meaningful error messages with context
 - ✅ Use JSDoc for all public functions
@@ -874,6 +892,7 @@ Define request/response types using JSDoc `@typedef`:
 - ❌ Never use `eval()` or `Function()` constructor
 - ❌ Never use raw paths (always use path.join)
 - ❌ Never make direct API calls using `makeApiCall` in new code (use `lib/api/` modules)
+- ❌ Never treat one-off edits in integration/ or builder/ as the permanent fix for a bug—update the generator or template that produces the file
 - ❌ Never skip type definitions for API request/response types
 - ❌ Never log authentication tokens or secrets in API calls
 

package/integration/hubspot/test.js

@@ -511,7 +511,7 @@ async function checkAppDirectory(appPath) {
  * @throws {Error} If required files are missing
  */
 async function validateRequiredFiles(appPath, entries) {
-  const requiredFiles = ['variables.yaml', 'env.template', 'README.md', 'deploy.sh', 'deploy.ps1'];
+  const requiredFiles = ['variables.yaml', 'env.template', 'README.md', 'deploy.js'];
   const missingFiles = [];
   for (const fileName of requiredFiles) {
     const filePath = path.join(appPath, fileName);

package/lib/api/wizard.api.js

@@ -311,7 +311,7 @@ async function testMcpConnection(dataplaneUrl, authConfig, serverUrl, token) {
 }
 
 /**
- * Get deployment documentation for a system
+ * Get deployment documentation for a system (from dataplane DB only)
  * GET /api/v1/wizard/deployment-docs/{systemKey}
  * @async
  * @function getDeploymentDocs
@@ -326,6 +326,28 @@ async function getDeploymentDocs(dataplaneUrl, authConfig, systemKey) {
   return await client.get(`/api/v1/wizard/deployment-docs/${systemKey}`);
 }
 
+/**
+ * Generate deployment documentation with variables.yaml and deploy JSON for better quality
+ * POST /api/v1/wizard/deployment-docs/{systemKey}
+ * Sends deployJson and variablesYaml in the request body so the dataplane can align README with the integration folder.
+ * @async
+ * @function postDeploymentDocs
+ * @param {string} dataplaneUrl - Dataplane base URL
+ * @param {Object} authConfig - Authentication configuration
+ * @param {string} systemKey - System key identifier
+ * @param {Object} [body] - Optional request body (WizardDeploymentDocsRequest)
+ * @param {Object} [body.deployJson] - Deploy JSON object (e.g. *-deploy.json content)
+ * @param {string} [body.variablesYaml] - variables.yaml file content as string
+ * @returns {Promise<Object>} Deployment documentation response (content, contentType, systemKey)
+ * @throws {Error} If request fails
+ */
+async function postDeploymentDocs(dataplaneUrl, authConfig, systemKey, body = null) {
+  const client = new ApiClient(dataplaneUrl, authConfig);
+  return await client.post(`/api/v1/wizard/deployment-docs/${systemKey}`, {
+    body: body || {}
+  });
+}
+
 /**
  * Get known wizard platforms from dataplane.
  * GET /api/v1/wizard/platforms
@@ -365,5 +387,6 @@ module.exports = {
   getPreview,
   testMcpConnection,
   getDeploymentDocs,
+  postDeploymentDocs,
   getWizardPlatforms
 };

package/lib/app/deploy.js

@@ -15,7 +15,7 @@ const yaml = require('js-yaml');
 const chalk = require('chalk');
 const pushUtils = require('../deployment/push');
 const logger = require('../utils/logger');
-const { detectAppType } = require('../utils/paths');
+const { detectAppType, getBuilderPath, getIntegrationPath } = require('../utils/paths');
 const { checkApplicationExists } = require('../utils/app-existence');
 const { loadDeploymentConfig } = require('./deploy-config');
 
@@ -219,15 +219,16 @@ function displayDeploymentResults(result) {
 }
 
 /**
- * Check if app is external and handle external deployment
+ * Check if app is external and handle external deployment.
+ * When options.type === 'external', forces deployment from integration/<app> (no app register needed).
  * @async
  * @function handleExternalDeployment
  * @param {string} appName - Application name
- * @param {Object} options - Deployment options
+ * @param {Object} options - Deployment options (type: 'external' to force integration/<app>)
  * @returns {Promise<Object|null>} Deployment result if external, null otherwise
  */
 async function handleExternalDeployment(appName, options) {
-  const { isExternal } = await detectAppType(appName);
+  const { isExternal } = await detectAppType(appName, options);
   if (isExternal) {
     const externalDeploy = require('../external-system/deploy');
     await externalDeploy.deployExternalSystem(appName, options);
@@ -317,6 +318,37 @@ async function executeStandardDeployment(appName, options) {
   }
 }
 
+/**
+ * Tries external deploy when builder/<app> does not exist but integration/<app> does.
+ * @async
+ * @param {string} appName - Application name
+ * @param {Object} options - Deployment options
+ * @returns {Promise<{usedExternalDeploy: boolean, result: Object|null}>}
+ */
+async function tryExternalDeployFallback(appName, options) {
+  const builderPath = getBuilderPath(appName);
+  const integrationPath = getIntegrationPath(appName);
+  let builderExists = false;
+  let integrationExists = false;
+  try {
+    await fs.access(builderPath);
+    builderExists = true;
+  } catch (e) {
+    if (e.code !== 'ENOENT') throw e;
+  }
+  try {
+    await fs.access(integrationPath);
+    integrationExists = true;
+  } catch (e) {
+    if (e.code !== 'ENOENT') throw e;
+  }
+  if (!builderExists && integrationExists) {
+    const fallbackResult = await handleExternalDeployment(appName, { ...options, type: 'external' });
+    if (fallbackResult) return { usedExternalDeploy: true, result: fallbackResult };
+  }
+  return { usedExternalDeploy: false, result: null };
+}
+
 /**
  * Deploys application to Miso Controller
  * Orchestrates manifest generation, key creation, and deployment
@@ -338,7 +370,7 @@ async function executeStandardDeployment(appName, options) {
  */
 async function deployApp(appName, options = {}) {
   let controllerUrl = null;
-  let usedExternalDeploy = false;
+  let usedExternalDeploy = options.type === 'external';
 
   try {
     if (!appName || typeof appName !== 'string' || appName.trim().length === 0) {
@@ -347,8 +379,12 @@ async function deployApp(appName, options = {}) {
     validateAppName(appName);
 
     const externalResult = await handleExternalDeployment(appName, options);
-    if (externalResult) {
-      return externalResult;
+    if (externalResult) return externalResult;
+
+    const fallback = await tryExternalDeployFallback(appName, options);
+    if (fallback.result) {
+      usedExternalDeploy = fallback.usedExternalDeploy;
+      return fallback.result;
     }
     usedExternalDeploy = false;
 

package/lib/app/list.js

@@ -164,9 +164,11 @@ function displayApplications(applications, environment, controllerUrl) {
 
   logger.log(chalk.bold(`\n📱 ${header}:\n`));
   applications.forEach((app) => {
+    const isExternal = app.configuration?.type === 'external';
+    const externalIcon = isExternal ? '🔗 ' : '';
     const hasPipeline = app.configuration?.pipeline?.isActive ? '✓' : '✗';
     const urlAndPort = formatUrlAndPort(app);
-    logger.log(`${hasPipeline} ${chalk.cyan(app.key)} - ${app.displayName} (${app.status || 'unknown'})${urlAndPort}`);
+    logger.log(`${externalIcon}${hasPipeline} ${chalk.cyan(app.key)} - ${app.displayName} (${app.status || 'unknown'})${urlAndPort}`);
   });
   logger.log(chalk.gray('  To show details for an app: aifabrix app show <appKey>\n'));
 }

package/lib/build/index.js

@@ -200,7 +200,7 @@ async function postBuildTasks(appName, buildConfig) {
 }
 
 /**
- * Check if app is external type and handle accordingly
+ * External apps have no Docker image; deploy JSON is generated by aifabrix json.
  * @async
  * @param {string} appName - Application name
  * @returns {Promise<boolean>} True if external (handled), false if should continue
@@ -208,9 +208,8 @@ async function postBuildTasks(appName, buildConfig) {
 async function checkExternalAppType(appName) {
   const variables = await loadVariablesYaml(appName);
   if (variables.app && variables.app.type === 'external') {
-    const generator = require('../generator');
-    const jsonPath = await generator.generateDeployJson(appName);
-    logger.log(chalk.green(`✓ Generated deployment JSON: ${jsonPath}`));
+    logger.log(chalk.blue(`External system: ${appName}`));
+    logger.log(chalk.gray('To regenerate deployment JSON, run: aifabrix json ' + appName));
     return true;
   }
   return false;

package/lib/cli/setup-app.js

@@ -197,6 +197,7 @@ See integration/hubspot/wizard-hubspot-e2e.yaml for an example.`)
 
   program.command('deploy <app>')
     .description('Deploy to Azure via Miso Controller')
+    .option('--type <type>', 'Application type: external to deploy from integration/<app> (no app register needed)')
     .option('--client-id <id>', 'Client ID (overrides config)')
     .option('--client-secret <secret>', 'Client Secret (overrides config)')
     .option('--poll', 'Poll for deployment status', true)

package/lib/cli/setup-external-system.js

@@ -66,6 +66,7 @@ function setupExternalSystemCommands(program) {
     .description('Run integration tests via dataplane pipeline API')
     .option('-d, --datasource <key>', 'Test specific datasource only')
     .option('-p, --payload <file>', 'Path to custom test payload file')
+    .option('--dataplane <url>', 'Dataplane URL (default: discovered from controller)')
     .option('-v, --verbose', 'Show detailed test output')
     .option('--timeout <ms>', 'Request timeout in milliseconds', '30000')
     .action(async(appName, options) => {

package/lib/cli/setup-utility.js

@@ -87,7 +87,7 @@ function setupUtilityCommands(program) {
     });
 
   program.command('json <app>')
-    .description('Generate deployment JSON (aifabrix-deploy.json for normal apps, application-schema.json for external systems)')
+    .description('Generate deployment JSON to disk (<app>-deploy.json). Use before commit so version control has the correct file.')
     .option('--type <type>', 'Application type (external) - if set, only checks integration folder')
     .action(async(appName, options) => {
       try {

package/lib/commands/up-common.js

@@ -30,6 +30,36 @@ async function ensureTemplateAtPath(appName, targetAppPath) {
   return true;
 }
 
+/**
+ * Patches variables.yaml to set build.envOutputPath to null for deploy-only (no local code).
+ * Use when running up-miso/up-platform so we do not copy .env to repo paths or show that message.
+ * Patches both primary builder path and cwd/builder if different.
+ *
+ * @param {string} appName - Application name (e.g. miso-controller, dataplane)
+ */
+function patchEnvOutputPathForDeployOnly(appName) {
+  if (!appName || typeof appName !== 'string') return;
+  const pathsToPatch = [pathsUtil.getBuilderPath(appName)];
+  const cwdBuilderPath = path.join(process.cwd(), 'builder', appName);
+  if (path.resolve(cwdBuilderPath) !== path.resolve(pathsToPatch[0])) {
+    pathsToPatch.push(cwdBuilderPath);
+  }
+  const envOutputPathLine = /^(\s*envOutputPath:)\s*.*$/m;
+  const replacement = '$1 null  # deploy only, no copy';
+  for (const appPath of pathsToPatch) {
+    const variablesPath = path.join(appPath, 'variables.yaml');
+    if (!fs.existsSync(variablesPath)) continue;
+    try {
+      let content = fs.readFileSync(variablesPath, 'utf8');
+      if (!envOutputPathLine.test(content)) continue;
+      content = content.replace(envOutputPathLine, replacement);
+      fs.writeFileSync(variablesPath, content, 'utf8');
+    } catch (err) {
+      logger.warn(chalk.yellow(`Could not patch envOutputPath in ${variablesPath}: ${err.message}`));
+    }
+  }
+}
+
 /**
  * Ensures builder app directory exists from template if variables.yaml is missing.
  * If builder/<appName>/variables.yaml does not exist, copies from templates/applications/<appName>.
@@ -69,4 +99,4 @@ async function ensureAppFromTemplate(appName) {
   return primaryCopied;
 }
 
-module.exports = { ensureAppFromTemplate };
+module.exports = { ensureAppFromTemplate, patchEnvOutputPathForDeployOnly };

package/lib/commands/up-miso.js

@@ -19,7 +19,7 @@ const secrets = require('../core/secrets');
 const infra = require('../infrastructure');
 const app = require('../app');
 const { saveLocalSecret } = require('../utils/local-secrets');
-const { ensureAppFromTemplate } = require('./up-common');
+const { ensureAppFromTemplate, patchEnvOutputPathForDeployOnly } = require('./up-common');
 
 /** Keycloak base port (from templates/applications/keycloak/variables.yaml) */
 const KEYCLOAK_BASE_PORT = 8082;
@@ -132,6 +132,10 @@ async function handleUpMiso(options = {}) {
   await ensureAppFromTemplate('keycloak');
   await ensureAppFromTemplate('miso-controller');
   await ensureAppFromTemplate('dataplane');
+  // Deploy-only: do not copy .env to repo paths; patch variables so envOutputPath is null
+  patchEnvOutputPathForDeployOnly('keycloak');
+  patchEnvOutputPathForDeployOnly('miso-controller');
+  patchEnvOutputPathForDeployOnly('dataplane');
   const developerId = await config.getDeveloperId();
   const devIdNum = typeof developerId === 'string' ? parseInt(developerId, 10) : developerId;
   await setMisoSecretsAndResolve(devIdNum);

package/lib/commands/wizard-core.js

@@ -18,7 +18,8 @@ const {
   detectType,
   generateConfig,
   validateWizardConfig,
-  getDeploymentDocs
+  getDeploymentDocs,
+  postDeploymentDocs
 } = require('../api/wizard.api');
 const { generateWizardFiles } = require('../generator/wizard');
 const {
@@ -353,23 +354,47 @@ async function handleFileSaving(appName, systemConfig, datasourceConfigs, system
   logger.log(chalk.blue('\n\uD83D\uDCCB Step 7: Save Files'));
   const spinner = ora('Saving files...').start();
   try {
-    let aiGeneratedReadme = null;
-    if (systemKey && dataplaneUrl && authConfig) {
+    const generatedFiles = await generateWizardFiles(appName, systemConfig, datasourceConfigs, systemKey, { aiGeneratedReadme: null });
+    if (systemKey && dataplaneUrl && authConfig && generatedFiles.appPath) {
       try {
-        const docsResponse = await getDeploymentDocs(dataplaneUrl, authConfig, systemKey);
-        if (docsResponse.success && docsResponse.data?.content) aiGeneratedReadme = docsResponse.data.content;
+        const appPath = generatedFiles.appPath;
+        const deployKey = appName;
+        const variablesPath = path.join(appPath, 'variables.yaml');
+        const deployPath = path.join(appPath, `${deployKey}-deploy.json`);
+        let variablesYaml = null;
+        let deployJson = null;
+        try {
+          variablesYaml = await fs.readFile(variablesPath, 'utf8');
+        } catch {
+          // optional
+        }
+        try {
+          const deployContent = await fs.readFile(deployPath, 'utf8');
+          deployJson = JSON.parse(deployContent);
+        } catch {
+          // optional
+        }
+        const body = (variablesYaml !== null && variablesYaml !== undefined) || (deployJson !== null && deployJson !== undefined) ? { variablesYaml: variablesYaml || null, deployJson: deployJson || null } : null;
+        const docsResponse = body
+          ? await postDeploymentDocs(dataplaneUrl, authConfig, systemKey, body)
+          : await getDeploymentDocs(dataplaneUrl, authConfig, systemKey);
+        const content = docsResponse?.data?.content ?? docsResponse?.content;
+        if (content && typeof content === 'string') {
+          const readmePath = path.join(appPath, 'README.md');
+          await fs.writeFile(readmePath, content, 'utf8');
+          logger.log(chalk.gray('  Updated README.md from deployment-docs API (variables.yaml + deploy JSON).'));
+        }
       } catch (e) {
         logger.log(chalk.gray(`  Could not fetch AI-generated README: ${e.message}`));
       }
     }
-    const generatedFiles = await generateWizardFiles(appName, systemConfig, datasourceConfigs, systemKey, { aiGeneratedReadme });
     spinner.stop();
     logger.log(chalk.green('\n\u2713 Wizard completed successfully!'));
     logger.log(chalk.green(`\nFiles created in: ${generatedFiles.appPath}`));
     logger.log(chalk.blue('\nNext steps:'));
     logger.log(chalk.gray(`  1. Review the generated files in integration/${appName}/`));
     logger.log(chalk.gray('  2. Update env.template with your authentication details'));
-    logger.log(chalk.gray(`  3. Deploy using: ./deploy.sh or aifabrix deploy ${appName}`));
+    logger.log(chalk.gray(`  3. Deploy using: node deploy.js or aifabrix deploy ${appName}`));
     return generatedFiles;
   } catch (error) {
     spinner.stop();

package/lib/deployment/deployer-status.js

@@ -0,0 +1,101 @@
+/**
+ * Deployment status and polling helpers for deployer.
+ *
+ * @fileoverview Deployment status checks and polling utilities
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+const chalk = require('chalk');
+const logger = require('../utils/logger');
+
+/**
+ * Checks if deployment status is terminal
+ * @param {string} status - Deployment status
+ * @returns {boolean} True if status is terminal
+ */
+function isTerminalStatus(status) {
+  return status === 'completed' || status === 'failed' || status === 'cancelled';
+}
+
+/**
+ * Convert authConfig to pipeline auth config format
+ * @param {Object} authConfig - Authentication configuration
+ * @returns {Object} Pipeline auth config
+ */
+function convertToPipelineAuthConfig(authConfig) {
+  return authConfig.type === 'bearer'
+    ? { type: 'bearer', token: authConfig.token }
+    : { type: 'client-credentials', clientId: authConfig.clientId, clientSecret: authConfig.clientSecret };
+}
+
+/**
+ * Handles error response from deployment status check
+ * @param {Object} response - API response
+ * @param {string} deploymentId - Deployment ID
+ * @throws {Error} Appropriate error message
+ */
+function handleDeploymentStatusError(response, deploymentId) {
+  if (response.status === 404) {
+    throw new Error(`Deployment ${deploymentId || response.deploymentId || 'unknown'} not found`);
+  }
+  throw new Error(`Status check failed: ${response.formattedError || response.error || 'Unknown error'}`);
+}
+
+/**
+ * Extracts deployment data from response
+ * @param {Object} response - API response
+ * @returns {Object} Deployment data
+ */
+function extractDeploymentData(response) {
+  const responseData = response.data;
+  return responseData.data || responseData;
+}
+
+/**
+ * Logs deployment progress
+ * @param {Object} deploymentData - Deployment data
+ * @param {number} attempt - Current attempt
+ * @param {number} maxAttempts - Maximum attempts
+ */
+function logDeploymentProgress(deploymentData, attempt, maxAttempts) {
+  const status = deploymentData.status;
+  const progress = deploymentData.progress || 0;
+  logger.log(chalk.blue(`   Status: ${status} (${progress}%) (attempt ${attempt + 1}/${maxAttempts})`));
+}
+
+/**
+ * Process deployment status response
+ * @param {Object} response - API response
+ * @param {number} attempt - Current attempt number
+ * @param {number} maxAttempts - Maximum attempts
+ * @param {number} interval - Polling interval
+ * @param {string} deploymentId - Deployment ID for error messages
+ * @returns {Promise<Object|null>} Deployment data if terminal, null if needs to continue polling
+ */
+async function processDeploymentStatusResponse(response, attempt, maxAttempts, interval, deploymentId) {
+  if (!response.success || !response.data) {
+    handleDeploymentStatusError(response, deploymentId);
+  }
+
+  const deploymentData = extractDeploymentData(response);
+  if (isTerminalStatus(deploymentData.status)) {
+    return deploymentData;
+  }
+
+  logDeploymentProgress(deploymentData, attempt, maxAttempts);
+  if (attempt < maxAttempts - 1) {
+    await new Promise(resolve => setTimeout(resolve, interval));
+  }
+
+  return null;
+}
+
+module.exports = {
+  isTerminalStatus,
+  convertToPipelineAuthConfig,
+  handleDeploymentStatusError,
+  extractDeploymentData,
+  logDeploymentProgress,
+  processDeploymentStatusResponse
+};