@aifabrix/builder 2.43.0 → 2.44.1

package/lib/certification/sync-system-certification.js

@@ -0,0 +1,197 @@
+/**
+ * @fileoverview Sync `certification` on *-system.json|yaml from dataplane active certificate(s).
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+'use strict';
+
+const path = require('path');
+const chalk = require('chalk');
+const logger = require('../utils/logger');
+const { getIntegrationPath } = require('../utils/paths');
+const { discoverIntegrationFiles } = require('../commands/repair-internal');
+const { loadConfigFile, writeConfigFile } = require('../utils/config-format');
+const { unwrapApiData } = require('../utils/external-system-readiness-core');
+const { getActiveIntegrationCertificate } = require('../api/certificates.api');
+const {
+  buildCertificationFromArtifact,
+  pickArtifactForCertificationMerge
+} = require('./merge-certification-from-artifact');
+
+/**
+ * @param {string} systemKey
+ * @returns {{ systemFilePath: string }|null}
+ */
+function resolvePrimarySystemFilePath(systemKey) {
+  const appPath = getIntegrationPath(systemKey);
+  const { systemFiles } = discoverIntegrationFiles(appPath);
+  if (!systemFiles || systemFiles.length === 0) return null;
+  return { systemFilePath: path.join(appPath, systemFiles[0]) };
+}
+
+/**
+ * @async
+ * @param {Object} ctx
+ * @returns {Promise<Array<import('../api/types/certificates.types').CertificateArtifactResponse>>}
+ */
+async function collectActiveArtifacts(ctx) {
+  const { dataplaneUrl, authConfig, systemKey, datasourceKeys } = ctx;
+  const out = [];
+  for (const dk of datasourceKeys || []) {
+    if (!dk || typeof dk !== 'string') continue;
+    try {
+      const res = await getActiveIntegrationCertificate(dataplaneUrl, authConfig, systemKey, dk);
+      if (res && res.success === false) continue;
+      const art = unwrapApiData(res);
+      if (art && typeof art === 'object') out.push(art);
+    } catch {
+      /* skip datasource */
+    }
+  }
+  return out;
+}
+
+/**
+ * @param {string} systemFilePath
+ * @returns {Object|null}
+ */
+function readSystemObject(systemFilePath) {
+  try {
+    return loadConfigFile(systemFilePath);
+  } catch (e) {
+    logger.log(chalk.yellow(`⚠ Certification sync: could not read system file: ${e.message}`));
+    return null;
+  }
+}
+
+/**
+ * @param {string} systemFilePath
+ * @param {Object} systemObj
+ * @param {Object} nextCert
+ * @returns {{ ok: true } | { ok: false }}
+ */
+function tryWriteSystemCertification(systemFilePath, systemObj, nextCert) {
+  try {
+    writeConfigFile(systemFilePath, { ...systemObj, certification: nextCert });
+    return { ok: true };
+  } catch (e) {
+    logger.log(chalk.yellow(`⚠ Certification sync: could not write system file: ${e.message}`));
+    return { ok: false };
+  }
+}
+
+/**
+ * @param {Object|null} chosen
+ * @param {'no_active'|'no_public_key'} detail
+ */
+function logSkippedCertification(chosen, detail) {
+  if (detail === 'no_active' || !chosen) {
+    logger.log(
+      chalk.yellow(
+        '⚠ Certification not written: dataplane has no active trusted certificate for this system/datasource scope yet. ' +
+          'Run a successful validation or E2E that issues a certificate, then upload or run cert sync again.'
+      )
+    );
+    return;
+  }
+  logger.log(
+    chalk.yellow(
+      '⚠ Certification not written: active certificate has no publicKey and your system file has none to merge. ' +
+        'The dataplane must return publicKey (or add certification.publicKey locally once) to satisfy the schema.'
+    )
+  );
+}
+
+/**
+ * Read system file, merge **certification** from dataplane, write back. Does not modify other top-level keys.
+ * @async
+ * @param {Object} params
+ * @param {string} params.systemKey
+ * @param {string} params.dataplaneUrl
+ * @param {Object} params.authConfig
+ * @param {string[]} params.datasourceKeys
+ * @returns {Promise<{ written: boolean, reason?: string }>}
+ */
+async function syncSystemCertificationFromDataplane(params) {
+  const { systemKey, dataplaneUrl, authConfig, datasourceKeys } = params;
+  const resolved = resolvePrimarySystemFilePath(systemKey);
+  if (!resolved) return { written: false, reason: 'no_system_file' };
+  if (!dataplaneUrl || !authConfig || !authConfig.token) {
+    return { written: false, reason: 'no_auth' };
+  }
+
+  const systemObj = readSystemObject(resolved.systemFilePath);
+  if (!systemObj || typeof systemObj !== 'object') {
+    return { written: false, reason: 'invalid_system' };
+  }
+
+  const existing = systemObj.certification;
+  const artifacts = await collectActiveArtifacts({
+    dataplaneUrl,
+    authConfig,
+    systemKey,
+    datasourceKeys
+  });
+  const chosen = pickArtifactForCertificationMerge(artifacts);
+  const nextCert = buildCertificationFromArtifact(chosen, existing);
+  if (!nextCert) {
+    const detail = chosen ? 'no_public_key' : 'no_active';
+    logSkippedCertification(chosen, detail);
+    return { written: false, reason: 'incomplete_certification', detail };
+  }
+
+  const writeResult = tryWriteSystemCertification(resolved.systemFilePath, systemObj, nextCert);
+  if (!writeResult.ok) return { written: false, reason: 'write_error' };
+  return { written: true };
+}
+
+/**
+ * Non-throwing entry for upload/deploy: failures are warnings only.
+ * @async
+ * @param {Object} params
+ * @param {string} [params.label] - e.g. "upload"
+ * @param {boolean} [params.noCertSync]
+ * @returns {Promise<void>}
+ */
+function logCertificationSyncNotWritten(r, label) {
+  const prefix = label ? ` (${label})` : '';
+  const map = {
+    no_system_file: `No *-system* file found under integration folder for this key${prefix}.`,
+    no_auth: `No Bearer token for dataplane${prefix}; run aifabrix login.`,
+    invalid_system: `Could not read or parse the primary system file${prefix}.`,
+    incomplete_certification: 'Could not build certification (see message above).',
+    write_error: 'Write to system file failed (see message above).'
+  };
+  const msg = (r && r.reason && map[r.reason]) || `Certification block not updated${prefix}.`;
+  logger.log(chalk.yellow(`⚠ ${msg}`));
+}
+
+async function maybeSyncSystemCertificationFromDataplane(params) {
+  const { label, noCertSync, systemKey, dataplaneUrl, authConfig, datasourceKeys } = params;
+  if (noCertSync === true) return;
+  try {
+    const r = await syncSystemCertificationFromDataplane({
+      systemKey,
+      dataplaneUrl,
+      authConfig,
+      datasourceKeys
+    });
+    if (r.written) {
+      logger.log(
+        chalk.gray(`Updated certification block from dataplane${label ? ` (${label})` : ''} in system file.`)
+      );
+    } else if (r.reason) {
+      logCertificationSyncNotWritten(r, label);
+    }
+  } catch (e) {
+    logger.log(chalk.yellow(`⚠ Certification sync skipped: ${e.message}`));
+  }
+}
+
+module.exports = {
+  syncSystemCertificationFromDataplane,
+  maybeSyncSystemCertificationFromDataplane,
+  resolvePrimarySystemFilePath,
+  collectActiveArtifacts
+};

package/lib/cli/index.js

@@ -17,6 +17,7 @@ const { setupEnvironmentCommands } = require('./setup-environment');
 const { setupUtilityCommands } = require('./setup-utility');
 const { setupDevCommands } = require('./setup-dev');
 const { setupSecretsCommands } = require('./setup-secrets');
+const { setupParametersCommands } = require('./setup-parameters');
 const { setupExternalSystemCommands } = require('./setup-external-system');
 const { setupAppCommands: setupAppManagementCommands } = require('../commands/app');
 const { setupDatasourceCommands } = require('../commands/datasource');
@@ -40,6 +41,7 @@ function setupCommands(program) {
   setupExternalSystemCommands(program);
   setupDevCommands(program);
   setupSecretsCommands(program);
+  setupParametersCommands(program);
 }
 
 module.exports = {

package/lib/cli/setup-app.help.js

@@ -0,0 +1,67 @@
+/**
+ * @fileoverview Help text blocks for app-level CLI commands.
+ */
+
+'use strict';
+
+const TEST_HELP_AFTER = `
+Examples:
+  # External system (integration/<systemKey>/) — local validation
+  $ aifabrix test hubspot
+  $ aifabrix test hubspot -v
+  $ aifabrix test hubspot -d
+
+  # Builder app (builder/<app>/) — runs in container
+  $ aifabrix test myapp -e dev
+  $ aifabrix test myapp -e tst
+
+Notes:
+  - To run unit test for one datasource, use:
+      aifabrix datasource test <datasourceKey>
+  - To run integration test, use:
+      aifabrix test-integration <app>
+  - Option --sync is not supported here (local validation only); use upload or dataplane test commands with --sync.
+`;
+
+const TEST_INTEGRATION_HELP_AFTER = `
+Examples:
+  # External system (integration/<systemKey>/) — integration health across datasources via dataplane
+  $ aifabrix test-integration hubspot
+  $ aifabrix test-integration hubspot -v
+  $ aifabrix test-integration hubspot -d
+
+  # Builder app (builder/<app>/) — runs in container
+  $ aifabrix test-integration myapp -e dev
+  $ aifabrix test-integration myapp -e tst
+
+Notes:
+  - To run integration test for one datasource, use:
+      aifabrix datasource test-integration <datasourceKey>
+  - To run E2E test, use:
+      aifabrix test-e2e <app>
+  - Optional --sync publishes local files to the dataplane first (external integration under integration/<systemKey>/ only).
+`;
+
+const TEST_E2E_HELP_AFTER = `
+Examples:
+  # External system (integration/<systemKey>/) — E2E across datasources via dataplane
+  $ aifabrix test-e2e hubspot
+  $ aifabrix test-e2e hubspot -v
+  $ aifabrix test-e2e hubspot -d
+
+  # Builder app (builder/<app>/) — runs in container
+  $ aifabrix test-e2e myapp -e dev
+  $ aifabrix test-e2e myapp -e tst
+
+Notes:
+  - To run E2E for one datasource, use:
+      aifabrix datasource test-e2e <datasourceKey>
+  - Optional --sync publishes local files to the dataplane first (external integration only).
+`;
+
+module.exports = {
+  TEST_HELP_AFTER,
+  TEST_INTEGRATION_HELP_AFTER,
+  TEST_E2E_HELP_AFTER
+};
+

package/lib/cli/setup-app.js

@@ -1,3 +1,4 @@
+const { formatSuccessParagraph } = require('../utils/cli-test-layout-chalk');
 /**
  * CLI application lifecycle command setup (create, wizard, build, run, push, deploy, dockerfile).
  *
@@ -11,6 +12,24 @@ const path = require('path');
 const app = require('../app');
 const logger = require('../utils/logger');
 const { handleCommandError } = require('../utils/cli-utils');
+const { setupInstallTestE2eLintCommands } = require('./setup-app.test-commands');
+
+const CREATE_HELP_AFTER = `
+Examples:
+  $ aifabrix create myapi --type webapp -l typescript
+  $ aifabrix create mycrm --wizard
+`;
+
+const DEPLOY_HELP_AFTER = `
+Examples:
+  $ aifabrix deploy myapp
+  $ aifabrix deploy myext --local
+`;
+
+const PUSH_HELP_AFTER = `
+Example:
+  $ aifabrix push myapp -t v1.0.0
+`;
 
 /**
  * Normalize options for external system creation
@@ -102,7 +121,8 @@ async function handleCreateCommand(appName, options) {
 
 function setupCreateCommand(program) {
   program.command('create <app>')
-    .description('Create new application with configuration files')
+    .description('Scaffold builder or external app (flags or --wizard)')
+    .addHelpText('after', CREATE_HELP_AFTER)
     .option('-p, --port <port>', 'Application port', '3000')
     .option('-d, --database', 'Requires database')
     .option('-r, --redis', 'Requires Redis')
@@ -142,15 +162,15 @@ Examples:
   $ aifabrix wizard --config wizard.yaml  Run headless from a wizard config file
   $ aifabrix wizard hubspot-test --debug  Enable debug output and save debug manifests on validation failure
 
-Config path: When appName is provided, integration/<appName>/wizard.yaml is used for load/save and error.log.
+Config path: When appName is provided, integration/<systemKey>/wizard.yaml is used for load/save and error.log.
 To change settings after a run, edit that file and run "aifabrix wizard <app>" again.
 Headless config must include: appName, mode (create-system|add-datasource), source (type + filePath/url/platform).
 See integration/hubspot-test/wizard-hubspot-e2e.yaml for an example.`;
   program.command('wizard [appName]')
-    .description('Create or extend external systems (OpenAPI, MCP, or known platforms like HubSpot) via guided steps or a config file')
+    .description('Guided external system setup (OpenAPI, MCP, HubSpot, …) or headless wizard.yaml')
     .option('-a, --app <app>', 'Application name (synonym for positional appName)')
     .option('--config <file>', 'Run headless using a wizard.yaml file (appName, mode, source, credential, preferences)')
-    .option('--silent', 'Run with saved integration/<app>/wizard.yaml only; no prompts (requires app name and existing wizard.yaml)')
+    .option('--silent', 'Run with saved integration/<systemKey>/wizard.yaml only; no prompts (requires app name and existing wizard.yaml)')
     .option('--debug', 'Enable debug output and save debug manifests on validation failure')
     .addHelpText('after', wizardHelp)
     .action(async(positionalAppName, options) => {
@@ -172,9 +192,10 @@ In dev: use --reload for sync and mount (requires remote server with Mutagen, or
 Examples:
   $ aifabrix run myapp
   $ aifabrix run myapp --env tst
+  $ aifabrix run myapp --tag v1.0.0
   $ aifabrix run myapp --reload`;
   program.command('run <app>')
-    .description('Run application locally (or remotely on your Docker host)')
+    .description('Run app locally or on remote Docker host')
     .option('-p, --port <port>', 'Override local port')
     .option('-d, --debug', 'Enable debug output with detailed container information')
     .option('-t, --tag <tag>', 'Image tag to run (e.g. v1.0.0); overrides application.yaml image.tag')
@@ -193,14 +214,15 @@ Examples:
 
 function setupBuildRunLogsDownCommands(program) {
   program.command('build <app>')
-    .description('Build container image (auto-detects runtime)')
+    .description('Build Docker image (auto-detect runtime)')
     .option('-l, --language <lang>', 'Override language detection')
     .option('-f, --force-template', 'Force rebuild from template')
+    .option('--no-cache', 'Full Docker rebuild (disable layer cache); use after Dockerfile or context fixes')
     .option('-t, --tag <tag>', 'Image tag (default: latest). Set image.tag in application.yaml to match for deploy.')
     .action(async(appName, options) => {
       try {
         const imageTag = await app.buildApp(appName, options);
-        logger.log(`✅ Built image: ${imageTag}`);
+        logger.log(`✔ Built image: ${imageTag}`);
       } catch (error) {
         handleCommandError(error, 'build');
         process.exit(1);
@@ -210,7 +232,7 @@ function setupBuildRunLogsDownCommands(program) {
   registerRunCommand(program);
 
   program.command('logs <app>')
-    .description('Show application container logs (and optional env summary with secrets masked)')
+    .description('Tail app container logs (optional env summary; secrets masked)')
     .option('-f', 'Follow log stream')
     .option('-t, --tail <lines>', 'Number of lines (default: 100); 0 = full list', '100')
     .option('-l, --level <level>', 'Show only logs at this level or above (debug|info|warn|error)')
@@ -227,7 +249,7 @@ function setupBuildRunLogsDownCommands(program) {
     });
 
   program.command('down-app <app>')
-    .description('Stop and remove application container; optionally remove volume and image')
+    .description('Stop and remove app container (--volumes removes data volume)')
     .option('--volumes', 'Remove application Docker volume')
     .action(async(appName, options) => {
       try {
@@ -242,7 +264,7 @@ function setupBuildRunLogsDownCommands(program) {
 
 function setupShellTestStopCommands(program) {
   program.command('stop <app>')
-    .description('Stop and remove application container (alias for down-app)')
+    .description('Alias for down-app: stop and remove container')
     .option('--volumes', 'Remove application Docker volume')
     .action(async(appName, options) => {
       try {
@@ -255,7 +277,7 @@ function setupShellTestStopCommands(program) {
     });
 
   program.command('shell <app>')
-    .description('Open interactive shell in the application container')
+    .description('Interactive shell in running or ephemeral container')
     .option('--env <env>', 'Environment (dev|tst); dev uses running container', 'dev')
     .action(async(appName, options) => {
       try {
@@ -266,105 +288,20 @@ function setupShellTestStopCommands(program) {
         process.exit(1);
       }
     });
-
-  program.command('test <app>')
-    .description('Run tests (builder app: in container; external system: local validation)')
-    .option('--env <env>', 'For builder app: dev (running container) or tst (ephemeral)', 'dev')
-    .option('-d, --datasource <key>', 'For external system: test specific datasource only')
-    .option('-v, --verbose', 'Verbose output')
-    .action(async(appName, options) => {
-      try {
-        const pathsUtil = require('../utils/paths');
-        const appType = await pathsUtil.detectAppType(appName).catch(() => null);
-        if (appType && appType.baseDir === 'integration') {
-          const test = require('../external-system/test');
-          const results = await test.testExternalSystem(appName, options);
-          test.displayTestResults(results, options.verbose);
-          if (!results.valid) process.exit(1);
-        } else {
-          const { runAppTest } = require('../commands/app-test');
-          await runAppTest(appName, { env: options.env });
-        }
-      } catch (error) {
-        handleCommandError(error, 'test');
-        process.exit(1);
-      }
-    });
 }
 
-async function runTestE2ECommand(appName, options) {
-  const pathsUtil = require('../utils/paths');
-  const appType = await pathsUtil.detectAppType(appName).catch(() => null);
-  if (appType && appType.baseDir === 'integration') {
-    const { runTestE2EForExternalSystem } = require('../commands/test-e2e-external');
-    const { success, results } = await runTestE2EForExternalSystem(appName, {
-      env: options.env,
-      debug: options.debug,
-      verbose: options.verbose,
-      async: options.async !== false
-    });
-    results.forEach(r => {
-      const icon = r.success ? chalk.green('✓') : chalk.red('✗');
-      const msg = r.error ? `${r.key}: ${r.error}` : r.key;
-      logger.log(`  ${icon} ${msg}`);
-    });
-    if (!success) process.exit(1);
-    return;
-  }
-  const { runAppTestE2e } = require('../commands/app-test');
-  await runAppTestE2e(appName, { env: options.env });
-}
-
-function setupInstallTestE2eLintCommands(program) {
-  program.command('install <app>')
-    .description('Install dependencies in container (builder apps only)')
-    .option('--env <env>', 'dev (running container) or tst (ephemeral with .env)', 'dev')
-    .action(async(appName, options) => {
-      try {
-        const pathsUtil = require('../utils/paths');
-        const appType = await pathsUtil.detectAppType(appName).catch(() => null);
-        if (appType && appType.baseDir === 'integration') {
-          logger.log(chalk.gray('Install is for builder applications only. Use aifabrix shell <app> to run commands in external setups.'));
-          return;
-        }
-        const { runAppInstall } = require('../commands/app-install');
-        await runAppInstall(appName, { env: options.env });
-      } catch (error) {
-        handleCommandError(error, 'install');
-        process.exit(1);
-      }
-    });
-
-  program.command('test-e2e <app>')
-    .description('Run e2e tests (builder: in container; external system: all datasources via dataplane)')
-    .option('-e, --env <env>', 'Environment: dev, tst, or pro (builder: dev/tst for container)')
-    .option('-v, --verbose', 'Show detailed step output and poll progress')
-    .option('--debug', 'Include debug output and write log to integration/<app>/logs/')
-    .option('--no-async', 'Use sync mode (no polling); single POST per datasource')
-    .action(async(appName, options) => {
-      try {
-        await runTestE2ECommand(appName, options);
-      } catch (error) {
-        handleCommandError(error, 'test-e2e');
-        process.exit(1);
-      }
-    });
-
-  program.command('lint <app>')
-    .description('Run lint in container (builder apps only)')
-    .option('--env <env>', 'dev (running container) or tst (ephemeral with .env)', 'dev')
+function setupDockerfileGenerateCommand(program) {
+  program.command('dockerfile <app>')
+    .description('Generate Dockerfile from detected runtime')
+    .option('-l, --language <lang>', 'Override language detection')
+    .option('-f, --force', 'Overwrite existing Dockerfile')
     .action(async(appName, options) => {
       try {
-        const pathsUtil = require('../utils/paths');
-        const appType = await pathsUtil.detectAppType(appName).catch(() => null);
-        if (appType && appType.baseDir === 'integration') {
-          logger.log(chalk.gray('lint is for builder applications only. Use aifabrix shell <app> then make lint or pnpm lint.'));
-          return;
-        }
-        const { runAppLint } = require('../commands/app-test');
-        await runAppLint(appName, { env: options.env });
+        const dockerfilePath = await app.generateDockerfileForApp(appName, options);
+        logger.log(formatSuccessParagraph('Dockerfile generated successfully!'));
+        logger.log(chalk.gray(`Location: ${dockerfilePath}`));
       } catch (error) {
-        handleCommandError(error, 'lint');
+        handleCommandError(error, 'dockerfile');
         process.exit(1);
       }
     });
@@ -373,6 +310,7 @@ function setupInstallTestE2eLintCommands(program) {
 function setupPushDeployDockerfileCommands(program) {
   program.command('push <app>')
     .description('Push image to Azure Container Registry')
+    .addHelpText('after', PUSH_HELP_AFTER)
     .option('-r, --registry <registry>', 'ACR registry URL (overrides application.yaml)')
     .option('-t, --tag <tag>', 'Image tag(s) - comma-separated for multiple (default: latest)')
     .action(async(appName, options) => {
@@ -385,15 +323,30 @@ function setupPushDeployDockerfileCommands(program) {
     });
 
   program.command('deploy <app>')
-    .description('Deploy to Azure or locally via Miso Controller')
+    .description('Deploy via Miso Controller (Azure or --local)')
+    .addHelpText('after', DEPLOY_HELP_AFTER)
     .option('--local', 'Send manifest to controller then run app locally (app: same as aifabrix run <app>; external: restart dataplane)')
     .option('--client-id <id>', 'Client ID (overrides config)')
     .option('--client-secret <secret>', 'Client Secret (overrides config)')
     .option('--poll', 'Poll for deployment status', true)
     .option('--no-poll', 'Do not poll for status')
+    .option('--probe', 'After external deploy, run dataplane runtime checks (validation/run); slower')
+    .option('--probe-timeout <ms>', 'Timeout for --probe on external deploy (default: 120000)', '120000')
+    .option(
+      '--no-cert-sync',
+      'Skip updating integration certification in the system file from the dataplane after external deploy'
+    )
     .action(async(appName, options) => {
       try {
-        const opts = { ...options, local: !!options.local };
+        const probeTimeout =
+          options.probeTimeout === undefined || options.probeTimeout === null
+            ? 120000
+            : Number(options.probeTimeout);
+        const opts = {
+          ...options,
+          local: !!options.local,
+          probeTimeout: Number.isFinite(probeTimeout) ? probeTimeout : 120000
+        };
         const outcome = await app.deployApp(appName, opts);
         if (opts.local && outcome) {
           if (outcome.usedExternalDeploy) await app.restartApp('dataplane');
@@ -405,20 +358,7 @@ function setupPushDeployDockerfileCommands(program) {
       }
     });
 
-  program.command('dockerfile <app>')
-    .description('Generate Dockerfile for an application')
-    .option('-l, --language <lang>', 'Override language detection')
-    .option('-f, --force', 'Overwrite existing Dockerfile')
-    .action(async(appName, options) => {
-      try {
-        const dockerfilePath = await app.generateDockerfileForApp(appName, options);
-        logger.log(chalk.green('\n✅ Dockerfile generated successfully!'));
-        logger.log(chalk.gray(`Location: ${dockerfilePath}`));
-      } catch (error) {
-        handleCommandError(error, 'dockerfile');
-        process.exit(1);
-      }
-    });
+  setupDockerfileGenerateCommand(program);
 }
 
 /**