@aifabrix/builder 2.44.6 → 2.45.5

package/lib/cli/setup-utility.js

@@ -10,45 +10,12 @@ const { formatSuccessParagraph } = require('../utils/cli-test-layout-chalk');
 const path = require('path');
 const fs = require('fs');
 const chalk = require('chalk');
-const secrets = require('../core/secrets');
 const generator = require('../generator');
 const logger = require('../utils/logger');
 const { handleCommandError, logOfflinePathWhenType } = require('../utils/cli-utils');
 const { detectAppType, getDeployJsonPath, getResolveAppPath } = require('../utils/paths');
-
-const JSON_HELP_AFTER = `
-Example:
-  $ aifabrix json myapp
-Generates *-deploy.json (or application-schema.json) for commit before deploy.
-`;
-
-const VALIDATE_HELP_AFTER = `
-Examples:
-  $ aifabrix validate myapp
-  $ aifabrix validate myapp --cert-sync
-  $ aifabrix validate --integration
-  $ aifabrix validate --builder
-`;
-
-const REPAIR_HELP_AFTER = `
-Examples:
-  $ aifabrix repair hubspot-demo
-    Align manifest, system/datasource files, RBAC extract, env.template, and deploy JSON under integration/<systemKey>/.
-
-  $ aifabrix repair hubspot-demo --dry-run
-    Show what would change without writing files.
-
-  $ aifabrix repair hubspot-demo --auth oauth2
-    Set authentication method; updates the system file and env.template (oauth2, aad, apikey, basic, …).
-
-  $ aifabrix repair hubspot-demo --rbac --expose --sync --test
-    Optional datasource fixes: RBAC roles/permissions, exposed.schema from attributes, default sync block, testPayload stubs.
-
-  $ aifabrix repair hubspot-demo --doc
-    Regenerate README.md from the current deployment manifest only (other drift fixes unchanged).
-
-  Before overwriting files, repair writes timestamped backups under integration/<systemKey>/backup/ (same as datasource capability copy). Use --no-backup to skip.
-`;
+const { setupResolveCommand } = require('./setup-utility-resolve');
+const { JSON_HELP_AFTER, VALIDATE_HELP_AFTER, REPAIR_HELP_AFTER } = require('./setup-utility-help-after');
 
 /**
  * Resolve app path and type for split-json (integration first, then builder).
@@ -121,46 +88,20 @@ function logSplitJsonResult(result) {
   }
 }
 
-function setupResolveCommand(program) {
-  program.command('resolve <app>')
-    .description('Generate .env from template; optional validate after')
-    .option('-f, --force', 'Generate missing secret keys in secrets file')
-    .option('--skip-validation', 'Skip file validation after generating .env')
-    .action(async(appName, options) => {
-      try {
-        const { appPath, envOnly } = await getResolveAppPath(appName);
-        const envPath = await secrets.generateEnvFile(
-          appName,
-          undefined,
-          'docker',
-          options.force,
-          { appPath, envOnly, skipOutputPath: false, preserveFromPath: null }
-        );
-        logger.log(`✔ Generated .env file: ${envPath}`);
-        if (envOnly) {
-          logger.log(chalk.gray('  (env-only mode: validation skipped; no application config file)'));
-        } else if (!options.skipValidation) {
-          const validate = require('../validation/validate');
-          const result = await validate.validateAppOrFile(appName);
-          validate.displayValidationResults(result);
-          if (!result.valid) {
-            logger.log(chalk.yellow('\n⚠  Validation found errors. Fix them before deploying.'));
-            process.exit(1);
-          }
-        }
-      } catch (error) {
-        handleCommandError(error, 'resolve');
-        process.exit(1);
-      }
-    });
-}
-
 function setupJsonCommand(program) {
-  program.command('json <app>')
+  program.command('json <appKey|systemKey>')
     .description('Write deployment JSON to disk for version control')
     .addHelpText('after', JSON_HELP_AFTER)
     .action(async(appName, options) => {
       try {
+        const resolved = await getResolveAppPath(appName);
+        const { emitManifestMetadataLineIfTTY } = require('../utils/manifest-source-emit');
+        emitManifestMetadataLineIfTTY(logger, {
+          appKey: appName,
+          appPath: resolved.appPath,
+          envOnly: resolved.envOnly,
+          json: false
+        });
         const result = await generator.generateDeployJsonWithValidation(appName, options);
         if (result.success) {
           const fileName = result.path.includes('application-schema.json') ? 'application-schema.json' : 'deployment JSON';
@@ -182,7 +123,7 @@ function setupJsonCommand(program) {
 }
 
 function setupSplitJsonCommand(program) {
-  program.command('split-json <app>')
+  program.command('split-json <appKey|systemKey>')
     .description('Split deploy JSON into env.template, application.yaml, rbac, README, …')
     .option('-o, --output <dir>', 'Output directory for component files (defaults to same directory as JSON)')
     .action(async(appName, options) => {
@@ -289,32 +230,54 @@ function setupRepairCommand(program) {
     });
 }
 
+async function resolveConvertTargetFormat(options) {
+  const config = require('../core/config');
+  const effectiveFormat = options.format || (await config.getFormat());
+  if (!effectiveFormat) {
+    throw new Error(
+      'Option --format is required and must be \'json\' or \'yaml\' (or set default with aifabrix dev set-format)'
+    );
+  }
+  const normalized = effectiveFormat.trim().toLowerCase();
+  if (normalized !== 'json' && normalized !== 'yaml') {
+    throw new Error('Option --format must be \'json\' or \'yaml\'');
+  }
+  return normalized;
+}
+
+function logConvertCommandResults(converted, deleted) {
+  logger.log(formatSuccessParagraph('Convert complete.'));
+  converted.forEach((p) => logger.log(`  • ${p}`));
+  if (deleted.length > 0) {
+    logger.log(chalk.gray('  Removed old files:'));
+    deleted.forEach((p) => logger.log(chalk.gray(`    ${p}`)));
+  }
+}
+
+async function runConvertCommandForApp(appName, options) {
+  const normalized = await resolveConvertTargetFormat(options);
+  const { isProtectionScope } = require('../protection/scope');
+  if (isProtectionScope(appName)) {
+    const { runConvertProtectionBatch } = require('../protection/convert-batch');
+    const { converted, deleted } = await runConvertProtectionBatch(normalized, {
+      force: options.force
+    });
+    logConvertCommandResults(converted, deleted);
+    return;
+  }
+  const { runConvert } = require('../commands/convert');
+  const { converted, deleted } = await runConvert(appName, { format: normalized, force: options.force });
+  logConvertCommandResults(converted, deleted);
+}
+
 function setupConvertCommand(program) {
-  program.command('convert <app>')
-    .description('Convert integration config files between JSON and YAML')
+  program.command('convert <appKey|systemKey>')
+    .description('Convert config files between JSON and YAML (builder/<appKey>/ or integration/<systemKey>/)')
     .option('--format <format>', 'Target format: json | yaml (required unless config format is set)')
     .option('-f, --force', 'Skip confirmation prompt')
     .action(async(appName, options) => {
       try {
-        const config = require('../core/config');
-        const effectiveFormat = options.format || (await config.getFormat());
-        if (!effectiveFormat) {
-          throw new Error(
-            'Option --format is required and must be \'json\' or \'yaml\' (or set default with aifabrix dev set-format)'
-          );
-        }
-        const normalized = effectiveFormat.trim().toLowerCase();
-        if (normalized !== 'json' && normalized !== 'yaml') {
-          throw new Error('Option --format must be \'json\' or \'yaml\'');
-        }
-        const { runConvert } = require('../commands/convert');
-        const { converted, deleted } = await runConvert(appName, { format: normalized, force: options.force });
-        logger.log(formatSuccessParagraph('Convert complete.'));
-        converted.forEach(p => logger.log(`  • ${p}`));
-        if (deleted.length > 0) {
-          logger.log(chalk.gray('  Removed old files:'));
-          deleted.forEach(p => logger.log(chalk.gray(`    ${p}`)));
-        }
+        await runConvertCommandForApp(appName, options);
       } catch (error) {
         handleCommandError(error, 'convert');
         process.exit(1);
@@ -323,8 +286,8 @@ function setupConvertCommand(program) {
 }
 
 function setupShowCommand(program) {
-  program.command('show <app>')
-    .description('Show app from local tree (default) or controller (--online)')
+  program.command('show <appKey|systemKey>')
+    .description('Show builder app or external system from local tree (default) or controller (--online)')
     .option('--online', 'Fetch application data from the controller')
     .option('--json', 'Output as JSON')
     .option(
@@ -353,52 +316,121 @@ function setupSplitJsonConvertShowCommands(program) {
   setupShowCommand(program);
 }
 
-async function runValidateCommand(appOrFile, options) {
-  const validate = require('../validation/validate');
-  const opts = options.opts ? options.opts() : options;
+function emitManifestAfterValidateIfApplicable(appOrFile, result, outFormat) {
+  if (outFormat === 'json' || !result?.appPath || !appOrFile || typeof appOrFile !== 'string') {
+    return;
+  }
+  try {
+    if (fs.existsSync(appOrFile) && fs.statSync(appOrFile).isFile()) {
+      return;
+    }
+    const { emitManifestMetadataLineIfTTY } = require('../utils/manifest-source-emit');
+    emitManifestMetadataLineIfTTY(logger, {
+      appKey: appOrFile,
+      appPath: result.appPath,
+      envOnly: false,
+      json: false
+    });
+  } catch {
+    /* ignore emit failures */
+  }
+}
+
+async function runValidateBatchBranch(validate, opts) {
   const integration = opts.integration === true;
   const builder = opts.builder === true;
+  if (!integration && !builder) {
+    return false;
+  }
   const outFormat = (opts.format || 'default').toLowerCase();
+  const batchResult = integration && builder
+    ? await validate.validateAll(opts)
+    : integration
+      ? await validate.validateAllIntegrations(opts)
+      : await validate.validateAllBuilderApps(opts);
+  if (outFormat === 'json') {
+    logger.log(JSON.stringify(batchResult, null, 2));
+  } else {
+    validate.displayBatchValidationResults(batchResult);
+  }
+  if (!batchResult.valid) process.exit(1);
+  return true;
+}
 
-  if (integration || builder) {
-    const batchResult = integration && builder
-      ? await validate.validateAll(opts)
-      : integration
-        ? await validate.validateAllIntegrations(opts)
-        : await validate.validateAllBuilderApps(opts);
-    if (outFormat === 'json') {
-      logger.log(JSON.stringify(batchResult, null, 2));
-    } else {
-      validate.displayBatchValidationResults(batchResult);
+async function runProtectionValidateBranch(appOrFile, opts) {
+  const { validateProtectionBatchCommand } = require('../protection/validate-batch');
+  const code = await validateProtectionBatchCommand(
+    {
+      env: opts.env,
+      json: (opts.format || '').toLowerCase() === 'json',
+      warningsAsErrors: opts.warningsAsErrors === true,
+      simulate: opts.simulate === true,
+      verbose: opts.verbose === true
+    },
+    logger
+  );
+  if (code !== 0) {
+    process.exit(code);
+  }
+}
+
+function buildValidateJsonPayload(result, appOrFile) {
+  let payload = result;
+  try {
+    if (result?.appPath && appOrFile && typeof appOrFile === 'string') {
+      if (!(fs.existsSync(appOrFile) && fs.statSync(appOrFile).isFile())) {
+        const { getManifestSourcePayload } = require('../utils/manifest-source-emit');
+        payload = { ...result, manifestSource: getManifestSourcePayload(appOrFile, result.appPath) };
+      }
     }
-    if (!batchResult.valid) process.exit(1);
+  } catch {
+    /* keep plain result */
+  }
+  return payload;
+}
+
+function displaySingleValidateResult(appOrFile, result, outFormat, validate) {
+  emitManifestAfterValidateIfApplicable(appOrFile, result, outFormat);
+  if (outFormat === 'json') {
+    logger.log(JSON.stringify(buildValidateJsonPayload(result, appOrFile), null, 2));
     return;
   }
+  validate.displayValidationResults(result);
+}
 
+async function runValidateCommand(appOrFile, options) {
+  const validate = require('../validation/validate');
+  const opts = options.opts ? options.opts() : options;
+  const { isProtectionScope } = require('../protection/scope');
+  if (isProtectionScope(appOrFile)) {
+    await runProtectionValidateBranch(appOrFile, opts);
+    return;
+  }
+  if (await runValidateBatchBranch(validate, opts)) {
+    return;
+  }
+  const outFormat = (opts.format || 'default').toLowerCase();
   if (!appOrFile || typeof appOrFile !== 'string') {
     logger.log(chalk.red('App name or file path is required, or use --integration / --builder'));
     process.exit(1);
   }
-
   const result = await validate.validateAppOrFile(appOrFile, {
     ...opts,
     certSync: opts.certSync === true
   });
-  if (outFormat === 'json') {
-    logger.log(JSON.stringify(result, null, 2));
-  } else {
-    validate.displayValidationResults(result);
-  }
+  displaySingleValidateResult(appOrFile, result, outFormat, validate);
   if (!result.valid) process.exit(1);
 }
 
 function setupValidateDiffCommands(program) {
-  program.command('validate [appOrFile]')
-    .description('Validate one app/file or all under integration/ or builder/')
+  program.command('validate [appKey|systemKey|file]')
+    .description('Validate appKey, systemKey, file path, .protection batch, or all under integration/ and builder/')
     .addHelpText('after', VALIDATE_HELP_AFTER)
     .option('--format <format>', 'Output format: json | default (human-readable)')
     .option('--integration', 'Validate all applications under integration/')
     .option('--builder', 'Validate all applications under builder/')
+    .option('--warnings-as-errors', 'For .protection: pass strict=true to dataplane validate')
+    .option('--simulate', 'For .protection: run simulate after validate')
     .option(
       '--cert-sync',
       'After successful validation of an external integration, refresh the system file certification block from the dataplane (requires login)'

package/lib/commands/app-logs.js

@@ -15,6 +15,7 @@ const containerHelpers = require('../utils/app-run-containers');
 const { validateAppName } = require('../app/push');
 
 const { execWithDockerEnv } = require('../utils/docker-exec');
+const { maskEnvLine } = require('../utils/log-redaction');
 
 /** Default number of log lines */
 const DEFAULT_TAIL_LINES = 100;
@@ -43,36 +44,11 @@ const LEVEL_JSON_NUMERIC_REGEX = /"level"\s*:\s*(\d+)/;
 /** Fallback: line contains whole-word "error" or "Error" when no other level detected (catches stack traces, "Error: msg", etc.) */
 const ERROR_WORD_FALLBACK_REGEX = /\berror\b/i;
 
-/** Env key patterns that indicate a secret (mask value) */
-const SECRET_KEY_PATTERN = /password|secret|token|credential|api[_-]?key/i;
+/** Keycloak / Java style: validation failure without the word "error" (would be dropped by `-l error` otherwise) */
+const VALIDATION_FAILURE_LINE_REGEX = /\bvalidation\s+failed\b/i;
 
-/** Prefixes to strip before checking key (avoids masking KEYCLOAK_SERVER_URL etc.) */
-const KEY_PREFIXES_TO_STRIP = /^KEYCLOAK_|^KEY_VAULT_/;
-
-/** URL with embedded credentials: scheme://user:password@host → scheme://user:***@host */
-const URL_CREDENTIAL_PATTERN = /(\w+:\/\/)([^:@]*):([^@]+)@/g;
-
-/**
- * Masks a single env line if the key looks like a secret or value contains URL credentials
- * @param {string} line - Line in form KEY=value
- * @returns {string} Same line or KEY=*** or value with masked URL credentials
- */
-function maskEnvLine(line) {
-  const eq = line.indexOf('=');
-  if (eq <= 0) return line;
-  const key = line.slice(0, eq);
-  const value = line.slice(eq + 1);
-
-  const keyForCheck = key.replace(KEY_PREFIXES_TO_STRIP, '');
-  const isSecretKey = SECRET_KEY_PATTERN.test(keyForCheck);
-
-  const maskedValue = value.replace(URL_CREDENTIAL_PATTERN, '$1$2:***@');
-  const hasUrlCredentials = maskedValue !== value;
-
-  if (isSecretKey) return `${key}=***`;
-  if (hasUrlCredentials) return `${key}=${maskedValue}`;
-  return line;
-}
+/** Other failure phrases treated as error severity for level filtering */
+const FAILURE_PHRASE_AS_ERROR_REGEX = /\b(fatal|failed to run|unable to (start|run)|exception in thread)\b/i;
 
 /** Normalize level string to canonical 'debug'|'info'|'warn'|'error'. */
 function normalizeLevel(raw) {
@@ -104,9 +80,66 @@ function getLogLevel(line) {
   const jsonNum = line.match(LEVEL_JSON_NUMERIC_REGEX);
   if (jsonNum) return numericLevelToName(parseInt(jsonNum[1], 10));
   if (ERROR_WORD_FALLBACK_REGEX.test(line)) return 'error';
+  if (VALIDATION_FAILURE_LINE_REGEX.test(line)) return 'error';
+  if (FAILURE_PHRASE_AS_ERROR_REGEX.test(line)) return 'error';
+  return null;
+}
+
+/** Lines after an `error`-level line often have no level prefix (e.g. Keycloak config validation details). */
+const ERROR_FILTER_CONTEXT_UNCLASSIFIED_MAX = 50;
+
+/**
+ * Whether to show a log line when `--level` filtering is on. After an emitted error line, includes
+ * up to {@link ERROR_FILTER_CONTEXT_UNCLASSIFIED_MAX} following lines with no parseable level so
+ * stack traces and multi-line validation output are not dropped.
+ *
+ * @param {string} line
+ * @param {string|null} minLevel - normalized LOG_LEVELS value or null
+ * @param {{ pendingAfterError: number }} state - mutable; only used when minLevel is `error`
+ * @returns {boolean}
+ */
+/**
+ * @param {string|null} level
+ * @param {string} minLevel
+ * @param {{ pendingAfterError: number }} state
+ * @returns {boolean|null} true if line shown by sticky rule; null to continue
+ */
+function tryStickyErrorContinuation(level, minLevel, state) {
+  if (minLevel !== 'error' || state.pendingAfterError <= 0) {
+    return null;
+  }
+  if (level === null || level === undefined) {
+    state.pendingAfterError -= 1;
+    return true;
+  }
+  if (level !== 'error') {
+    state.pendingAfterError = 0;
+  }
   return null;
 }
 
+function shouldShowFilteredLogLine(line, minLevel, state) {
+  if (minLevel === null || minLevel === undefined || minLevel === '' || !LOG_LEVELS.includes(minLevel)) {
+    return true;
+  }
+  if (line.trim() === '' && minLevel === 'error' && state.pendingAfterError > 0) {
+    return true;
+  }
+  const level = getLogLevel(line);
+  const sticky = tryStickyErrorContinuation(level, minLevel, state);
+  if (sticky === true) {
+    return true;
+  }
+
+  if (passesLevelFilter(level, minLevel)) {
+    if (minLevel === 'error' && level === 'error') {
+      state.pendingAfterError = ERROR_FILTER_CONTEXT_UNCLASSIFIED_MAX;
+    }
+    return true;
+  }
+  return false;
+}
+
 /**
  * Whether a line's level passes the minimum level filter (show this level and above).
  * @param {string|null} lineLevel - Level from getLogLevel (null treated as 'info')
@@ -143,7 +176,12 @@ async function dumpMaskedEnv(containerName) {
     lines.forEach((line) => logger.log(maskEnvLine(line)));
     logger.log(chalk.gray('\n--- Logs ---\n'));
   } catch (err) {
-    logger.log(chalk.gray('(Could not read container env; container may be stopped)\n'));
+    logger.log(
+      chalk.gray(
+        '(Could not read container env — the container may be stopped, restarting, or not ready yet. ' +
+          'Docker log output below is still shown.)\n'
+      )
+    );
   }
 }
 
@@ -175,8 +213,9 @@ async function runDockerLogs(containerName, options) {
     const proc = spawn('docker', args, { stdio: ['inherit', 'pipe', 'pipe'], env: dockerEnv });
     proc.on('error', reject);
 
+    const filterState = { pendingAfterError: 0 };
     function onLine(line) {
-      if (passesLevelFilter(getLogLevel(line), minLevel)) {
+      if (shouldShowFilteredLogLine(line, minLevel, filterState)) {
         process.stdout.write(line + '\n');
       }
     }
@@ -241,8 +280,11 @@ async function runDockerLogsFollow(containerName, tail, minLevel) {
     logger.log(chalk.red(`Error: ${err.message}`));
     process.exit(1);
   });
+  const filterState = { pendingAfterError: 0 };
   function onLine(line) {
-    if (passesLevelFilter(getLogLevel(line), level)) process.stdout.write(line + '\n');
+    if (shouldShowFilteredLogLine(line, level, filterState)) {
+      process.stdout.write(line + '\n');
+    }
   }
   const rlOut = readline.createInterface({ input: proc.stdout, crlfDelay: Infinity });
   rlOut.on('line', onLine);
@@ -301,4 +343,10 @@ async function runAppLogs(appKey, options = {}) {
   }
 }
 
-module.exports = { runAppLogs, maskEnvLine, getLogLevel, passesLevelFilter };
+module.exports = {
+  runAppLogs,
+  maskEnvLine,
+  getLogLevel,
+  passesLevelFilter,
+  shouldShowFilteredLogLine
+};