@aifabrix/builder 2.39.2 → 2.40.0

package/lib/commands/app-logs.js

@@ -8,6 +8,7 @@
 
 const chalk = require('chalk');
 const { exec, spawn } = require('child_process');
+const readline = require('readline');
 const { promisify } = require('util');
 const logger = require('../utils/logger');
 const config = require('../core/config');
@@ -19,6 +20,30 @@ const execAsync = promisify(exec);
 /** Default number of log lines */
 const DEFAULT_TAIL_LINES = 100;
 
+/** Allowed log levels for --level filter (lowest to highest severity) */
+const LOG_LEVELS = ['debug', 'info', 'warn', 'error'];
+
+/** Severity rank: higher = more severe. Used for "show this level and above". */
+const LEVEL_RANK = { debug: 0, info: 1, warn: 2, error: 3 };
+
+/** Prefix pattern: INFO:, ERROR:, WARN:, WARNING:, DEBUG: at start of line */
+const LEVEL_PREFIX_REGEX = /^(DEBUG|INFO|WARN|WARNING|ERROR)\s*[:-\s]/i;
+
+/** Level after timestamp or space (e.g. "2026-02-11 08:51:01 error: msg" or "  error: msg") */
+const LEVEL_AFTER_PREFIX_REGEX = /(?:^|\s)(DEBUG|INFO|WARN|WARNING|ERROR)\s*:/i;
+
+/** Level after word boundary (e.g. "[pino]error: msg" or BOM + "error:") so "error:" is detected anywhere */
+const LEVEL_WORD_BOUNDARY_REGEX = /\b(DEBUG|INFO|WARN|WARNING|ERROR)\s*:/i;
+
+/** JSON "level" field pattern (string: "error", "info", etc.) */
+const LEVEL_JSON_REGEX = /"level"\s*:\s*"(\w+)"/i;
+
+/** JSON "level" numeric (pino/bunyan: 50=error, 60=fatal, 40=warn, 30=info, 20=debug, 10=trace) */
+const LEVEL_JSON_NUMERIC_REGEX = /"level"\s*:\s*(\d+)/;
+
+/** Fallback: line contains whole-word "error" or "Error" when no other level detected (catches stack traces, "Error: msg", etc.) */
+const ERROR_WORD_FALLBACK_REGEX = /\berror\b/i;
+
 /** Env key patterns that indicate a secret (mask value) */
 const SECRET_KEY_PATTERN = /password|secret|token|credential|api[_-]?key/i;
 
@@ -50,6 +75,55 @@ function maskEnvLine(line) {
   return line;
 }
 
+/** Normalize level string to canonical 'debug'|'info'|'warn'|'error'. */
+function normalizeLevel(raw) {
+  const s = (raw || '').toLowerCase();
+  return s === 'warning' ? 'warn' : s;
+}
+
+/** Map pino/bunyan numeric level (10–60) to level string. */
+function numericLevelToName(num) {
+  if (num >= 50) return 'error';
+  if (num >= 40) return 'warn';
+  if (num >= 30) return 'info';
+  return 'debug';
+}
+
+/**
+ * Extract log level from a line (prefix like INFO:/error: or JSON "level":"info").
+ * Supports: INFO:, ERROR:, error:, info: (miso-controller/pino), WARN:, DEBUG:, and "level":"x" in JSON.
+ * @param {string} line - Single log line
+ * @returns {string|null} One of 'debug'|'info'|'warn'|'error', or null if not parseable
+ */
+function getLogLevel(line) {
+  if (!line || typeof line !== 'string') return null;
+  const prefixRe = [LEVEL_PREFIX_REGEX, LEVEL_AFTER_PREFIX_REGEX, LEVEL_WORD_BOUNDARY_REGEX, LEVEL_JSON_REGEX];
+  for (const re of prefixRe) {
+    const m = line.match(re);
+    if (m) return normalizeLevel(m[1]);
+  }
+  const jsonNum = line.match(LEVEL_JSON_NUMERIC_REGEX);
+  if (jsonNum) return numericLevelToName(parseInt(jsonNum[1], 10));
+  if (ERROR_WORD_FALLBACK_REGEX.test(line)) return 'error';
+  return null;
+}
+
+/**
+ * Whether a line's level passes the minimum level filter (show this level and above).
+ * @param {string|null} lineLevel - Level from getLogLevel (null treated as 'info')
+ * @param {string|undefined|null} minLevel - Minimum level (e.g. 'error', 'info')
+ * @returns {boolean} True if line should be shown
+ */
+function passesLevelFilter(lineLevel, minLevel) {
+  if (minLevel === null || minLevel === undefined || minLevel === '') return true;
+  const normalized = (minLevel || '').toString().trim().toLowerCase();
+  if (!LOG_LEVELS.includes(normalized)) return true;
+  const lineRank =
+    lineLevel === null || lineLevel === undefined ? LEVEL_RANK.info : (LEVEL_RANK[lineLevel] ?? LEVEL_RANK.info);
+  const minRank = LEVEL_RANK[normalized];
+  return lineRank >= minRank;
+}
+
 /**
  * Dump container env (masked) and print to logger
  * @async
@@ -75,33 +149,102 @@ async function dumpMaskedEnv(containerName) {
 }
 
 /**
- * Run docker logs (non-follow): tail N lines or full (tail 0)
+ * Run docker logs (non-follow): tail N lines or full (tail 0). If options.level is set, stdout is piped and filtered.
  * @async
  * @param {string} containerName - Docker container name
- * @param {Object} options - { tail: number } (0 = full, no limit)
+ * @param {Object} options - { tail: number, level?: string }
  * @returns {Promise<void>}
  */
 async function runDockerLogs(containerName, options) {
   const args = options.tail === 0 ? ['logs', containerName] : ['logs', '--tail', String(options.tail), containerName];
+  const minLevel =
+    options.level !== undefined && options.level !== null && options.level !== ''
+      ? String(options.level).trim().toLowerCase()
+      : null;
+
+  if (minLevel === null || minLevel === undefined || !LOG_LEVELS.includes(minLevel)) {
+    return new Promise((resolve, reject) => {
+      const proc = spawn('docker', args, { stdio: 'inherit' });
+      proc.on('error', reject);
+      proc.on('close', (code) => (code === 0 ? resolve() : reject(new Error(`docker logs exited with ${code}`))));
+    });
+  }
+
   return new Promise((resolve, reject) => {
-    const proc = spawn('docker', args, { stdio: 'inherit' });
+    const proc = spawn('docker', args, { stdio: ['inherit', 'pipe', 'pipe'] });
     proc.on('error', reject);
-    proc.on('close', (code) => (code === 0 ? resolve() : reject(new Error(`docker logs exited with ${code}`))));
+
+    function onLine(line) {
+      if (passesLevelFilter(getLogLevel(line), minLevel)) {
+        process.stdout.write(line + '\n');
+      }
+    }
+
+    const rlOut = readline.createInterface({ input: proc.stdout, crlfDelay: Infinity });
+    rlOut.on('line', onLine);
+    const rlErr = readline.createInterface({ input: proc.stderr, crlfDelay: Infinity });
+    rlErr.on('line', onLine);
+
+    let streamsClosed = 0;
+    let exitCode = null;
+    function tryFinish() {
+      if (streamsClosed < 2 || exitCode === null) return;
+      if (exitCode !== 0) reject(new Error(`docker logs exited with ${exitCode}`));
+      else resolve();
+    }
+    rlOut.on('close', () => {
+      streamsClosed += 1;
+      tryFinish();
+    });
+    rlErr.on('close', () => {
+      streamsClosed += 1;
+      tryFinish();
+    });
+    proc.on('close', (code) => {
+      exitCode = code;
+      tryFinish();
+    });
   });
 }
 
 /**
- * Run docker logs --follow (stream), optionally with tail
+ * Run docker logs --follow (stream), optionally with tail. If minLevel is set, stdout is piped and filtered.
  * @param {string} containerName - Docker container name
  * @param {number} [tail] - Lines to show (0 = full, omit --tail)
+ * @param {string|null} [minLevel] - Minimum log level to show (debug|info|warn|error)
  */
-function runDockerLogsFollow(containerName, tail) {
+function runDockerLogsFollow(containerName, tail, minLevel) {
   const args = tail === 0 ? ['logs', '-f', containerName] : ['logs', '-f', '--tail', String(tail), containerName];
-  const proc = spawn('docker', args, { stdio: 'inherit' });
+  const level =
+    minLevel !== undefined && minLevel !== null && minLevel !== ''
+      ? String(minLevel).trim().toLowerCase()
+      : null;
+  const useFilter = level !== null && level !== undefined && LOG_LEVELS.includes(level);
+
+  if (!useFilter) {
+    const proc = spawn('docker', args, { stdio: 'inherit' });
+    proc.on('error', (err) => {
+      logger.log(chalk.red(`Error: ${err.message}`));
+      process.exit(1);
+    });
+    proc.on('close', (code) => {
+      if (code !== 0 && code !== null) process.exit(code);
+    });
+    return;
+  }
+
+  const proc = spawn('docker', args, { stdio: ['inherit', 'pipe', 'pipe'] });
   proc.on('error', (err) => {
     logger.log(chalk.red(`Error: ${err.message}`));
     process.exit(1);
   });
+  function onLine(line) {
+    if (passesLevelFilter(getLogLevel(line), level)) process.stdout.write(line + '\n');
+  }
+  const rlOut = readline.createInterface({ input: proc.stdout, crlfDelay: Infinity });
+  rlOut.on('line', onLine);
+  const rlErr = readline.createInterface({ input: proc.stderr, crlfDelay: Infinity });
+  rlErr.on('line', onLine);
   proc.on('close', (code) => {
     if (code !== 0 && code !== null) process.exit(code);
   });
@@ -114,10 +257,22 @@ function runDockerLogsFollow(containerName, tail) {
  * @param {Object} options - CLI options
  * @param {boolean} [options.follow] - Follow log stream (-f)
  * @param {number} [options.tail] - Number of lines (default 100; 0 = full list)
+ * @param {string} [options.level] - Show only logs at this level or above (debug|info|warn|error)
  * @returns {Promise<void>}
  */
 async function runAppLogs(appKey, options = {}) {
   validateAppName(appKey);
+  const rawLevel =
+    options.level !== undefined && options.level !== null && options.level !== ''
+      ? String(options.level).trim()
+      : undefined;
+  const level = rawLevel ? rawLevel.toLowerCase() : undefined;
+  if (level !== undefined && level !== null && !LOG_LEVELS.includes(level)) {
+    throw new Error(
+      `Invalid log level '${rawLevel}'; use one of: ${LOG_LEVELS.join(', ')}`
+    );
+  }
+
   const developerId = await config.getDeveloperId();
   const containerName = containerHelpers.getContainerName(appKey, developerId);
 
@@ -131,16 +286,16 @@ async function runAppLogs(appKey, options = {}) {
   }
 
   if (follow) {
-    runDockerLogsFollow(containerName, tail);
+    runDockerLogsFollow(containerName, tail, level);
     return;
   }
 
   try {
-    await runDockerLogs(containerName, { tail });
+    await runDockerLogs(containerName, { tail, level });
   } catch (err) {
     logger.log(chalk.red(`Error: ${err.message}`));
     throw new Error(`Failed to show logs: ${err.message}`);
   }
 }
 
-module.exports = { runAppLogs, maskEnvLine };
+module.exports = { runAppLogs, maskEnvLine, getLogLevel, passesLevelFilter };

package/lib/commands/app.js

@@ -17,21 +17,11 @@ const { rotateSecret } = require('../app/rotate-secret');
 const { showApp } = require('../app/show');
 const { runAppDeploymentList } = require('./deployment-list');
 
-/**
- * Setup application management commands
- * @param {Command} program - Commander program instance
- */
-function setupAppCommands(program) {
-  const app = program
-    .command('app')
-    .description('Manage applications');
-
-  // Register command (controller and environment from config.yaml)
-  app
-    .command('register <appKey>')
+function setupAppRegisterCommand(app) {
+  app.command('register <appKey>')
     .description('Register application and get pipeline credentials')
-    .option('-p, --port <port>', 'Application port (default: from variables.yaml)')
-    .option('-u, --url <url>', 'Application URL. If omitted: app.url, deployment.dataplaneUrl or deployment.appUrl in variables.yaml; else http://localhost:{build.localPort or port}')
+    .option('-p, --port <port>', 'Application port (default: from application.yaml)')
+    .option('-u, --url <url>', 'Application URL. If omitted: app.url, deployment.dataplaneUrl or deployment.appUrl in application.yaml; else http://localhost:{build.localPort or port}')
     .option('-n, --name <name>', 'Override display name')
     .option('-d, --description <desc>', 'Override description')
     .action(async(appKey, options) => {
@@ -42,10 +32,10 @@ function setupAppCommands(program) {
         process.exit(1);
       }
     });
+}
 
-  // List command (controller and environment from config.yaml)
-  app
-    .command('list')
+function setupAppListCommand(app) {
+  app.command('list')
     .description('List applications')
     .action(async(options) => {
       try {
@@ -55,10 +45,10 @@ function setupAppCommands(program) {
         process.exit(1);
       }
     });
+}
 
-  // Rotate secret command (controller and environment from config.yaml)
-  app
-    .command('rotate-secret <appKey>')
+function setupAppRotateSecretCommand(app) {
+  app.command('rotate-secret <appKey>')
     .description('Rotate pipeline ClientSecret for an application')
     .action(async(appKey, options) => {
       try {
@@ -68,11 +58,12 @@ function setupAppCommands(program) {
         process.exit(1);
       }
     });
+}
 
-  // Show command: show application from controller (online). Same as aifabrix show <appKey> --online.
-  app
-    .command('show <appKey>')
+function setupAppShowCommand(app) {
+  app.command('show <appKey>')
     .description('Show application from controller (online). Same as aifabrix show <appKey> --online')
+    .option('--online', 'Fetch from controller (default for this command)')
     .option('--json', 'Output as JSON')
     .option('--permissions', 'Show only list of permissions')
     .action(async(appKey, options) => {
@@ -83,10 +74,10 @@ function setupAppCommands(program) {
         process.exit(1);
       }
     });
+}
 
-  // Deployment list for an application
-  app
-    .command('deployment <appKey>')
+function setupAppDeploymentCommand(app) {
+  app.command('deployment <appKey>')
     .description('List last N deployments for an application in current environment (default pageSize=50)')
     .option('--controller <url>', 'Controller URL (default: from config)')
     .option('--environment <env>', 'Environment key (default: from config)')
@@ -106,5 +97,18 @@ function setupAppCommands(program) {
     });
 }
 
+/**
+ * Setup application management commands
+ * @param {Command} program - Commander program instance
+ */
+function setupAppCommands(program) {
+  const app = program.command('app').description('Manage applications');
+  setupAppRegisterCommand(app);
+  setupAppListCommand(app);
+  setupAppRotateSecretCommand(app);
+  setupAppShowCommand(app);
+  setupAppDeploymentCommand(app);
+}
+
 module.exports = { setupAppCommands };
 

package/lib/commands/convert.js

@@ -0,0 +1,202 @@
+/**
+ * Convert command: convert integration/external system and datasource config files between JSON and YAML.
+ *
+ * Process: validate first, then (unless --force) prompt for confirmation, then convert (write new files),
+ * update application config links, then delete old files.
+ *
+ * @fileoverview Convert config format (JSON/YAML) for external integration files
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+'use strict';
+
+const path = require('path');
+const fs = require('fs');
+const readline = require('readline');
+const { detectAppType } = require('../utils/paths');
+const { logOfflinePathWhenType } = require('../utils/cli-utils');
+const { resolveApplicationConfigPath } = require('../utils/app-config-resolver');
+const { loadConfigFile, writeConfigFile } = require('../utils/config-format');
+
+const TARGET_EXT = { yaml: '.yaml', json: '.json' };
+const APP_CONFIG_NAMES = { yaml: 'application.yaml', json: 'application.json' };
+
+/**
+ * Prompts the user for confirmation (y/N).
+ *
+ * @param {string} message - Prompt message
+ * @returns {Promise<boolean>} True if user confirms (y/yes), false otherwise
+ */
+function promptConfirm(message) {
+  const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
+  return new Promise(resolve => {
+    rl.question(message, answer => {
+      rl.close();
+      const normalized = (answer || '').trim().toLowerCase();
+      resolve(normalized === 'y' || normalized === 'yes');
+    });
+  });
+}
+
+/**
+ * Returns the target filename for a given file path and format (same base name, new extension).
+ *
+ * @param {string} filePath - Current file path
+ * @param {string} format - Target format: 'json' or 'yaml'
+ * @returns {string} New filename (basename only)
+ */
+function targetFileName(filePath, format) {
+  const base = path.basename(filePath, path.extname(filePath));
+  const ext = TARGET_EXT[format] || (format === 'json' ? '.json' : '.yaml');
+  return base + ext;
+}
+
+/**
+ * Converts a single config file to the target format: writes to new path, returns old and new paths.
+ *
+ * @param {string} sourcePath - Absolute path to existing file
+ * @param {string} targetPath - Absolute path for new file
+ * @param {string} format - 'json' or 'yaml'
+ * @returns {{ oldPath: string, newPath: string }}
+ */
+function convertOneFile(sourcePath, targetPath, format) {
+  const obj = loadConfigFile(sourcePath);
+  writeConfigFile(targetPath, obj, format);
+  return { oldPath: sourcePath, newPath: targetPath };
+}
+
+/**
+ * Converts a list of config files (system or datasource) in a directory; skips missing files.
+ *
+ * @param {string} schemaBasePath - Base directory for files
+ * @param {string[]} fileNames - List of filenames
+ * @param {string} format - 'json' or 'yaml'
+ * @returns {{ converted: string[], toDelete: string[], newNames: string[] }}
+ */
+function convertFileList(schemaBasePath, fileNames, format) {
+  const converted = [];
+  const toDelete = [];
+  const newNames = [];
+  for (const fileName of fileNames) {
+    const sourcePath = path.join(schemaBasePath, fileName);
+    if (!fs.existsSync(sourcePath)) {
+      newNames.push(fileName);
+      continue;
+    }
+    const newFileName = targetFileName(sourcePath, format);
+    const targetPath = path.join(schemaBasePath, newFileName);
+    convertOneFile(sourcePath, targetPath, format);
+    converted.push(targetPath);
+    newNames.push(newFileName);
+    if (path.normalize(sourcePath) !== path.normalize(targetPath)) {
+      toDelete.push(sourcePath);
+    }
+  }
+  return { converted, toDelete, newNames };
+}
+
+/**
+ * Validates app and prompts for confirmation unless opts.force.
+ *
+ * @param {Object} opts - Options
+ * @param {string} opts.appName - Application name
+ * @param {Object} opts.cmdOptions - Command options (force, type)
+ * @param {string} opts.schemaBasePath - Base path for external files
+ * @param {string[]} opts.systemFiles - System file names
+ * @param {string[]} opts.datasourceFiles - Datasource file names
+ * @param {string} opts.configPath - Current application config path
+ * @param {string} opts.format - Target format
+ * @throws {Error} If validation fails or user cancels
+ */
+async function validateAndPrompt(opts) {
+  const validate = require('../validation/validate');
+  const result = await validate.validateAppOrFile(opts.appName, opts.cmdOptions);
+  if (!result.valid) {
+    validate.displayValidationResults(result);
+    throw new Error('Validation failed. Fix errors before converting.');
+  }
+  const { configPath, format, schemaBasePath, systemFiles, datasourceFiles } = opts;
+  const appConfigName = APP_CONFIG_NAMES[format];
+  const targetConfigPath = path.join(path.dirname(configPath), appConfigName);
+  const willConvertAppConfig = path.normalize(configPath) !== path.normalize(targetConfigPath) ||
+    path.extname(configPath) !== (format === 'json' ? '.json' : '.yaml');
+  const summaryLines = [...systemFiles, ...datasourceFiles]
+    .filter(Boolean)
+    .map(f => `  • ${f} → ${targetFileName(path.join(schemaBasePath, f), format)}`);
+  if (willConvertAppConfig) summaryLines.push(`  • application config → ${appConfigName}`);
+  summaryLines.push('  Old files will be removed after writing new ones.');
+  if (!opts.cmdOptions.force) {
+    const confirmed = await promptConfirm(`Convert the following to ${format}?\n${summaryLines.join('\n')}\nAre you sure? (y/N) `);
+    if (!confirmed) throw new Error('Convert cancelled.');
+  }
+}
+
+/**
+ * Converts files, updates application config, and deletes old files.
+ *
+ * @param {Object} variables - Current application config
+ * @param {string} configPath - Current application config path
+ * @param {string} schemaBasePath - Base path for external files
+ * @param {string[]} systemFiles - System file names
+ * @param {string[]} datasourceFiles - Datasource file names
+ * @param {string} format - Target format
+ * @returns {{ converted: string[], deleted: string[] }}
+ */
+function executeConversion(variables, configPath, schemaBasePath, systemFiles, datasourceFiles, format) {
+  const sys = convertFileList(schemaBasePath, systemFiles, format);
+  const ds = convertFileList(schemaBasePath, datasourceFiles, format);
+  const converted = [...sys.converted, ...ds.converted];
+  const toDelete = [...sys.toDelete, ...ds.toDelete];
+  const updatedVariables = { ...variables };
+  if (updatedVariables.externalIntegration) {
+    updatedVariables.externalIntegration = { ...updatedVariables.externalIntegration };
+    updatedVariables.externalIntegration.systems = sys.newNames;
+    updatedVariables.externalIntegration.dataSources = ds.newNames;
+  }
+  const appConfigName = APP_CONFIG_NAMES[format];
+  const targetConfigPath = path.join(path.dirname(configPath), appConfigName);
+  writeConfigFile(targetConfigPath, updatedVariables, format);
+  converted.push(targetConfigPath);
+  if (path.normalize(configPath) !== path.normalize(targetConfigPath)) toDelete.push(configPath);
+  toDelete.forEach(oldPath => {
+    if (fs.existsSync(oldPath)) fs.unlinkSync(oldPath);
+  });
+  return { converted, deleted: toDelete };
+}
+
+/**
+ * Runs conversion: validate first, optional prompt, convert files, update app config links, delete old files.
+ *
+ * @param {string} appName - Application name
+ * @param {Object} options - Command options
+ * @param {string} options.format - Target format: 'json' or 'yaml'
+ * @param {boolean} [options.force] - Skip confirmation prompt
+ *
+ * @returns {Promise<{ converted: string[], deleted: string[] }>} Lists of converted and deleted file paths
+ * @throws {Error} If validation fails, user aborts, or conversion fails
+ */
+async function runConvert(appName, options) {
+  const format = (options.format || '').toLowerCase();
+  if (format !== 'json' && format !== 'yaml') {
+    throw new Error('Option --format is required and must be \'json\' or \'yaml\'');
+  }
+  const { appPath } = await detectAppType(appName);
+  logOfflinePathWhenType(appPath);
+  const configPath = resolveApplicationConfigPath(appPath);
+  const variables = loadConfigFile(configPath);
+  const schemaBasePath = path.resolve(path.dirname(configPath), variables.externalIntegration?.schemaBasePath || './');
+  const systemFiles = variables.externalIntegration?.systems || [];
+  const datasourceFiles = variables.externalIntegration?.dataSources || [];
+  await validateAndPrompt({
+    appName, cmdOptions: options, schemaBasePath, systemFiles, datasourceFiles, configPath, format
+  });
+  return executeConversion(variables, configPath, schemaBasePath, systemFiles, datasourceFiles, format);
+}
+
+module.exports = {
+  runConvert,
+  promptConfirm,
+  targetFileName,
+  convertOneFile
+};