@aifabrix/builder 2.59.0 → 2.61.2

package/lib/commands/setup-prompts.js

@@ -1,23 +1,12 @@
 /**
  * Inquirer prompts for `aifabrix setup`.
  *
- * Three small prompt helpers:
+ * Setup prompt helpers:
  *   1. {@link promptModeSelection} - Mode menu shown when infra is already up.
  *   2. {@link promptAdminCredentials} - Admin email + password (fresh install only); email default
  *      from {@link module:lib/core/config.getAdminEmail} when `adminEmail` exists in config.yaml.
- *   3. {@link promptAiTool} - AI tool provider + keys, only when the merged
- *      secret cascade (user-local + shared) does not already provide the keys.
  *
- * Reads merged secrets via `loadSecrets()` from `lib/core/secrets.js` so the
- * resolution honors both user-local secrets and the shared `aifabrix-secrets`
- * store (file or remote API) when configured.
- *
- * Writes use `setSecretInStore(...)` (primary user `secrets.local.yaml` only; shared is human via
- * `aifabrix secret set --shared`) so the
- * installer can write either to user-local secrets or to the shared store,
- * depending on config.
- *
- * @fileoverview Setup prompts (mode menu, admin creds, AI tool)
+ * @fileoverview Setup prompts (mode menu, admin credentials, builder conflicts)
  * @author AI Fabrix Team
  * @version 2.0.0
  */
@@ -26,11 +15,6 @@
 
 const path = require('path');
 const inquirer = require('inquirer');
-const chalk = require('chalk');
-const logger = require('../utils/logger');
-const secretsCore = require('../core/secrets');
-const secretsEnsure = require('../core/secrets-ensure');
-const { formatSuccessLine, infoLine } = require('../utils/cli-test-layout-chalk');
 const { validateEmail, validatePassword } = require('./setup-prompts-validators');
 const {
   promptAdminCredentialsWithProfile,
@@ -45,13 +29,6 @@ const MODE = Object.freeze({
   UPDATE_IMAGES: 'update-images'
 });
 
-/** AI tool key names in `secrets.local.yaml` / shared secrets. */
-const AI_KEYS = Object.freeze({
-  OPENAI_API_KEY: 'secrets-openaiApiKeyVault',
-  AZURE_OPENAI_URL: 'azure-openaiapi-urlKeyVault',
-  AZURE_OPENAI_API_KEY: 'secrets-azureOpenaiApiKeyVault'
-});
-
 /** Builder folder conflict actions (setup preflight). */
 const BUILDER_DIR_ACTION = Object.freeze({
   KEEP: 'keep',
@@ -131,158 +108,6 @@ async function promptAdminCredentials(options = {}) {
   };
 }
 
-/**
- * @typedef {Object} AiToolStatus
- * @property {boolean} openAiConfigured - True when OpenAI key resolves non-empty
- * @property {boolean} azureOpenAiConfigured - True when Azure URL + key resolve non-empty
- */
-
-/**
- * Detect whether AI tool keys are already present in the merged secret view.
- * Treats values surrounded by Handlebars-style placeholders (e.g. `{{...}}`)
- * as not configured.
- *
- * @async
- * @returns {Promise<AiToolStatus>}
- */
-async function detectAiToolStatus() {
-  let merged = {};
-  try {
-    merged = await secretsCore.loadSecrets(undefined);
-  } catch {
-    merged = {};
-  }
-  const isSet = (key) => {
-    const raw = merged ? merged[key] : undefined;
-    if (raw === undefined || raw === null) return false;
-    const value = String(raw).trim();
-    if (!value) return false;
-    if (value.startsWith('{{') && value.endsWith('}}')) return false;
-    return true;
-  };
-  return {
-    openAiConfigured: isSet(AI_KEYS.OPENAI_API_KEY),
-    azureOpenAiConfigured:
-      isSet(AI_KEYS.AZURE_OPENAI_URL) && isSet(AI_KEYS.AZURE_OPENAI_API_KEY)
-  };
-}
-
-/**
- * Ask which AI tool the user wants to configure when none is configured yet.
- * @async
- * @returns {Promise<'openai'|'azure'|'skip'>}
- */
-async function askAiToolChoice() {
-  const { choice } = await inquirer.prompt([
-    {
-      type: 'list',
-      name: 'choice',
-      message: 'Which AI tool do you want to configure for the dataplane?',
-      choices: [
-        { name: 'OpenAI (api.openai.com)', value: 'openai' },
-        { name: 'Azure OpenAI (your Azure resource)', value: 'azure' },
-        { name: 'Skip for now (set later via aifabrix secret set)', value: 'skip' }
-      ]
-    }
-  ]);
-  return choice;
-}
-
-/**
- * Collect and persist OpenAI key.
- * @async
- * @returns {Promise<void>}
- */
-async function collectAndSaveOpenAiKey() {
-  const { apiKey } = await inquirer.prompt([
-    {
-      type: 'password',
-      name: 'apiKey',
-      message: 'OpenAI API key:',
-      mask: '*',
-      validate(input) {
-        return String(input || '').trim() ? true : 'API key is required';
-      }
-    }
-  ]);
-  await secretsEnsure.setSecretInStore(AI_KEYS.OPENAI_API_KEY, String(apiKey).trim());
-  logger.log(formatSuccessLine('OpenAI API key saved'));
-}
-
-/**
- * Collect and persist Azure OpenAI URL + key.
- * @async
- * @returns {Promise<void>}
- */
-async function collectAndSaveAzureOpenAi() {
-  const answers = await inquirer.prompt([
-    {
-      type: 'input',
-      name: 'url',
-      message: 'Azure OpenAI endpoint URL (e.g. https://my-aoai.openai.azure.com):',
-      validate(input) {
-        const value = String(input || '').trim();
-        if (!value) return 'Endpoint URL is required';
-        if (!/^https?:\/\//.test(value)) return 'URL must start with http(s)://';
-        return true;
-      }
-    },
-    {
-      type: 'password',
-      name: 'apiKey',
-      message: 'Azure OpenAI API key:',
-      mask: '*',
-      validate(input) {
-        return String(input || '').trim() ? true : 'API key is required';
-      }
-    }
-  ]);
-  await secretsEnsure.setSecretInStore(AI_KEYS.AZURE_OPENAI_URL, String(answers.url).trim());
-  await secretsEnsure.setSecretInStore(AI_KEYS.AZURE_OPENAI_API_KEY, String(answers.apiKey).trim());
-  logger.log(formatSuccessLine('Azure OpenAI endpoint and API key saved'));
-}
-
-/**
- * Run the AI tool prompt sequence.
- * Skips silently when either provider is already configured (in any source).
- *
- * @async
- * @param {{ silentIfConfigured?: boolean }} [options]
- * @returns {Promise<void>}
- */
-async function promptAiTool(options = {}) {
-  const silentIfConfigured = options && options.silentIfConfigured === true;
-  const status = await detectAiToolStatus();
-  if (status.openAiConfigured) {
-    if (!silentIfConfigured) {
-      logger.log(infoLine('OpenAI key already configured - skipping AI tool prompt.'));
-    }
-    return;
-  }
-  if (status.azureOpenAiConfigured) {
-    if (!silentIfConfigured) {
-      logger.log(infoLine('Azure OpenAI endpoint and key already configured - skipping AI tool prompt.'));
-    }
-    return;
-  }
-  const choice = await askAiToolChoice();
-  if (choice === 'openai') {
-    await collectAndSaveOpenAiKey();
-    return;
-  }
-  if (choice === 'azure') {
-    await collectAndSaveAzureOpenAi();
-    return;
-  }
-  logger.log(
-    chalk.gray(
-      'Skipped. Set later: aifabrix secret set ' +
-        AI_KEYS.OPENAI_API_KEY +
-        ' <key>  (or the two azure-openaiapi-urlKeyVault / secrets-azureOpenaiApiKeyVault keys)'
-    )
-  );
-}
-
 /**
  * Absolute directory paths (with trailing sep) shown when setup will replace platform apps.
  *
@@ -353,7 +178,6 @@ async function promptBuilderDirConflict(info) {
 
 module.exports = {
   MODE,
-  AI_KEYS,
   BUILDER_DIR_ACTION,
   INSTALLATION_PROFILE,
   formatBuilderPlatformReplaceLines,
@@ -361,8 +185,6 @@ module.exports = {
   confirmDestructiveMode,
   promptInstallationProfile,
   promptAdminCredentials,
-  promptAiTool,
-  detectAiToolStatus,
   promptBuilderDirConflict,
   validateEmail,
   validatePassword

package/lib/commands/verify-operations-skip-e2e.js

@@ -105,7 +105,8 @@ function shouldSkipE2eForStaticPackage(systemKey, options) {
     if (storage.length === 0) {
       return false;
     }
-    return storage.every((ds) => String(ds.syncBridge || 'disabled').toLowerCase() === 'disabled');
+    return storage.every((ds) => !isKeyedDocumentDatasource(ds) &&
+      String(ds.syncBridge || 'disabled').toLowerCase() === 'disabled');
   } catch {
     return false;
   }
@@ -124,7 +125,30 @@ function shouldSkipOperationsE2e(systemKey, options) {
     || shouldSkipE2eForStaticPackage(systemKey, options);
 }
 
+/** @param {Object} ds @returns {boolean} */
+function isKeyedDocumentDatasource(ds) {
+  const ops = ds?.execution?.cip?.operations;
+  return ds?.entityType === 'documentStorage' && !ops?.list &&
+    !ds.openapi?.operations?.list && !ds.capabilities?.some(cap => (cap.key || cap) === 'list') &&
+    Array.isArray(ops?.get?.steps) && ops.get.steps.some(step => step?.output?.mode === 'documents');
+}
+
+/** @param {string} systemKey @param {Object} options @returns {string[]} */
+function keyedDocumentDatasourceKeys(systemKey, options) {
+  if (options.skipE2e === true || options.minimal === true) return [];
+  try {
+    const { resolveAppWorkspace } = require('../utils/app-path-resolve');
+    const { appPath } = resolveAppWorkspace(systemKey, options);
+    const storage = loadStorageDatasources(appPath);
+    return storage.length && storage.every(isKeyedDocumentDatasource) ? storage.map(ds => ds.key) : [];
+  } catch {
+    return [];
+  }
+}
+
 module.exports = {
+  isKeyedDocumentDatasource,
+  keyedDocumentDatasourceKeys,
   shouldSkipOperationsE2e,
   shouldSkipE2eForRoleAssistant,
   shouldSkipE2eForStaticPackage

package/lib/commands/verify-operations-steps.js

@@ -4,6 +4,9 @@
 
 'use strict';
 
+const { keyedDocumentDatasourceKeys } = require('./verify-operations-skip-e2e');
+const { hasAllKeyedDocumentPersistence } = require('../utils/datasource-binary-evidence');
+
 const { syncLocalIfRequested } = require('./test-e2e-external');
 const { cliOptsSkipPublish } = require('../utils/cli-sync-options');
 const { buildOperationsVerifyStepPlan } = require('./verify-operations-step-plan');
@@ -99,10 +102,19 @@ async function runOperationsVerifySteps(systemKey, options) {
     }
   }
 
+  const keyedKeys = keyedDocumentDatasourceKeys(systemKey, options);
   const e2eLabel = STEP_PROGRESS.e2e;
   stepResults.e2e = await runWithVerifyStepProgress(
     e2eLabel,
-    async(progress) => runVerifyStep(() => plan.e2e.run(progress), plan.e2e.ok),
+    async(progress) => {
+      const result = await plan.e2e.run(progress);
+      if (keyedKeys.length) {
+        stepResults.keyedDocumentGet = true;
+        stepResults.documentPersistence = hasAllKeyedDocumentPersistence(result, keyedKeys);
+        return plan.e2e.ok(result) && stepResults.documentPersistence;
+      }
+      return plan.e2e.ok(result);
+    },
     options
   );
   return { stepResults, hardFailure: !stepResults.e2e, stoppedEarly: false };

package/lib/commands/wizard-config-normalizer.js

@@ -4,6 +4,8 @@
  * @version 2.0.0
  */
 
+const { repairKeyedDocument } = require('./repair-datasource-keyed-document');
+
 const ENTITY_TYPE_FALLBACK = 'record-storage';
 const VALID_ENTITY_TYPES = new Set([
   'document-storage',
@@ -56,7 +58,7 @@ function normalizeCipOutputModes(datasourceConfig) {
   for (const operation of Object.values(operations)) {
     if (!operation || !Array.isArray(operation.steps)) continue;
     for (const step of operation.steps) {
-      if (step?.output?.mode && step.output.mode !== 'records') {
+      if (step?.output?.mode && !['records', 'documents'].includes(step.output.mode)) {
         step.output.mode = 'records';
       }
     }
@@ -80,7 +82,7 @@ function normalizeWizardTestPayload(datasourceConfig) {
  * @param {Object} datasourceConfig - Datasource config
  * @returns {Object} Normalized config
  */
-function normalizeDatasourceConfig(datasourceConfig) {
+function normalizeDatasourceConfig(datasourceConfig, openapiSpec) {
   if (!datasourceConfig || typeof datasourceConfig !== 'object') {
     return datasourceConfig;
   }
@@ -89,6 +91,7 @@ function normalizeDatasourceConfig(datasourceConfig) {
   }
   normalizePortalInput(datasourceConfig);
   normalizeCipOutputModes(datasourceConfig);
+  repairKeyedDocument(datasourceConfig, { openapiSpec });
   normalizeWizardTestPayload(datasourceConfig);
   return datasourceConfig;
 }
@@ -100,10 +103,10 @@ function normalizeDatasourceConfig(datasourceConfig) {
  * @param {Object|Object[]} datasourceConfigs - Datasource config(s)
  * @returns {{ systemConfig: Object, datasourceConfigs: Object[] }} Normalized configs
  */
-function normalizeWizardConfigs(systemConfig, datasourceConfigs) {
+function normalizeWizardConfigs(systemConfig, datasourceConfigs, openapiSpec) {
   const normalizedSystem = normalizeSystemConfig(systemConfig);
   const configs = Array.isArray(datasourceConfigs) ? datasourceConfigs : [datasourceConfigs];
-  const normalizedDatasources = configs.map(normalizeDatasourceConfig);
+  const normalizedDatasources = configs.map(config => normalizeDatasourceConfig(config, openapiSpec));
   return { systemConfig: normalizedSystem, datasourceConfigs: normalizedDatasources };
 }
 

package/lib/commands/wizard-core.js

@@ -3,28 +3,24 @@
  * @author AI Fabrix Team
  * @version 2.0.0
  */
-/* eslint-disable max-lines */
 
+const path = require('path');
 const chalk = require('chalk');
 const ora = require('ora');
-const path = require('path');
-const fs = require('fs').promises;
 const logger = require('../utils/logger');
 const { getDeploymentAuth, requireBearerForDataplanePipeline } = require('../utils/token-manager');
 const { resolveControllerUrl } = require('../utils/controller-url');
 const { normalizeWizardConfigs } = require('./wizard-config-normalizer');
-const { formatSuccessLine, formatSuccessParagraph } = require('../utils/cli-layout-chalk');
+const { formatSuccessLine } = require('../utils/cli-layout-chalk');
 const {
   createWizardSession,
   updateWizardSession,
   detectType,
   getPlatformConfig,
   generateConfig,
-  validateWizardConfig,
-  getDeploymentDocs,
-  postDeploymentDocs
+  validateWizardConfig
 } = require('../api/wizard.api');
-const { generateWizardFiles } = require('../generator/wizard');
+const { handleFileSaving, validateAndCheckAppDirectory } = require('./wizard-file-saving');
 const {
   parseOpenApiSource,
   testMcpServerConnection,
@@ -44,38 +40,6 @@ const {
 const { validateDatasourceKeysBeforePlatformConfig } = require('../validation/validate-datasource-keys-api');
 const { handleEntitySelection } = require('./wizard-entity-selection');
 
-/**
- * Validate app name and check if directory exists
- * @async
- * @function validateAndCheckAppDirectory
- * @param {string} appName - Application name
- * @param {boolean} [interactive=true] - Whether to prompt for confirmation
- * @returns {Promise<boolean>} True if should continue, false if cancelled
- */
-async function validateAndCheckAppDirectory(appName, interactive = true) {
-  if (!/^[a-z0-9-_]+$/.test(appName)) {
-    throw new Error('Application name must contain only lowercase letters, numbers, hyphens, and underscores');
-  }
-  const appPath = path.join(process.cwd(), 'integration', appName);
-  try {
-    await fs.access(appPath);
-    if (interactive) {
-      const { overwrite } = await require('inquirer').prompt([{
-        type: 'confirm', name: 'overwrite',
-        message: `Directory ${appPath} already exists. Overwrite?`, default: false
-      }]);
-      if (!overwrite) {
-        logger.log(chalk.yellow('Wizard cancelled.')); return false;
-      }
-    } else {
-      logger.log(chalk.yellow(`Warning: Directory ${appPath} exists. Overwriting...`));
-    }
-  } catch (error) {
-    if (error.code !== 'ENOENT') throw error;
-  }
-  return true;
-}
-
 /**
  * Extract session ID from response data
  * @function extractSessionId
@@ -393,7 +357,7 @@ async function handleConfigurationGeneration(dataplaneUrl, authConfig, options)
     }
     const result = extractConfigurationFromResponse(generateResponse);
     const systemConfig = resolveSystemConfigForWizardMode(result.systemConfig, options);
-    const normalized = normalizeWizardConfigs(systemConfig, result.datasourceConfigs);
+    const normalized = normalizeWizardConfigs(systemConfig, result.datasourceConfigs, options.openapiSpec);
     await maybeWriteWizardDebugLog(generateResponse, prefs, options);
     logger.log(formatSuccessLine('Configuration generated successfully'));
     return {
@@ -446,122 +410,6 @@ async function validateWizardConfiguration(dataplaneUrl, authConfig, systemConfi
   }
 }
 
-/**
- * Fetches deployment docs and writes README.md when application config and deploy JSON are available.
- * @async
- * @param {string} appPath - Application path
- * @param {string} appName - Application name
- * @param {string} dataplaneUrl - Dataplane URL
- * @param {Object} authConfig - Authentication configuration
- * @param {string} systemKey - System key
- */
-async function tryUpdateReadmeFromDeploymentDocs(appPath, appName, dataplaneUrl, authConfig, systemKey) {
-  const { resolveApplicationConfigPath } = require('../utils/app-config-resolver');
-  const deployPath = path.join(appPath, `${appName}-deploy.json`);
-  let variablesYaml = null;
-  let deployJson = null;
-  try {
-    const configPath = resolveApplicationConfigPath(appPath);
-    variablesYaml = await fs.readFile(configPath, 'utf8');
-  } catch {
-    // optional
-  }
-  try {
-    const deployContent = await fs.readFile(deployPath, 'utf8');
-    deployJson = JSON.parse(deployContent);
-  } catch {
-    // optional
-  }
-  const hasBody = variablesYaml !== null || deployJson !== null;
-  const body = hasBody ? { variablesYaml: variablesYaml || null, deployJson: deployJson || null } : null;
-  const docsResponse = body
-    ? await postDeploymentDocs(dataplaneUrl, authConfig, systemKey, body)
-    : await getDeploymentDocs(dataplaneUrl, authConfig, systemKey);
-  const content = docsResponse?.data?.content ?? docsResponse?.content;
-  // Only overwrite README when the API returns substantial content; otherwise keep the template README
-  const MIN_README_LENGTH = 400;
-  if (content && typeof content === 'string' && content.trim().length >= MIN_README_LENGTH) {
-    const readmePath = path.join(appPath, 'README.md');
-    await fs.writeFile(readmePath, content, 'utf8');
-    logger.log(chalk.gray('  Updated README.md from deployment-docs API (application config + deploy JSON).'));
-  }
-}
-
-/**
- * Writes rbac.yaml / rbac.json from datasource resourceType + capabilities (same as `af repair --rbac`).
- * @param {Object} generatedFiles - Result of generateWizardFiles (appPath, systemFilePath, datasourceFilePaths)
- * @param {string} format - Project format: yaml | json
- */
-function logWizardFileSaveFooter(appName, generatedFiles) {
-  logger.log(formatSuccessParagraph('Wizard completed successfully!'));
-  logger.log(chalk.green(`\nFiles created in: ${generatedFiles.appPath}`));
-  logger.log(chalk.blue('\nNext steps:'));
-  logger.log(chalk.gray(`  1. Review the generated files in integration/${appName}/`));
-  logger.log(chalk.gray('  2. Update env.template with your authentication details'));
-  logger.log(chalk.gray(`  3. Deploy using: node deploy.js or aifabrix deploy ${appName}`));
-}
-
-function mergeRbacAfterWizardFilesWritten(generatedFiles, format) {
-  const { mergeRbacFromDatasources, extractRbacFromSystem } = require('./repair-rbac');
-  const { loadConfigFile } = require('../utils/config-format');
-  const systemParsedForRbac = loadConfigFile(generatedFiles.systemFilePath);
-  const datasourceFileNames = (generatedFiles.datasourceFilePaths || []).map((p) => path.basename(p));
-  const rbacChanges = [];
-  const rbacUpdated = mergeRbacFromDatasources(
-    generatedFiles.appPath,
-    systemParsedForRbac,
-    datasourceFileNames,
-    extractRbacFromSystem,
-    { format: format === 'json' ? 'json' : 'yaml', dryRun: false, changes: rbacChanges }
-  );
-  if (rbacUpdated && rbacChanges.length) {
-    rbacChanges.forEach((c) => logger.log(chalk.gray(`  RBAC: ${c}`)));
-    logger.log(chalk.green('  RBAC file updated from datasource capabilities (enableRBAC).'));
-  }
-}
-
-/**
- * Handle file saving step
- * @async
- * @function handleFileSaving
- * @param {string} appName - Application name
- * @param {Object} systemConfig - System configuration
- * @param {Object[]} datasourceConfigs - Datasource configurations
- * @param {string} systemKey - System key
- * @param {{ dataplaneUrl: string, authConfig: Object, enableRBAC?: boolean }} ctx - Dataplane auth + optional RBAC generation
- * @returns {Promise<Object>} Generated files information
- */
-async function handleFileSaving(appName, systemConfig, datasourceConfigs, systemKey, ctx) {
-  const { dataplaneUrl, authConfig, enableRBAC = false } = ctx || {};
-  logger.log(chalk.blue('\n\uD83D\uDCCB Step 7: Save Files'));
-  const spinner = ora('Saving files...').start();
-  try {
-    const config = require('../core/config');
-    const format = ctx?.format || (await config.getFormat()) || 'yaml';
-    const generatedFiles = await generateWizardFiles(appName, systemConfig, datasourceConfigs, systemKey, { aiGeneratedReadme: null, format });
-    if (enableRBAC && generatedFiles.appPath && generatedFiles.systemFilePath) {
-      try {
-        mergeRbacAfterWizardFilesWritten(generatedFiles, format);
-      } catch (e) {
-        logger.log(chalk.yellow(`  Could not generate RBAC file: ${e.message}`));
-      }
-    }
-    if (systemKey && dataplaneUrl && authConfig && generatedFiles.appPath) {
-      try {
-        await tryUpdateReadmeFromDeploymentDocs(generatedFiles.appPath, appName, dataplaneUrl, authConfig, systemKey);
-      } catch (e) {
-        logger.log(chalk.gray(`  Could not fetch AI-generated README: ${e.message}`));
-      }
-    }
-    spinner.stop();
-    logWizardFileSaveFooter(appName, generatedFiles);
-    return generatedFiles;
-  } catch (error) {
-    spinner.stop();
-    throw error;
-  }
-}
-
 /**
  * Setup dataplane URL and authentication
  * @async