@aifabrix/builder 2.60.0 → 2.61.2

package/lib/commands/repair-datasource-keyed-document.js

@@ -0,0 +1,122 @@
+/**
+ * @fileoverview Generate keyed document retrieval from vendor OpenAPI response media types.
+ * @author AI Fabrix Team
+ */
+'use strict';
+
+const fs = require('fs');
+const { toCamelCaseKey } = require('./repair-datasource-auto-rbac');
+const { loadConfigFile } = require('../utils/config-format');
+const { documentKeyToLocalOpenApiPath } = require('./repair-openapi-export');
+const { findVendorOperation } = require('../validation/openapi-contract-vendor-validation');
+
+/** @param {Object} ds @param {Object} options @returns {Object|null} */
+function loadVendorSpec(ds, options) {
+  if (options.openapiSpec) return options.openapiSpec;
+  if (!options.appPath || !ds.openapi?.documentKey) return null;
+  const file = documentKeyToLocalOpenApiPath(options.appPath, ds.systemKey || '', ds.openapi.documentKey);
+  return fs.existsSync(file) ? loadConfigFile(file) : null;
+}
+
+/** @param {Object} operation @returns {Object[]} */
+function binaryResponses(operation) {
+  return Object.entries(operation?.responses || {})
+    .filter(([status]) => /^2(?:\d\d|XX)$/i.test(status))
+    .map(([, response]) => response)
+    .filter(response => Object.keys(response?.content || {}).some(type =>
+      /^(application\/(octet-stream|pdf)|image\/[^;]+)(;|$)/i.test(type)));
+}
+
+/** @param {Object} ds @param {Object} operation @param {Object[]} responses */
+function addMetadataMappings(ds, operation, responses) {
+  ds.fieldMappings = ds.fieldMappings || {};
+  ds.fieldMappings.attributes = ds.fieldMappings.attributes || {};
+  const attributes = ds.fieldMappings.attributes;
+  for (const response of responses) {
+    for (const name of Object.keys(response.headers || {})) {
+      addMappingStub(attributes, name, 'headers');
+    }
+  }
+  for (const param of operation.parameters || []) {
+    if (param.required && ['query', 'path'].includes(param.in) && param.name) {
+      addMappingStub(attributes, param.name, 'params');
+    }
+  }
+}
+
+/** @param {Object} attributes @param {string} name @param {string} source */
+function addMappingStub(attributes, name, source) {
+  const key = toCamelCaseKey(name);
+  if (!/^[a-z][a-zA-Z0-9]*$/.test(key) || Object.hasOwn(attributes, key)) return;
+  attributes[key] = { expression: `{{raw.${source}.${name}}}` };
+}
+
+/** @param {Object} ds @param {string} ref @returns {Object[]} */
+function ensureGetSteps(ds, ref) {
+  ds.execution = ds.execution || {};
+  ds.execution.engine = 'cip';
+  ds.execution.cip = ds.execution.cip || { version: '1.0', operations: {} };
+  ds.execution.cip.operations = ds.execution.cip.operations || {};
+  const operations = ds.execution.cip.operations;
+  if (!operations.get) {
+    operations.get = { steps: [
+      { fetch: { source: 'openapi', openapiRef: ref, responseType: 'binary' } },
+      { map: { useFieldMappings: true } },
+      { filter: { abac: { mode: 'mandatory' } } },
+      { output: { mode: 'documents' } }
+    ] };
+  }
+  return Array.isArray(operations.get.steps) ? operations.get.steps : [];
+}
+
+/**
+ * Leave list-based document sync and author-owned unrelated operations unchanged.
+ * @param {Object} ds - Datasource, mutated
+ * @param {Object} options - Vendor spec or local appPath
+ * @param {string[]} changes - Repair log
+ * @returns {boolean}
+ */
+function repairKeyedDocument(ds, options = {}, changes = []) {
+  const candidate = keyedDocumentCandidate(ds, options);
+  if (!candidate) return false;
+  const { ref, vendor, responses } = candidate;
+  const before = JSON.stringify(ds);
+  const getSteps = ensureGetSteps(ds, ref);
+  if (!getSteps.some(step => step?.fetch) || !getSteps.some(step => step?.output)) return false;
+  for (const step of getSteps) {
+    if (step?.fetch && (step.fetch.openapiRef || 'get') === ref) step.fetch.responseType = 'binary';
+    if (step?.output) step.output.mode = 'documents';
+  }
+  ds.documentStorage = { enabled: true, ingestAfterSync: true,
+    ...ds.documentStorage, responseType: 'binary', binaryOperationRef: 'get' };
+  addMetadataMappings(ds, vendor, responses);
+  if (JSON.stringify(ds) === before) return false;
+  changes.push('Configured keyed binary get with documents output and header/parameter mappings');
+  return true;
+}
+
+/** @param {Object} ds @param {Object} options @returns {Object|null} */
+function keyedDocumentCandidate(ds, options) {
+  if (!['documentStorage', 'document-storage'].includes(ds?.entityType)) return null;
+  if (!canGenerateKeyedDocument(ds)) return null;
+  const operations = ds.execution?.cip?.operations || {};
+  const steps = operations.get?.steps || [];
+  const fetch = steps.find(step => step?.fetch)?.fetch;
+  if (fetch && !['openapi', undefined].includes(fetch.source)) return null;
+  const ref = fetch?.openapiRef || 'get';
+  const binding = ds.openapi?.operations?.[ref];
+  const vendor = findVendorOperation(loadVendorSpec(ds, options), binding);
+  const responses = binaryResponses(vendor);
+  return responses.length ? { ref, vendor, responses } : null;
+}
+
+/** @param {Object} ds @returns {boolean} */
+function canGenerateKeyedDocument(ds) {
+  if (ds.execution?.cip?.operations?.list || ds.openapi?.operations?.list) return false;
+  if (ds.capabilities?.some(cap => (cap.key || cap) === 'list')) return false;
+  if (ds.documentStorage?.enabled === false) return false;
+  if (ds.execution?.engine && ds.execution.engine !== 'cip') return false;
+  return !ds.documentStorage?.binaryOperationRef || ds.documentStorage.binaryOperationRef === 'get';
+}
+
+module.exports = { repairKeyedDocument };

package/lib/commands/repair-datasource-run.js

@@ -37,6 +37,7 @@ function runDatasourceRepairs(appPath, datasourceFiles, options, dryRun, changes
     try {
       const parsed = loadConfigFile(filePath);
       const { updated: fileUpdated, changes: fileChanges } = repairDatasourceFile(parsed, {
+        appPath,
         expose: options.expose,
         sync: options.sync,
         test: options.test,

package/lib/commands/setup-modes.js

@@ -322,8 +322,7 @@ function removeUserLocalSecrets() {
 }
 
 /**
- * Fresh install: wizard collects admin email + password, then AI tool keys,
- * then up-infra + up-platform.
+ * Fresh install: wizard collects admin credentials, then runs up-infra + up-platform.
  *
  * @async
  * @param {{ adminEmail: string, adminPassword?: string, passwordBundle?: object }} adminCreds
@@ -345,9 +344,6 @@ async function runFreshInstall(adminCreds) {
   });
   stopSpinnerSuccess(infraSpin, 'Infrastructure ready');
   await runSetupImageRefresh({ quiet: true });
-  // Bring up infra first, then collect optional platform-level secrets (AI tool)
-  // so the flow feels like a product installer: infra → platform configuration → platform services.
-  await setupPrompts.promptAiTool({ silentIfConfigured: true });
   // Fresh-install should reset dev defaults (env + controller URL) the same way
   // as other setup recovery paths, otherwise the platform can come up pointed at
   // stale configuration.
@@ -382,7 +378,6 @@ async function runReinstall() {
   await runSetupImageRefresh({ quiet: true });
   const adminCreds = await setupPrompts.promptAdminCredentials();
   await config.setAdminEmail(adminCreds.adminEmail);
-  await setupPrompts.promptAiTool({ silentIfConfigured: true });
   const infraSpin = startSpinner('Starting infrastructure...');
   await startInfraFromConfig({
     adminEmail: adminCreds.adminEmail,
@@ -436,7 +431,6 @@ async function runWipeData() {
   });
   stopSpinnerSuccess(wipeSpin, 'Databases reset');
   await runSetupImageRefresh({ quiet: true });
-  await setupPrompts.promptAiTool({ silentIfConfigured: true });
   const infraSpin = startSpinner('Starting infrastructure...');
   await startInfraFromConfig({});
   stopSpinnerSuccess(infraSpin, 'Infrastructure ready');
@@ -460,7 +454,6 @@ async function runWipeData() {
  */
 async function runUpdateImages() {
   await runSetupImageRefresh({ quiet: true });
-  await setupPrompts.promptAiTool({ silentIfConfigured: true });
   const infraSpin = startSpinner('Starting infrastructure...');
   await startInfraFromConfig({});
   stopSpinnerSuccess(infraSpin, 'Infrastructure ready');

package/lib/commands/setup-prompts.js

@@ -1,23 +1,12 @@
 /**
  * Inquirer prompts for `aifabrix setup`.
  *
- * Three small prompt helpers:
+ * Setup prompt helpers:
  *   1. {@link promptModeSelection} - Mode menu shown when infra is already up.
  *   2. {@link promptAdminCredentials} - Admin email + password (fresh install only); email default
  *      from {@link module:lib/core/config.getAdminEmail} when `adminEmail` exists in config.yaml.
- *   3. {@link promptAiTool} - AI tool provider + keys, only when the merged
- *      secret cascade (user-local + shared) does not already provide the keys.
  *
- * Reads merged secrets via `loadSecrets()` from `lib/core/secrets.js` so the
- * resolution honors both user-local secrets and the shared `aifabrix-secrets`
- * store (file or remote API) when configured.
- *
- * Writes use `setSecretInStore(...)` (primary user `secrets.local.yaml` only; shared is human via
- * `aifabrix secret set --shared`) so the
- * installer can write either to user-local secrets or to the shared store,
- * depending on config.
- *
- * @fileoverview Setup prompts (mode menu, admin creds, AI tool)
+ * @fileoverview Setup prompts (mode menu, admin credentials, builder conflicts)
  * @author AI Fabrix Team
  * @version 2.0.0
  */
@@ -26,11 +15,6 @@
 
 const path = require('path');
 const inquirer = require('inquirer');
-const chalk = require('chalk');
-const logger = require('../utils/logger');
-const secretsCore = require('../core/secrets');
-const secretsEnsure = require('../core/secrets-ensure');
-const { formatSuccessLine, infoLine } = require('../utils/cli-test-layout-chalk');
 const { validateEmail, validatePassword } = require('./setup-prompts-validators');
 const {
   promptAdminCredentialsWithProfile,
@@ -45,13 +29,6 @@ const MODE = Object.freeze({
   UPDATE_IMAGES: 'update-images'
 });
 
-/** AI tool key names in `secrets.local.yaml` / shared secrets. */
-const AI_KEYS = Object.freeze({
-  OPENAI_API_KEY: 'secrets-openaiApiKeyVault',
-  AZURE_OPENAI_URL: 'azure-openaiapi-urlKeyVault',
-  AZURE_OPENAI_API_KEY: 'secrets-azureOpenaiApiKeyVault'
-});
-
 /** Builder folder conflict actions (setup preflight). */
 const BUILDER_DIR_ACTION = Object.freeze({
   KEEP: 'keep',
@@ -131,158 +108,6 @@ async function promptAdminCredentials(options = {}) {
   };
 }
 
-/**
- * @typedef {Object} AiToolStatus
- * @property {boolean} openAiConfigured - True when OpenAI key resolves non-empty
- * @property {boolean} azureOpenAiConfigured - True when Azure URL + key resolve non-empty
- */
-
-/**
- * Detect whether AI tool keys are already present in the merged secret view.
- * Treats values surrounded by Handlebars-style placeholders (e.g. `{{...}}`)
- * as not configured.
- *
- * @async
- * @returns {Promise<AiToolStatus>}
- */
-async function detectAiToolStatus() {
-  let merged = {};
-  try {
-    merged = await secretsCore.loadSecrets(undefined);
-  } catch {
-    merged = {};
-  }
-  const isSet = (key) => {
-    const raw = merged ? merged[key] : undefined;
-    if (raw === undefined || raw === null) return false;
-    const value = String(raw).trim();
-    if (!value) return false;
-    if (value.startsWith('{{') && value.endsWith('}}')) return false;
-    return true;
-  };
-  return {
-    openAiConfigured: isSet(AI_KEYS.OPENAI_API_KEY),
-    azureOpenAiConfigured:
-      isSet(AI_KEYS.AZURE_OPENAI_URL) && isSet(AI_KEYS.AZURE_OPENAI_API_KEY)
-  };
-}
-
-/**
- * Ask which AI tool the user wants to configure when none is configured yet.
- * @async
- * @returns {Promise<'openai'|'azure'|'skip'>}
- */
-async function askAiToolChoice() {
-  const { choice } = await inquirer.prompt([
-    {
-      type: 'list',
-      name: 'choice',
-      message: 'Which AI tool do you want to configure for the dataplane?',
-      choices: [
-        { name: 'OpenAI (api.openai.com)', value: 'openai' },
-        { name: 'Azure OpenAI (your Azure resource)', value: 'azure' },
-        { name: 'Skip for now (set later via aifabrix secret set)', value: 'skip' }
-      ]
-    }
-  ]);
-  return choice;
-}
-
-/**
- * Collect and persist OpenAI key.
- * @async
- * @returns {Promise<void>}
- */
-async function collectAndSaveOpenAiKey() {
-  const { apiKey } = await inquirer.prompt([
-    {
-      type: 'password',
-      name: 'apiKey',
-      message: 'OpenAI API key:',
-      mask: '*',
-      validate(input) {
-        return String(input || '').trim() ? true : 'API key is required';
-      }
-    }
-  ]);
-  await secretsEnsure.setSecretInStore(AI_KEYS.OPENAI_API_KEY, String(apiKey).trim());
-  logger.log(formatSuccessLine('OpenAI API key saved'));
-}
-
-/**
- * Collect and persist Azure OpenAI URL + key.
- * @async
- * @returns {Promise<void>}
- */
-async function collectAndSaveAzureOpenAi() {
-  const answers = await inquirer.prompt([
-    {
-      type: 'input',
-      name: 'url',
-      message: 'Azure OpenAI endpoint URL (e.g. https://my-aoai.openai.azure.com):',
-      validate(input) {
-        const value = String(input || '').trim();
-        if (!value) return 'Endpoint URL is required';
-        if (!/^https?:\/\//.test(value)) return 'URL must start with http(s)://';
-        return true;
-      }
-    },
-    {
-      type: 'password',
-      name: 'apiKey',
-      message: 'Azure OpenAI API key:',
-      mask: '*',
-      validate(input) {
-        return String(input || '').trim() ? true : 'API key is required';
-      }
-    }
-  ]);
-  await secretsEnsure.setSecretInStore(AI_KEYS.AZURE_OPENAI_URL, String(answers.url).trim());
-  await secretsEnsure.setSecretInStore(AI_KEYS.AZURE_OPENAI_API_KEY, String(answers.apiKey).trim());
-  logger.log(formatSuccessLine('Azure OpenAI endpoint and API key saved'));
-}
-
-/**
- * Run the AI tool prompt sequence.
- * Skips silently when either provider is already configured (in any source).
- *
- * @async
- * @param {{ silentIfConfigured?: boolean }} [options]
- * @returns {Promise<void>}
- */
-async function promptAiTool(options = {}) {
-  const silentIfConfigured = options && options.silentIfConfigured === true;
-  const status = await detectAiToolStatus();
-  if (status.openAiConfigured) {
-    if (!silentIfConfigured) {
-      logger.log(infoLine('OpenAI key already configured - skipping AI tool prompt.'));
-    }
-    return;
-  }
-  if (status.azureOpenAiConfigured) {
-    if (!silentIfConfigured) {
-      logger.log(infoLine('Azure OpenAI endpoint and key already configured - skipping AI tool prompt.'));
-    }
-    return;
-  }
-  const choice = await askAiToolChoice();
-  if (choice === 'openai') {
-    await collectAndSaveOpenAiKey();
-    return;
-  }
-  if (choice === 'azure') {
-    await collectAndSaveAzureOpenAi();
-    return;
-  }
-  logger.log(
-    chalk.gray(
-      'Skipped. Set later: aifabrix secret set ' +
-        AI_KEYS.OPENAI_API_KEY +
-        ' <key>  (or the two azure-openaiapi-urlKeyVault / secrets-azureOpenaiApiKeyVault keys)'
-    )
-  );
-}
-
 /**
  * Absolute directory paths (with trailing sep) shown when setup will replace platform apps.
  *
@@ -353,7 +178,6 @@ async function promptBuilderDirConflict(info) {
 
 module.exports = {
   MODE,
-  AI_KEYS,
   BUILDER_DIR_ACTION,
   INSTALLATION_PROFILE,
   formatBuilderPlatformReplaceLines,
@@ -361,8 +185,6 @@ module.exports = {
   confirmDestructiveMode,
   promptInstallationProfile,
   promptAdminCredentials,
-  promptAiTool,
-  detectAiToolStatus,
   promptBuilderDirConflict,
   validateEmail,
   validatePassword

package/lib/commands/verify-operations-skip-e2e.js

@@ -105,7 +105,8 @@ function shouldSkipE2eForStaticPackage(systemKey, options) {
     if (storage.length === 0) {
       return false;
     }
-    return storage.every((ds) => String(ds.syncBridge || 'disabled').toLowerCase() === 'disabled');
+    return storage.every((ds) => !isKeyedDocumentDatasource(ds) &&
+      String(ds.syncBridge || 'disabled').toLowerCase() === 'disabled');
   } catch {
     return false;
   }
@@ -124,7 +125,30 @@ function shouldSkipOperationsE2e(systemKey, options) {
     || shouldSkipE2eForStaticPackage(systemKey, options);
 }
 
+/** @param {Object} ds @returns {boolean} */
+function isKeyedDocumentDatasource(ds) {
+  const ops = ds?.execution?.cip?.operations;
+  return ds?.entityType === 'documentStorage' && !ops?.list &&
+    !ds.openapi?.operations?.list && !ds.capabilities?.some(cap => (cap.key || cap) === 'list') &&
+    Array.isArray(ops?.get?.steps) && ops.get.steps.some(step => step?.output?.mode === 'documents');
+}
+
+/** @param {string} systemKey @param {Object} options @returns {string[]} */
+function keyedDocumentDatasourceKeys(systemKey, options) {
+  if (options.skipE2e === true || options.minimal === true) return [];
+  try {
+    const { resolveAppWorkspace } = require('../utils/app-path-resolve');
+    const { appPath } = resolveAppWorkspace(systemKey, options);
+    const storage = loadStorageDatasources(appPath);
+    return storage.length && storage.every(isKeyedDocumentDatasource) ? storage.map(ds => ds.key) : [];
+  } catch {
+    return [];
+  }
+}
+
 module.exports = {
+  isKeyedDocumentDatasource,
+  keyedDocumentDatasourceKeys,
   shouldSkipOperationsE2e,
   shouldSkipE2eForRoleAssistant,
   shouldSkipE2eForStaticPackage

package/lib/commands/verify-operations-steps.js

@@ -4,6 +4,9 @@
 
 'use strict';
 
+const { keyedDocumentDatasourceKeys } = require('./verify-operations-skip-e2e');
+const { hasAllKeyedDocumentPersistence } = require('../utils/datasource-binary-evidence');
+
 const { syncLocalIfRequested } = require('./test-e2e-external');
 const { cliOptsSkipPublish } = require('../utils/cli-sync-options');
 const { buildOperationsVerifyStepPlan } = require('./verify-operations-step-plan');
@@ -99,10 +102,19 @@ async function runOperationsVerifySteps(systemKey, options) {
     }
   }
 
+  const keyedKeys = keyedDocumentDatasourceKeys(systemKey, options);
   const e2eLabel = STEP_PROGRESS.e2e;
   stepResults.e2e = await runWithVerifyStepProgress(
     e2eLabel,
-    async(progress) => runVerifyStep(() => plan.e2e.run(progress), plan.e2e.ok),
+    async(progress) => {
+      const result = await plan.e2e.run(progress);
+      if (keyedKeys.length) {
+        stepResults.keyedDocumentGet = true;
+        stepResults.documentPersistence = hasAllKeyedDocumentPersistence(result, keyedKeys);
+        return plan.e2e.ok(result) && stepResults.documentPersistence;
+      }
+      return plan.e2e.ok(result);
+    },
     options
   );
   return { stepResults, hardFailure: !stepResults.e2e, stoppedEarly: false };

package/lib/commands/wizard-config-normalizer.js

@@ -4,6 +4,8 @@
  * @version 2.0.0
  */
 
+const { repairKeyedDocument } = require('./repair-datasource-keyed-document');
+
 const ENTITY_TYPE_FALLBACK = 'record-storage';
 const VALID_ENTITY_TYPES = new Set([
   'document-storage',
@@ -56,7 +58,7 @@ function normalizeCipOutputModes(datasourceConfig) {
   for (const operation of Object.values(operations)) {
     if (!operation || !Array.isArray(operation.steps)) continue;
     for (const step of operation.steps) {
-      if (step?.output?.mode && step.output.mode !== 'records') {
+      if (step?.output?.mode && !['records', 'documents'].includes(step.output.mode)) {
         step.output.mode = 'records';
       }
     }
@@ -80,7 +82,7 @@ function normalizeWizardTestPayload(datasourceConfig) {
  * @param {Object} datasourceConfig - Datasource config
  * @returns {Object} Normalized config
  */
-function normalizeDatasourceConfig(datasourceConfig) {
+function normalizeDatasourceConfig(datasourceConfig, openapiSpec) {
   if (!datasourceConfig || typeof datasourceConfig !== 'object') {
     return datasourceConfig;
   }
@@ -89,6 +91,7 @@ function normalizeDatasourceConfig(datasourceConfig) {
   }
   normalizePortalInput(datasourceConfig);
   normalizeCipOutputModes(datasourceConfig);
+  repairKeyedDocument(datasourceConfig, { openapiSpec });
   normalizeWizardTestPayload(datasourceConfig);
   return datasourceConfig;
 }
@@ -100,10 +103,10 @@ function normalizeDatasourceConfig(datasourceConfig) {
  * @param {Object|Object[]} datasourceConfigs - Datasource config(s)
  * @returns {{ systemConfig: Object, datasourceConfigs: Object[] }} Normalized configs
  */
-function normalizeWizardConfigs(systemConfig, datasourceConfigs) {
+function normalizeWizardConfigs(systemConfig, datasourceConfigs, openapiSpec) {
   const normalizedSystem = normalizeSystemConfig(systemConfig);
   const configs = Array.isArray(datasourceConfigs) ? datasourceConfigs : [datasourceConfigs];
-  const normalizedDatasources = configs.map(normalizeDatasourceConfig);
+  const normalizedDatasources = configs.map(config => normalizeDatasourceConfig(config, openapiSpec));
   return { systemConfig: normalizedSystem, datasourceConfigs: normalizedDatasources };
 }