@aifabrix/builder 2.44.3 → 2.44.4

package/lib/datasource/test-e2e.js

@@ -33,9 +33,8 @@ function buildUnifiedE2eRunOptions(options, timeoutMs, pk) {
     verbose: options.verbose,
     async: options.async !== false,
     noAsync: options.async === false,
-    testCrud: options.testCrud,
-    recordId: options.recordId,
     cleanup: options.cleanup,
+    runScenarios: options.runScenarios,
     primaryKeyValue: pk,
     minVectorHits: options.minVectorHits,
     minProcessed: options.minProcessed,
@@ -72,6 +71,59 @@ function e2eIntegrationLogDir(appKey) {
   return path.dirname(getIntegrationPath(appKey));
 }
 
+/**
+ * @param {Object} options
+ * @returns {number}
+ */
+function resolveE2ePollTimeoutMs(options) {
+  const raw =
+    options.timeout !== undefined && options.timeout !== null && options.timeout !== ''
+      ? parseInt(String(options.timeout), 10)
+      : options.pollTimeoutMs;
+  return Number.isFinite(raw) && raw > 0 ? raw : DEFAULT_POLL_TIMEOUT_MS;
+}
+
+/**
+ * @param {string} datasourceKey
+ * @param {Object} options
+ * @param {string|Object|null} pk
+ * @returns {Object}
+ */
+function buildE2eRequestMeta(datasourceKey, options, pk) {
+  return {
+    datasourceKey,
+    runType: 'e2e',
+    includeDebug: includeDebugForRequest(options.debug),
+    cleanup: options.cleanup,
+    runScenarios: options.runScenarios,
+    primaryKeyValue: pk !== undefined && pk !== null,
+    minVectorHits: options.minVectorHits,
+    minProcessed: options.minProcessed,
+    minRecordCount: options.minRecordCount
+  };
+}
+
+/**
+ * Write JSON debug log and print paths + CLI wall clock when --debug.
+ * @param {string} appKey
+ * @param {Object} requestMeta
+ * @param {Object} envelope
+ * @param {number} cliWallSeconds
+ * @returns {Promise<void>}
+ */
+async function logE2eDebugSuccess(appKey, requestMeta, envelope, cliWallSeconds) {
+  const logPath = await writeTestLog(
+    appKey,
+    { request: { ...requestMeta, cliWallSeconds }, response: envelope },
+    'test-e2e',
+    e2eIntegrationLogDir(appKey)
+  );
+  logger.log(chalk.gray(`  Debug log: ${logPath}`));
+  logger.log(
+    chalk.gray(`  CLI wall time: ${cliWallSeconds}s (URL resolve, POST, poll, log write)`)
+  );
+}
+
 /**
  * Throw when unified run failed, timed out, or needs async; optionally write debug log.
  * @returns {Promise<void>}
@@ -112,42 +164,19 @@ async function throwIfUnifiedE2EBlocked(unifiedResult, appKey, options, requestM
 }
 
 /**
- * Run E2E test for one datasource (unified validation API; deployment auth like test-integration).
- * @async
- * @param {string} datasourceKey
- * @param {Object} options
- * @param {boolean} [options.sync] - Publish local datasource JSON before validation when true
- * @returns {Promise<Object>} Shape compatible with displayE2EResults (steps, success, status)
+ * Run E2E for one datasource (unified validation API).
+ * @returns {Promise<Object>} displayE2EResults-compatible shape
  */
 async function runDatasourceTestE2E(datasourceKey, options = {}) {
   if (!datasourceKey || typeof datasourceKey !== 'string') {
     throw new Error('Datasource key is required');
   }
+  const cliWallStartedAt = Date.now();
   const { appKey } = await resolveAppKeyForDatasource(datasourceKey, options.app);
-
   logE2eDatasourceBanner(datasourceKey, options.verbose);
-
   const pk = await resolvePrimaryKeyValue(options.primaryKeyValue);
-  const timeoutRaw =
-    options.timeout !== undefined && options.timeout !== null && options.timeout !== ''
-      ? parseInt(String(options.timeout), 10)
-      : options.pollTimeoutMs;
-  const timeoutMs =
-    Number.isFinite(timeoutRaw) && timeoutRaw > 0 ? timeoutRaw : DEFAULT_POLL_TIMEOUT_MS;
-
-  const requestMeta = {
-    datasourceKey,
-    runType: 'e2e',
-    includeDebug: includeDebugForRequest(options.debug),
-    testCrud: options.testCrud,
-    recordId: options.recordId,
-    cleanup: options.cleanup,
-    primaryKeyValue: pk !== undefined && pk !== null,
-    minVectorHits: options.minVectorHits,
-    minProcessed: options.minProcessed,
-    minRecordCount: options.minRecordCount
-  };
-
+  const timeoutMs = resolveE2ePollTimeoutMs(options);
+  const requestMeta = buildE2eRequestMeta(datasourceKey, options, pk);
   const unifiedResult = await runUnifiedDatasourceValidation(
     datasourceKey,
     buildUnifiedE2eRunOptions(options, timeoutMs, pk)
@@ -158,14 +187,13 @@ async function runDatasourceTestE2E(datasourceKey, options = {}) {
   const display = e2eShapeFromEnvelope(unifiedResult.envelope);
   Object.assign(display, { datasourceTestRun: unifiedResult.envelope });
 
+  const cliWallSeconds = Math.round((Date.now() - cliWallStartedAt) / 10) / 100;
+  if (unifiedResult.envelope && typeof unifiedResult.envelope === 'object') {
+    unifiedResult.envelope.cliWallSeconds = cliWallSeconds;
+  }
+
   if (options.debug) {
-    const logPath = await writeTestLog(
-      appKey,
-      { request: requestMeta, response: unifiedResult.envelope },
-      'test-e2e',
-      e2eIntegrationLogDir(appKey)
-    );
-    logger.log(chalk.gray(`  Debug log: ${logPath}`));
+    await logE2eDebugSuccess(appKey, requestMeta, unifiedResult.envelope, cliWallSeconds);
   }
 
   return display;

package/lib/datasource/unified-validation-run-body.js

@@ -42,10 +42,9 @@ async function buildUnifiedValidationBody(params) {
       ? buildE2eOptionsFromCli({
         debug: options.debug,
         verbose: options.verbose,
-        testCrud: options.testCrud,
-        recordId: options.recordId,
         cleanup: options.cleanup,
         primaryKeyValue: options.primaryKeyValue,
+        runScenarios: options.runScenarios,
         minVectorHits: options.minVectorHits,
         minProcessed: options.minProcessed,
         minRecordCount: options.minRecordCount,

package/lib/datasource/validate.js

@@ -15,6 +15,7 @@ const { loadExternalDataSourceSchema } = require('../utils/schema-loader');
 const { formatValidationErrors } = require('../utils/error-formatter');
 const { validateFieldReferences } = require('./field-reference-validator');
 const { validateAbac } = require('./abac-validator');
+const { buildDatasourceValidateSummary } = require('./datasource-validate-summary');
 
 const EXCLUDE_JSON_NAMES = new Set(['application.json', 'rbac.json', 'package.json', 'package-lock.json']);
 
@@ -161,7 +162,8 @@ function resolveValidateInputPath(identifier) {
  * @async
  * @function validateDatasourceFile
  * @param {string} filePathOrKey - Path to the datasource JSON file, or datasource `key` resolved under integration/<app>/
- * @returns {Promise<Object>} Validation result with errors, warnings, and `resolvedPath` (absolute JSON path)
+ * @returns {Promise<Object>} Validation result: `valid`, `errors`, `warnings`, `resolvedPath`, and `summary`
+ *   (from `buildDatasourceValidateSummary` when JSON parses; `null` on JSON syntax errors).
  * @throws {Error} If file cannot be read or parsed
  *
  * @example
@@ -185,19 +187,23 @@ async function validateDatasourceFile(filePathOrKey) {
       valid: false,
       errors: [`Invalid JSON syntax: ${error.message}`],
       warnings: [],
-      resolvedPath: filePath
+      resolvedPath: filePath,
+      summary: null
     };
   }
 
+  const summary = buildDatasourceValidateSummary(parsed);
+
   const validate = loadExternalDataSourceSchema();
   const schemaValid = validate(parsed);
 
   if (!schemaValid) {
     return {
       valid: false,
-      errors: formatValidationErrors(validate.errors),
+      errors: formatValidationErrors(validate.errors, { rootData: parsed, dedupe: true }),
       warnings: [],
-      resolvedPath: filePath
+      resolvedPath: filePath,
+      summary
     };
   }
 
@@ -209,7 +215,8 @@ async function validateDatasourceFile(filePathOrKey) {
       valid: false,
       errors: postSchemaErrors,
       warnings: [],
-      resolvedPath: filePath
+      resolvedPath: filePath,
+      summary
     };
   }
 
@@ -217,7 +224,8 @@ async function validateDatasourceFile(filePathOrKey) {
     valid: true,
     errors: [],
     warnings: [],
-    resolvedPath: filePath
+    resolvedPath: filePath,
+    summary
   };
 }
 

package/lib/external-system/test.js

@@ -154,12 +154,21 @@ async function loadDatasourceFiles(datasourceFiles, schemaBasePath, appPath) {
 async function loadExternalSystemFiles(appName) {
   const { appPath } = await detectAppType(appName);
   const { resolveApplicationConfigPath } = require('../utils/app-config-resolver');
+  const { orderDatasourceFileNamesBySystemKeys } = require('../utils/schema-resolver');
   const configPath = resolveApplicationConfigPath(appPath);
   const variables = await loadVariablesYamlFile(configPath);
 
   const schemaBasePath = variables.externalIntegration.schemaBasePath || './';
   const systemFiles = variables.externalIntegration.systems || [];
-  const datasourceFiles = variables.externalIntegration.dataSources || [];
+  const rawDatasourceFiles = variables.externalIntegration.dataSources || [];
+  const schemaResolved = path.isAbsolute(schemaBasePath)
+    ? schemaBasePath
+    : path.resolve(appPath, schemaBasePath);
+  const datasourceFiles = orderDatasourceFileNamesBySystemKeys(
+    schemaResolved,
+    systemFiles,
+    rawDatasourceFiles
+  );
 
   const systemJsonFiles = await loadSystemFiles(systemFiles, schemaBasePath, appPath);
   const datasourceJsonFiles = await loadDatasourceFiles(datasourceFiles, schemaBasePath, appPath);
@@ -198,10 +207,10 @@ function validateDatasourceSchema(datasource, systemKey, externalDataSourceSchem
 /**
  * Test datasource with payload template
  * @param {Object} datasource - Datasource configuration
- * @param {boolean} verbose - Show detailed output
+ * @param {boolean} _verbose - Show detailed output (reserved; not used in this path yet)
  * @returns {Object} Test results
  */
-function testDatasourceWithPayload(datasource, verbose) {
+function testDatasourceWithPayload(datasource, _verbose) {
   const errors = [];
   const warnings = [];
 
@@ -223,11 +232,6 @@ function testDatasourceWithPayload(datasource, verbose) {
     warnings.push(...metadataSchemaResults.warnings);
   }
 
-  // Compare with expectedResult if provided
-  if (datasource.testPayload.expectedResult && fieldMappingResults.mappedFields && verbose) {
-    warnings.push('expectedResult validation not yet implemented (requires transformation engine)');
-  }
-
   return {
     fieldMappingResults,
     metadataSchemaResults,

package/lib/generator/external-controller-manifest.js

@@ -12,6 +12,7 @@
 const path = require('path');
 const { detectAppType } = require('../utils/paths');
 const { resolveApplicationConfigPath, resolveRbacPath } = require('../utils/app-config-resolver');
+const { orderDatasourceFileNamesBySystemKeys } = require('../utils/schema-resolver');
 const { loadSystemFile, loadDatasourceFiles } = require('./external');
 const { loadVariables, loadRbac } = require('./helpers');
 
@@ -124,9 +125,18 @@ async function generateControllerManifest(appName, options = {}) {
   if (systemFiles.length === 0) {
     throw new Error('No system files specified in externalIntegration.systems');
   }
+  const schemaResolved = path.isAbsolute(schemaBasePath)
+    ? schemaBasePath
+    : path.resolve(appPath, schemaBasePath);
+  const rawDatasourceNames = variables.externalIntegration.dataSources || [];
+  const orderedDatasourceNames = orderDatasourceFileNamesBySystemKeys(
+    schemaResolved,
+    systemFiles,
+    rawDatasourceNames
+  );
   const [systemJson, datasourceJsons] = await Promise.all([
     loadSystemWithRbac(appPath, schemaBasePath, systemFiles[0]),
-    loadDatasourceFiles(appPath, schemaBasePath, variables.externalIntegration.dataSources || [], {
+    loadDatasourceFiles(appPath, schemaBasePath, orderedDatasourceNames, {
       skipMissingDatasourceFiles: options.skipMissingDatasourceFiles
     })
   ]);
@@ -134,7 +144,7 @@ async function generateControllerManifest(appName, options = {}) {
   const externalIntegration = {
     schemaBasePath,
     systems: systemFiles,
-    dataSources: variables.externalIntegration.dataSources || [],
+    dataSources: orderedDatasourceNames,
     autopublish: variables.externalIntegration.autopublish !== false,
     version: appVersion
   };

package/lib/schema/cip-capacity-display.fallback.json

@@ -0,0 +1,7 @@
+{
+  "$comment": "Used when external-datasource.schema.json is not on disk. standardOperationOrder must match $defs.cipDefinition.properties.operations.properties key order in dataplane app/schemas/json/external-datasource.schema.json.",
+  "standardOperationOrder": ["list", "get", "create", "update", "delete"],
+  "displayAliases": {
+    "create": "insert (create)"
+  }
+}

package/lib/schema/datasource-test-run.schema.json

@@ -9,7 +9,7 @@
     "reportVersion": {
       "type": "string",
       "description": "Semver-style contract version for this envelope. Policy: additive fields => minor bump within same major; removed/renamed fields => major bump. Route migrations must cite target reportVersion.",
-      "example": "1.1.0"
+      "example": "1.3.0"
     },
     "runType": {
       "type": "string",
@@ -51,6 +51,10 @@
     "certificate": {
       "$ref": "#/$defs/CertificateResult"
     },
+    "certificateIssuance": {
+      "$ref": "#/$defs/CertificateIssuanceResult",
+      "description": "Integration-certificate auto-issue outcome for this datasource scope (VALIDATION_AUTO_ISSUE_INTEGRATION_CERTIFICATE)."
+    },
     "capabilitySummary": {
       "$ref": "#/$defs/CapabilitySummary"
     },
@@ -69,9 +73,50 @@
     },
     "developer": {
       "$ref": "#/$defs/DeveloperSummary"
+    },
+    "datasourceSummaries": {
+      "type": "array",
+      "description": "Optional per-datasource rollups when the underlying run evaluated multiple datasources (e.g. validationScope=externalSystem).",
+      "items": {
+        "$ref": "#/$defs/DatasourceSummary"
+      }
     }
   },
   "$defs": {
+    "DatasourceSummary": {
+      "type": "object",
+      "required": ["datasourceKey", "status"],
+      "additionalProperties": false,
+      "description": "Lightweight per-datasource rollup for system-scoped runs.",
+      "properties": {
+        "datasourceKey": {
+          "type": "string",
+          "description": "Datasource key within the system."
+        },
+        "status": {
+          "type": "string",
+          "description": "Datasource rollup: ok | warn | fail | skipped.",
+          "enum": ["ok", "warn", "fail", "skipped"]
+        },
+        "validationStatus": {
+          "type": "string",
+          "description": "Validation layer rollup for this datasource.",
+          "enum": ["ok", "warn", "fail"]
+        },
+        "certificateStatus": {
+          "type": "string",
+          "description": "Certification rollup for this datasource.",
+          "enum": ["passed", "not_passed"]
+        },
+        "issues": {
+          "type": "array",
+          "description": "Blocking/warning issues for this datasource.",
+          "items": {
+            "$ref": "#/$defs/Issue"
+          }
+        }
+      }
+    },
     "ValidationLayerResult": {
       "type": "object",
       "required": ["status"],
@@ -161,6 +206,39 @@
         }
       }
     },
+    "CertificateIssuanceResult": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["status", "systemKey", "datasourceKey"],
+      "description": "Structured result when the dataplane evaluates integration-certificate auto-issue (safe for clients and logs).",
+      "properties": {
+        "status": {
+          "type": "string",
+          "enum": ["issued", "reusedActive", "skipped", "failed"],
+          "description": "issued | reusedActive | skipped | failed."
+        },
+        "reasonCode": {
+          "type": "string",
+          "description": "Stable machine code when status is skipped or failed."
+        },
+        "message": {
+          "type": "string",
+          "description": "Human-readable summary."
+        },
+        "validationHint": {
+          "type": "string",
+          "description": "Optional remediation hint."
+        },
+        "systemKey": {
+          "type": "string",
+          "description": "External system key for this issuance scope."
+        },
+        "datasourceKey": {
+          "type": "string",
+          "description": "Datasource key for this issuance scope."
+        }
+      }
+    },
     "CertificateResult": {
       "type": "object",
       "required": ["status"],

package/lib/schema/external-datasource.schema.json

@@ -7,12 +7,12 @@
         "key":"external-datasource-schema",
         "name":"External Data Source Configuration Schema",
         "description":"JSON schema for validating ExternalDataSource configuration files",
-        "version":"2.4.5",
+    "version":"2.4.6",
         "type":"schema",
         "category":"integration",
         "author":"AI Fabrix Team",
         "createdAt":"2024-01-01T00:00:00Z",
-        "updatedAt":"2026-04-07T00:00:00Z",
+        "updatedAt":"2026-04-20T00:00:00Z",
         "compatibility":{
            "minVersion":"1.0.0",
            "maxVersion":"3.0.0",
@@ -175,6 +175,14 @@
               "exposed.profiles object profiles: attributes may be an object map (property names = field keys; values ignored), matching runtime exposed_profile_field_names and CIP/ExposedFieldsValidator behavior."
            ],
            "breaking":false
+        },
+        {
+           "version":"2.4.6",
+           "date":"2026-04-24T00:00:00Z",
+           "changes":[
+              "Added optional fetch.httpResponseNormalization for manifest-driven GET response shaping (302 Location follow, JSON redirect envelopes, nested JSON string URL follow-up)."
+           ],
+           "breaking":false
         }
      ]
   },
@@ -705,6 +713,10 @@
                     "input":{
                        "type":"object",
                        "additionalProperties":true
+                    },
+                    "order":{
+                       "type":"integer",
+                       "description":"Optional stable sort hint for scenario execution / diagnostics."
                     }
                  },
                  "additionalProperties":false
@@ -950,6 +962,83 @@
      }
   ],
   "$defs":{
+     "httpResponseNormalization":{
+        "type":"object",
+        "description":"Optional GET response shaping for miso-client adapter dicts and native 302 Location follows. Omitted = disabled.",
+        "additionalProperties":false,
+        "properties":{
+           "applyWhenUrlContains":{
+              "type":"array",
+              "items":{
+                 "type":"string",
+                 "minLength":1
+              },
+              "description":"When non-empty, normalization applies only if the request URL (path plus query, lowercased) contains at least one substring (case-insensitive)."
+           },
+           "followRedirectJsonEnvelope":{
+              "type":"object",
+              "additionalProperties":false,
+              "properties":{
+                 "enabled":{
+                    "type":"boolean",
+                    "default":false
+                 },
+                 "locationHostSuffixAllowlist":{
+                    "type":"array",
+                    "items":{
+                       "type":"string",
+                       "minLength":1
+                    },
+                    "description":"Location header URL must contain one of these substrings (case-insensitive) before an unauthenticated follow-up GET."
+                 }
+              }
+           },
+           "followNestedJsonStringUrl":{
+              "type":"object",
+              "additionalProperties":false,
+              "properties":{
+                 "enabled":{
+                    "type":"boolean",
+                    "default":false
+                 },
+                 "jsonKeys":{
+                    "type":"array",
+                    "items":{
+                       "type":"string",
+                       "minLength":1
+                    },
+                    "description":"JSON object key names searched depth-first for a non-empty string URL."
+                 },
+                 "targetUrlHostSuffixAllowlist":{
+                    "type":"array",
+                    "items":{
+                       "type":"string",
+                       "minLength":1
+                    },
+                    "description":"Discovered URL must contain one of these substrings before an unauthenticated follow-up GET."
+                 }
+              }
+           },
+           "followNativeHttpRedirect302":{
+              "type":"object",
+              "additionalProperties":false,
+              "properties":{
+                 "enabled":{
+                    "type":"boolean",
+                    "default":false
+                 },
+                 "locationHostSuffixAllowlist":{
+                    "type":"array",
+                    "items":{
+                       "type":"string",
+                       "minLength":1
+                    },
+                    "description":"Native httpx 302 Location URL must contain one of these substrings before follow-up GET."
+                 }
+              }
+           }
+        }
+     },
      "metadataSchemaNode":{
         "type":"object",
         "description":"Strict supported subset of JSON Schema for metadataSchema.",
@@ -1829,6 +1918,9 @@
                     "additionalProperties":{
                        "type":"string"
                     }
+                 },
+                 "httpResponseNormalization":{
+                    "$ref":"#/$defs/httpResponseNormalization"
                  }
               },
               "allOf":[

package/lib/schema/flag-map-validation-run.json

@@ -11,8 +11,7 @@
     { "cliFlag": "--timeout <ms>", "requestField": null, "note": "Client aggregate HTTP budget (POST + polls)" },
     { "cliFlag": "--debug [level]", "requestField": "includeDebug", "default": false, "note": "summary|full|raw affects TTY appendix only; any presence sets includeDebug" },
     { "cliFlag": "--no-async", "requestField": "asyncRun", "note": "When true on CLI, omit asyncRun or set false; sync-only POST" },
-    { "cliFlag": "--test-crud", "requestField": "e2eOptions.testCrud", "runType": "e2e" },
-    { "cliFlag": "--record-id", "requestField": "e2eOptions.recordId", "runType": "e2e" },
+    { "cliFlag": "--no-run-scenarios", "requestField": "e2eOptions.runScenarios", "runType": "e2e" },
     { "cliFlag": "--no-cleanup", "requestField": "e2eOptions.cleanup", "runType": "e2e" },
     { "cliFlag": "--primary-key-value", "requestField": "e2eOptions.primaryKeyValue", "runType": "e2e" },
     { "cliFlag": "--capability <key>", "requestField": "e2eOptions.capabilityKeys", "runType": "e2e", "note": "Single key forwarded as capabilityKeys array in request body when set" },