@aifabrix/builder 2.44.3 → 2.44.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.npmrc.token +1 -1
- package/integration/roundtrip-test-local/README.md +1 -2
- package/integration/roundtrip-test-local2/README.md +1 -2
- package/jest.projects.js +31 -15
- package/lib/api/certificates.api.js +21 -3
- package/lib/api/types/wizard.types.js +2 -1
- package/lib/certification/post-unified-cert-sync.js +13 -2
- package/lib/certification/sync-after-external-command.js +6 -3
- package/lib/certification/sync-system-certification.js +60 -14
- package/lib/cli/setup-app.help.js +1 -1
- package/lib/cli/setup-app.test-commands.js +75 -39
- package/lib/cli/setup-infra.js +6 -2
- package/lib/cli/setup-utility.js +20 -1
- package/lib/commands/datasource-unified-test-cli.js +81 -46
- package/lib/commands/datasource-unified-test-cli.options.js +4 -2
- package/lib/commands/datasource.js +3 -31
- package/lib/commands/repair-datasource-keys.js +1 -1
- package/lib/commands/repair-datasource-openapi.js +57 -0
- package/lib/commands/repair-datasource.js +5 -0
- package/lib/commands/repair-internal.js +2 -4
- package/lib/commands/repair-rbac.js +25 -2
- package/lib/commands/repair.js +2 -19
- package/lib/commands/test-e2e-external.js +9 -9
- package/lib/commands/up-common.js +25 -0
- package/lib/commands/upload.js +18 -4
- package/lib/commands/wizard-core.js +53 -11
- package/lib/commands/wizard-dataplane.js +14 -6
- package/lib/commands/wizard-entity-selection.js +71 -14
- package/lib/commands/wizard-headless.js +5 -2
- package/lib/commands/wizard-helpers.js +13 -1
- package/lib/commands/wizard.js +208 -60
- package/lib/datasource/datasource-validate-display.js +162 -0
- package/lib/datasource/datasource-validate-summary.js +194 -0
- package/lib/datasource/test-e2e.js +65 -37
- package/lib/datasource/unified-validation-run-body.js +1 -2
- package/lib/datasource/validate.js +14 -6
- package/lib/external-system/test.js +12 -8
- package/lib/generator/external-controller-manifest.js +12 -2
- package/lib/generator/wizard-prompts.js +7 -1
- package/lib/generator/wizard.js +34 -0
- package/lib/schema/cip-capacity-display.fallback.json +7 -0
- package/lib/schema/datasource-test-run.schema.json +79 -1
- package/lib/schema/external-datasource.schema.json +94 -2
- package/lib/schema/flag-map-validation-run.json +1 -2
- package/lib/schema/type/document-storage.json +83 -3
- package/lib/schema/wizard-config.schema.json +1 -1
- package/lib/utils/configuration-env-resolver.js +38 -0
- package/lib/utils/dataplane-resolver.js +3 -2
- package/lib/utils/datasource-test-run-capacity-operations.js +149 -0
- package/lib/utils/datasource-test-run-debug-display.js +143 -1
- package/lib/utils/datasource-test-run-display.js +46 -33
- package/lib/utils/datasource-test-run-tty-log.js +6 -2
- package/lib/utils/datasource-test-run-tty-meta-lines.js +123 -0
- package/lib/utils/error-formatter.js +32 -2
- package/lib/utils/external-readme.js +47 -3
- package/lib/utils/external-system-readiness-core.js +39 -0
- package/lib/utils/external-system-readiness-deploy-display.js +2 -3
- package/lib/utils/external-system-readiness-display-internals.js +3 -2
- package/lib/utils/external-system-system-test-tty.js +33 -9
- package/lib/utils/external-system-validators.js +62 -5
- package/lib/utils/load-cip-capacity-display-config.js +130 -0
- package/lib/utils/paths.js +10 -3
- package/lib/utils/schema-resolver.js +98 -2
- package/lib/utils/urls-local-registry.js +52 -10
- package/lib/utils/validation-run-poll.js +15 -4
- package/lib/utils/validation-run-request.js +4 -6
- package/lib/validation/dimension-display-helpers.js +60 -0
- package/lib/validation/validate-display-log-helpers.js +39 -0
- package/lib/validation/validate-display.js +89 -83
- package/package.json +1 -1
- package/templates/applications/miso-controller/env.template +6 -6
- package/templates/external-system/README.md.hbs +58 -32
|
@@ -133,8 +133,8 @@ function countByStatus(rows) {
|
|
|
133
133
|
return counts;
|
|
134
134
|
}
|
|
135
135
|
|
|
136
|
-
function
|
|
137
|
-
|
|
136
|
+
function rollupRowsWithKeys(rows) {
|
|
137
|
+
return rows
|
|
138
138
|
.map(r => {
|
|
139
139
|
const env = r && r.datasourceTestRun;
|
|
140
140
|
const st = rollupRowStatus(r);
|
|
@@ -146,15 +146,37 @@ function pickBlockingDatasourceKey(rows) {
|
|
|
146
146
|
: '';
|
|
147
147
|
return { key, st };
|
|
148
148
|
})
|
|
149
|
-
.filter(x => x.key)
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
149
|
+
.filter(x => x.key)
|
|
150
|
+
.sort((a, b) => {
|
|
151
|
+
const d = statusRank(a.st) - statusRank(b.st);
|
|
152
|
+
if (d !== 0) return d;
|
|
153
|
+
return a.key.localeCompare(b.key);
|
|
154
|
+
});
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* Worst-first datasource key for suggested CLI drill-down (always returned when any row exists).
|
|
159
|
+
* @param {Array} rows
|
|
160
|
+
* @returns {string|null}
|
|
161
|
+
*/
|
|
162
|
+
function pickDrillDownDatasourceKey(rows) {
|
|
163
|
+
const keys = rollupRowsWithKeys(rows);
|
|
155
164
|
return keys.length ? keys[0].key : null;
|
|
156
165
|
}
|
|
157
166
|
|
|
167
|
+
/**
|
|
168
|
+
* Datasource key to label as "blocking" only when at least one row is not fully OK (fail/warn/skipped).
|
|
169
|
+
* When every row is OK, returns null so we do not imply a blocker when the system is all-green.
|
|
170
|
+
* @param {Array} rows
|
|
171
|
+
* @returns {string|null}
|
|
172
|
+
*/
|
|
173
|
+
function pickBlockingDatasourceKey(rows) {
|
|
174
|
+
const keys = rollupRowsWithKeys(rows);
|
|
175
|
+
if (keys.length === 0) return null;
|
|
176
|
+
if (keys.every(x => x.st === 'ok')) return null;
|
|
177
|
+
return keys[0].key;
|
|
178
|
+
}
|
|
179
|
+
|
|
158
180
|
function issueKey(issue) {
|
|
159
181
|
const code = issue && issue.code ? String(issue.code) : '';
|
|
160
182
|
const msg = issue && issue.message ? String(issue.message) : '';
|
|
@@ -390,6 +412,7 @@ function displaySystemAggregateDatasourceTestRuns(results, opts) {
|
|
|
390
412
|
const systemStatus = deriveSystemStatus(rows);
|
|
391
413
|
const counts = countByStatus(rows);
|
|
392
414
|
const blocking = pickBlockingDatasourceKey(rows);
|
|
415
|
+
const drillDownKey = pickDrillDownDatasourceKey(rows);
|
|
393
416
|
logSystemHeader(results, runType, systemStatus);
|
|
394
417
|
logVerdictAndSummary(runType, systemStatus, systemCertStatus(rows), counts);
|
|
395
418
|
logDataQualityAndReadiness(rows);
|
|
@@ -400,7 +423,7 @@ function displaySystemAggregateDatasourceTestRuns(results, opts) {
|
|
|
400
423
|
logCapabilitiesOverviewSection(rows, ttyIo);
|
|
401
424
|
logIntegrationHealthSectionBlock(rows, runType, ttyIo);
|
|
402
425
|
logCertificationSection(rows);
|
|
403
|
-
logUseAndFooter(results, runType, systemStatus,
|
|
426
|
+
logUseAndFooter(results, runType, systemStatus, drillDownKey);
|
|
404
427
|
}
|
|
405
428
|
|
|
406
429
|
module.exports = {
|
|
@@ -410,6 +433,7 @@ module.exports = {
|
|
|
410
433
|
deriveSystemReadiness,
|
|
411
434
|
rollupRowStatus,
|
|
412
435
|
pickBlockingDatasourceKey,
|
|
436
|
+
pickDrillDownDatasourceKey,
|
|
413
437
|
collectKeyIssues,
|
|
414
438
|
systemCertStatus,
|
|
415
439
|
drillDownCommand
|
|
@@ -39,7 +39,21 @@ function validateFieldMappingExpression(expression) {
|
|
|
39
39
|
|
|
40
40
|
// Validate transformations (optional)
|
|
41
41
|
const transformations = expression.split('|').slice(1).map(t => t.trim());
|
|
42
|
-
|
|
42
|
+
// Keep in sync with dataplane field-mapping function registry (pipe DSL after {{path}})
|
|
43
|
+
const validTransformations = [
|
|
44
|
+
'toUpper',
|
|
45
|
+
'toLower',
|
|
46
|
+
'trim',
|
|
47
|
+
'default',
|
|
48
|
+
'toNumber',
|
|
49
|
+
'toString',
|
|
50
|
+
'coalesce',
|
|
51
|
+
'substring',
|
|
52
|
+
'replace',
|
|
53
|
+
'join',
|
|
54
|
+
'split',
|
|
55
|
+
'normalizeWhitespace'
|
|
56
|
+
];
|
|
43
57
|
for (const trans of transformations) {
|
|
44
58
|
const transName = trans.split('(')[0].trim();
|
|
45
59
|
if (!validTransformations.includes(transName)) {
|
|
@@ -67,6 +81,40 @@ function validateFieldMappingExpression(expression) {
|
|
|
67
81
|
* @param {Object} payloadTemplate - Payload template
|
|
68
82
|
* @param {Object} results - Results object to update
|
|
69
83
|
*/
|
|
84
|
+
/**
|
|
85
|
+
* Root object used to verify {{...}} paths exist in the test payload.
|
|
86
|
+
* HubSpot-style configs map from `{{raw.id}}` while `payloadTemplate` often holds the
|
|
87
|
+
* API record at the top level; wrap virtually so path checks match CIP input shape
|
|
88
|
+
* without duplicating the record under `raw` in JSON (metadataSchema validation still
|
|
89
|
+
* uses the unwrapped `payloadTemplate`).
|
|
90
|
+
* @param {Object} payloadTemplate - testPayload.payloadTemplate
|
|
91
|
+
* @param {Object} attributes - fieldMappings.attributes
|
|
92
|
+
* @returns {Object}
|
|
93
|
+
*/
|
|
94
|
+
function fieldMappingPathRoot(payloadTemplate, attributes) {
|
|
95
|
+
if (!payloadTemplate || typeof payloadTemplate !== 'object') {
|
|
96
|
+
return payloadTemplate;
|
|
97
|
+
}
|
|
98
|
+
if ('raw' in payloadTemplate) {
|
|
99
|
+
return payloadTemplate;
|
|
100
|
+
}
|
|
101
|
+
const usesRawPrefix =
|
|
102
|
+
attributes &&
|
|
103
|
+
typeof attributes === 'object' &&
|
|
104
|
+
!Array.isArray(attributes) &&
|
|
105
|
+
Object.values(attributes).some(
|
|
106
|
+
cfg =>
|
|
107
|
+
cfg &&
|
|
108
|
+
typeof cfg === 'object' &&
|
|
109
|
+
typeof cfg.expression === 'string' &&
|
|
110
|
+
/\{\{\s*raw\./.test(cfg.expression)
|
|
111
|
+
);
|
|
112
|
+
if (usesRawPrefix) {
|
|
113
|
+
return { raw: payloadTemplate };
|
|
114
|
+
}
|
|
115
|
+
return payloadTemplate;
|
|
116
|
+
}
|
|
117
|
+
|
|
70
118
|
function validateSingleFieldMapping(fieldName, fieldConfig, payloadTemplate, results) {
|
|
71
119
|
if (!fieldConfig.expression) {
|
|
72
120
|
results.errors.push(`Field '${fieldName}' missing expression`);
|
|
@@ -124,8 +172,13 @@ function checkPathExistsInPayload(fieldPath, payloadTemplate) {
|
|
|
124
172
|
* @returns {Object} Map suitable for validateDimensions
|
|
125
173
|
*/
|
|
126
174
|
function getDimensionsMapForValidation(datasource) {
|
|
127
|
-
|
|
128
|
-
|
|
175
|
+
if (!datasource || !Object.prototype.hasOwnProperty.call(datasource, 'dimensions')) {
|
|
176
|
+
return undefined;
|
|
177
|
+
}
|
|
178
|
+
const root = datasource.dimensions;
|
|
179
|
+
if (!root || typeof root !== 'object' || Array.isArray(root)) {
|
|
180
|
+
return {};
|
|
181
|
+
}
|
|
129
182
|
const out = {};
|
|
130
183
|
for (const [dimKey, binding] of Object.entries(root)) {
|
|
131
184
|
if (binding && typeof binding === 'object' && typeof binding.field === 'string') {
|
|
@@ -238,7 +291,10 @@ function validateFieldMappings(datasource, testPayload) {
|
|
|
238
291
|
return results;
|
|
239
292
|
}
|
|
240
293
|
|
|
241
|
-
|
|
294
|
+
const dimensionsForValidation = getDimensionsMapForValidation(datasource);
|
|
295
|
+
if (dimensionsForValidation !== undefined) {
|
|
296
|
+
validateDimensions(dimensionsForValidation, results);
|
|
297
|
+
}
|
|
242
298
|
|
|
243
299
|
// Validate attributes structure (required in new schema)
|
|
244
300
|
const attributes = validateAttributesStructure(datasource.fieldMappings.attributes, results);
|
|
@@ -248,8 +304,9 @@ function validateFieldMappings(datasource, testPayload) {
|
|
|
248
304
|
|
|
249
305
|
// Validate each attribute
|
|
250
306
|
const payloadTemplate = testPayload.payloadTemplate || testPayload;
|
|
307
|
+
const pathCheckRoot = fieldMappingPathRoot(payloadTemplate, attributes);
|
|
251
308
|
for (const [attributeName, attributeConfig] of Object.entries(attributes)) {
|
|
252
|
-
validateSingleAttribute(attributeName, attributeConfig,
|
|
309
|
+
validateSingleAttribute(attributeName, attributeConfig, pathCheckRoot, results);
|
|
253
310
|
}
|
|
254
311
|
|
|
255
312
|
return results;
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Resolve CIP standard operation order for CLI display from the
|
|
3
|
+
* same source as `external-datasource.schema.json` ($defs.cipDefinition.properties.operations.properties).
|
|
4
|
+
* Falls back to `lib/schema/cip-capacity-display.fallback.json` when the schema file is unavailable.
|
|
5
|
+
* @author AI Fabrix Team
|
|
6
|
+
* @version 1.0.0
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
'use strict';
|
|
10
|
+
|
|
11
|
+
const fs = require('fs');
|
|
12
|
+
const path = require('path');
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* @param {object|null} schema - Parsed external-datasource.schema.json
|
|
16
|
+
* @returns {string[]|null}
|
|
17
|
+
*/
|
|
18
|
+
function extractStandardOperationOrderFromSchema(schema) {
|
|
19
|
+
const props =
|
|
20
|
+
schema &&
|
|
21
|
+
schema.$defs &&
|
|
22
|
+
schema.$defs.cipDefinition &&
|
|
23
|
+
schema.$defs.cipDefinition.properties &&
|
|
24
|
+
schema.$defs.cipDefinition.properties.operations &&
|
|
25
|
+
schema.$defs.cipDefinition.properties.operations.properties;
|
|
26
|
+
if (!props || typeof props !== 'object') return null;
|
|
27
|
+
return Object.keys(props);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* @param {string} p
|
|
32
|
+
* @returns {object|null}
|
|
33
|
+
*/
|
|
34
|
+
function tryReadJsonFile(p) {
|
|
35
|
+
try {
|
|
36
|
+
if (!p || !fs.existsSync(p)) return null;
|
|
37
|
+
return JSON.parse(fs.readFileSync(p, 'utf8'));
|
|
38
|
+
} catch {
|
|
39
|
+
return null;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Resolve path to dataplane external-datasource.schema.json (monorepo / env).
|
|
45
|
+
* Set `AIFABRIX_EXTERNAL_DATASOURCE_SCHEMA` to override.
|
|
46
|
+
* @returns {string|null}
|
|
47
|
+
*/
|
|
48
|
+
function resolveExternalDatasourceSchemaPath() {
|
|
49
|
+
if (process.env.AIFABRIX_EXTERNAL_DATASOURCE_SCHEMA) {
|
|
50
|
+
const envPath = String(process.env.AIFABRIX_EXTERNAL_DATASOURCE_SCHEMA).trim();
|
|
51
|
+
if (envPath && fs.existsSync(envPath)) return envPath;
|
|
52
|
+
}
|
|
53
|
+
const candidates = [
|
|
54
|
+
path.join(process.cwd(), 'app/schemas/json/external-datasource.schema.json'),
|
|
55
|
+
path.join(process.cwd(), 'aifabrix-dataplane/app/schemas/json/external-datasource.schema.json'),
|
|
56
|
+
path.join(process.cwd(), '..', 'aifabrix-dataplane', 'app/schemas/json/external-datasource.schema.json'),
|
|
57
|
+
path.join(__dirname, '../../../aifabrix-dataplane/app/schemas/json/external-datasource.schema.json')
|
|
58
|
+
];
|
|
59
|
+
for (const c of candidates) {
|
|
60
|
+
if (c && fs.existsSync(c)) return c;
|
|
61
|
+
}
|
|
62
|
+
return null;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const FALLBACK_PATH = path.join(__dirname, '../schema/cip-capacity-display.fallback.json');
|
|
66
|
+
|
|
67
|
+
/** @type {{ standardOrder: string[], aliases: Record<string, string> }|null} */
|
|
68
|
+
let _cache = null;
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* @returns {{ standardOrder: string[], aliases: Record<string, string> }}
|
|
72
|
+
*/
|
|
73
|
+
function getCipCapacityDisplayConfig() {
|
|
74
|
+
if (_cache) return _cache;
|
|
75
|
+
const schemaPath = resolveExternalDatasourceSchemaPath();
|
|
76
|
+
const schema = schemaPath ? tryReadJsonFile(schemaPath) : null;
|
|
77
|
+
const fromSchema = extractStandardOperationOrderFromSchema(schema);
|
|
78
|
+
const fallback = tryReadJsonFile(FALLBACK_PATH) || {};
|
|
79
|
+
const fbOrder = Array.isArray(fallback.standardOperationOrder)
|
|
80
|
+
? fallback.standardOperationOrder
|
|
81
|
+
: [];
|
|
82
|
+
const standardOrder = fromSchema && fromSchema.length ? fromSchema : fbOrder;
|
|
83
|
+
const aliases =
|
|
84
|
+
fallback.displayAliases && typeof fallback.displayAliases === 'object'
|
|
85
|
+
? fallback.displayAliases
|
|
86
|
+
: { create: 'insert (create)' };
|
|
87
|
+
_cache = { standardOrder, aliases };
|
|
88
|
+
return _cache;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* @param {string[]} standardOrder - from schema `cipDefinition.operations.properties` keys
|
|
93
|
+
* @param {string} op
|
|
94
|
+
* @returns {number}
|
|
95
|
+
*/
|
|
96
|
+
function standardOperationRank(standardOrder, op) {
|
|
97
|
+
const idx = standardOrder.indexOf(op);
|
|
98
|
+
if (idx >= 0) return idx;
|
|
99
|
+
return 500;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* @param {string} capacityKey e.g. capacity:myOp#3 or capacity:update#1
|
|
104
|
+
* @returns {{ op: string, index: number }|null}
|
|
105
|
+
*/
|
|
106
|
+
function parseCapacityDetailKey(capacityKey) {
|
|
107
|
+
const s = String(capacityKey);
|
|
108
|
+
const m = s.match(/^capacity:([^#]+)#(\d+)$/i);
|
|
109
|
+
if (m) {
|
|
110
|
+
return { op: String(m[1]).toLowerCase(), index: parseInt(m[2], 10) };
|
|
111
|
+
}
|
|
112
|
+
const m2 = s.match(/^capacity:([^#]+)$/i);
|
|
113
|
+
if (m2) {
|
|
114
|
+
return { op: String(m2[1]).toLowerCase(), index: 0 };
|
|
115
|
+
}
|
|
116
|
+
return null;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
function clearCipCapacityDisplayConfigCacheForTests() {
|
|
120
|
+
_cache = null;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
module.exports = {
|
|
124
|
+
extractStandardOperationOrderFromSchema,
|
|
125
|
+
resolveExternalDatasourceSchemaPath,
|
|
126
|
+
getCipCapacityDisplayConfig,
|
|
127
|
+
standardOperationRank,
|
|
128
|
+
parseCapacityDetailKey,
|
|
129
|
+
clearCipCapacityDisplayConfigCacheForTests
|
|
130
|
+
};
|
package/lib/utils/paths.js
CHANGED
|
@@ -615,8 +615,10 @@ async function detectAppType(appName, _options = {}) {
|
|
|
615
615
|
}
|
|
616
616
|
|
|
617
617
|
/**
|
|
618
|
-
* Resolve-specific app path:
|
|
619
|
-
* If integration/<systemKey>/env.template exists
|
|
618
|
+
* Resolve-specific app path: integration + env.template without application config → env-only mode.
|
|
619
|
+
* If integration/<systemKey>/env.template exists but there is no application.yaml, application.json,
|
|
620
|
+
* application.yml, or variables.yaml, use that directory with envOnly true (kv resolve only).
|
|
621
|
+
* When any application config file exists in that folder, use full resolve (envOnly false).
|
|
620
622
|
* Otherwise fall back to detectAppType (integration or builder with full config).
|
|
621
623
|
*
|
|
622
624
|
* @param {string} appName - Application name
|
|
@@ -630,7 +632,12 @@ async function getResolveAppPath(appName) {
|
|
|
630
632
|
const integrationPath = getIntegrationPath(appName);
|
|
631
633
|
const envTemplatePath = path.join(integrationPath, 'env.template');
|
|
632
634
|
if (fs.existsSync(integrationPath) && fs.existsSync(envTemplatePath)) {
|
|
633
|
-
|
|
635
|
+
try {
|
|
636
|
+
resolveApplicationConfigPath(integrationPath);
|
|
637
|
+
return { appPath: integrationPath, envOnly: false };
|
|
638
|
+
} catch {
|
|
639
|
+
return { appPath: integrationPath, envOnly: true };
|
|
640
|
+
}
|
|
634
641
|
}
|
|
635
642
|
const result = await detectAppType(appName);
|
|
636
643
|
return { appPath: result.appPath, envOnly: false };
|
|
@@ -175,6 +175,96 @@ function resolveDatasourceFiles(schemaBasePath, datasourceFiles) {
|
|
|
175
175
|
return datasourceFiles.map(fileName => resolveSingleExternalFile(schemaBasePath, fileName, 'datasource'));
|
|
176
176
|
}
|
|
177
177
|
|
|
178
|
+
/**
|
|
179
|
+
* @param {string} schemaBasePath
|
|
180
|
+
* @param {string[]} systemFileNames
|
|
181
|
+
* @returns {string[]|null}
|
|
182
|
+
*/
|
|
183
|
+
function readSystemDatasourceKeyOrder(schemaBasePath, systemFileNames) {
|
|
184
|
+
if (!Array.isArray(systemFileNames) || systemFileNames.length === 0) {
|
|
185
|
+
return null;
|
|
186
|
+
}
|
|
187
|
+
const systemPath = path.join(schemaBasePath, systemFileNames[0]);
|
|
188
|
+
if (!fs.existsSync(systemPath)) {
|
|
189
|
+
return null;
|
|
190
|
+
}
|
|
191
|
+
try {
|
|
192
|
+
const systemParsed = loadConfigFile(systemPath);
|
|
193
|
+
const orderKeys = Array.isArray(systemParsed.dataSources) ? systemParsed.dataSources : null;
|
|
194
|
+
return orderKeys && orderKeys.length > 0 ? orderKeys : null;
|
|
195
|
+
} catch {
|
|
196
|
+
return null;
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/**
|
|
201
|
+
* @param {string} schemaBasePath
|
|
202
|
+
* @param {string[]} datasourceFileNames
|
|
203
|
+
* @returns {Map<string, string>}
|
|
204
|
+
*/
|
|
205
|
+
function mapDatasourceKeysToFilenames(schemaBasePath, datasourceFileNames) {
|
|
206
|
+
const byKey = new Map();
|
|
207
|
+
for (const fileName of datasourceFileNames) {
|
|
208
|
+
if (typeof fileName !== 'string') {
|
|
209
|
+
continue;
|
|
210
|
+
}
|
|
211
|
+
const fp = path.join(schemaBasePath, fileName);
|
|
212
|
+
if (!fs.existsSync(fp)) {
|
|
213
|
+
continue;
|
|
214
|
+
}
|
|
215
|
+
try {
|
|
216
|
+
const d = loadConfigFile(fp);
|
|
217
|
+
const k = d && typeof d.key === 'string' ? d.key.trim() : '';
|
|
218
|
+
if (k) {
|
|
219
|
+
byKey.set(k, fileName);
|
|
220
|
+
}
|
|
221
|
+
} catch {
|
|
222
|
+
/* tail via original list */
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
return byKey;
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
/**
|
|
229
|
+
* Reorder datasource filenames to match external system JSON `dataSources` key order when present.
|
|
230
|
+
* Keeps application.yaml list membership but fixes FK-safe sequencing when filenames were sorted or drifted.
|
|
231
|
+
*
|
|
232
|
+
* @param {string} schemaBasePath
|
|
233
|
+
* @param {string[]} systemFileNames - externalIntegration.systems
|
|
234
|
+
* @param {string[]} datasourceFileNames - externalIntegration.dataSources filenames
|
|
235
|
+
* @returns {string[]} Same filenames, system-declaration order first, then any leftovers in original order
|
|
236
|
+
*/
|
|
237
|
+
function orderDatasourceFileNamesBySystemKeys(
|
|
238
|
+
schemaBasePath,
|
|
239
|
+
systemFileNames,
|
|
240
|
+
datasourceFileNames
|
|
241
|
+
) {
|
|
242
|
+
if (!Array.isArray(datasourceFileNames) || datasourceFileNames.length <= 1) {
|
|
243
|
+
return datasourceFileNames;
|
|
244
|
+
}
|
|
245
|
+
const orderKeys = readSystemDatasourceKeyOrder(schemaBasePath, systemFileNames);
|
|
246
|
+
if (!orderKeys) {
|
|
247
|
+
return datasourceFileNames;
|
|
248
|
+
}
|
|
249
|
+
const byKey = mapDatasourceKeysToFilenames(schemaBasePath, datasourceFileNames);
|
|
250
|
+
const out = [];
|
|
251
|
+
const used = new Set();
|
|
252
|
+
for (const k of orderKeys) {
|
|
253
|
+
const ks = typeof k === 'string' ? k.trim() : '';
|
|
254
|
+
const f = ks ? byKey.get(ks) : null;
|
|
255
|
+
if (f && !used.has(f)) {
|
|
256
|
+
out.push(f);
|
|
257
|
+
used.add(f);
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
for (const fileName of datasourceFileNames) {
|
|
261
|
+
if (fileName && !used.has(fileName)) {
|
|
262
|
+
out.push(fileName);
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
return out.length ? out : datasourceFileNames;
|
|
266
|
+
}
|
|
267
|
+
|
|
178
268
|
/**
|
|
179
269
|
* Loads and validates application config
|
|
180
270
|
* @async
|
|
@@ -206,13 +296,19 @@ async function resolveExternalFiles(appName, options = {}) {
|
|
|
206
296
|
|
|
207
297
|
const schemaBasePath = await resolveSchemaBasePath(appName, options);
|
|
208
298
|
const systemFiles = resolveSystemFiles(schemaBasePath, variables.externalIntegration.systems);
|
|
209
|
-
const
|
|
299
|
+
const orderedNames = orderDatasourceFileNamesBySystemKeys(
|
|
300
|
+
schemaBasePath,
|
|
301
|
+
variables.externalIntegration.systems,
|
|
302
|
+
variables.externalIntegration.dataSources || []
|
|
303
|
+
);
|
|
304
|
+
const datasourceFiles = resolveDatasourceFiles(schemaBasePath, orderedNames);
|
|
210
305
|
|
|
211
306
|
return [...systemFiles, ...datasourceFiles];
|
|
212
307
|
}
|
|
213
308
|
|
|
214
309
|
module.exports = {
|
|
215
310
|
resolveSchemaBasePath,
|
|
216
|
-
resolveExternalFiles
|
|
311
|
+
resolveExternalFiles,
|
|
312
|
+
orderDatasourceFileNamesBySystemKeys
|
|
217
313
|
};
|
|
218
314
|
|
|
@@ -175,6 +175,54 @@ function mergeBuilderDirIntoRegistry(merged, builderDir) {
|
|
|
175
175
|
}
|
|
176
176
|
}
|
|
177
177
|
|
|
178
|
+
/**
|
|
179
|
+
* True when getBuilderRoot() resolves to the same path as AIFABRIX_BUILDER_DIR (authoritative override).
|
|
180
|
+
* @param {string|null} resolvedEffective
|
|
181
|
+
* @param {string|null} envResolved
|
|
182
|
+
* @returns {boolean}
|
|
183
|
+
*/
|
|
184
|
+
function effectiveBuilderMatchesEnvVar(resolvedEffective, envResolved) {
|
|
185
|
+
return Boolean(envResolved && resolvedEffective && resolvedEffective === envResolved);
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* Builder dirs to scan in order; later merges overwrite registry keys from earlier dirs.
|
|
190
|
+
* When `AIFABRIX_BUILDER_DIR` is set and differs from projectRoot/builder, env builder root is merged last.
|
|
191
|
+
* Otherwise projectRoot/builder is merged last so explicit refresh roots override getBuilderRoot().
|
|
192
|
+
*
|
|
193
|
+
* @param {string} root - Resolved project root passed to refresh
|
|
194
|
+
* @param {string|null} effectiveBuilderDir - pathsUtil.getBuilderRoot()
|
|
195
|
+
* @returns {string[]}
|
|
196
|
+
*/
|
|
197
|
+
function getOrderedBuilderDirsForRegistryScan(root, effectiveBuilderDir) {
|
|
198
|
+
const legacyBuilderDir = path.join(root, 'builder');
|
|
199
|
+
let resolvedLegacy;
|
|
200
|
+
let resolvedEffective;
|
|
201
|
+
try {
|
|
202
|
+
resolvedLegacy = path.resolve(legacyBuilderDir);
|
|
203
|
+
resolvedEffective = effectiveBuilderDir ? path.resolve(effectiveBuilderDir) : null;
|
|
204
|
+
} catch {
|
|
205
|
+
return [legacyBuilderDir];
|
|
206
|
+
}
|
|
207
|
+
const envRaw = process.env.AIFABRIX_BUILDER_DIR && String(process.env.AIFABRIX_BUILDER_DIR).trim();
|
|
208
|
+
const envResolved = envRaw ? path.resolve(envRaw) : null;
|
|
209
|
+
// Only treat env as authoritative when getBuilderRoot() is actually that path. Otherwise a stray
|
|
210
|
+
// AIFABRIX_BUILDER_DIR on CI (or Jest mocking getBuilderRoot to a temp dir) must not force
|
|
211
|
+
// [legacy, effective] — that order lets the real checkout builder overwrite the mocked root last.
|
|
212
|
+
const effectiveMatchesEnvVar = effectiveBuilderMatchesEnvVar(resolvedEffective, envResolved);
|
|
213
|
+
|
|
214
|
+
if (effectiveBuilderDir && resolvedEffective && resolvedEffective === resolvedLegacy) {
|
|
215
|
+
return [legacyBuilderDir];
|
|
216
|
+
}
|
|
217
|
+
if (effectiveMatchesEnvVar && effectiveBuilderDir && resolvedEffective && resolvedEffective !== resolvedLegacy) {
|
|
218
|
+
return [legacyBuilderDir, effectiveBuilderDir];
|
|
219
|
+
}
|
|
220
|
+
if (effectiveBuilderDir && resolvedEffective && resolvedEffective !== resolvedLegacy) {
|
|
221
|
+
return [effectiveBuilderDir, legacyBuilderDir];
|
|
222
|
+
}
|
|
223
|
+
return [legacyBuilderDir];
|
|
224
|
+
}
|
|
225
|
+
|
|
178
226
|
/**
|
|
179
227
|
* Merge scan results into registry (does not remove stale keys).
|
|
180
228
|
* @param {string|null} projectRoot - getProjectRoot() or null (same semantics as projectRoot || getProjectRoot())
|
|
@@ -188,19 +236,13 @@ function refreshUrlsLocalRegistryFromBuilder(projectRoot) {
|
|
|
188
236
|
}
|
|
189
237
|
// Published npm tarball omits builder/ under the package root (.npmignore). Global installs must
|
|
190
238
|
// still refresh from the real builder tree (AIFABRIX_BUILDER_DIR or integration base + builder).
|
|
191
|
-
|
|
192
|
-
const effectiveBuilderDir = pathsUtil.getBuilderRoot();
|
|
193
|
-
const builderDirs = [legacyBuilderDir];
|
|
239
|
+
let effectiveBuilderDir = null;
|
|
194
240
|
try {
|
|
195
|
-
|
|
196
|
-
effectiveBuilderDir &&
|
|
197
|
-
path.resolve(effectiveBuilderDir) !== path.resolve(legacyBuilderDir)
|
|
198
|
-
) {
|
|
199
|
-
builderDirs.push(effectiveBuilderDir);
|
|
200
|
-
}
|
|
241
|
+
effectiveBuilderDir = pathsUtil.getBuilderRoot();
|
|
201
242
|
} catch {
|
|
202
|
-
|
|
243
|
+
effectiveBuilderDir = null;
|
|
203
244
|
}
|
|
245
|
+
const builderDirs = getOrderedBuilderDirsForRegistryScan(root, effectiveBuilderDir);
|
|
204
246
|
for (const builderDir of builderDirs) {
|
|
205
247
|
mergeBuilderDirIntoRegistry(merged, builderDir);
|
|
206
248
|
}
|
|
@@ -8,16 +8,25 @@ const chalk = require('chalk');
|
|
|
8
8
|
const logger = require('./logger');
|
|
9
9
|
const { getValidationRunWithTransportRetry } = require('./validation-run-post-retry');
|
|
10
10
|
|
|
11
|
-
|
|
11
|
+
/** Short fixed delay while waiting for async E2E (typically a few seconds). */
|
|
12
|
+
const FAST_POLL_COUNT = 24;
|
|
13
|
+
const FAST_POLL_INTERVAL_MS = 400;
|
|
14
|
+
/** After fast phase, exponential backoff to limit load on long runs. */
|
|
15
|
+
const SLOW_BASE_INTERVAL_MS = 2000;
|
|
12
16
|
const MAX_INTERVAL_MS = 15000;
|
|
13
17
|
|
|
14
18
|
/**
|
|
15
|
-
* Delay between polls after attempt `n` (0-based)
|
|
19
|
+
* Delay between polls after attempt `n` (0-based).
|
|
20
|
+
* Fast phase (~10s of 400ms gaps) so CLI detects completion quickly; then 2s, 4s, … cap 15s.
|
|
16
21
|
* @param {number} attemptIndex - Zero-based poll index after initial POST
|
|
17
22
|
* @returns {number}
|
|
18
23
|
*/
|
|
19
24
|
function nextPollDelayMs(attemptIndex) {
|
|
20
|
-
|
|
25
|
+
if (attemptIndex < FAST_POLL_COUNT) {
|
|
26
|
+
return FAST_POLL_INTERVAL_MS;
|
|
27
|
+
}
|
|
28
|
+
const slowIndex = attemptIndex - FAST_POLL_COUNT;
|
|
29
|
+
const raw = SLOW_BASE_INTERVAL_MS * 2 ** Math.max(0, slowIndex);
|
|
21
30
|
return Math.min(raw, MAX_INTERVAL_MS);
|
|
22
31
|
}
|
|
23
32
|
|
|
@@ -104,7 +113,9 @@ async function pollValidationRunUntilComplete(opts) {
|
|
|
104
113
|
}
|
|
105
114
|
|
|
106
115
|
module.exports = {
|
|
107
|
-
|
|
116
|
+
FAST_POLL_COUNT,
|
|
117
|
+
FAST_POLL_INTERVAL_MS,
|
|
118
|
+
SLOW_BASE_INTERVAL_MS,
|
|
108
119
|
MAX_INTERVAL_MS,
|
|
109
120
|
nextPollDelayMs,
|
|
110
121
|
pollValidationRunUntilComplete,
|
|
@@ -36,9 +36,8 @@ function assignOptionalNonNegativeInt(e2e, key, raw) {
|
|
|
36
36
|
* @param {Object} options - CLI-derived options
|
|
37
37
|
* @param {boolean} [options.debug]
|
|
38
38
|
* @param {boolean} [options.verbose]
|
|
39
|
-
* @param {boolean} [options.testCrud]
|
|
40
|
-
* @param {string} [options.recordId]
|
|
41
39
|
* @param {boolean} [options.cleanup]
|
|
40
|
+
* @param {boolean} [options.runScenarios] - false when --no-run-scenarios
|
|
42
41
|
* @param {string|Object} [options.primaryKeyValue]
|
|
43
42
|
* @param {Object} [options.e2eOptionsExtra] - Shallow-merged last (e.g. server-specific drill-down fields)
|
|
44
43
|
* @returns {Object}
|
|
@@ -47,15 +46,14 @@ function buildE2eOptionsFromCli(options = {}) {
|
|
|
47
46
|
const e2e = {};
|
|
48
47
|
if (options.debug) e2e.includeDebug = true;
|
|
49
48
|
if (options.verbose) e2e.audit = true;
|
|
50
|
-
if (options.testCrud === true) e2e.testCrud = true;
|
|
51
|
-
if (options.recordId !== undefined && options.recordId !== null && options.recordId !== '') {
|
|
52
|
-
e2e.recordId = String(options.recordId);
|
|
53
|
-
}
|
|
54
49
|
if (options.cleanup === false) e2e.cleanup = false;
|
|
55
50
|
else if (options.cleanup === true) e2e.cleanup = true;
|
|
56
51
|
if (options.primaryKeyValue !== undefined && options.primaryKeyValue !== null) {
|
|
57
52
|
e2e.primaryKeyValue = options.primaryKeyValue;
|
|
58
53
|
}
|
|
54
|
+
if (options.runScenarios === false) {
|
|
55
|
+
e2e.runScenarios = false;
|
|
56
|
+
}
|
|
59
57
|
assignOptionalNonNegativeInt(e2e, 'minVectorHits', options.minVectorHits);
|
|
60
58
|
assignOptionalNonNegativeInt(e2e, 'minProcessed', options.minProcessed);
|
|
61
59
|
assignOptionalNonNegativeInt(e2e, 'minRecordCount', options.minRecordCount);
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Flatten datasource root `dimensions` for validate CLI output.
|
|
3
|
+
* @author AI Fabrix Team
|
|
4
|
+
* @version 1.0.0
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* @param {Array<{ fk?: string, dimension?: string }>} via - dimensionBinding.via
|
|
9
|
+
* @returns {string}
|
|
10
|
+
*/
|
|
11
|
+
function formatFkViaChain(via) {
|
|
12
|
+
return via
|
|
13
|
+
.map(v =>
|
|
14
|
+
v && typeof v.fk === 'string' && typeof v.dimension === 'string' ? `${v.fk}→${v.dimension}` : ''
|
|
15
|
+
)
|
|
16
|
+
.filter(Boolean)
|
|
17
|
+
.join(', ');
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* @param {unknown} actor - dimensionBinding.actor
|
|
22
|
+
* @returns {string}
|
|
23
|
+
*/
|
|
24
|
+
function formatFkActorSuffix(actor) {
|
|
25
|
+
return typeof actor === 'string' && actor.length > 0 ? ` (actor: ${actor})` : '';
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Flatten root dimensions: local → metadata.<field>, fk → fk:<chain> (actor: ...).
|
|
30
|
+
* @param {Record<string, unknown>|null|undefined} root - datasource.dimensions
|
|
31
|
+
* @returns {Record<string, string>}
|
|
32
|
+
*/
|
|
33
|
+
function flattenRootDimensionsForDisplay(root) {
|
|
34
|
+
const rootFlat = {};
|
|
35
|
+
if (!root || typeof root !== 'object' || Array.isArray(root)) {
|
|
36
|
+
return rootFlat;
|
|
37
|
+
}
|
|
38
|
+
for (const [dimKey, binding] of Object.entries(root)) {
|
|
39
|
+
if (!binding || typeof binding !== 'object') {
|
|
40
|
+
continue;
|
|
41
|
+
}
|
|
42
|
+
if (typeof binding.field === 'string') {
|
|
43
|
+
rootFlat[dimKey] = `metadata.${binding.field}`;
|
|
44
|
+
continue;
|
|
45
|
+
}
|
|
46
|
+
if (binding.type !== 'fk' || !Array.isArray(binding.via)) {
|
|
47
|
+
continue;
|
|
48
|
+
}
|
|
49
|
+
const chain = formatFkViaChain(binding.via);
|
|
50
|
+
if (!chain) {
|
|
51
|
+
continue;
|
|
52
|
+
}
|
|
53
|
+
rootFlat[dimKey] = `fk:${chain}${formatFkActorSuffix(binding.actor)}`;
|
|
54
|
+
}
|
|
55
|
+
return rootFlat;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
module.exports = {
|
|
59
|
+
flattenRootDimensionsForDisplay
|
|
60
|
+
};
|