@aifabrix/builder 2.51.0 → 2.51.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (162) hide show
  1. package/jest.isolated-projects.js +77 -1
  2. package/jest.projects.js +5 -426
  3. package/lib/api/assistants.api.js +67 -0
  4. package/lib/api/datasources-extended.api.js +6 -6
  5. package/lib/api/evidence-authoring-helpers.js +267 -0
  6. package/lib/api/evidence-authoring.api.js +337 -0
  7. package/lib/api/evidence.api.js +57 -1
  8. package/lib/api/index.js +24 -7
  9. package/lib/api/pipeline.api.js +43 -3
  10. package/lib/api/records-bulk.api.js +28 -3
  11. package/lib/api/types/evidence.types.js +156 -8
  12. package/lib/api/work.api.js +283 -0
  13. package/lib/app/run-deploy-job.js +108 -0
  14. package/lib/app/run-helpers.js +10 -3
  15. package/lib/app/run.js +8 -0
  16. package/lib/channels/add-channel.js +54 -23
  17. package/lib/channels/approval-guides/chatgpt.js +8 -7
  18. package/lib/channels/approval-guides/microsoft-copilot.js +7 -8
  19. package/lib/channels/channel-artifact-generator.js +14 -2
  20. package/lib/channels/channel-credential.js +3 -1
  21. package/lib/channels/channel-probe-evidence.js +79 -0
  22. package/lib/channels/channel-probe.js +265 -13
  23. package/lib/channels/channel-runtime-resolver.js +120 -29
  24. package/lib/channels/channel-status-display.js +44 -0
  25. package/lib/channels/channel-targets.js +7 -0
  26. package/lib/channels/chatgpt-runtime-contract.js +29 -0
  27. package/lib/channels/microsoft-copilot-package.js +280 -0
  28. package/lib/channels/runtime-tool-catalog.js +165 -0
  29. package/lib/cli/index.js +2 -0
  30. package/lib/cli/setup-app.js +4 -0
  31. package/lib/cli/setup-external-system-help.js +8 -2
  32. package/lib/cli/setup-external-system.js +12 -0
  33. package/lib/commands/app-test.js +2 -14
  34. package/lib/commands/datasource-load-export-cli.js +6 -1
  35. package/lib/commands/deploy-gate-failure-details.js +19 -17
  36. package/lib/commands/dev-down.js +2 -6
  37. package/lib/commands/evidence-kind.js +65 -15
  38. package/lib/commands/governance-generate-external.js +56 -21
  39. package/lib/commands/lifecycle-external.js +10 -4
  40. package/lib/commands/repair-datasource-apply.js +10 -1
  41. package/lib/commands/repair-datasource-run.js +4 -2
  42. package/lib/commands/repair-datasource-test-payload.js +80 -19
  43. package/lib/commands/repair-datasource-webhooks-sync-bridge.js +42 -3
  44. package/lib/commands/repair-internal.js +9 -1
  45. package/lib/commands/repair-openapi-sync.js +34 -17
  46. package/lib/commands/role-assistant.js +313 -0
  47. package/lib/commands/upload-data-sync-finalize.js +5 -4
  48. package/lib/commands/upload-sidecar-followups.js +144 -47
  49. package/lib/commands/upload.js +43 -6
  50. package/lib/commands/verify-governance-combined-external.js +73 -9
  51. package/lib/commands/verify-governance-db.js +80 -25
  52. package/lib/commands/verify-governance-external.js +29 -5
  53. package/lib/commands/wizard-core-helpers.js +7 -3
  54. package/lib/commands/wizard-core.js +78 -9
  55. package/lib/datasource/bulk-loader-run-batches.js +4 -2
  56. package/lib/datasource/bulk-loader-service.js +5 -1
  57. package/lib/datasource/datasource-validate-summary.js +2 -2
  58. package/lib/datasource/validate.js +13 -6
  59. package/lib/external-system/deploy-helpers.js +35 -1
  60. package/lib/external-system/deploy.js +23 -10
  61. package/lib/external-system/download-helpers.js +111 -2
  62. package/lib/external-system/download-role-assistant.js +55 -0
  63. package/lib/external-system/download.js +45 -26
  64. package/lib/external-system/test-helpers.js +1 -1
  65. package/lib/external-system/test.js +39 -3
  66. package/lib/generator/builders.js +3 -0
  67. package/lib/generator/wizard.js +51 -4
  68. package/lib/lifecycle/access-control-abac.js +121 -0
  69. package/lib/lifecycle/lifecycle-governance-options.js +4 -1
  70. package/lib/lifecycle/product-model.js +3 -2
  71. package/lib/lifecycle/report-display.js +14 -2
  72. package/lib/programmatic/capability-registry.js +20 -1
  73. package/lib/programmatic/command-options.js +1 -0
  74. package/lib/programmatic/index.js +4 -0
  75. package/lib/programmatic/manifest-patch-section.js +48 -9
  76. package/lib/programmatic/manifest-section-registry-data.js +3 -2
  77. package/lib/programmatic/openapi-apply-result-schemas.js +44 -0
  78. package/lib/programmatic/openapi-component-schemas.js +16 -1
  79. package/lib/programmatic/openapi-description-enricher.js +3 -1
  80. package/lib/programmatic/openapi-descriptions.js +55 -5
  81. package/lib/programmatic/openapi-evidence-schemas.js +161 -0
  82. package/lib/programmatic/openapi-manifest-schemas.js +6 -2
  83. package/lib/programmatic/openapi-operation-bindings.js +99 -0
  84. package/lib/programmatic/openapi-request-schemas.js +36 -1
  85. package/lib/programmatic/openapi-result-schemas.js +1 -1
  86. package/lib/programmatic/openapi-schema-descriptions-certify.js +5 -1
  87. package/lib/programmatic/openapi-schema-descriptions-core.js +1 -1
  88. package/lib/programmatic/openapi-schema-descriptions-evidence.js +156 -0
  89. package/lib/programmatic/openapi-schema-descriptions-manifest.js +4 -3
  90. package/lib/programmatic/openapi-schema-descriptions-operations.js +70 -0
  91. package/lib/programmatic/openapi-spec.js +47 -28
  92. package/lib/programmatic/orchestration-openapi-meta.js +5 -0
  93. package/lib/programmatic/request-timeout.js +3 -0
  94. package/lib/programmatic/route-handler-map.js +89 -2
  95. package/lib/programmatic/run-certify.js +15 -2
  96. package/lib/programmatic/run-deploy.js +74 -0
  97. package/lib/programmatic/run-evidence-authoring.js +80 -0
  98. package/lib/programmatic/run-protection.js +124 -0
  99. package/lib/programmatic/run-template-workspace.js +28 -1
  100. package/lib/programmatic/workspace-context.js +6 -0
  101. package/lib/protection/auth-context.js +48 -6
  102. package/lib/protection/paths.js +22 -0
  103. package/lib/protection/run-protection-create-helpers.js +22 -1
  104. package/lib/protection/run-protection-create.js +1 -1
  105. package/lib/protection/sync-after-upload.js +1 -1
  106. package/lib/role-assistant/evidence-sync.js +450 -0
  107. package/lib/role-assistant/github-workflow.js +96 -0
  108. package/lib/role-assistant/knowledge-sync.js +465 -0
  109. package/lib/role-assistant/settings.js +126 -0
  110. package/lib/role-assistant/test-cases-evaluate.js +170 -0
  111. package/lib/role-assistant/test-cases-honesty.js +184 -0
  112. package/lib/role-assistant/test-cases.js +364 -0
  113. package/lib/role-assistant/test-runner-workhub-answers.js +203 -0
  114. package/lib/role-assistant/test-runner-workhub.js +385 -0
  115. package/lib/role-assistant/test-runner.js +500 -0
  116. package/lib/schema/application-schema.json +16 -2
  117. package/lib/schema/deployment-rules.yaml +1 -0
  118. package/lib/schema/external-datasource.schema.json +38 -4
  119. package/lib/schema/external-system.schema.json +4 -4
  120. package/lib/schema/infra.parameter.yaml +20 -0
  121. package/lib/schema/role-assistant-settings.schema.json +108 -0
  122. package/lib/utils/api.js +52 -27
  123. package/lib/utils/app-path-resolve.js +72 -24
  124. package/lib/utils/authentication-shape.js +3 -1
  125. package/lib/utils/deployment-validation.js +28 -3
  126. package/lib/utils/docker-run-ephemeral.js +55 -0
  127. package/lib/utils/enterprise-knowledge-paths.js +29 -12
  128. package/lib/utils/env-copy-keycloak.js +196 -0
  129. package/lib/utils/env-copy.js +34 -1
  130. package/lib/utils/external-system-readiness-compact-display.js +3 -0
  131. package/lib/utils/help-builder.js +1 -0
  132. package/lib/utils/paths.js +67 -23
  133. package/lib/utils/pipeline-upload-auth-retry.js +15 -3
  134. package/lib/utils/role-assistant-paths.js +98 -0
  135. package/lib/utils/schema-loader.js +41 -0
  136. package/lib/utils/schema-resolver.js +36 -4
  137. package/lib/utils/token-manager.js +38 -29
  138. package/lib/utils/url-declarative-resolve-load-doc.js +38 -1
  139. package/lib/validation/external-manifest-validator.js +31 -8
  140. package/lib/validation/validate-external-file.js +8 -1
  141. package/package.json +2 -1
  142. package/scripts/export-builder-api-evidence.js +1 -1
  143. package/templates/applications/dataplane/application.yaml +23 -15
  144. package/templates/applications/dataplane/env.template +1 -1
  145. package/templates/applications/dataplane/rbac.yaml +14 -0
  146. package/templates/applications/miso-controller/application.yaml +1 -1
  147. package/templates/applications/miso-controller/env.template +3 -1
  148. package/templates/channels/chatgpt/install.template.md.hbs +42 -24
  149. package/templates/channels/chatgpt/integration/README.md +10 -7
  150. package/templates/channels/chatgpt/integration/application.json +2 -2
  151. package/templates/channels/chatgpt/integration/chatgpt-system.json +3 -3
  152. package/templates/channels/chatgpt/integration/env.template +1 -1
  153. package/templates/channels/chatgpt/manifest.template.json.hbs +36 -5
  154. package/templates/channels/microsoft-copilot/ai-plugin.template.json.hbs +28 -0
  155. package/templates/channels/microsoft-copilot/declarative-agent.template.json.hbs +22 -0
  156. package/templates/channels/microsoft-copilot/install.template.md.hbs +32 -22
  157. package/templates/channels/microsoft-copilot/integration/README.md +9 -7
  158. package/templates/channels/microsoft-copilot/integration/application.json +2 -2
  159. package/templates/channels/microsoft-copilot/integration/env.template +1 -1
  160. package/templates/channels/microsoft-copilot/integration/microsoft-copilot-system.json +7 -6
  161. package/templates/channels/microsoft-copilot/manifest.template.json.hbs +34 -6
  162. /package/.cursor/plans/{157.1-security-quality-violations-by-type.csv → Done/157.1-security-quality-violations-by-type.csv} +0 -0
@@ -10,6 +10,56 @@ const {
10
10
  buildSubjectsFromOptions,
11
11
  resolveGovernanceIntegrationContext
12
12
  } = require('./governance-cli-shared');
13
+ const { VERDICT } = require('../lifecycle/product-model');
14
+ const {
15
+ allDatasourcesAbacNotApplicable,
16
+ filterAbacRequiredDatasourceKeys
17
+ } = require('../lifecycle/access-control-abac');
18
+ const { runSystemLifecycle } = require('../api/lifecycle.api');
19
+
20
+ /**
21
+ * @param {string} resolvedKey
22
+ * @param {string[]} keys
23
+ * @param {Object} [extra]
24
+ * @returns {Object}
25
+ */
26
+ function buildNotApplicableGovernanceResult(resolvedKey, keys, extra = {}) {
27
+ return {
28
+ systemKey: resolvedKey,
29
+ command: 'verify-governance',
30
+ verdict: VERDICT.NOT_APPLICABLE,
31
+ policyCoveragePercent: 100,
32
+ dimensionCoveragePercent: 100,
33
+ enforcementScenarios: { passed: 0, total: 0 },
34
+ datasourceRows: keys.map(datasourceKey => ({
35
+ datasourceKey,
36
+ verdict: VERDICT.NOT_APPLICABLE
37
+ })),
38
+ ...extra
39
+ };
40
+ }
41
+
42
+ /**
43
+ * @param {string} dataplaneUrl
44
+ * @param {Object} authConfig
45
+ * @param {string} resolvedKey
46
+ * @param {string[]} keys
47
+ * @param {Object} [extra]
48
+ * @returns {Promise<Object>}
49
+ */
50
+ async function closeAllGovernanceAsNotApplicable(
51
+ dataplaneUrl,
52
+ authConfig,
53
+ resolvedKey,
54
+ keys,
55
+ extra = {}
56
+ ) {
57
+ await runSystemLifecycle(dataplaneUrl, authConfig, resolvedKey, {
58
+ steps: ['governance'],
59
+ datasourceKeys: keys
60
+ });
61
+ return buildNotApplicableGovernanceResult(resolvedKey, keys, extra);
62
+ }
13
63
 
14
64
  /**
15
65
  * @async
@@ -23,22 +73,32 @@ async function runVerifyGovernanceCombined(systemKey, options = {}) {
23
73
  }
24
74
 
25
75
  const subjects = buildSubjectsFromOptions(options);
76
+ const { resolvedKey, keys, authConfig, dataplaneUrl } =
77
+ await resolveGovernanceIntegrationContext(systemKey, options);
78
+
26
79
  if (subjects.length === 0) {
27
- throw new Error(
28
- 'At least one --subject-user-id or --subject-email is required for verify-governance'
29
- );
80
+ if (!allDatasourcesAbacNotApplicable(resolvedKey, keys)) {
81
+ throw new Error(
82
+ 'At least one --subject-user-id or --subject-email is required for verify-governance'
83
+ );
84
+ }
85
+ return closeAllGovernanceAsNotApplicable(dataplaneUrl, authConfig, resolvedKey, keys);
30
86
  }
31
87
 
32
- const { resolvedKey, authConfig, dataplaneUrl } = await resolveGovernanceIntegrationContext(
33
- systemKey,
34
- options
35
- );
88
+ const requiredKeys = filterAbacRequiredDatasourceKeys(resolvedKey, keys);
89
+ if (requiredKeys.length === 0) {
90
+ return closeAllGovernanceAsNotApplicable(dataplaneUrl, authConfig, resolvedKey, keys, {
91
+ generated: 0,
92
+ generateResults: []
93
+ });
94
+ }
36
95
 
37
96
  const pipelineOpts = {
38
97
  ...options,
39
98
  noPublish: true,
40
99
  authConfig,
41
- dataplaneUrl
100
+ dataplaneUrl,
101
+ datasourceKeys: requiredKeys
42
102
  };
43
103
 
44
104
  const generateResult = await runGovernanceGenerateForExternalSystem(
@@ -46,7 +106,11 @@ async function runVerifyGovernanceCombined(systemKey, options = {}) {
46
106
  pipelineOpts
47
107
  );
48
108
 
49
- const verifyResult = await runVerifyGovernanceForExternalSystem(resolvedKey, pipelineOpts);
109
+ // Verify all datasources so notApplicable keys close as NOT_APPLICABLE.
110
+ const verifyResult = await runVerifyGovernanceForExternalSystem(resolvedKey, {
111
+ ...pipelineOpts,
112
+ datasourceKeys: keys
113
+ });
50
114
 
51
115
  return {
52
116
  ...verifyResult,
@@ -27,6 +27,58 @@ function attachUploadAuthToOptions(options, uploadCtx) {
27
27
  options.silentResolve = true;
28
28
  }
29
29
 
30
+ /**
31
+ * @param {string} key
32
+ * @param {Object|null} config
33
+ * @returns {Object}
34
+ */
35
+ function buildNotApplicableGovernanceRow(key, config) {
36
+ const {
37
+ parseAccessControlAbac
38
+ } = require('../lifecycle/access-control-abac');
39
+ const abac = parseAccessControlAbac(config);
40
+ return {
41
+ datasourceKey: key,
42
+ verdict: VERDICT.NOT_APPLICABLE,
43
+ scenarios: { passed: 0, total: 0 },
44
+ policyCoveragePercent: 100,
45
+ dimensionCoveragePercent: 100,
46
+ rationale: abac.rationale
47
+ };
48
+ }
49
+
50
+ /**
51
+ * @param {string} key
52
+ * @param {Object} ctx
53
+ * @returns {Promise<Object|null>}
54
+ */
55
+ async function runRequiredGovernanceDatasource(key, ctx) {
56
+ try {
57
+ const packRow = await getDatasourceGovernancePack(ctx.dataplaneUrl, ctx.authConfig, key);
58
+ if (!packRow || !packRow.pack) {
59
+ return null;
60
+ }
61
+ const result = await runDatasourceGovernanceScenarios(ctx.dataplaneUrl, ctx.authConfig, key);
62
+ const summary = result.summary || {};
63
+ return {
64
+ datasourceKey: key,
65
+ verdict: summary.failed > 0 ? VERDICT.FAILED : VERDICT.VERIFIED,
66
+ scenarios: { passed: summary.passed || 0, total: summary.total || 0 },
67
+ policyCoveragePercent: result.policyCoveragePercent ?? 100,
68
+ dimensionCoveragePercent: result.dimensionCoveragePercent ?? 100,
69
+ packDocument: packRow.pack,
70
+ result
71
+ };
72
+ } catch (err) {
73
+ return {
74
+ datasourceKey: key,
75
+ verdict: VERDICT.FAILED,
76
+ error: err.message,
77
+ scenarios: { passed: 0, total: 0 }
78
+ };
79
+ }
80
+ }
81
+
30
82
  /**
31
83
  * @async
32
84
  * @param {string[]} keys
@@ -35,35 +87,27 @@ function attachUploadAuthToOptions(options, uploadCtx) {
35
87
  * @returns {Promise<Array>}
36
88
  */
37
89
  async function runGovernanceForDatasourceKeys(keys, ctx, progress = {}) {
90
+ const {
91
+ filterAbacRequiredDatasourceKeys,
92
+ loadDatasourceConfigsByKey
93
+ } = require('../lifecycle/access-control-abac');
94
+ const configs = loadDatasourceConfigsByKey(ctx.systemKey || '');
95
+ const required = new Set(
96
+ filterAbacRequiredDatasourceKeys(ctx.systemKey || '', keys)
97
+ );
38
98
  const rows = [];
39
99
  for (let index = 0; index < keys.length; index += 1) {
40
100
  const key = keys[index];
41
101
  if (typeof progress.setLabel === 'function') {
42
102
  progress.setLabel(`Running governance scenarios: ${key} (${index + 1}/${keys.length})`);
43
103
  }
44
- try {
45
- const packRow = await getDatasourceGovernancePack(ctx.dataplaneUrl, ctx.authConfig, key);
46
- if (!packRow || !packRow.pack) {
47
- continue;
48
- }
49
- const result = await runDatasourceGovernanceScenarios(ctx.dataplaneUrl, ctx.authConfig, key);
50
- const summary = result.summary || {};
51
- rows.push({
52
- datasourceKey: key,
53
- verdict: summary.failed > 0 ? VERDICT.FAILED : VERDICT.VERIFIED,
54
- scenarios: { passed: summary.passed || 0, total: summary.total || 0 },
55
- policyCoveragePercent: result.policyCoveragePercent ?? 100,
56
- dimensionCoveragePercent: result.dimensionCoveragePercent ?? 100,
57
- packDocument: packRow.pack,
58
- result
59
- });
60
- } catch (err) {
61
- rows.push({
62
- datasourceKey: key,
63
- verdict: VERDICT.FAILED,
64
- error: err.message,
65
- scenarios: { passed: 0, total: 0 }
66
- });
104
+ if (!required.has(key)) {
105
+ rows.push(buildNotApplicableGovernanceRow(key, configs.get(key) || null));
106
+ continue;
107
+ }
108
+ const row = await runRequiredGovernanceDatasource(key, ctx);
109
+ if (row) {
110
+ rows.push(row);
67
111
  }
68
112
  }
69
113
  return rows;
@@ -76,7 +120,12 @@ async function runGovernanceForDatasourceKeys(keys, ctx, progress = {}) {
76
120
  * @returns {Promise<Object>}
77
121
  */
78
122
  async function runVerifyGovernanceFromDbPacks(systemKey, options) {
79
- const { systemKey: resolvedKey, keys } = resolveExternalIntegrationContext(systemKey);
123
+ const { systemKey: resolvedKey, keys: discoveredKeys } =
124
+ resolveExternalIntegrationContext(systemKey);
125
+ const keys =
126
+ Array.isArray(options.datasourceKeys) && options.datasourceKeys.length
127
+ ? options.datasourceKeys
128
+ : discoveredKeys;
80
129
  if (keys.length === 0) {
81
130
  return { resolvedKey, datasourceRows: [] };
82
131
  }
@@ -87,10 +136,16 @@ async function runVerifyGovernanceFromDbPacks(systemKey, options) {
87
136
  }
88
137
 
89
138
  const { authConfig, dataplaneUrl } = await resolveIntegrationAuth(resolvedKey, options);
139
+ options.authConfig = authConfig;
140
+ options.dataplaneUrl = dataplaneUrl;
90
141
  const datasourceRows = await runWithVerifyStepProgress(
91
142
  'Running governance scenarios',
92
143
  async(progress) =>
93
- runGovernanceForDatasourceKeys(keys, { authConfig, dataplaneUrl }, progress),
144
+ runGovernanceForDatasourceKeys(
145
+ keys,
146
+ { authConfig, dataplaneUrl, systemKey: resolvedKey },
147
+ progress
148
+ ),
94
149
  options
95
150
  );
96
151
  return { resolvedKey, datasourceRows };
@@ -8,7 +8,7 @@
8
8
 
9
9
  const { runTestGovernanceForExternalSystem } = require('./test-governance-external');
10
10
  const { runVerifyGovernanceFromDbPacks } = require('./verify-governance-db');
11
- const { VERDICT, verdictFromPassed } = require('../lifecycle/product-model');
11
+ const { VERDICT } = require('../lifecycle/product-model');
12
12
  const {
13
13
  sumEnforcementScenarios,
14
14
  aggregateCoverage,
@@ -20,14 +20,26 @@ const {
20
20
  * @returns {{ passed: number, total: number }}
21
21
  */
22
22
  function buildDbPackResult(resolvedKey, datasourceRows) {
23
- const allVerified = datasourceRows.every(r => r.verdict === VERDICT.VERIFIED);
23
+ const hasFailed = datasourceRows.some(r => r.verdict === VERDICT.FAILED);
24
+ const allNa =
25
+ datasourceRows.length > 0
26
+ && datasourceRows.every(r => r.verdict === VERDICT.NOT_APPLICABLE);
27
+ const allClosed = datasourceRows.every(
28
+ r => r.verdict === VERDICT.VERIFIED || r.verdict === VERDICT.NOT_APPLICABLE
29
+ );
30
+ let verdict = VERDICT.FAILED;
31
+ if (allNa) {
32
+ verdict = VERDICT.NOT_APPLICABLE;
33
+ } else if (allClosed && !hasFailed) {
34
+ verdict = VERDICT.VERIFIED;
35
+ }
24
36
  const coverage = aggregateCoverage(datasourceRows);
25
37
  return {
26
38
  systemKey: resolvedKey,
27
39
  command: 'verify-governance',
28
- verdict: verdictFromPassed(allVerified),
29
- policyCoveragePercent: coverage.policy ?? (allVerified ? 100 : 0),
30
- dimensionCoveragePercent: coverage.dimension ?? (allVerified ? 100 : 0),
40
+ verdict,
41
+ policyCoveragePercent: coverage.policy ?? (allClosed ? 100 : 0),
42
+ dimensionCoveragePercent: coverage.dimension ?? (allClosed ? 100 : 0),
31
43
  enforcementScenarios: sumEnforcementScenarios(datasourceRows),
32
44
  datasourceRows
33
45
  };
@@ -66,6 +78,18 @@ async function runVerifyGovernanceForExternalSystem(systemKey, options = {}) {
66
78
  };
67
79
  }
68
80
 
81
+ const naKeys = datasourceRows
82
+ .filter(row => row.verdict === VERDICT.NOT_APPLICABLE)
83
+ .map(row => row.datasourceKey)
84
+ .filter(Boolean);
85
+ if (naKeys.length > 0 && options.authConfig && options.dataplaneUrl) {
86
+ const { runSystemLifecycle } = require('../api/lifecycle.api');
87
+ await runSystemLifecycle(options.dataplaneUrl, options.authConfig, resolvedKey, {
88
+ steps: ['governance'],
89
+ datasourceKeys: naKeys
90
+ });
91
+ }
92
+
69
93
  return buildDbPackResult(resolvedKey, datasourceRows);
70
94
  }
71
95
 
@@ -232,15 +232,19 @@ function buildPlatformConfigPayload({ credentialIdOrKey, datasourceKeys, configu
232
232
  }
233
233
 
234
234
  /**
235
- * Extract configuration from API response
235
+ * Extract configuration from API response.
236
+ * add-datasource generation returns datasourceConfig(s) only (systemConfig is null by design).
237
+ * Callers must load the existing workspace system when systemConfig is missing.
236
238
  * @param {Object} generateResponse - API response
237
239
  * @returns {Object} Extracted configuration
238
240
  */
239
241
  function extractConfigurationFromResponse(generateResponse) {
240
- const systemConfig = generateResponse.data?.systemConfig;
242
+ const systemConfig = generateResponse.data?.systemConfig || null;
241
243
  const datasourceConfigs = generateResponse.data?.datasourceConfigs ||
242
244
  (generateResponse.data?.datasourceConfig ? [generateResponse.data.datasourceConfig] : []);
243
- if (!systemConfig) throw new Error('System configuration not found');
245
+ if (!systemConfig && (!Array.isArray(datasourceConfigs) || datasourceConfigs.length === 0)) {
246
+ throw new Error('System configuration not found');
247
+ }
244
248
  return { systemConfig, datasourceConfigs, systemKey: generateResponse.data?.systemKey };
245
249
  }
246
250
 
@@ -312,6 +312,72 @@ async function callGenerateApi(dataplaneUrl, authConfig, options, prefs) {
312
312
  return await generateConfig(dataplaneUrl, authConfig, configPayload);
313
313
  }
314
314
 
315
+ /**
316
+ * Load existing system JSON/YAML from the integration workspace (add-datasource).
317
+ * @param {string} appName - Integration app / system key
318
+ * @returns {Object|null}
319
+ */
320
+ function loadExistingWorkspaceSystemConfig(appName) {
321
+ if (!appName || typeof appName !== 'string') {
322
+ return null;
323
+ }
324
+ const { getIntegrationPath } = require('../utils/paths');
325
+ const { loadConfigFile } = require('../utils/config-format');
326
+ const fsSync = require('fs');
327
+ const appPath = getIntegrationPath(appName);
328
+ for (const ext of ['.json', '.yaml', '.yml']) {
329
+ const candidate = path.join(appPath, `${appName}-system${ext}`);
330
+ if (fsSync.existsSync(candidate)) {
331
+ try {
332
+ return loadConfigFile(candidate);
333
+ } catch {
334
+ return null;
335
+ }
336
+ }
337
+ }
338
+ return null;
339
+ }
340
+
341
+ /**
342
+ * @param {Object|null|undefined} systemConfig
343
+ * @param {Object} options
344
+ * @returns {Object}
345
+ */
346
+ function resolveSystemConfigForWizardMode(systemConfig, options) {
347
+ if (systemConfig) {
348
+ return systemConfig;
349
+ }
350
+ if (options.mode === 'add-datasource') {
351
+ const existing = loadExistingWorkspaceSystemConfig(
352
+ options.appName || options.systemIdOrKey
353
+ );
354
+ if (!existing) {
355
+ throw new Error(
356
+ 'System configuration not found for add-datasource. ' +
357
+ 'Run create-system wizard first so the workspace has a system file.'
358
+ );
359
+ }
360
+ return existing;
361
+ }
362
+ throw new Error('System configuration not found');
363
+ }
364
+
365
+ /**
366
+ * @param {Object} generateResponse
367
+ * @param {Object} prefs
368
+ * @param {Object} options
369
+ * @returns {Promise<void>}
370
+ */
371
+ async function maybeWriteWizardDebugLog(generateResponse, prefs, options) {
372
+ if (!prefs.debug || !options.appName) {
373
+ return;
374
+ }
375
+ const debugLog = generateResponse.data?.debugLog;
376
+ if (debugLog && typeof debugLog === 'string') {
377
+ await writeDebugLog(options.appName, debugLog);
378
+ }
379
+ }
380
+
315
381
  async function handleConfigurationGeneration(dataplaneUrl, authConfig, options) {
316
382
  logger.log(chalk.blue('\n\uD83D\uDCCB Step 5: Generate Configuration'));
317
383
  const prefs = buildConfigPreferences(options.configPrefs);
@@ -320,18 +386,21 @@ async function handleConfigurationGeneration(dataplaneUrl, authConfig, options)
320
386
  const generateResponse = await callGenerateApi(dataplaneUrl, authConfig, options, prefs);
321
387
  spinner.stop();
322
388
  if (!generateResponse.success) {
323
- await saveDebugManifestOnErrorAndThrow(generateResponse, { enableDebug: prefs.debug, appName: options.appName });
389
+ await saveDebugManifestOnErrorAndThrow(generateResponse, {
390
+ enableDebug: prefs.debug,
391
+ appName: options.appName
392
+ });
324
393
  }
325
394
  const result = extractConfigurationFromResponse(generateResponse);
326
- const normalized = normalizeWizardConfigs(result.systemConfig, result.datasourceConfigs);
327
- if (prefs.debug && options.appName) {
328
- const debugLog = generateResponse.data?.debugLog;
329
- if (debugLog && typeof debugLog === 'string') {
330
- await writeDebugLog(options.appName, debugLog);
331
- }
332
- }
395
+ const systemConfig = resolveSystemConfigForWizardMode(result.systemConfig, options);
396
+ const normalized = normalizeWizardConfigs(systemConfig, result.datasourceConfigs);
397
+ await maybeWriteWizardDebugLog(generateResponse, prefs, options);
333
398
  logger.log(formatSuccessLine('Configuration generated successfully'));
334
- return { systemConfig: normalized.systemConfig, datasourceConfigs: normalized.datasourceConfigs, systemKey: result.systemKey };
399
+ return {
400
+ systemConfig: normalized.systemConfig,
401
+ datasourceConfigs: normalized.datasourceConfigs,
402
+ systemKey: result.systemKey || options.systemIdOrKey || options.appName
403
+ };
335
404
  } catch (error) {
336
405
  spinner.stop();
337
406
  throw error;
@@ -70,19 +70,21 @@ function recordBatchSuccess(state, index, response, batchRecords) {
70
70
  * @returns {Promise<{ batches: Object[], totals: Object }>}
71
71
  */
72
72
  async function uploadRecordBatches(params) {
73
- const { chunks, syncType, dataplaneUrl, datasourceKey, authConfig } = params;
73
+ const { chunks, syncType, dataplaneUrl, datasourceKey, authConfig, timeoutMs } = params;
74
74
  const state = {
75
75
  batches: [],
76
76
  totals: { insertedCount: 0, updatedCount: 0, failedCount: 0 },
77
77
  chunkCount: chunks.length
78
78
  };
79
+ const transportOpts =
80
+ timeoutMs !== undefined && timeoutMs !== null ? { timeoutMs } : {};
79
81
 
80
82
  for (let i = 0; i < chunks.length; i += 1) {
81
83
  const batchRecords = chunks[i];
82
84
  const body = { syncType, sync: false, records: batchRecords };
83
85
  try {
84
86
  const response = await withTransientRetry(() =>
85
- bulkSyncRecords(dataplaneUrl, datasourceKey, authConfig, body)
87
+ bulkSyncRecords(dataplaneUrl, datasourceKey, authConfig, body, transportOpts)
86
88
  );
87
89
  recordBatchSuccess(state, i, response, batchRecords);
88
90
  } catch (err) {
@@ -81,12 +81,16 @@ async function executeDatasourceLoadWork(params) {
81
81
  return { ...result, context: ctx };
82
82
  }
83
83
 
84
+ const timeoutMs = options.timeoutMs
85
+ ? parseInt(String(options.timeoutMs), 10)
86
+ : undefined;
84
87
  const { batches, totals } = await uploadRecordBatches({
85
88
  chunks: chunkRecords(records, batchSize),
86
89
  syncType,
87
90
  dataplaneUrl: ctx.dataplaneUrl,
88
91
  datasourceKey: ctx.datasourceKey,
89
- authConfig: ctx.authConfig
92
+ authConfig: ctx.authConfig,
93
+ timeoutMs: Number.isFinite(timeoutMs) && timeoutMs > 0 ? timeoutMs : undefined
90
94
  });
91
95
 
92
96
  return { ...result, batches, totals, context: ctx };
@@ -167,10 +167,10 @@ function assembleSummary(parsed) {
167
167
  foreignKeys: fks,
168
168
  dimensionKeys,
169
169
  dimensions,
170
- exposedProfileNames: summarizeExposedProfileNames(parsed),
171
- openapiLine: oa.openapiLine,
172
170
  hasOpenapi: oa.hasOpenapi,
171
+ openapiLine: oa.openapiLine,
173
172
  capabilityKeys: oa.capabilityKeys,
173
+ exposedProfileNames: summarizeExposedProfileNames(parsed),
174
174
  testPayloadLine: summarizeTestPayloadLine(parsed),
175
175
  hasTestPayload: !!parsed.testPayload && typeof parsed.testPayload === 'object',
176
176
  syncLine: summarizeSyncLine(sync)
@@ -242,9 +242,7 @@ async function validateDatasourceFile(filePathOrKey) {
242
242
  };
243
243
  }
244
244
 
245
- const fieldRefErrors = validateFieldReferences(parsed);
246
- const abacErrors = validateAbac(parsed);
247
- const postSchemaErrors = [...fieldRefErrors, ...abacErrors];
245
+ const postSchemaErrors = collectPostSchemaErrors(parsed);
248
246
  if (postSchemaErrors.length > 0) {
249
247
  return {
250
248
  valid: false,
@@ -264,6 +262,17 @@ async function validateDatasourceFile(filePathOrKey) {
264
262
  };
265
263
  }
266
264
 
265
+ /**
266
+ * @param {object} parsed
267
+ * @returns {string[]}
268
+ */
269
+ function collectPostSchemaErrors(parsed) {
270
+ return [
271
+ ...validateFieldReferences(parsed),
272
+ ...validateAbac(parsed)
273
+ ];
274
+ }
275
+
267
276
  /**
268
277
  * Validates parsed datasource JSON (same rules as {@link validateDatasourceFile}, without reading a file).
269
278
  *
@@ -294,9 +303,7 @@ function validateDatasourceParsed(parsed) {
294
303
  };
295
304
  }
296
305
 
297
- const fieldRefErrors = validateFieldReferences(parsed);
298
- const abacErrors = validateAbac(parsed);
299
- const postSchemaErrors = [...fieldRefErrors, ...abacErrors];
306
+ const postSchemaErrors = collectPostSchemaErrors(parsed);
300
307
  if (postSchemaErrors.length > 0) {
301
308
  return {
302
309
  valid: false,
@@ -133,12 +133,46 @@ function extractSystemKey(systemFilePath) {
133
133
  return systemFileName.replace(/-deploy$/, '');
134
134
  }
135
135
 
136
+ /**
137
+ * Role Assistant (digitalWorker) deploy uses upload, not controller deploy.
138
+ * @async
139
+ * @param {string} appName - Application name
140
+ * @param {Object} options - Deploy options
141
+ * @param {boolean} compact - Quiet UI when true
142
+ * @param {Object} manifest - Controller manifest
143
+ * @returns {Promise<Object|null>} Upload result, or null when not a digital worker
144
+ */
145
+ async function uploadDigitalWorkerIfNeeded(appName, options, compact, manifest) {
146
+ const systemKind =
147
+ manifest && manifest.system && typeof manifest.system.systemKind === 'string'
148
+ ? manifest.system.systemKind.trim()
149
+ : '';
150
+ if (systemKind !== 'digitalWorker') {
151
+ return null;
152
+ }
153
+ if (!compact) {
154
+ const { formatSuccessParagraph } = require('../utils/cli-test-layout-chalk');
155
+ const logger = require('../utils/logger');
156
+ logger.log(
157
+ formatSuccessParagraph(
158
+ 'Role Assistant deployment uses upload pipeline (controller deploy is skipped).'
159
+ )
160
+ );
161
+ }
162
+ const { uploadExternalSystem } = require('../commands/upload');
163
+ return uploadExternalSystem(appName, {
164
+ ...options,
165
+ uploadStatus: 'published'
166
+ });
167
+ }
168
+
136
169
  module.exports = {
137
170
  loadVariablesYaml,
138
171
  validateSingleSystemFile,
139
172
  validateSystemFiles,
140
173
  validateSingleDatasourceFile,
141
174
  validateDatasourceFiles,
142
- extractSystemKey
175
+ extractSystemKey,
176
+ uploadDigitalWorkerIfNeeded
143
177
  };
144
178
 
@@ -30,6 +30,7 @@ const { displayValidationResults } = require('../validation/validate-display');
30
30
  const { syncDeployJsonFromSources } = require('./sync-deploy-manifest');
31
31
  const { runDeployGateOrchestrator } = require('../commands/deploy-gate-orchestrator');
32
32
  const { runPostPublishDataSync } = require('../commands/run-post-publish-data-sync');
33
+ const { uploadDigitalWorkerIfNeeded } = require('./deploy-helpers');
33
34
 
34
35
  /**
35
36
  * Validates manifest on dataplane (dry-run) before controller deploy.
@@ -342,8 +343,20 @@ async function executeDeployAndDisplay(manifest, controllerUrl, environment, aut
342
343
  * @param {Object} options - Deployment options
343
344
  * @returns {Promise<Object>} Deployment configuration
344
345
  */
345
- async function prepareDeploymentConfig(appName, _options) {
346
+ async function prepareDeploymentConfig(appName, options = {}) {
346
347
  const { resolveEnvironment } = require('../core/config');
348
+ if (options.authConfig && (options.authConfig.token || options.authConfig.clientId)) {
349
+ const environment = options.environment || (await resolveEnvironment());
350
+ let controllerUrl = options.controllerUrl || '';
351
+ if (!controllerUrl) {
352
+ controllerUrl = await resolveControllerUrl();
353
+ }
354
+ if (!controllerUrl) {
355
+ throw new Error('Controller URL is required. Run "aifabrix login" first.');
356
+ }
357
+ return { environment, controllerUrl, authConfig: options.authConfig };
358
+ }
359
+
347
360
  const environment = await resolveEnvironment();
348
361
  const controllerUrl = await resolveControllerUrl();
349
362
  const authConfig = await getDeploymentAuth(controllerUrl, environment, appName);
@@ -413,24 +426,26 @@ async function runExternalSystemDeployFlow(appName, options) {
413
426
  const compact = options.verbose !== true;
414
427
  const { appPath } = await detectAppType(appName);
415
428
  logOfflinePathWhenType(appPath);
416
-
417
429
  if (!compact) {
418
430
  logger.log(chalk.blue(`\nšŸš€ Deploying external system: ${appName}`));
419
431
  }
420
-
421
432
  await validateExternalSystemForDeploy(appName, options, compact);
422
-
423
433
  const gateOutcomes = await runDeployGateOrchestrator(appName, {
424
434
  ...options,
425
435
  quietDeployUi: compact
426
436
  });
427
-
428
437
  await syncDeployJsonFromSources(appName);
429
-
430
438
  const manifest = await generateControllerManifest(appName, options);
431
-
439
+ const digitalWorkerResult = await uploadDigitalWorkerIfNeeded(
440
+ appName,
441
+ options,
442
+ compact,
443
+ manifest
444
+ );
445
+ if (digitalWorkerResult) {
446
+ return digitalWorkerResult;
447
+ }
432
448
  const { environment, controllerUrl, authConfig } = await prepareDeploymentConfig(appName, options);
433
-
434
449
  let dataplaneUrl;
435
450
  try {
436
451
  dataplaneUrl = await resolveDataplaneUrl(controllerUrl, environment, authConfig, {
@@ -439,9 +454,7 @@ async function runExternalSystemDeployFlow(appName, options) {
439
454
  } catch (err) {
440
455
  throw new Error(`Failed to resolve dataplane URL before deploy: ${err.message}`);
441
456
  }
442
-
443
457
  await validateDataplaneManifestForDeploy(manifest, authConfig, dataplaneUrl, compact);
444
-
445
458
  return executeDeployAndDisplay(manifest, controllerUrl, environment, authConfig, {
446
459
  ...options,
447
460
  gateOutcomes,