@aifabrix/builder 2.50.0 → 2.51.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (236) hide show
  1. package/.cursor/plans/Done/157.1-security-quality-violations-by-type.csv +374 -0
  2. package/.cursor/rules/assistant-driven-integration-build.mdc +40 -0
  3. package/integration/hubspot-test/create-hubspot.js +1 -16
  4. package/jest.isolated-projects.js +77 -1
  5. package/jest.projects.js +5 -426
  6. package/lib/api/assistants.api.js +67 -0
  7. package/lib/api/datasources-extended.api.js +6 -6
  8. package/lib/api/evidence-authoring-helpers.js +267 -0
  9. package/lib/api/evidence-authoring.api.js +337 -0
  10. package/lib/api/evidence-kinds.api.js +128 -0
  11. package/lib/api/evidence.api.js +57 -1
  12. package/lib/api/index.js +24 -7
  13. package/lib/api/pipeline.api.js +47 -3
  14. package/lib/api/records-bulk.api.js +28 -3
  15. package/lib/api/resource-types.api.js +4 -4
  16. package/lib/api/roles.api.js +121 -0
  17. package/lib/api/types/evidence-kinds.types.js +42 -0
  18. package/lib/api/types/evidence.types.js +156 -8
  19. package/lib/api/types/roles.types.js +36 -0
  20. package/lib/api/work.api.js +283 -0
  21. package/lib/app/certification-verify-rows.js +1 -1
  22. package/lib/app/run-deploy-job.js +108 -0
  23. package/lib/app/run-helpers.js +10 -3
  24. package/lib/app/run.js +8 -0
  25. package/lib/app/show.js +2 -2
  26. package/lib/certification/merge-certification-from-artifact.js +1 -1
  27. package/lib/channels/add-channel.js +54 -23
  28. package/lib/channels/approval-guides/chatgpt.js +8 -7
  29. package/lib/channels/approval-guides/microsoft-copilot.js +7 -8
  30. package/lib/channels/channel-artifact-generator.js +16 -3
  31. package/lib/channels/channel-credential.js +3 -1
  32. package/lib/channels/channel-probe-evidence.js +79 -0
  33. package/lib/channels/channel-probe.js +265 -13
  34. package/lib/channels/channel-runtime-resolver.js +120 -29
  35. package/lib/channels/channel-status-display.js +44 -0
  36. package/lib/channels/channel-targets.js +7 -0
  37. package/lib/channels/chatgpt-runtime-contract.js +29 -0
  38. package/lib/channels/microsoft-copilot-package.js +280 -0
  39. package/lib/channels/runtime-tool-catalog.js +165 -0
  40. package/lib/cli/index.js +6 -0
  41. package/lib/cli/setup-app.js +26 -5
  42. package/lib/cli/setup-dev.js +23 -7
  43. package/lib/cli/setup-external-system-help.js +8 -2
  44. package/lib/cli/setup-external-system.js +12 -0
  45. package/lib/cli/setup-utility-help-after.js +12 -0
  46. package/lib/cli/setup-utility-validate.js +41 -2
  47. package/lib/commands/app-test.js +2 -14
  48. package/lib/commands/datasource-load-export-cli.js +6 -1
  49. package/lib/commands/deploy-gate-failure-details.js +23 -0
  50. package/lib/commands/deploy-gate-orchestrator.js +64 -18
  51. package/lib/commands/dev-cli-handlers.js +6 -3
  52. package/lib/commands/dev-down.js +2 -6
  53. package/lib/commands/dev-init.js +7 -109
  54. package/lib/commands/dev-refresh.js +197 -0
  55. package/lib/commands/evidence-kind.js +478 -0
  56. package/lib/commands/governance-command-action.js +4 -2
  57. package/lib/commands/governance-generate-external.js +56 -21
  58. package/lib/commands/lifecycle-external.js +10 -4
  59. package/lib/commands/repair-datasource-apply.js +10 -1
  60. package/lib/commands/repair-datasource-run.js +4 -2
  61. package/lib/commands/repair-datasource-test-payload.js +80 -19
  62. package/lib/commands/repair-datasource-webhooks-sync-bridge.js +42 -3
  63. package/lib/commands/repair-internal.js +9 -1
  64. package/lib/commands/repair-openapi-export.js +1 -1
  65. package/lib/commands/repair-openapi-sync.js +43 -18
  66. package/lib/commands/resource-type.js +101 -18
  67. package/lib/commands/role-assistant.js +313 -0
  68. package/lib/commands/role.js +452 -0
  69. package/lib/commands/setup.js +1 -1
  70. package/lib/commands/upload-data-sync-finalize.js +5 -4
  71. package/lib/commands/upload-sidecar-followups.js +144 -47
  72. package/lib/commands/upload.js +43 -6
  73. package/lib/commands/validate-json.js +230 -0
  74. package/lib/commands/verify-governance-combined-external.js +73 -9
  75. package/lib/commands/verify-governance-db.js +80 -25
  76. package/lib/commands/verify-governance-external.js +29 -5
  77. package/lib/commands/verify-operations-step-plan.js +78 -40
  78. package/lib/commands/wizard-core-helpers.js +7 -3
  79. package/lib/commands/wizard-core.js +78 -9
  80. package/lib/core/audit-logger.js +1 -1
  81. package/lib/core/secrets-load.js +1 -3
  82. package/lib/datasource/bulk-loader-run-batches.js +4 -2
  83. package/lib/datasource/bulk-loader-service.js +5 -1
  84. package/lib/datasource/capability/relate-operations.js +1 -1
  85. package/lib/datasource/datasource-validate-summary.js +2 -2
  86. package/lib/datasource/validate.js +13 -6
  87. package/lib/external-system/deploy-helpers.js +35 -1
  88. package/lib/external-system/deploy.js +23 -10
  89. package/lib/external-system/download-helpers.js +111 -2
  90. package/lib/external-system/download-role-assistant.js +55 -0
  91. package/lib/external-system/download.js +45 -26
  92. package/lib/external-system/test-helpers.js +1 -1
  93. package/lib/external-system/test.js +39 -3
  94. package/lib/generator/builders.js +3 -0
  95. package/lib/generator/helpers.js +13 -5
  96. package/lib/generator/split-variables.js +9 -3
  97. package/lib/generator/wizard.js +51 -4
  98. package/lib/infrastructure/helpers-docker-check.js +6 -0
  99. package/lib/infrastructure/helpers.js +1 -1
  100. package/lib/lifecycle/access-control-abac.js +121 -0
  101. package/lib/lifecycle/lifecycle-governance-options.js +4 -1
  102. package/lib/lifecycle/product-model.js +3 -2
  103. package/lib/lifecycle/report-display.js +14 -2
  104. package/lib/programmatic/auth-config.js +3 -2
  105. package/lib/programmatic/bearer-auth.js +1 -1
  106. package/lib/programmatic/capability-registry.js +20 -1
  107. package/lib/programmatic/command-options.js +1 -0
  108. package/lib/programmatic/enterprise-mcp-client.js +18 -2
  109. package/lib/programmatic/index.js +4 -0
  110. package/lib/programmatic/manifest-patch-section.js +48 -9
  111. package/lib/programmatic/manifest-section-registry-data.js +3 -2
  112. package/lib/programmatic/manifest-section-registry.js +18 -1
  113. package/lib/programmatic/openapi-apply-result-schemas.js +44 -0
  114. package/lib/programmatic/openapi-component-schemas.js +16 -1
  115. package/lib/programmatic/openapi-description-enricher.js +3 -1
  116. package/lib/programmatic/openapi-descriptions.js +55 -5
  117. package/lib/programmatic/openapi-evidence-schemas.js +161 -0
  118. package/lib/programmatic/openapi-manifest-schemas.js +6 -2
  119. package/lib/programmatic/openapi-operation-bindings.js +99 -0
  120. package/lib/programmatic/openapi-request-schemas.js +36 -1
  121. package/lib/programmatic/openapi-result-schemas.js +29 -12
  122. package/lib/programmatic/openapi-schema-descriptions-certify.js +5 -1
  123. package/lib/programmatic/openapi-schema-descriptions-core.js +6 -3
  124. package/lib/programmatic/openapi-schema-descriptions-evidence.js +156 -0
  125. package/lib/programmatic/openapi-schema-descriptions-manifest.js +4 -3
  126. package/lib/programmatic/openapi-schema-descriptions-operations.js +70 -0
  127. package/lib/programmatic/openapi-schema-helpers.js +16 -1
  128. package/lib/programmatic/openapi-spec.js +48 -29
  129. package/lib/programmatic/orchestration-openapi-meta.js +5 -0
  130. package/lib/programmatic/request-timeout.js +3 -0
  131. package/lib/programmatic/route-handler-map.js +89 -2
  132. package/lib/programmatic/run-certify.js +15 -2
  133. package/lib/programmatic/run-deploy.js +74 -0
  134. package/lib/programmatic/run-evidence-authoring.js +80 -0
  135. package/lib/programmatic/run-protection.js +124 -0
  136. package/lib/programmatic/run-repository-emcp.js +11 -3
  137. package/lib/programmatic/run-template-workspace.js +42 -2
  138. package/lib/programmatic/workspace-context.js +6 -0
  139. package/lib/protection/auth-context.js +48 -6
  140. package/lib/protection/paths.js +22 -0
  141. package/lib/protection/run-protection-create-helpers.js +22 -1
  142. package/lib/protection/run-protection-create.js +1 -1
  143. package/lib/protection/sync-after-upload.js +1 -1
  144. package/lib/resolvers/evidence-kind-file.js +230 -0
  145. package/lib/resolvers/resource-type-file.js +36 -1
  146. package/lib/resolvers/role-file.js +227 -0
  147. package/lib/role-assistant/evidence-sync.js +450 -0
  148. package/lib/role-assistant/github-workflow.js +96 -0
  149. package/lib/role-assistant/knowledge-sync.js +465 -0
  150. package/lib/role-assistant/settings.js +126 -0
  151. package/lib/role-assistant/test-cases-evaluate.js +170 -0
  152. package/lib/role-assistant/test-cases-honesty.js +184 -0
  153. package/lib/role-assistant/test-cases.js +364 -0
  154. package/lib/role-assistant/test-runner-workhub-answers.js +203 -0
  155. package/lib/role-assistant/test-runner-workhub.js +385 -0
  156. package/lib/role-assistant/test-runner.js +500 -0
  157. package/lib/schema/application-schema.json +36 -2
  158. package/lib/schema/deployment-rules.yaml +1 -0
  159. package/lib/schema/external-datasource.schema.json +121 -4
  160. package/lib/schema/external-system.schema.json +20 -6
  161. package/lib/schema/infra.parameter.yaml +45 -1
  162. package/lib/schema/role-assistant-settings.schema.json +108 -0
  163. package/lib/schema/type/credential-field-registry.json +19 -3
  164. package/lib/utils/api.js +52 -21
  165. package/lib/utils/app-path-resolve.js +72 -24
  166. package/lib/utils/app-register-config.js +4 -3
  167. package/lib/utils/authentication-shape.js +3 -1
  168. package/lib/utils/cli-sync-options.js +59 -3
  169. package/lib/utils/cli-utils.js +11 -3
  170. package/lib/utils/deployment-validation.js +28 -3
  171. package/lib/utils/dev-ca-install.js +5 -6
  172. package/lib/utils/dev-config.js +2 -2
  173. package/lib/utils/dev-list-sort.js +145 -0
  174. package/lib/utils/dev-platform-tls.js +0 -63
  175. package/lib/utils/docker-build.js +46 -27
  176. package/lib/utils/docker-daemon-tls-ca.js +1 -1
  177. package/lib/utils/docker-exec.js +91 -6
  178. package/lib/utils/docker-run-ephemeral.js +55 -0
  179. package/lib/utils/enterprise-knowledge-paths.js +29 -12
  180. package/lib/utils/env-copy-keycloak.js +196 -0
  181. package/lib/utils/env-copy.js +35 -4
  182. package/lib/utils/error-formatters/error-parser.js +0 -3
  183. package/lib/utils/external-env-template.js +1 -1
  184. package/lib/utils/external-system-readiness-compact-blocks.js +28 -1
  185. package/lib/utils/external-system-readiness-compact-display.js +3 -0
  186. package/lib/utils/health-check-request.js +2 -3
  187. package/lib/utils/help-builder.js +4 -0
  188. package/lib/utils/image-name.js +22 -4
  189. package/lib/utils/logger.js +37 -4
  190. package/lib/utils/manifest-source-emit.js +1 -1
  191. package/lib/utils/paths.js +98 -26
  192. package/lib/utils/pipeline-upload-auth-retry.js +19 -3
  193. package/lib/utils/postgres-platform-bootstrap.js +24 -17
  194. package/lib/utils/postgres-wipe.js +5 -2
  195. package/lib/utils/remote-dev-auth.js +39 -4
  196. package/lib/utils/resolve-docker-image-ref.js +36 -6
  197. package/lib/utils/role-assistant-paths.js +98 -0
  198. package/lib/utils/schema-loader.js +41 -0
  199. package/lib/utils/schema-resolver.js +36 -4
  200. package/lib/utils/token-manager.js +47 -22
  201. package/lib/utils/upload-sync-options.js +17 -3
  202. package/lib/utils/url-declarative-resolve-load-doc.js +38 -1
  203. package/lib/validation/external-manifest-validator.js +31 -8
  204. package/lib/validation/openapi-contract-surface-validator.js +4 -0
  205. package/lib/validation/validate-external-file.js +8 -1
  206. package/lib/validation/validator.js +6 -5
  207. package/package.json +35 -12
  208. package/scripts/export-builder-api-evidence.js +3 -3
  209. package/scripts/lib/plan-actor.js +2 -2
  210. package/scripts/lib/plan-meta.js +1 -1
  211. package/scripts/lib/plan-parse.js +1 -1
  212. package/scripts/lib/plan-prevalidate-gates.js +12 -1
  213. package/scripts/lib/plan-run-report.js +13 -2
  214. package/scripts/pnpm/lib/resolve-aifabrix-bin.mjs +10 -10
  215. package/templates/applications/dataplane/application.yaml +47 -17
  216. package/templates/applications/dataplane/env.template +28 -9
  217. package/templates/applications/dataplane/rbac.yaml +78 -2
  218. package/templates/applications/miso-controller/application.yaml +10 -1
  219. package/templates/applications/miso-controller/env.template +13 -8
  220. package/templates/applications/miso-controller/rbac.yaml +72 -0
  221. package/templates/channels/chatgpt/install.template.md.hbs +42 -24
  222. package/templates/channels/chatgpt/integration/README.md +10 -7
  223. package/templates/channels/chatgpt/integration/application.json +2 -2
  224. package/templates/channels/chatgpt/integration/chatgpt-system.json +3 -3
  225. package/templates/channels/chatgpt/integration/env.template +1 -1
  226. package/templates/channels/chatgpt/manifest.template.json.hbs +36 -5
  227. package/templates/channels/microsoft-copilot/ai-plugin.template.json.hbs +28 -0
  228. package/templates/channels/microsoft-copilot/declarative-agent.template.json.hbs +22 -0
  229. package/templates/channels/microsoft-copilot/install.template.md.hbs +32 -22
  230. package/templates/channels/microsoft-copilot/integration/README.md +9 -7
  231. package/templates/channels/microsoft-copilot/integration/application.json +2 -2
  232. package/templates/channels/microsoft-copilot/integration/env.template +1 -1
  233. package/templates/channels/microsoft-copilot/integration/microsoft-copilot-system.json +7 -6
  234. package/templates/channels/microsoft-copilot/manifest.template.json.hbs +34 -6
  235. package/templates/channels/slack/integration/slack-system.json +60 -2
  236. package/templates/python/requirements.txt +1 -1
@@ -27,6 +27,58 @@ function attachUploadAuthToOptions(options, uploadCtx) {
27
27
  options.silentResolve = true;
28
28
  }
29
29
 
30
+ /**
31
+ * @param {string} key
32
+ * @param {Object|null} config
33
+ * @returns {Object}
34
+ */
35
+ function buildNotApplicableGovernanceRow(key, config) {
36
+ const {
37
+ parseAccessControlAbac
38
+ } = require('../lifecycle/access-control-abac');
39
+ const abac = parseAccessControlAbac(config);
40
+ return {
41
+ datasourceKey: key,
42
+ verdict: VERDICT.NOT_APPLICABLE,
43
+ scenarios: { passed: 0, total: 0 },
44
+ policyCoveragePercent: 100,
45
+ dimensionCoveragePercent: 100,
46
+ rationale: abac.rationale
47
+ };
48
+ }
49
+
50
+ /**
51
+ * @param {string} key
52
+ * @param {Object} ctx
53
+ * @returns {Promise<Object|null>}
54
+ */
55
+ async function runRequiredGovernanceDatasource(key, ctx) {
56
+ try {
57
+ const packRow = await getDatasourceGovernancePack(ctx.dataplaneUrl, ctx.authConfig, key);
58
+ if (!packRow || !packRow.pack) {
59
+ return null;
60
+ }
61
+ const result = await runDatasourceGovernanceScenarios(ctx.dataplaneUrl, ctx.authConfig, key);
62
+ const summary = result.summary || {};
63
+ return {
64
+ datasourceKey: key,
65
+ verdict: summary.failed > 0 ? VERDICT.FAILED : VERDICT.VERIFIED,
66
+ scenarios: { passed: summary.passed || 0, total: summary.total || 0 },
67
+ policyCoveragePercent: result.policyCoveragePercent ?? 100,
68
+ dimensionCoveragePercent: result.dimensionCoveragePercent ?? 100,
69
+ packDocument: packRow.pack,
70
+ result
71
+ };
72
+ } catch (err) {
73
+ return {
74
+ datasourceKey: key,
75
+ verdict: VERDICT.FAILED,
76
+ error: err.message,
77
+ scenarios: { passed: 0, total: 0 }
78
+ };
79
+ }
80
+ }
81
+
30
82
  /**
31
83
  * @async
32
84
  * @param {string[]} keys
@@ -35,35 +87,27 @@ function attachUploadAuthToOptions(options, uploadCtx) {
35
87
  * @returns {Promise<Array>}
36
88
  */
37
89
  async function runGovernanceForDatasourceKeys(keys, ctx, progress = {}) {
90
+ const {
91
+ filterAbacRequiredDatasourceKeys,
92
+ loadDatasourceConfigsByKey
93
+ } = require('../lifecycle/access-control-abac');
94
+ const configs = loadDatasourceConfigsByKey(ctx.systemKey || '');
95
+ const required = new Set(
96
+ filterAbacRequiredDatasourceKeys(ctx.systemKey || '', keys)
97
+ );
38
98
  const rows = [];
39
99
  for (let index = 0; index < keys.length; index += 1) {
40
100
  const key = keys[index];
41
101
  if (typeof progress.setLabel === 'function') {
42
102
  progress.setLabel(`Running governance scenarios: ${key} (${index + 1}/${keys.length})`);
43
103
  }
44
- try {
45
- const packRow = await getDatasourceGovernancePack(ctx.dataplaneUrl, ctx.authConfig, key);
46
- if (!packRow || !packRow.pack) {
47
- continue;
48
- }
49
- const result = await runDatasourceGovernanceScenarios(ctx.dataplaneUrl, ctx.authConfig, key);
50
- const summary = result.summary || {};
51
- rows.push({
52
- datasourceKey: key,
53
- verdict: summary.failed > 0 ? VERDICT.FAILED : VERDICT.VERIFIED,
54
- scenarios: { passed: summary.passed || 0, total: summary.total || 0 },
55
- policyCoveragePercent: result.policyCoveragePercent ?? 100,
56
- dimensionCoveragePercent: result.dimensionCoveragePercent ?? 100,
57
- packDocument: packRow.pack,
58
- result
59
- });
60
- } catch (err) {
61
- rows.push({
62
- datasourceKey: key,
63
- verdict: VERDICT.FAILED,
64
- error: err.message,
65
- scenarios: { passed: 0, total: 0 }
66
- });
104
+ if (!required.has(key)) {
105
+ rows.push(buildNotApplicableGovernanceRow(key, configs.get(key) || null));
106
+ continue;
107
+ }
108
+ const row = await runRequiredGovernanceDatasource(key, ctx);
109
+ if (row) {
110
+ rows.push(row);
67
111
  }
68
112
  }
69
113
  return rows;
@@ -76,7 +120,12 @@ async function runGovernanceForDatasourceKeys(keys, ctx, progress = {}) {
76
120
  * @returns {Promise<Object>}
77
121
  */
78
122
  async function runVerifyGovernanceFromDbPacks(systemKey, options) {
79
- const { systemKey: resolvedKey, keys } = resolveExternalIntegrationContext(systemKey);
123
+ const { systemKey: resolvedKey, keys: discoveredKeys } =
124
+ resolveExternalIntegrationContext(systemKey);
125
+ const keys =
126
+ Array.isArray(options.datasourceKeys) && options.datasourceKeys.length
127
+ ? options.datasourceKeys
128
+ : discoveredKeys;
80
129
  if (keys.length === 0) {
81
130
  return { resolvedKey, datasourceRows: [] };
82
131
  }
@@ -87,10 +136,16 @@ async function runVerifyGovernanceFromDbPacks(systemKey, options) {
87
136
  }
88
137
 
89
138
  const { authConfig, dataplaneUrl } = await resolveIntegrationAuth(resolvedKey, options);
139
+ options.authConfig = authConfig;
140
+ options.dataplaneUrl = dataplaneUrl;
90
141
  const datasourceRows = await runWithVerifyStepProgress(
91
142
  'Running governance scenarios',
92
143
  async(progress) =>
93
- runGovernanceForDatasourceKeys(keys, { authConfig, dataplaneUrl }, progress),
144
+ runGovernanceForDatasourceKeys(
145
+ keys,
146
+ { authConfig, dataplaneUrl, systemKey: resolvedKey },
147
+ progress
148
+ ),
94
149
  options
95
150
  );
96
151
  return { resolvedKey, datasourceRows };
@@ -8,7 +8,7 @@
8
8
 
9
9
  const { runTestGovernanceForExternalSystem } = require('./test-governance-external');
10
10
  const { runVerifyGovernanceFromDbPacks } = require('./verify-governance-db');
11
- const { VERDICT, verdictFromPassed } = require('../lifecycle/product-model');
11
+ const { VERDICT } = require('../lifecycle/product-model');
12
12
  const {
13
13
  sumEnforcementScenarios,
14
14
  aggregateCoverage,
@@ -20,14 +20,26 @@ const {
20
20
  * @returns {{ passed: number, total: number }}
21
21
  */
22
22
  function buildDbPackResult(resolvedKey, datasourceRows) {
23
- const allVerified = datasourceRows.every(r => r.verdict === VERDICT.VERIFIED);
23
+ const hasFailed = datasourceRows.some(r => r.verdict === VERDICT.FAILED);
24
+ const allNa =
25
+ datasourceRows.length > 0
26
+ && datasourceRows.every(r => r.verdict === VERDICT.NOT_APPLICABLE);
27
+ const allClosed = datasourceRows.every(
28
+ r => r.verdict === VERDICT.VERIFIED || r.verdict === VERDICT.NOT_APPLICABLE
29
+ );
30
+ let verdict = VERDICT.FAILED;
31
+ if (allNa) {
32
+ verdict = VERDICT.NOT_APPLICABLE;
33
+ } else if (allClosed && !hasFailed) {
34
+ verdict = VERDICT.VERIFIED;
35
+ }
24
36
  const coverage = aggregateCoverage(datasourceRows);
25
37
  return {
26
38
  systemKey: resolvedKey,
27
39
  command: 'verify-governance',
28
- verdict: verdictFromPassed(allVerified),
29
- policyCoveragePercent: coverage.policy ?? (allVerified ? 100 : 0),
30
- dimensionCoveragePercent: coverage.dimension ?? (allVerified ? 100 : 0),
40
+ verdict,
41
+ policyCoveragePercent: coverage.policy ?? (allClosed ? 100 : 0),
42
+ dimensionCoveragePercent: coverage.dimension ?? (allClosed ? 100 : 0),
31
43
  enforcementScenarios: sumEnforcementScenarios(datasourceRows),
32
44
  datasourceRows
33
45
  };
@@ -66,6 +78,18 @@ async function runVerifyGovernanceForExternalSystem(systemKey, options = {}) {
66
78
  };
67
79
  }
68
80
 
81
+ const naKeys = datasourceRows
82
+ .filter(row => row.verdict === VERDICT.NOT_APPLICABLE)
83
+ .map(row => row.datasourceKey)
84
+ .filter(Boolean);
85
+ if (naKeys.length > 0 && options.authConfig && options.dataplaneUrl) {
86
+ const { runSystemLifecycle } = require('../api/lifecycle.api');
87
+ await runSystemLifecycle(options.dataplaneUrl, options.authConfig, resolvedKey, {
88
+ steps: ['governance'],
89
+ datasourceKeys: naKeys
90
+ });
91
+ }
92
+
69
93
  return buildDbPackResult(resolvedKey, datasourceRows);
70
94
  }
71
95
 
@@ -8,58 +8,96 @@ const { validateExternalSystemComplete } = require('../validation/validate');
8
8
  const { testExternalSystem, testExternalSystemIntegration } = require('../external-system/test');
9
9
  const { runTestE2EForExternalSystem } = require('./test-e2e-external');
10
10
 
11
+ /**
12
+ * @param {string} systemKey
13
+ * @returns {Array}
14
+ */
15
+ function buildBeforeSyncSteps(systemKey) {
16
+ return [
17
+ {
18
+ key: 'validation',
19
+ run: () => validateExternalSystemComplete(systemKey, { type: 'external' }),
20
+ ok: (result) => result.valid === true
21
+ },
22
+ {
23
+ key: 'unit',
24
+ run: () => testExternalSystem(systemKey, { verbose: false }),
25
+ ok: (result) => result.valid === true
26
+ }
27
+ ];
28
+ }
29
+
11
30
  /**
12
31
  * @param {string} systemKey
13
32
  * @param {Object} options
14
- * @returns {{ beforeSync: Array, afterSync: Array, e2e: Object }}
33
+ * @returns {Array}
15
34
  */
16
- function buildOperationsVerifyStepPlan(systemKey, options) {
17
- return {
18
- beforeSync: [
19
- {
20
- key: 'validation',
21
- run: () => validateExternalSystemComplete(systemKey, { type: 'external' }),
22
- ok: (result) => result.valid === true
23
- },
24
- {
25
- key: 'unit',
26
- run: () => testExternalSystem(systemKey, { verbose: false }),
27
- ok: (result) => result.valid === true
28
- }
29
- ],
30
- afterSync: [
31
- {
32
- key: 'integration',
33
- run: () =>
34
- testExternalSystemIntegration(systemKey, {
35
- environment: options.env,
36
- noPublish: true,
37
- verbose: false,
38
- debug: options.debug === true
39
- }),
40
- ok: (result) => result.success === true
41
- }
42
- ],
43
- e2e: {
44
- key: 'e2e',
45
- run: (progress) =>
46
- runTestE2EForExternalSystem(systemKey, {
47
- env: options.env,
35
+ function buildAfterSyncSteps(systemKey, options) {
36
+ return [
37
+ {
38
+ key: 'integration',
39
+ run: () =>
40
+ testExternalSystemIntegration(systemKey, {
41
+ environment: options.env || options.environment,
48
42
  noPublish: true,
49
43
  verbose: false,
50
44
  debug: options.debug === true,
51
- async: options.async !== false,
45
+ // Token-forward (builder-api certify): same auth as E2E step — do not
46
+ // fall back to getDeploymentAuth inside the container.
52
47
  authConfig: options.authConfig,
53
48
  dataplaneUrl: options.dataplaneUrl,
54
- silentResolve: options.silentResolve === true,
55
- quietPollUi: true,
56
- onDatasourceProgress: progress?.setLabel
57
- ? ({ key, index, total }) =>
58
- progress.setLabel(`Running E2E: ${key} (${index + 1}/${total})`)
59
- : undefined
49
+ silentResolve: options.silentResolve === true
60
50
  }),
61
51
  ok: (result) => result.success === true
62
52
  }
53
+ ];
54
+ }
55
+
56
+ /**
57
+ * @param {string} systemKey
58
+ * @param {Object} options
59
+ * @returns {Object}
60
+ */
61
+ function buildE2eStep(systemKey, options) {
62
+ if (options.skipE2e === true || options.minimal === true) {
63
+ return {
64
+ key: 'e2e',
65
+ run: async() => ({ success: true, skipped: true, reason: 'minimal' }),
66
+ ok: () => true
67
+ };
68
+ }
69
+ return {
70
+ key: 'e2e',
71
+ run: (progress) =>
72
+ runTestE2EForExternalSystem(systemKey, {
73
+ env: options.env,
74
+ noPublish: true,
75
+ verbose: false,
76
+ debug: options.debug === true,
77
+ async: options.async !== false,
78
+ authConfig: options.authConfig,
79
+ dataplaneUrl: options.dataplaneUrl,
80
+ silentResolve: options.silentResolve === true,
81
+ quietPollUi: true,
82
+ onDatasourceProgress: progress?.setLabel
83
+ ? ({ key, index, total }) =>
84
+ progress.setLabel(`Running E2E: ${key} (${index + 1}/${total})`)
85
+ : undefined
86
+ }),
87
+ ok: (result) => result.success === true
88
+ };
89
+ }
90
+
91
+ /**
92
+ * @param {string} systemKey
93
+ * @param {Object} options
94
+ * @returns {{ beforeSync: Array, afterSync: Array, e2e: Object }}
95
+ */
96
+ function buildOperationsVerifyStepPlan(systemKey, options) {
97
+ return {
98
+ beforeSync: buildBeforeSyncSteps(systemKey),
99
+ afterSync: buildAfterSyncSteps(systemKey, options),
100
+ e2e: buildE2eStep(systemKey, options)
63
101
  };
64
102
  }
65
103
 
@@ -232,15 +232,19 @@ function buildPlatformConfigPayload({ credentialIdOrKey, datasourceKeys, configu
232
232
  }
233
233
 
234
234
  /**
235
- * Extract configuration from API response
235
+ * Extract configuration from API response.
236
+ * add-datasource generation returns datasourceConfig(s) only (systemConfig is null by design).
237
+ * Callers must load the existing workspace system when systemConfig is missing.
236
238
  * @param {Object} generateResponse - API response
237
239
  * @returns {Object} Extracted configuration
238
240
  */
239
241
  function extractConfigurationFromResponse(generateResponse) {
240
- const systemConfig = generateResponse.data?.systemConfig;
242
+ const systemConfig = generateResponse.data?.systemConfig || null;
241
243
  const datasourceConfigs = generateResponse.data?.datasourceConfigs ||
242
244
  (generateResponse.data?.datasourceConfig ? [generateResponse.data.datasourceConfig] : []);
243
- if (!systemConfig) throw new Error('System configuration not found');
245
+ if (!systemConfig && (!Array.isArray(datasourceConfigs) || datasourceConfigs.length === 0)) {
246
+ throw new Error('System configuration not found');
247
+ }
244
248
  return { systemConfig, datasourceConfigs, systemKey: generateResponse.data?.systemKey };
245
249
  }
246
250
 
@@ -312,6 +312,72 @@ async function callGenerateApi(dataplaneUrl, authConfig, options, prefs) {
312
312
  return await generateConfig(dataplaneUrl, authConfig, configPayload);
313
313
  }
314
314
 
315
+ /**
316
+ * Load existing system JSON/YAML from the integration workspace (add-datasource).
317
+ * @param {string} appName - Integration app / system key
318
+ * @returns {Object|null}
319
+ */
320
+ function loadExistingWorkspaceSystemConfig(appName) {
321
+ if (!appName || typeof appName !== 'string') {
322
+ return null;
323
+ }
324
+ const { getIntegrationPath } = require('../utils/paths');
325
+ const { loadConfigFile } = require('../utils/config-format');
326
+ const fsSync = require('fs');
327
+ const appPath = getIntegrationPath(appName);
328
+ for (const ext of ['.json', '.yaml', '.yml']) {
329
+ const candidate = path.join(appPath, `${appName}-system${ext}`);
330
+ if (fsSync.existsSync(candidate)) {
331
+ try {
332
+ return loadConfigFile(candidate);
333
+ } catch {
334
+ return null;
335
+ }
336
+ }
337
+ }
338
+ return null;
339
+ }
340
+
341
+ /**
342
+ * @param {Object|null|undefined} systemConfig
343
+ * @param {Object} options
344
+ * @returns {Object}
345
+ */
346
+ function resolveSystemConfigForWizardMode(systemConfig, options) {
347
+ if (systemConfig) {
348
+ return systemConfig;
349
+ }
350
+ if (options.mode === 'add-datasource') {
351
+ const existing = loadExistingWorkspaceSystemConfig(
352
+ options.appName || options.systemIdOrKey
353
+ );
354
+ if (!existing) {
355
+ throw new Error(
356
+ 'System configuration not found for add-datasource. ' +
357
+ 'Run create-system wizard first so the workspace has a system file.'
358
+ );
359
+ }
360
+ return existing;
361
+ }
362
+ throw new Error('System configuration not found');
363
+ }
364
+
365
+ /**
366
+ * @param {Object} generateResponse
367
+ * @param {Object} prefs
368
+ * @param {Object} options
369
+ * @returns {Promise<void>}
370
+ */
371
+ async function maybeWriteWizardDebugLog(generateResponse, prefs, options) {
372
+ if (!prefs.debug || !options.appName) {
373
+ return;
374
+ }
375
+ const debugLog = generateResponse.data?.debugLog;
376
+ if (debugLog && typeof debugLog === 'string') {
377
+ await writeDebugLog(options.appName, debugLog);
378
+ }
379
+ }
380
+
315
381
  async function handleConfigurationGeneration(dataplaneUrl, authConfig, options) {
316
382
  logger.log(chalk.blue('\n\uD83D\uDCCB Step 5: Generate Configuration'));
317
383
  const prefs = buildConfigPreferences(options.configPrefs);
@@ -320,18 +386,21 @@ async function handleConfigurationGeneration(dataplaneUrl, authConfig, options)
320
386
  const generateResponse = await callGenerateApi(dataplaneUrl, authConfig, options, prefs);
321
387
  spinner.stop();
322
388
  if (!generateResponse.success) {
323
- await saveDebugManifestOnErrorAndThrow(generateResponse, { enableDebug: prefs.debug, appName: options.appName });
389
+ await saveDebugManifestOnErrorAndThrow(generateResponse, {
390
+ enableDebug: prefs.debug,
391
+ appName: options.appName
392
+ });
324
393
  }
325
394
  const result = extractConfigurationFromResponse(generateResponse);
326
- const normalized = normalizeWizardConfigs(result.systemConfig, result.datasourceConfigs);
327
- if (prefs.debug && options.appName) {
328
- const debugLog = generateResponse.data?.debugLog;
329
- if (debugLog && typeof debugLog === 'string') {
330
- await writeDebugLog(options.appName, debugLog);
331
- }
332
- }
395
+ const systemConfig = resolveSystemConfigForWizardMode(result.systemConfig, options);
396
+ const normalized = normalizeWizardConfigs(systemConfig, result.datasourceConfigs);
397
+ await maybeWriteWizardDebugLog(generateResponse, prefs, options);
333
398
  logger.log(formatSuccessLine('Configuration generated successfully'));
334
- return { systemConfig: normalized.systemConfig, datasourceConfigs: normalized.datasourceConfigs, systemKey: result.systemKey };
399
+ return {
400
+ systemConfig: normalized.systemConfig,
401
+ datasourceConfigs: normalized.datasourceConfigs,
402
+ systemKey: result.systemKey || options.systemIdOrKey || options.appName
403
+ };
335
404
  } catch (error) {
336
405
  spinner.stop();
337
406
  throw error;
@@ -238,7 +238,7 @@ async function logApiCall({ url, options, statusCode, duration, success, errorIn
238
238
  const path = extractPathFromUrl(url);
239
239
 
240
240
  // Extract controller URL from full URL
241
- let controllerUrl = 'unknown';
241
+ let controllerUrl;
242
242
  try {
243
243
  const urlObj = new URL(url);
244
244
  controllerUrl = `${urlObj.protocol}//${urlObj.host}`;
@@ -133,9 +133,7 @@ function mergeUserWithConfigFile(userSecrets, resolvedConfigPath, keySources) {
133
133
  for (const key of Object.keys(configSecrets)) {
134
134
  if (!(key in merged) || merged[key] === undefined || merged[key] === null || merged[key] === '') {
135
135
  merged[key] = configSecrets[key];
136
- if (keySources) {
137
- keySources[key] = resolvedConfigPath;
138
- }
136
+ keySources[key] = resolvedConfigPath;
139
137
  }
140
138
  }
141
139
  return merged;
@@ -70,19 +70,21 @@ function recordBatchSuccess(state, index, response, batchRecords) {
70
70
  * @returns {Promise<{ batches: Object[], totals: Object }>}
71
71
  */
72
72
  async function uploadRecordBatches(params) {
73
- const { chunks, syncType, dataplaneUrl, datasourceKey, authConfig } = params;
73
+ const { chunks, syncType, dataplaneUrl, datasourceKey, authConfig, timeoutMs } = params;
74
74
  const state = {
75
75
  batches: [],
76
76
  totals: { insertedCount: 0, updatedCount: 0, failedCount: 0 },
77
77
  chunkCount: chunks.length
78
78
  };
79
+ const transportOpts =
80
+ timeoutMs !== undefined && timeoutMs !== null ? { timeoutMs } : {};
79
81
 
80
82
  for (let i = 0; i < chunks.length; i += 1) {
81
83
  const batchRecords = chunks[i];
82
84
  const body = { syncType, sync: false, records: batchRecords };
83
85
  try {
84
86
  const response = await withTransientRetry(() =>
85
- bulkSyncRecords(dataplaneUrl, datasourceKey, authConfig, body)
87
+ bulkSyncRecords(dataplaneUrl, datasourceKey, authConfig, body, transportOpts)
86
88
  );
87
89
  recordBatchSuccess(state, i, response, batchRecords);
88
90
  } catch (err) {
@@ -81,12 +81,16 @@ async function executeDatasourceLoadWork(params) {
81
81
  return { ...result, context: ctx };
82
82
  }
83
83
 
84
+ const timeoutMs = options.timeoutMs
85
+ ? parseInt(String(options.timeoutMs), 10)
86
+ : undefined;
84
87
  const { batches, totals } = await uploadRecordBatches({
85
88
  chunks: chunkRecords(records, batchSize),
86
89
  syncType,
87
90
  dataplaneUrl: ctx.dataplaneUrl,
88
91
  datasourceKey: ctx.datasourceKey,
89
- authConfig: ctx.authConfig
92
+ authConfig: ctx.authConfig,
93
+ timeoutMs: Number.isFinite(timeoutMs) && timeoutMs > 0 ? timeoutMs : undefined
90
94
  });
91
95
 
92
96
  return { ...result, batches, totals, context: ctx };
@@ -23,7 +23,7 @@ function _asObject(x) {
23
23
  function _uniqStrings(list) {
24
24
  const out = [];
25
25
  const seen = new Set();
26
- for (const x of list || []) {
26
+ for (const x of list) {
27
27
  const s = String(x || '').trim();
28
28
  if (!s || seen.has(s)) continue;
29
29
  seen.add(s);
@@ -167,10 +167,10 @@ function assembleSummary(parsed) {
167
167
  foreignKeys: fks,
168
168
  dimensionKeys,
169
169
  dimensions,
170
- exposedProfileNames: summarizeExposedProfileNames(parsed),
171
- openapiLine: oa.openapiLine,
172
170
  hasOpenapi: oa.hasOpenapi,
171
+ openapiLine: oa.openapiLine,
173
172
  capabilityKeys: oa.capabilityKeys,
173
+ exposedProfileNames: summarizeExposedProfileNames(parsed),
174
174
  testPayloadLine: summarizeTestPayloadLine(parsed),
175
175
  hasTestPayload: !!parsed.testPayload && typeof parsed.testPayload === 'object',
176
176
  syncLine: summarizeSyncLine(sync)
@@ -242,9 +242,7 @@ async function validateDatasourceFile(filePathOrKey) {
242
242
  };
243
243
  }
244
244
 
245
- const fieldRefErrors = validateFieldReferences(parsed);
246
- const abacErrors = validateAbac(parsed);
247
- const postSchemaErrors = [...fieldRefErrors, ...abacErrors];
245
+ const postSchemaErrors = collectPostSchemaErrors(parsed);
248
246
  if (postSchemaErrors.length > 0) {
249
247
  return {
250
248
  valid: false,
@@ -264,6 +262,17 @@ async function validateDatasourceFile(filePathOrKey) {
264
262
  };
265
263
  }
266
264
 
265
+ /**
266
+ * @param {object} parsed
267
+ * @returns {string[]}
268
+ */
269
+ function collectPostSchemaErrors(parsed) {
270
+ return [
271
+ ...validateFieldReferences(parsed),
272
+ ...validateAbac(parsed)
273
+ ];
274
+ }
275
+
267
276
  /**
268
277
  * Validates parsed datasource JSON (same rules as {@link validateDatasourceFile}, without reading a file).
269
278
  *
@@ -294,9 +303,7 @@ function validateDatasourceParsed(parsed) {
294
303
  };
295
304
  }
296
305
 
297
- const fieldRefErrors = validateFieldReferences(parsed);
298
- const abacErrors = validateAbac(parsed);
299
- const postSchemaErrors = [...fieldRefErrors, ...abacErrors];
306
+ const postSchemaErrors = collectPostSchemaErrors(parsed);
300
307
  if (postSchemaErrors.length > 0) {
301
308
  return {
302
309
  valid: false,
@@ -133,12 +133,46 @@ function extractSystemKey(systemFilePath) {
133
133
  return systemFileName.replace(/-deploy$/, '');
134
134
  }
135
135
 
136
+ /**
137
+ * Role Assistant (digitalWorker) deploy uses upload, not controller deploy.
138
+ * @async
139
+ * @param {string} appName - Application name
140
+ * @param {Object} options - Deploy options
141
+ * @param {boolean} compact - Quiet UI when true
142
+ * @param {Object} manifest - Controller manifest
143
+ * @returns {Promise<Object|null>} Upload result, or null when not a digital worker
144
+ */
145
+ async function uploadDigitalWorkerIfNeeded(appName, options, compact, manifest) {
146
+ const systemKind =
147
+ manifest && manifest.system && typeof manifest.system.systemKind === 'string'
148
+ ? manifest.system.systemKind.trim()
149
+ : '';
150
+ if (systemKind !== 'digitalWorker') {
151
+ return null;
152
+ }
153
+ if (!compact) {
154
+ const { formatSuccessParagraph } = require('../utils/cli-test-layout-chalk');
155
+ const logger = require('../utils/logger');
156
+ logger.log(
157
+ formatSuccessParagraph(
158
+ 'Role Assistant deployment uses upload pipeline (controller deploy is skipped).'
159
+ )
160
+ );
161
+ }
162
+ const { uploadExternalSystem } = require('../commands/upload');
163
+ return uploadExternalSystem(appName, {
164
+ ...options,
165
+ uploadStatus: 'published'
166
+ });
167
+ }
168
+
136
169
  module.exports = {
137
170
  loadVariablesYaml,
138
171
  validateSingleSystemFile,
139
172
  validateSystemFiles,
140
173
  validateSingleDatasourceFile,
141
174
  validateDatasourceFiles,
142
- extractSystemKey
175
+ extractSystemKey,
176
+ uploadDigitalWorkerIfNeeded
143
177
  };
144
178