@aifabrix/builder 2.54.1 → 2.55.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (210) hide show
  1. package/README.md +8 -0
  2. package/lib/agent-kit/apply-ops.js +82 -0
  3. package/lib/agent-kit/checksums.js +28 -0
  4. package/lib/agent-kit/classify.js +113 -0
  5. package/lib/agent-kit/claude-shim.js +61 -0
  6. package/lib/agent-kit/constants.js +55 -0
  7. package/lib/agent-kit/file-list.js +103 -0
  8. package/lib/agent-kit/host-approve.js +130 -0
  9. package/lib/agent-kit/hosting/github.js +63 -0
  10. package/lib/agent-kit/hosts.js +71 -0
  11. package/lib/agent-kit/init.js +319 -0
  12. package/lib/agent-kit/kit-root.js +17 -0
  13. package/lib/agent-kit/managed-section.js +79 -0
  14. package/lib/agent-kit/package-load.js +45 -0
  15. package/lib/agent-kit/plan-ops.js +69 -0
  16. package/lib/agent-kit/remote-prompts.js +239 -0
  17. package/lib/agent-kit/render-host-markdown.js +40 -0
  18. package/lib/agent-kit/repo-guard.js +47 -0
  19. package/lib/agent-kit/report.js +107 -0
  20. package/lib/agent-kit/run.js +82 -0
  21. package/lib/agent-kit/scm/git.js +122 -0
  22. package/lib/agent-kit/state-file.js +83 -0
  23. package/lib/agent-kit/status.js +50 -0
  24. package/lib/agent-kit/v1-detect.js +81 -0
  25. package/lib/agent-kit/validate-kit.js +87 -0
  26. package/lib/agent-kit/workspace-name.js +40 -0
  27. package/lib/agent-kit/workspace-skeleton.js +103 -0
  28. package/lib/api/role-assistant-test-pack.api.js +86 -0
  29. package/lib/api/work-search.api.js +31 -4
  30. package/lib/api/work.api.js +19 -12
  31. package/lib/app/list-external-systems.js +2 -1
  32. package/lib/app/show-display.js +33 -6
  33. package/lib/app/show.js +25 -2
  34. package/lib/channels/add-channel.js +1 -0
  35. package/lib/channels/approval-guides/claude.js +2 -2
  36. package/lib/channels/channel-add-failure.js +1 -0
  37. package/lib/channels/channel-add-remaining.js +15 -9
  38. package/lib/channels/channel-definition-status.js +115 -0
  39. package/lib/channels/channel-description.js +2 -1
  40. package/lib/channels/channel-install-envelope.js +4 -2
  41. package/lib/channels/channel-integration-generator.js +2 -0
  42. package/lib/channels/channel-runtime-resolver.js +14 -0
  43. package/lib/channels/channel-targets.js +31 -5
  44. package/lib/channels/claude-setup-values.js +4 -1
  45. package/lib/channels/slack-channel-add.js +59 -6
  46. package/lib/channels/slack-mcp-callback.js +51 -0
  47. package/lib/channels/slack-mcp-runtime.js +156 -0
  48. package/lib/channels/slack-provisioning/install-cli.js +14 -2
  49. package/lib/channels/slack-provisioning/with-curl-on-path.js +6 -1
  50. package/lib/cli/index.js +4 -0
  51. package/lib/cli/setup-certification-commands.js +2 -2
  52. package/lib/cli/setup-certification.help.js +3 -1
  53. package/lib/commands/agent-kit-help.js +79 -0
  54. package/lib/commands/agent-kit.js +346 -0
  55. package/lib/commands/channel-add-help.js +5 -5
  56. package/lib/commands/datasource-capability-relate-cli.js +5 -4
  57. package/lib/commands/repair-auth-apply.js +1 -1
  58. package/lib/commands/repair-datasource-key-skip.js +39 -0
  59. package/lib/commands/repair-datasource-keys.js +24 -12
  60. package/lib/commands/repair-rbac.js +16 -10
  61. package/lib/commands/role-assistant-copy-output.js +61 -0
  62. package/lib/commands/role-assistant-copy.js +155 -0
  63. package/lib/commands/role-assistant-test-output.js +8 -9
  64. package/lib/commands/role-assistant-test-result-rows.js +42 -0
  65. package/lib/commands/role-assistant.js +117 -48
  66. package/lib/commands/search.js +143 -0
  67. package/lib/commands/test-trust-external.js +7 -5
  68. package/lib/commands/upload-sidecar-followups.js +4 -1
  69. package/lib/commands/verify-operations-skip-e2e.js +131 -0
  70. package/lib/commands/verify-operations-step-plan.js +6 -33
  71. package/lib/commands/verify-operations-steps.js +3 -1
  72. package/lib/datasource/capability/relate-operations.js +28 -10
  73. package/lib/datasource/capability/run-capability-relate.js +48 -25
  74. package/lib/datasource/datasource-identifier.js +13 -5
  75. package/lib/deployment/deployer-poll-run.js +117 -35
  76. package/lib/external-system/deploy-dataplane-published.js +70 -0
  77. package/lib/external-system/deploy.js +12 -2
  78. package/lib/external-system/download-role-assistant.js +41 -15
  79. package/lib/external-system/download.js +7 -1
  80. package/lib/generator/wizard-datasource-key.js +54 -0
  81. package/lib/generator/wizard.js +55 -54
  82. package/lib/integration-definition/apply-datasource-keys.js +75 -0
  83. package/lib/integration-definition/apply-relationships.js +195 -0
  84. package/lib/integration-definition/apply-static-enterprise.js +229 -0
  85. package/lib/integration-definition/apply-system-file.js +71 -0
  86. package/lib/integration-definition/apply-test-endpoint.js +89 -0
  87. package/lib/integration-definition/apply-viewpoint-defaults.js +157 -0
  88. package/lib/integration-definition/apply.js +78 -59
  89. package/lib/integration-definition/enrich-openapi-operations.js +36 -10
  90. package/lib/integration-definition/enrich-openapi-pick.js +87 -0
  91. package/lib/lifecycle/role-assistant-governance.js +56 -21
  92. package/lib/programmatic/capability-registry.js +1 -0
  93. package/lib/programmatic/index.js +2 -0
  94. package/lib/programmatic/openapi-apply-result-schemas.js +8 -0
  95. package/lib/programmatic/openapi-channel-install-schemas.js +4 -1
  96. package/lib/programmatic/openapi-component-schemas.js +1 -0
  97. package/lib/programmatic/openapi-descriptions.js +9 -3
  98. package/lib/programmatic/openapi-envelope-examples.js +7 -0
  99. package/lib/programmatic/openapi-operation-bindings.js +5 -0
  100. package/lib/programmatic/openapi-request-schemas.js +5 -0
  101. package/lib/programmatic/openapi-schema-descriptions-channel.js +8 -5
  102. package/lib/programmatic/openapi-schema-descriptions-operations.js +19 -0
  103. package/lib/programmatic/openapi-spec.js +1 -0
  104. package/lib/programmatic/request-timeout.js +3 -0
  105. package/lib/programmatic/route-handler-map.js +7 -0
  106. package/lib/programmatic/run-channel-catalog.js +50 -8
  107. package/lib/programmatic/run-channel-install-options.js +5 -1
  108. package/lib/programmatic/run-download.js +41 -0
  109. package/lib/protection/auth-context.js +3 -1
  110. package/lib/protection/protection-display.js +3 -2
  111. package/lib/protection/report-exit.js +50 -0
  112. package/lib/protection/run-commands.js +2 -0
  113. package/lib/role-assistant/package-validate.js +175 -0
  114. package/lib/role-assistant/test-case-assertion-row.js +36 -0
  115. package/lib/role-assistant/test-case-copy-bind.js +77 -0
  116. package/lib/role-assistant/test-case-copy-expected.js +127 -0
  117. package/lib/role-assistant/test-case-copy-from-work.js +90 -0
  118. package/lib/role-assistant/test-case-copy-interactions.js +67 -0
  119. package/lib/role-assistant/test-case-copy.js +119 -0
  120. package/lib/role-assistant/test-case-expected-catalog.js +135 -0
  121. package/lib/role-assistant/test-case-validate.js +88 -0
  122. package/lib/role-assistant/test-cases-bid.js +117 -4
  123. package/lib/role-assistant/test-cases-evaluate.js +137 -30
  124. package/lib/role-assistant/test-cases-honesty.js +155 -6
  125. package/lib/role-assistant/test-cases-lifecycle.js +291 -0
  126. package/lib/role-assistant/test-cases-search-graph.js +122 -0
  127. package/lib/role-assistant/test-cases-search.js +20 -2
  128. package/lib/role-assistant/test-cases.js +34 -168
  129. package/lib/role-assistant/test-pack-map.js +180 -0
  130. package/lib/role-assistant/test-pack-sync.js +187 -0
  131. package/lib/role-assistant/test-runner-debug.js +54 -0
  132. package/lib/role-assistant/test-runner-respond-body.js +36 -0
  133. package/lib/role-assistant/test-runner-search.js +20 -5
  134. package/lib/role-assistant/test-runner-start-payload.js +63 -10
  135. package/lib/role-assistant/test-runner-workhub-answers.js +45 -16
  136. package/lib/role-assistant/test-runner-workhub.js +45 -21
  137. package/lib/role-assistant/test-runner.js +50 -10
  138. package/lib/schema/external-datasource.schema.json +36 -5
  139. package/lib/schema/external-system.schema.json +20 -2
  140. package/lib/schema/role-assistant-test-case.schema.json +408 -0
  141. package/lib/search/dataplane-context.js +23 -0
  142. package/lib/search/search-args.js +211 -0
  143. package/lib/search/search-display.js +238 -0
  144. package/lib/search/search-errors.js +50 -0
  145. package/lib/search/search-prompts.js +119 -0
  146. package/lib/search/search-roles.js +85 -0
  147. package/lib/search/search-run.js +241 -0
  148. package/lib/search/search-skills.js +198 -0
  149. package/lib/utils/authentication-slot-show.js +153 -0
  150. package/lib/utils/credential-display.js +43 -1
  151. package/lib/utils/external-system-readiness-compact-blocks.js +5 -1
  152. package/lib/utils/external-system-readiness-compact-display.js +30 -1
  153. package/lib/utils/external-system-validators.js +2 -1
  154. package/lib/utils/help-builder.js +3 -1
  155. package/lib/utils/schema-loader.js +37 -0
  156. package/lib/validation/validate-external-file.js +5 -1
  157. package/lib/validation/validate.js +3 -2
  158. package/package.json +1 -1
  159. package/templates/agent-kit/adapters/claude/nested.CLAUDE.md +1 -0
  160. package/templates/agent-kit/adapters/claude/root.CLAUDE.md +5 -0
  161. package/templates/agent-kit/adapters/cursor/commands/cust-0-plan.md +5 -0
  162. package/templates/agent-kit/adapters/cursor/commands/cust-1-validate.md +5 -0
  163. package/templates/agent-kit/adapters/cursor/commands/cust-2-build.md +5 -0
  164. package/templates/agent-kit/adapters/cursor/commands/cust-3-push.md +5 -0
  165. package/templates/agent-kit/adapters/cursor/commands/cust-biz-1-validate.md +5 -0
  166. package/templates/agent-kit/adapters/cursor/commands/cust-biz-2-build.md +5 -0
  167. package/templates/agent-kit/adapters/cursor/commands/cust-biz-3-prove.md +5 -0
  168. package/templates/agent-kit/agent-kit.yaml +45 -0
  169. package/templates/agent-kit/instructions/AGENTKIT.md +83 -0
  170. package/templates/agent-kit/instructions/integration.AGENTS.md +25 -0
  171. package/templates/agent-kit/instructions/role-assistant.AGENTS.md +22 -0
  172. package/templates/agent-kit/instructions/root.AGENTS.md +37 -0
  173. package/templates/agent-kit/parity-inventory.yaml +188 -0
  174. package/templates/agent-kit/skills/aifabrix-connected-system/SKILL.md +58 -0
  175. package/templates/agent-kit/skills/aifabrix-connected-system/references/contract-checklist.md +40 -0
  176. package/templates/agent-kit/skills/aifabrix-connected-system/references/delivery-gates.md +54 -0
  177. package/templates/agent-kit/skills/aifabrix-connected-system/references/production-safety.md +35 -0
  178. package/templates/agent-kit/skills/aifabrix-connected-system/scripts/reserved-system-key.js +19 -0
  179. package/templates/agent-kit/skills/aifabrix-plan/SKILL.md +51 -0
  180. package/templates/agent-kit/skills/aifabrix-plan/references/challenge-checklist.md +108 -0
  181. package/templates/agent-kit/skills/aifabrix-plan/references/interaction.md +34 -0
  182. package/templates/agent-kit/skills/aifabrix-plan/references/plan-template.md +182 -0
  183. package/templates/agent-kit/skills/aifabrix-prove/SKILL.md +50 -0
  184. package/templates/agent-kit/skills/aifabrix-prove/references/demo-template.md +65 -0
  185. package/templates/agent-kit/skills/aifabrix-prove/references/evidence-lifecycle.md +68 -0
  186. package/templates/agent-kit/skills/aifabrix-prove/references/learning-template.md +66 -0
  187. package/templates/agent-kit/skills/aifabrix-role-assistant/SKILL.md +42 -0
  188. package/templates/agent-kit/skills/aifabrix-role-assistant/references/package-boundary.md +21 -0
  189. package/templates/agent-kit/skills/shared/hosts.md +78 -0
  190. package/templates/agent-kit/workspace/.cursor/plans/README.md +22 -0
  191. package/templates/agent-kit/workspace/LICENSE.mit +21 -0
  192. package/templates/agent-kit/workspace/README.md +89 -0
  193. package/templates/applications/builder-api/Dockerfile +3 -1
  194. package/templates/applications/dataplane/application.yaml +2 -2
  195. package/templates/applications/dataplane/env.template +1 -1
  196. package/templates/applications/dataplane/rbac.yaml +28 -0
  197. package/templates/applications/keycloak/env.template +3 -3
  198. package/templates/applications/miso-controller/application.yaml +1 -1
  199. package/templates/applications/miso-controller/env.template +8 -9
  200. package/templates/channels/claude/TROUBLESHOOTING.md +3 -0
  201. package/templates/channels/claude/channel.json +1 -1
  202. package/templates/channels/claude/install.template.md.hbs +1 -0
  203. package/templates/channels/claude/integration/claude-system.json +1 -1
  204. package/templates/channels/slack/channel.json +11 -0
  205. package/templates/channels/slack/install.template.md.hbs +9 -0
  206. package/templates/channels/slack/integration/slack-system.json +11 -0
  207. package/templates/channels/slack/manifest.template.json.hbs +6 -3
  208. package/templates/channels/teams-bot/channel.json +1 -0
  209. package/templates/python/docker-compose.hbs +14 -0
  210. package/templates/typescript/docker-compose.hbs +14 -0
@@ -5,6 +5,8 @@
5
5
 
6
6
  'use strict';
7
7
 
8
+ const { skipRow } = require('./test-case-assertion-row');
9
+
8
10
  function asArray(value) {
9
11
  return Array.isArray(value) ? value : [];
10
12
  }
@@ -71,10 +73,71 @@ function collectBidSchemaAssertions(expectedBid, observed, assertions, soft) {
71
73
  }
72
74
  }
73
75
 
74
- function assertDefinitionHash(expectedBid, observed, assertions, soft) {
75
- if (expectedBid.requireDefinitionHash !== true) {
76
+ function isDatasourceRecord(value) {
77
+ const key = String(value.key || value.datasourceKey || '').trim();
78
+ return Boolean(key && (Array.isArray(value.capabilities) || value.entityType || value.resourceType));
79
+ }
80
+
81
+ function nestedBidContainers(value) {
82
+ const nested = [];
83
+ if (Array.isArray(value.datasources)) {
84
+ nested.push(value.datasources);
85
+ }
86
+ if (value.spec && typeof value.spec === 'object') {
87
+ nested.push(value.spec);
88
+ }
89
+ if (value.emitted && typeof value.emitted === 'object') {
90
+ nested.push(value.emitted);
91
+ }
92
+ if (value.bid && typeof value.bid === 'object') {
93
+ nested.push(value.bid);
94
+ }
95
+ return nested;
96
+ }
97
+
98
+ function collectBidObjects(value, acc) {
99
+ if (Array.isArray(value)) {
100
+ value.forEach((item) => collectBidObjects(item, acc));
76
101
  return;
77
102
  }
103
+ if (!value || typeof value !== 'object') {
104
+ return;
105
+ }
106
+ nestedBidContainers(value).forEach((child) => collectBidObjects(child, acc));
107
+ if (isDatasourceRecord(value)) {
108
+ acc.push(value);
109
+ }
110
+ }
111
+
112
+ function bidBlob(observed) {
113
+ return observed.emitted || observed.emitRow || observed.applyRow || {};
114
+ }
115
+
116
+ function datasourceKeysFromBid(observed) {
117
+ const objects = [];
118
+ collectBidObjects(bidBlob(observed), objects);
119
+ return new Set(objects.map((row) => String(row.key || row.datasourceKey || '').trim()).filter(Boolean));
120
+ }
121
+
122
+ function datasourceHasCapability(observed, datasource, capKey) {
123
+ const objects = [];
124
+ collectBidObjects(bidBlob(observed), objects);
125
+ const match = objects.find((row) => String(row.key || row.datasourceKey || '') === datasource);
126
+ const caps = asArray(match && match.capabilities);
127
+ return caps.some((cap) => String((cap && cap.key) || cap || '') === capKey);
128
+ }
129
+
130
+ function pushBooleanBidFlag(expectedBid, field, observed, assertions, soft, assertFn) {
131
+ if (expectedBid[field] === false) {
132
+ assertions.push(skipRow(`expected.bid.${field}`, `${field} is false — check skipped.`));
133
+ return;
134
+ }
135
+ if (expectedBid[field] === true) {
136
+ assertFn(expectedBid, observed, assertions, soft);
137
+ }
138
+ }
139
+
140
+ function assertDefinitionHash(expectedBid, observed, assertions, soft) {
78
141
  const { emitRow, applyRow, emitted } = observed;
79
142
  const hash =
80
143
  (emitted && emitted.definitionHash) ||
@@ -112,11 +175,60 @@ function assertSuiteRunId(expectedBid, observed, assertions, soft) {
112
175
  }
113
176
 
114
177
  function collectBidProvenanceAssertions(expectedBid, observed, assertions, soft) {
115
- assertDefinitionHash(expectedBid, observed, assertions, soft);
116
- assertSuiteRunId(expectedBid, observed, assertions, soft);
178
+ pushBooleanBidFlag(expectedBid, 'requireDefinitionHash', observed, assertions, soft, assertDefinitionHash);
179
+ pushBooleanBidFlag(expectedBid, 'requireSuiteRunId', observed, assertions, soft, assertSuiteRunId);
180
+ }
181
+
182
+ function collectBidDatasourceAssertions(expectedBid, observed, assertions, soft) {
183
+ if (Array.isArray(expectedBid.minDatasources)) {
184
+ collectMinDatasources(expectedBid.minDatasources, observed, assertions, soft);
185
+ }
186
+ if (expectedBid.requireCapability && typeof expectedBid.requireCapability === 'object') {
187
+ collectRequireCapability(expectedBid.requireCapability, observed, assertions, soft);
188
+ }
189
+ }
190
+
191
+ function collectMinDatasources(requiredKeys, observed, assertions, soft) {
192
+ if (requiredKeys.length === 0) {
193
+ assertions.push(skipRow('expected.bid.minDatasources', 'minDatasources is empty — nothing to assert.'));
194
+ return;
195
+ }
196
+ const actual = datasourceKeysFromBid(observed);
197
+ for (const raw of requiredKeys) {
198
+ const key = String(raw || '').trim();
199
+ const passed = hasEmitOrApply(observed) && actual.has(key);
200
+ pushBidAssertion(
201
+ assertions,
202
+ soft,
203
+ `expected.bid.minDatasources:${key}`,
204
+ passed,
205
+ `BID lists datasource "${key}".`,
206
+ `BID does not list required datasource "${key}".`
207
+ );
208
+ }
209
+ }
210
+
211
+ function collectRequireCapability(spec, observed, assertions, soft) {
212
+ const datasource = String(spec.datasource || '').trim();
213
+ const capKey = String(spec.key || '').trim();
214
+ const passed = hasEmitOrApply(observed) && datasourceHasCapability(observed, datasource, capKey);
215
+ pushBidAssertion(
216
+ assertions,
217
+ soft,
218
+ 'expected.bid.requireCapability',
219
+ passed,
220
+ `BID datasource "${datasource}" lists capability "${capKey}".`,
221
+ `BID datasource "${datasource}" does not list capability "${capKey}".`
222
+ );
117
223
  }
118
224
 
119
225
  function collectBidPathAssertions(expectedBid, observed, assertions, soft) {
226
+ if (expectedBid.forbidGoldPaths === false) {
227
+ assertions.push(
228
+ skipRow('expected.bid.forbidGoldPaths', 'forbidGoldPaths is false — gold-path check skipped.')
229
+ );
230
+ return;
231
+ }
120
232
  if (expectedBid.forbidGoldPaths === true) {
121
233
  const { emitRow, applyRow, emitted } = observed;
122
234
  const blob = JSON.stringify(emitted || emitRow || applyRow || {});
@@ -146,6 +258,7 @@ function collectBidAssertions(expectedBid, observation, options = {}) {
146
258
  const soft = options.soft === true;
147
259
  collectBidSchemaAssertions(expectedBid, observed, assertions, soft);
148
260
  collectBidProvenanceAssertions(expectedBid, observed, assertions, soft);
261
+ collectBidDatasourceAssertions(expectedBid, observed, assertions, soft);
149
262
  collectBidPathAssertions(expectedBid, observed, assertions, soft);
150
263
  return assertions;
151
264
  }
@@ -18,12 +18,41 @@ const {
18
18
  evaluateSearchExpectations,
19
19
  evaluateSkillsExpectations
20
20
  } = require('./test-cases-search');
21
+ const { notesRow, withKind } = require('./test-case-assertion-row');
22
+
23
+ const UNSAFE_FACT_SEGMENTS = new Set(['__proto__', 'prototype', 'constructor']);
24
+ const WILDCARD_FACT_SEGMENTS = new Set(['*', '**']);
25
+
26
+ function splitDottedPath(dottedPath) {
27
+ return String(dottedPath || '').split('.').filter((part) => part.length > 0);
28
+ }
29
+
30
+ function factPathSafetyMessage(parts) {
31
+ for (const part of parts) {
32
+ if (UNSAFE_FACT_SEGMENTS.has(part)) {
33
+ return `Fact path rejects prototype segment "${part}" before lookup.`;
34
+ }
35
+ if (WILDCARD_FACT_SEGMENTS.has(part)) {
36
+ return `Fact path rejects wildcard segment "${part}".`;
37
+ }
38
+ if (part === 'latest') {
39
+ return 'Fact path rejects latest-global record lookup.';
40
+ }
41
+ }
42
+ return null;
43
+ }
21
44
 
22
45
  function getValueAtPath(obj, dottedPath) {
23
- const parts = String(dottedPath || '').split('.').filter(Boolean);
46
+ const parts = splitDottedPath(dottedPath);
47
+ if (factPathSafetyMessage(parts)) {
48
+ return undefined;
49
+ }
24
50
  let cursor = obj;
25
51
  for (const key of parts) {
26
- if (!cursor || typeof cursor !== 'object' || !(key in cursor)) {
52
+ if (cursor === null || cursor === undefined || typeof cursor !== 'object') {
53
+ return undefined;
54
+ }
55
+ if (!Object.prototype.hasOwnProperty.call(cursor, key)) {
27
56
  return undefined;
28
57
  }
29
58
  cursor = cursor[key];
@@ -31,6 +60,10 @@ function getValueAtPath(obj, dottedPath) {
31
60
  return cursor;
32
61
  }
33
62
 
63
+ function stringifyObserved(value) {
64
+ return value === undefined ? 'missing' : JSON.stringify(value);
65
+ }
66
+
34
67
  function evaluateLifecycleStateExpectation(expectedState, observedState) {
35
68
  if (!expectedState) {
36
69
  return null;
@@ -46,7 +79,7 @@ function evaluateLifecycleStateExpectation(expectedState, observedState) {
46
79
  };
47
80
  }
48
81
 
49
- function evaluateCapabilityExpectations(requiredList, observedCapabilities) {
82
+ function evaluateCapabilityExpectations(kind, requiredList, observedCapabilities) {
50
83
  if (!Array.isArray(requiredList) || requiredList.length === 0) {
51
84
  return [];
52
85
  }
@@ -55,29 +88,39 @@ function evaluateCapabilityExpectations(requiredList, observedCapabilities) {
55
88
  const key = String(capability || '').trim();
56
89
  const passed = observedSet.has(key);
57
90
  return {
58
- id: `expected.capabilities.required:${key}`,
91
+ id: `expected.capabilities.${kind}:${key}`,
59
92
  passed,
60
93
  message: passed
61
- ? `Required capability "${key}" observed.`
62
- : `Required capability "${key}" was not observed.`
94
+ ? `${kind === 'allowed' ? 'Allowed' : 'Required'} capability "${key}" observed.`
95
+ : `${kind === 'allowed' ? 'Allowed' : 'Required'} capability "${key}" was not observed.`
63
96
  };
64
97
  });
65
98
  }
66
99
 
100
+ function evaluateOneFact(factPath, expectedValue, observation) {
101
+ const parts = splitDottedPath(factPath);
102
+ const safety = factPathSafetyMessage(parts);
103
+ const id = `expected.facts:${factPath}`;
104
+ if (safety) {
105
+ return { id, passed: false, message: safety };
106
+ }
107
+ const actualValue = getValueAtPath(observation, factPath);
108
+ const passed = JSON.stringify(actualValue) === JSON.stringify(expectedValue);
109
+ return {
110
+ id,
111
+ passed,
112
+ message: passed
113
+ ? `Fact "${factPath}" matched expected value.`
114
+ : `Fact "${factPath}" expected ${JSON.stringify(expectedValue)} but got ${stringifyObserved(actualValue)}.`
115
+ };
116
+ }
117
+
67
118
  function evaluateFactExpectations(expectedFacts, observation) {
68
119
  if (!expectedFacts || typeof expectedFacts !== 'object' || Array.isArray(expectedFacts)) {
69
120
  return [];
70
121
  }
71
122
  return Object.entries(expectedFacts).map(([factPath, expectedValue]) => {
72
- const actualValue = getValueAtPath(observation, factPath);
73
- const passed = JSON.stringify(actualValue) === JSON.stringify(expectedValue);
74
- return {
75
- id: `expected.facts:${factPath}`,
76
- passed,
77
- message: passed
78
- ? `Fact "${factPath}" matched expected value.`
79
- : `Fact "${factPath}" expected ${JSON.stringify(expectedValue)} but got ${JSON.stringify(actualValue)}.`
80
- };
123
+ return evaluateOneFact(factPath, expectedValue, observation);
81
124
  });
82
125
  }
83
126
 
@@ -86,6 +129,11 @@ function evaluateProhibitedExpectations(prohibited, observation) {
86
129
  return [];
87
130
  }
88
131
  return prohibited.map((factPath) => {
132
+ const parts = splitDottedPath(factPath);
133
+ const safety = factPathSafetyMessage(parts);
134
+ if (safety) {
135
+ return { id: `expected.prohibited:${factPath}`, passed: false, message: safety };
136
+ }
89
137
  const actualValue = getValueAtPath(observation, factPath);
90
138
  const passed = actualValue === undefined || actualValue === null || actualValue === false;
91
139
  return {
@@ -93,7 +141,7 @@ function evaluateProhibitedExpectations(prohibited, observation) {
93
141
  passed,
94
142
  message: passed
95
143
  ? `Prohibited marker "${factPath}" was not observed.`
96
- : `Prohibited marker "${factPath}" observed: ${JSON.stringify(actualValue)}.`
144
+ : `Prohibited marker "${factPath}" observed: ${stringifyObserved(actualValue)}.`
97
145
  };
98
146
  });
99
147
  }
@@ -152,7 +200,60 @@ function softWrapFailedAssertion(assertion, soft) {
152
200
  return softWrapLifecycleAssertion(assertion, soft);
153
201
  }
154
202
 
155
- function collectCaseAssertions(expected, observation) {
203
+ function collectSoftModifierRows(expected) {
204
+ const rows = [];
205
+ if (Object.prototype.hasOwnProperty.call(expected, 'soft')) {
206
+ rows.push(
207
+ notesRow(
208
+ 'expected.soft',
209
+ expected.soft === true
210
+ ? 'soft: true — failures become warnings'
211
+ : 'soft: false — failures are blocking'
212
+ )
213
+ );
214
+ }
215
+ const notes = Array.isArray(expected.softNotes) ? expected.softNotes : [];
216
+ notes.forEach((item, index) => {
217
+ const text = typeof item === 'string'
218
+ ? item
219
+ : String((item && (item.description || item.note)) || JSON.stringify(item));
220
+ rows.push(notesRow(`expected.softNotes[${index}]`, text));
221
+ });
222
+ return rows;
223
+ }
224
+
225
+ function collectEvidenceContextRows(spec, prefix) {
226
+ if (!spec || typeof spec !== 'object' || Array.isArray(spec)) {
227
+ return [];
228
+ }
229
+ const rows = [];
230
+ if (spec.roleKey) {
231
+ rows.push(notesRow(`${prefix}.roleKey`, `Evidence roleKey "${spec.roleKey}" (preflight context).`));
232
+ }
233
+ if (spec.resourceType) {
234
+ rows.push(
235
+ notesRow(`${prefix}.resourceType`, `Evidence resourceType "${spec.resourceType}" (preflight context).`)
236
+ );
237
+ }
238
+ if (spec.requireStatus) {
239
+ rows.push(
240
+ notesRow(`${prefix}.requireStatus`, `Evidence requireStatus "${spec.requireStatus}" (preflight context).`)
241
+ );
242
+ }
243
+ return rows;
244
+ }
245
+
246
+ function collectCapabilityAssertions(capabilities, observedCapabilities, soft) {
247
+ const spec = capabilities && typeof capabilities === 'object' ? capabilities : {};
248
+ const rows = [
249
+ ...evaluateCapabilityExpectations('required', spec.required, observedCapabilities),
250
+ ...evaluateCapabilityExpectations('allowed', spec.allowed, observedCapabilities),
251
+ ...evaluateForbiddenCapabilities(spec.forbidden, observedCapabilities)
252
+ ];
253
+ return rows.map((row) => softWrapFailedAssertion(row, soft));
254
+ }
255
+
256
+ function collectCaseAssertions(expected, observation, preflight = {}) {
156
257
  const soft = expected.soft === true;
157
258
  const observedCapabilities = observation.capabilities?.observed || [];
158
259
  const assertions = [];
@@ -163,31 +264,35 @@ function collectCaseAssertions(expected, observation) {
163
264
  if (lifecycleAssertion) {
164
265
  assertions.push(lifecycleAssertion);
165
266
  }
267
+ assertions.push(...collectSoftModifierRows(expected));
268
+ assertions.push(...collectCapabilityAssertions(expected.capabilities, observedCapabilities, soft));
166
269
  assertions.push(
167
- ...evaluateCapabilityExpectations(
168
- expected?.capabilities?.required,
169
- observedCapabilities
170
- ).map((row) => softWrapFailedAssertion(row, soft))
270
+ ...evaluateFactExpectations(expected.facts, observation).map((row) =>
271
+ softWrapFailedAssertion(row, soft)
272
+ )
171
273
  );
172
274
  assertions.push(
173
- ...evaluateForbiddenCapabilities(
174
- expected?.capabilities?.forbidden,
175
- observedCapabilities
176
- ).map((row) => softWrapFailedAssertion(row, soft))
275
+ ...evaluateProhibitedExpectations(expected.prohibited, observation).map((row) =>
276
+ softWrapFailedAssertion(row, soft)
277
+ )
177
278
  );
178
- assertions.push(...evaluateFactExpectations(expected.facts, observation));
179
- assertions.push(...evaluateProhibitedExpectations(expected.prohibited, observation));
180
279
  assertions.push(...evaluateHonestyExpectations(expected.honesty, observation, { soft }));
181
280
  assertions.push(...collectBidAssertions(expected.bid, observation, { soft }));
182
281
  assertions.push(...evaluateEvidenceExpectations(expected.evidence, observation));
282
+ assertions.push(...collectEvidenceContextRows(expected.evidence, 'expected.evidence'));
283
+ assertions.push(...collectEvidenceContextRows(preflight.evidence, 'preflight.evidence'));
183
284
  assertions.push(...evaluateSearchExpectations(expected.search, observation));
184
285
  assertions.push(...evaluateSkillsExpectations(expected.skills, observation));
185
- return assertions;
286
+ return assertions.map(withKind);
186
287
  }
187
288
 
188
289
  function evaluateRoleAssistantTestCase(testCase, result, lifecycleVerdict, portalUrl) {
189
290
  const observation = buildCaseObservation(result, lifecycleVerdict, portalUrl);
190
- const assertions = collectCaseAssertions(testCase.expected || {}, observation);
291
+ const assertions = collectCaseAssertions(
292
+ testCase.expected || {},
293
+ observation,
294
+ testCase.preflight || {}
295
+ );
191
296
  const failed = assertions.filter((row) => !row.passed);
192
297
  return {
193
298
  caseId: testCase.caseId,
@@ -202,5 +307,7 @@ function evaluateRoleAssistantTestCase(testCase, result, lifecycleVerdict, porta
202
307
  }
203
308
 
204
309
  module.exports = {
205
- evaluateRoleAssistantTestCase
310
+ evaluateRoleAssistantTestCase,
311
+ evaluateFactExpectations,
312
+ getValueAtPath
206
313
  };
@@ -4,6 +4,8 @@
4
4
 
5
5
  'use strict';
6
6
 
7
+ const { skipRow, notesRow } = require('./test-case-assertion-row');
8
+
7
9
  const HONESTY_COMPLETION_STATES = new Set([
8
10
  'notEvaluated',
9
11
  'waiting',
@@ -26,6 +28,14 @@ const HONESTY_CONTRIB_STATUSES = new Set([
26
28
  'notRealized',
27
29
  'partial'
28
30
  ]);
31
+ const OBJECTIVE_TEXT_KEYS = new Set([
32
+ 'objective',
33
+ 'goal',
34
+ 'planObjective',
35
+ 'objectiveSummary',
36
+ 'acceptedObjective'
37
+ ]);
38
+ const FORBIDDEN_BUILDER_TOKEN = /\b(?:knownPlatform|gold-package|[a-z][a-z0-9-]*\.[a-z][a-zA-Z0-9]+)\b/i;
29
39
 
30
40
  function emptyHonestyObservation() {
31
41
  return {
@@ -83,7 +93,59 @@ function pushHonestyAssertion(assertions, id, passed, message, options = {}) {
83
93
  assertions.push({ id, passed, message });
84
94
  }
85
95
 
86
- function assertWorkResultPresent(assertions, honesty, observation) {
96
+ function collectObjectiveStrings(value, acc) {
97
+ if (Array.isArray(value)) {
98
+ value.forEach((item) => collectObjectiveStrings(item, acc));
99
+ return;
100
+ }
101
+ if (!value || typeof value !== 'object') {
102
+ return;
103
+ }
104
+ for (const [key, child] of Object.entries(value)) {
105
+ if (OBJECTIVE_TEXT_KEYS.has(key) && typeof child === 'string') {
106
+ acc.push(child);
107
+ } else if (child && typeof child === 'object') {
108
+ collectObjectiveStrings(child, acc);
109
+ }
110
+ }
111
+ }
112
+
113
+ function interactionObjectiveText(row) {
114
+ if (!row || typeof row !== 'object') {
115
+ return '';
116
+ }
117
+ const role = String(row.role || row.type || '').toLowerCase();
118
+ if (role === 'user' || role === 'human') {
119
+ return '';
120
+ }
121
+ return [row.message, row.content, row.text, row.objective]
122
+ .filter((item) => typeof item === 'string')
123
+ .join('\n');
124
+ }
125
+
126
+ function governedObjectiveHaystack(observation) {
127
+ const parts = [];
128
+ collectObjectiveStrings(observation.workResult || {}, parts);
129
+ collectObjectiveStrings(observation.emittedBid || {}, parts);
130
+ for (const row of Array.isArray(observation.interactions) ? observation.interactions : []) {
131
+ const text = interactionObjectiveText(row);
132
+ if (text) {
133
+ parts.push(text);
134
+ }
135
+ }
136
+ return parts.join('\n');
137
+ }
138
+
139
+ function assertWorkResultPresent(assertions, honestySpec, honesty, observation, soft) {
140
+ if (honestySpec.requireWorkResult === false) {
141
+ assertions.push(
142
+ skipRow(
143
+ 'expected.honesty.requireWorkResult',
144
+ 'requireWorkResult is false — work.result is not required.'
145
+ )
146
+ );
147
+ return;
148
+ }
87
149
  const present = honesty.present === true;
88
150
  const missingSuffix = observation.workResultError
89
151
  ? `: ${observation.workResultError}`
@@ -94,7 +156,8 @@ function assertWorkResultPresent(assertions, honesty, observation) {
94
156
  present,
95
157
  present
96
158
  ? 'work.result Output was present.'
97
- : `work.result Output missing${missingSuffix}`
159
+ : `work.result Output missing${missingSuffix}`,
160
+ { soft }
98
161
  );
99
162
  }
100
163
 
@@ -102,6 +165,15 @@ function assertSchemaVersion(assertions, honestySpec, honesty, soft) {
102
165
  if (!honestySpec.schemaVersion) {
103
166
  return;
104
167
  }
168
+ if (!honesty.present) {
169
+ assertions.push(
170
+ skipRow(
171
+ 'expected.honesty.schemaVersion',
172
+ 'schemaVersion skipped because work.result is absent.'
173
+ )
174
+ );
175
+ return;
176
+ }
105
177
  const expected = String(honestySpec.schemaVersion);
106
178
  const passed = String(honesty.schemaVersion || '') === expected;
107
179
  pushHonestyAssertion(
@@ -117,6 +189,12 @@ function assertSchemaVersion(assertions, honestySpec, honesty, soft) {
117
189
 
118
190
  function assertCompletion(assertions, honestySpec, honesty, soft) {
119
191
  if (honestySpec.requireCompletion === false) {
192
+ assertions.push(
193
+ skipRow(
194
+ 'expected.honesty.requireCompletion',
195
+ 'requireCompletion is false — completion check skipped.'
196
+ )
197
+ );
120
198
  return;
121
199
  }
122
200
  const stateOk = HONESTY_COMPLETION_STATES.has(String(honesty.completionState || ''));
@@ -134,6 +212,12 @@ function assertCompletion(assertions, honestySpec, honesty, soft) {
134
212
 
135
213
  function assertContribution(assertions, honestySpec, honesty, soft) {
136
214
  if (honestySpec.requireContributionObservation === false) {
215
+ assertions.push(
216
+ skipRow(
217
+ 'expected.honesty.requireContributionObservation',
218
+ 'requireContributionObservation is false — contribution check skipped.'
219
+ )
220
+ );
137
221
  return;
138
222
  }
139
223
  const ok = HONESTY_CONTRIB_STATUSES.has(String(honesty.contributionStatus || ''));
@@ -149,9 +233,28 @@ function assertContribution(assertions, honestySpec, honesty, soft) {
149
233
  }
150
234
 
151
235
  function assertMinResourceChanges(assertions, honestySpec, honesty, soft) {
236
+ if (typeof honestySpec.softResourceChanges === 'boolean') {
237
+ assertions.push(
238
+ notesRow(
239
+ 'expected.honesty.softResourceChanges',
240
+ honestySpec.softResourceChanges
241
+ ? 'softResourceChanges: true — minResourceChanges misses are warnings.'
242
+ : 'softResourceChanges: false — minResourceChanges misses are blocking.'
243
+ )
244
+ );
245
+ }
152
246
  if (typeof honestySpec.minResourceChanges !== 'number') {
153
247
  return;
154
248
  }
249
+ if (!honesty.present) {
250
+ assertions.push(
251
+ skipRow(
252
+ 'expected.honesty.minResourceChanges',
253
+ 'minResourceChanges skipped because work.result is absent.'
254
+ )
255
+ );
256
+ return;
257
+ }
155
258
  const count = Number(honesty.resourceChangeCount || 0);
156
259
  const passed = count >= honestySpec.minResourceChanges;
157
260
  const softChange = soft || honestySpec.softResourceChanges === true;
@@ -171,6 +274,53 @@ function assertMinResourceChanges(assertions, honestySpec, honesty, soft) {
171
274
  }
172
275
  }
173
276
 
277
+ function assertForbidBuilderKeys(assertions, honestySpec, observation, soft) {
278
+ if (honestySpec.forbidBuilderKeysInObjective === false) {
279
+ assertions.push(
280
+ skipRow(
281
+ 'expected.honesty.forbidBuilderKeysInObjective',
282
+ 'forbidBuilderKeysInObjective is false — Builder-key check skipped.'
283
+ )
284
+ );
285
+ return;
286
+ }
287
+ if (honestySpec.forbidBuilderKeysInObjective !== true) {
288
+ return;
289
+ }
290
+ const haystack = governedObjectiveHaystack(observation);
291
+ const match = haystack.match(FORBIDDEN_BUILDER_TOKEN);
292
+ const passed = !match;
293
+ pushHonestyAssertion(
294
+ assertions,
295
+ 'expected.honesty.forbidBuilderKeysInObjective',
296
+ passed,
297
+ passed
298
+ ? 'Objective text has no Builder capability keys or generation tokens.'
299
+ : `Objective text contains forbidden token "${match[0]}".`,
300
+ { soft }
301
+ );
302
+ }
303
+
304
+ function emitPresentFlagRows(assertions, honestySpec, honesty, observation, soft) {
305
+ if (honestySpec.requireCompletion === false || honestySpec.requireCompletion === true) {
306
+ assertCompletion(assertions, honestySpec, honesty, soft);
307
+ }
308
+ if (
309
+ honestySpec.requireContributionObservation === false ||
310
+ honestySpec.requireContributionObservation === true
311
+ ) {
312
+ assertContribution(assertions, honestySpec, honesty, soft);
313
+ }
314
+ if (
315
+ typeof honestySpec.minResourceChanges === 'number' ||
316
+ typeof honestySpec.softResourceChanges === 'boolean'
317
+ ) {
318
+ assertMinResourceChanges(assertions, honestySpec, honesty, soft);
319
+ }
320
+ assertSchemaVersion(assertions, honestySpec, honesty, soft);
321
+ assertForbidBuilderKeys(assertions, honestySpec, observation, soft);
322
+ }
323
+
174
324
  function evaluateHonestyExpectations(honestySpec, observation, options = {}) {
175
325
  if (!honestySpec || typeof honestySpec !== 'object' || Array.isArray(honestySpec)) {
176
326
  return [];
@@ -178,17 +328,16 @@ function evaluateHonestyExpectations(honestySpec, observation, options = {}) {
178
328
  const soft = options.soft === true;
179
329
  const honesty = observation.honesty || {};
180
330
  const assertions = [];
181
- if (honestySpec.requireWorkResult !== false) {
182
- assertWorkResultPresent(assertions, honesty, observation);
183
- }
331
+ assertWorkResultPresent(assertions, honestySpec, honesty, observation, soft);
184
332
  if (!honesty.present) {
185
- // Missing Output is always a hard failure when required — soft does not waive it.
333
+ emitPresentFlagRows(assertions, honestySpec, honesty, observation, soft);
186
334
  return assertions;
187
335
  }
188
336
  assertSchemaVersion(assertions, honestySpec, honesty, soft);
189
337
  assertCompletion(assertions, honestySpec, honesty, soft);
190
338
  assertContribution(assertions, honestySpec, honesty, soft);
191
339
  assertMinResourceChanges(assertions, honestySpec, honesty, soft);
340
+ assertForbidBuilderKeys(assertions, honestySpec, observation, soft);
192
341
  return assertions;
193
342
  }
194
343