@provartesting/provardx-cli 1.4.7 → 1.5.0-dev
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +264 -9
- package/lib/commands/provar/automation/config/validate.js.map +1 -1
- package/lib/commands/provar/automation/project/validate.d.ts +14 -0
- package/lib/commands/provar/automation/project/validate.js +69 -0
- package/lib/commands/provar/automation/project/validate.js.map +1 -0
- package/lib/commands/provar/mcp/start.d.ts +16 -0
- package/lib/commands/provar/mcp/start.js +62 -0
- package/lib/commands/provar/mcp/start.js.map +1 -0
- package/lib/commands/provar/quality-hub/connect.d.ts +5 -0
- package/lib/commands/provar/quality-hub/connect.js +12 -0
- package/lib/commands/provar/quality-hub/connect.js.map +1 -0
- package/lib/commands/provar/quality-hub/display.d.ts +5 -0
- package/lib/commands/provar/quality-hub/display.js +12 -0
- package/lib/commands/provar/quality-hub/display.js.map +1 -0
- package/lib/commands/provar/quality-hub/open.d.ts +5 -0
- package/lib/commands/provar/quality-hub/open.js +12 -0
- package/lib/commands/provar/quality-hub/open.js.map +1 -0
- package/lib/commands/provar/quality-hub/test/run/abort.d.ts +5 -0
- package/lib/commands/provar/quality-hub/test/run/abort.js +12 -0
- package/lib/commands/provar/quality-hub/test/run/abort.js.map +1 -0
- package/lib/commands/provar/quality-hub/test/run/report.d.ts +5 -0
- package/lib/commands/provar/quality-hub/test/run/report.js +12 -0
- package/lib/commands/provar/quality-hub/test/run/report.js.map +1 -0
- package/lib/commands/provar/quality-hub/test/run.d.ts +5 -0
- package/lib/commands/provar/quality-hub/test/run.js +12 -0
- package/lib/commands/provar/quality-hub/test/run.js.map +1 -0
- package/lib/commands/provar/quality-hub/testcase/retrieve.d.ts +5 -0
- package/lib/commands/provar/quality-hub/testcase/retrieve.js +12 -0
- package/lib/commands/provar/quality-hub/testcase/retrieve.js.map +1 -0
- package/lib/mcp/licensing/algasClient.d.ts +19 -0
- package/lib/mcp/licensing/algasClient.js +144 -0
- package/lib/mcp/licensing/algasClient.js.map +1 -0
- package/lib/mcp/licensing/ideDetection.d.ts +34 -0
- package/lib/mcp/licensing/ideDetection.js +179 -0
- package/lib/mcp/licensing/ideDetection.js.map +1 -0
- package/lib/mcp/licensing/index.d.ts +5 -0
- package/lib/mcp/licensing/index.js +10 -0
- package/lib/mcp/licensing/index.js.map +1 -0
- package/lib/mcp/licensing/licenseCache.d.ts +20 -0
- package/lib/mcp/licensing/licenseCache.js +79 -0
- package/lib/mcp/licensing/licenseCache.js.map +1 -0
- package/lib/mcp/licensing/licenseError.d.ts +4 -0
- package/lib/mcp/licensing/licenseError.js +15 -0
- package/lib/mcp/licensing/licenseError.js.map +1 -0
- package/lib/mcp/licensing/licenseValidator.d.ts +33 -0
- package/lib/mcp/licensing/licenseValidator.js +103 -0
- package/lib/mcp/licensing/licenseValidator.js.map +1 -0
- package/lib/mcp/logging/logger.d.ts +7 -0
- package/lib/mcp/logging/logger.js +22 -0
- package/lib/mcp/logging/logger.js.map +1 -0
- package/lib/mcp/rules/page_object_validation_rules.json +344 -0
- package/lib/mcp/rules/provar_best_practices_rules.json +3192 -0
- package/lib/mcp/schemas/common.d.ts +20 -0
- package/lib/mcp/schemas/common.js +16 -0
- package/lib/mcp/schemas/common.js.map +1 -0
- package/lib/mcp/security/pathPolicy.d.ts +14 -0
- package/lib/mcp/security/pathPolicy.js +38 -0
- package/lib/mcp/security/pathPolicy.js.map +1 -0
- package/lib/mcp/server.d.ts +5 -0
- package/lib/mcp/server.js +59 -0
- package/lib/mcp/server.js.map +1 -0
- package/lib/mcp/tools/antTools.d.ts +21 -0
- package/lib/mcp/tools/antTools.js +602 -0
- package/lib/mcp/tools/antTools.js.map +1 -0
- package/lib/mcp/tools/automationTools.d.ts +14 -0
- package/lib/mcp/tools/automationTools.js +386 -0
- package/lib/mcp/tools/automationTools.js.map +1 -0
- package/lib/mcp/tools/bestPracticesEngine.d.ts +30 -0
- package/lib/mcp/tools/bestPracticesEngine.js +632 -0
- package/lib/mcp/tools/bestPracticesEngine.js.map +1 -0
- package/lib/mcp/tools/defectTools.d.ts +15 -0
- package/lib/mcp/tools/defectTools.js +199 -0
- package/lib/mcp/tools/defectTools.js.map +1 -0
- package/lib/mcp/tools/hierarchyValidate.d.ts +139 -0
- package/lib/mcp/tools/hierarchyValidate.js +540 -0
- package/lib/mcp/tools/hierarchyValidate.js.map +1 -0
- package/lib/mcp/tools/pageObjectGenerate.d.ts +3 -0
- package/lib/mcp/tools/pageObjectGenerate.js +153 -0
- package/lib/mcp/tools/pageObjectGenerate.js.map +1 -0
- package/lib/mcp/tools/pageObjectValidate.d.ts +18 -0
- package/lib/mcp/tools/pageObjectValidate.js +420 -0
- package/lib/mcp/tools/pageObjectValidate.js.map +1 -0
- package/lib/mcp/tools/projectInspect.d.ts +3 -0
- package/lib/mcp/tools/projectInspect.js +694 -0
- package/lib/mcp/tools/projectInspect.js.map +1 -0
- package/lib/mcp/tools/projectValidateFromPath.d.ts +3 -0
- package/lib/mcp/tools/projectValidateFromPath.js +153 -0
- package/lib/mcp/tools/projectValidateFromPath.js.map +1 -0
- package/lib/mcp/tools/propertiesTools.d.ts +7 -0
- package/lib/mcp/tools/propertiesTools.js +314 -0
- package/lib/mcp/tools/propertiesTools.js.map +1 -0
- package/lib/mcp/tools/qualityHubTools.d.ts +8 -0
- package/lib/mcp/tools/qualityHubTools.js +178 -0
- package/lib/mcp/tools/qualityHubTools.js.map +1 -0
- package/lib/mcp/tools/rcaTools.d.ts +4 -0
- package/lib/mcp/tools/rcaTools.js +620 -0
- package/lib/mcp/tools/rcaTools.js.map +1 -0
- package/lib/mcp/tools/sfSpawn.d.ts +28 -0
- package/lib/mcp/tools/sfSpawn.js +50 -0
- package/lib/mcp/tools/sfSpawn.js.map +1 -0
- package/lib/mcp/tools/testCaseGenerate.d.ts +3 -0
- package/lib/mcp/tools/testCaseGenerate.js +221 -0
- package/lib/mcp/tools/testCaseGenerate.js.map +1 -0
- package/lib/mcp/tools/testCaseValidate.d.ts +20 -0
- package/lib/mcp/tools/testCaseValidate.js +227 -0
- package/lib/mcp/tools/testCaseValidate.js.map +1 -0
- package/lib/mcp/tools/testPlanTools.d.ts +6 -0
- package/lib/mcp/tools/testPlanTools.js +311 -0
- package/lib/mcp/tools/testPlanTools.js.map +1 -0
- package/lib/mcp/tools/testPlanValidate.d.ts +2 -0
- package/lib/mcp/tools/testPlanValidate.js +75 -0
- package/lib/mcp/tools/testPlanValidate.js.map +1 -0
- package/lib/mcp/tools/testSuiteValidate.d.ts +2 -0
- package/lib/mcp/tools/testSuiteValidate.js +63 -0
- package/lib/mcp/tools/testSuiteValidate.js.map +1 -0
- package/lib/services/projectValidation.d.ts +119 -0
- package/lib/services/projectValidation.js +678 -0
- package/lib/services/projectValidation.js.map +1 -0
- package/messages/sf.provar.automation.project.validate.md +52 -0
- package/messages/sf.provar.mcp.start.md +74 -0
- package/oclif.manifest.json +1298 -1
- package/package.json +29 -15
|
@@ -0,0 +1,3192 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schemaVersion": "1.0",
|
|
3
|
+
"name": "Provar Test Case Quality Best Practices",
|
|
4
|
+
"description": "Programmatic ruleset for validating and scoring Provar Automation test case quality based on the Best Practices project, README, and best-practices documentation.",
|
|
5
|
+
"artifactTypes": [
|
|
6
|
+
"Repository",
|
|
7
|
+
"Project",
|
|
8
|
+
"Folder",
|
|
9
|
+
"TestCase",
|
|
10
|
+
"Step",
|
|
11
|
+
"Parameter",
|
|
12
|
+
"PageObject",
|
|
13
|
+
"Field",
|
|
14
|
+
"DataSource"
|
|
15
|
+
],
|
|
16
|
+
"scoring": {
|
|
17
|
+
"defaultMaxScore": 100,
|
|
18
|
+
"ruleWeighting": "weightedSum",
|
|
19
|
+
"categories": {
|
|
20
|
+
"NamingConventions": 0.2,
|
|
21
|
+
"StructureAndGrouping": 0.2,
|
|
22
|
+
"DataDrivenTesting": 0.2,
|
|
23
|
+
"ReusabilityAndCallables": 0.15,
|
|
24
|
+
"ConnectionsAndEnvironments": 0.1,
|
|
25
|
+
"TestCaseDesign": 0.1,
|
|
26
|
+
"MaintenanceAndFolders": 0.05,
|
|
27
|
+
"BuildAndCI": 0.0
|
|
28
|
+
},
|
|
29
|
+
"notes": "BuildAndCI contributes to repository health but is excluded from per-testCase quality score."
|
|
30
|
+
},
|
|
31
|
+
"rules": [
|
|
32
|
+
{
|
|
33
|
+
"id": "SCHEMA-ROOT-001",
|
|
34
|
+
"category": "XMLSchema",
|
|
35
|
+
"name": "Test case root element must be testCase",
|
|
36
|
+
"description": "The root XML element must be <testCase>. Any other root element prevents the test from being imported into Provar.",
|
|
37
|
+
"appliesTo": [
|
|
38
|
+
"TestCase"
|
|
39
|
+
],
|
|
40
|
+
"severity": "critical",
|
|
41
|
+
"weight": 10,
|
|
42
|
+
"recommendation": "Ensure the root element is <testCase> with proper namespace and attributes.",
|
|
43
|
+
"check": {
|
|
44
|
+
"type": "disabled",
|
|
45
|
+
"comment": "Handled by validator handler's validate_schema_compliance() for early detection",
|
|
46
|
+
"target": "testCase"
|
|
47
|
+
},
|
|
48
|
+
"source": "Provar XML Schema: Test Case Structure"
|
|
49
|
+
},
|
|
50
|
+
{
|
|
51
|
+
"id": "SCHEMA-ID-001",
|
|
52
|
+
"category": "XMLSchema",
|
|
53
|
+
"name": "Test case must have valid identifier",
|
|
54
|
+
"description": "Test case must have one of: 'id' (recommended), 'guid' (Provar V3), or 'registryId' (legacy) attribute. Missing identifier prevents test case import.",
|
|
55
|
+
"appliesTo": [
|
|
56
|
+
"TestCase"
|
|
57
|
+
],
|
|
58
|
+
"severity": "critical",
|
|
59
|
+
"weight": 10,
|
|
60
|
+
"recommendation": "Add 'id' attribute to <testCase> element (e.g., id=\"1\"), or use 'guid' for Provar V3 compatibility.",
|
|
61
|
+
"check": {
|
|
62
|
+
"type": "disabled",
|
|
63
|
+
"comment": "Handled by validator handler's validate_schema_compliance() for early detection",
|
|
64
|
+
"target": "testCase"
|
|
65
|
+
},
|
|
66
|
+
"source": "Provar XML Schema: Test Case Identification"
|
|
67
|
+
},
|
|
68
|
+
{
|
|
69
|
+
"id": "STRUCT-ATTR-001",
|
|
70
|
+
"category": "XMLSchema",
|
|
71
|
+
"name": "Test case should have failureBehaviour attribute",
|
|
72
|
+
"description": "The 'failureBehaviour' attribute is recommended for new tests (default: 'Continue'). While not required, it makes test behavior explicit.",
|
|
73
|
+
"appliesTo": [
|
|
74
|
+
"TestCase"
|
|
75
|
+
],
|
|
76
|
+
"severity": "info",
|
|
77
|
+
"weight": 1,
|
|
78
|
+
"recommendation": "Add failureBehaviour=\"Continue\" to <testCase> element for explicit failure handling.",
|
|
79
|
+
"check": {
|
|
80
|
+
"type": "disabled",
|
|
81
|
+
"comment": "Handled by validator handler's validate_schema_compliance() for early detection",
|
|
82
|
+
"target": "testCase"
|
|
83
|
+
},
|
|
84
|
+
"source": "Provar XML Schema: Test Case Attributes"
|
|
85
|
+
},
|
|
86
|
+
{
|
|
87
|
+
"id": "SCHEMA-LEGACY-001",
|
|
88
|
+
"category": "XMLSchema",
|
|
89
|
+
"name": "Consider migrating from registryId to id or guid",
|
|
90
|
+
"description": "Using legacy 'registryId' attribute. Consider migrating to 'id' (simple) or 'guid' (Provar V3) for better compatibility.",
|
|
91
|
+
"appliesTo": [
|
|
92
|
+
"TestCase"
|
|
93
|
+
],
|
|
94
|
+
"severity": "info",
|
|
95
|
+
"weight": 1,
|
|
96
|
+
"recommendation": "Replace registryId with 'id' or 'guid' attribute for Provar V3 compatibility.",
|
|
97
|
+
"check": {
|
|
98
|
+
"type": "disabled",
|
|
99
|
+
"comment": "Handled by validator handler's validate_schema_compliance() for early detection",
|
|
100
|
+
"target": "testCase"
|
|
101
|
+
},
|
|
102
|
+
"source": "Provar XML Schema: Version Compatibility"
|
|
103
|
+
},
|
|
104
|
+
{
|
|
105
|
+
"id": "SCHEMA-VALUE-001",
|
|
106
|
+
"category": "XMLSchema",
|
|
107
|
+
"name": "Value elements must not use text attribute",
|
|
108
|
+
"description": "Content must be inside <value> element, not as text=\"...\" attribute. Using text attribute causes XML parsing errors.",
|
|
109
|
+
"appliesTo": [
|
|
110
|
+
"Step"
|
|
111
|
+
],
|
|
112
|
+
"severity": "critical",
|
|
113
|
+
"weight": 10,
|
|
114
|
+
"recommendation": "Change <value text=\"content\"/> to <value>content</value>",
|
|
115
|
+
"check": {
|
|
116
|
+
"type": "disabled",
|
|
117
|
+
"comment": "Handled by validator handler's validate_schema_compliance() for early detection",
|
|
118
|
+
"target": "testCase"
|
|
119
|
+
},
|
|
120
|
+
"source": "Provar XML Schema: Value Element Format"
|
|
121
|
+
},
|
|
122
|
+
{
|
|
123
|
+
"id": "SCHEMA-URI-001",
|
|
124
|
+
"category": "XMLSchema",
|
|
125
|
+
"name": "URI attributes must properly encode ampersands",
|
|
126
|
+
"description": "Ampersands in URI attributes must be encoded as & for valid XML. Unencoded & causes XML parsing errors.",
|
|
127
|
+
"appliesTo": [
|
|
128
|
+
"Step"
|
|
129
|
+
],
|
|
130
|
+
"severity": "critical",
|
|
131
|
+
"weight": 10,
|
|
132
|
+
"recommendation": "Replace & with & in all uri=\"...\" attributes (e.g., uri=\"sf:ui:binding:object?object=Lead&action=View\")",
|
|
133
|
+
"check": {
|
|
134
|
+
"type": "disabled",
|
|
135
|
+
"comment": "Handled by validator handler's validate_schema_compliance() for early detection",
|
|
136
|
+
"target": "testCase"
|
|
137
|
+
},
|
|
138
|
+
"source": "XML Specification: Entity Encoding"
|
|
139
|
+
},
|
|
140
|
+
{
|
|
141
|
+
"id": "SCHEMA-STEPS-001",
|
|
142
|
+
"category": "XMLSchema",
|
|
143
|
+
"name": "Test case must have steps element",
|
|
144
|
+
"description": "Test case must contain a <steps> element with at least one apiCall. Missing steps element prevents test case from loading.",
|
|
145
|
+
"appliesTo": [
|
|
146
|
+
"TestCase"
|
|
147
|
+
],
|
|
148
|
+
"severity": "critical",
|
|
149
|
+
"weight": 10,
|
|
150
|
+
"recommendation": "Add <steps> element containing apiCall steps to the test case.",
|
|
151
|
+
"check": {
|
|
152
|
+
"type": "disabled",
|
|
153
|
+
"comment": "Handled by validator handler's validate_schema_compliance() for early detection",
|
|
154
|
+
"target": "testCase"
|
|
155
|
+
},
|
|
156
|
+
"source": "Provar XML Schema: Test Case Structure"
|
|
157
|
+
},
|
|
158
|
+
{
|
|
159
|
+
"id": "SCHEMA-EMPTY-001",
|
|
160
|
+
"category": "XMLSchema",
|
|
161
|
+
"name": "Test case should not be empty",
|
|
162
|
+
"description": "Test case <steps> element should contain at least one <apiCall>. Empty test cases have no executable logic.",
|
|
163
|
+
"appliesTo": [
|
|
164
|
+
"TestCase"
|
|
165
|
+
],
|
|
166
|
+
"severity": "minor",
|
|
167
|
+
"weight": 2,
|
|
168
|
+
"recommendation": "Add apiCall steps to implement test logic.",
|
|
169
|
+
"check": {
|
|
170
|
+
"type": "disabled",
|
|
171
|
+
"comment": "Handled by validator handler's validate_schema_compliance() for early detection",
|
|
172
|
+
"target": "testCase"
|
|
173
|
+
},
|
|
174
|
+
"source": "Provar Test Design: Test Content"
|
|
175
|
+
},
|
|
176
|
+
{
|
|
177
|
+
"id": "API-UNKNOWN-001",
|
|
178
|
+
"category": "XMLSchema",
|
|
179
|
+
"name": "API identifier must be a valid Provar API",
|
|
180
|
+
"description": "The apiId attribute must reference a real Provar API. Unknown or hallucinated apiIds (like 'ApexDescribeObject') cause complete test step failures because Provar cannot find the API implementation.",
|
|
181
|
+
"appliesTo": [
|
|
182
|
+
"Step"
|
|
183
|
+
],
|
|
184
|
+
"severity": "critical",
|
|
185
|
+
"weight": 10,
|
|
186
|
+
"recommendation": "Use only valid Provar apiIds. Common corrections: ApexDescribeObject→ApexReadObject, ApexQueryObject→ApexSoqlQuery, ApexInsertObject→ApexCreateObject. Refer to provar_test_step_schema.json for the complete list of valid APIs.",
|
|
187
|
+
"check": {
|
|
188
|
+
"type": "unknownApiId"
|
|
189
|
+
},
|
|
190
|
+
"notes": "CRITICAL severity - unknown apiIds cause complete test step failures. This rule catches LLM hallucinations where non-existent APIs are generated.",
|
|
191
|
+
"source": "Provar API Schema: Valid API Identifiers"
|
|
192
|
+
},
|
|
193
|
+
{
|
|
194
|
+
"id": "NC-FOLDER-001",
|
|
195
|
+
"category": "NamingConventions",
|
|
196
|
+
"name": "Folder names are modular and title-cased",
|
|
197
|
+
"description": "Test folders must be concise, modular, alphabetic, and use Title Case with spaces between words.",
|
|
198
|
+
"appliesTo": [
|
|
199
|
+
"Folder",
|
|
200
|
+
"TestCase"
|
|
201
|
+
],
|
|
202
|
+
"severity": "major",
|
|
203
|
+
"weight": 5,
|
|
204
|
+
"recommendation": "Rename folder to be concise, alphabetic only, and in Title Case (e.g. 'Test Case Design').",
|
|
205
|
+
"check": {
|
|
206
|
+
"type": "regex",
|
|
207
|
+
"target": "folder.name",
|
|
208
|
+
"pattern": "^[A-Z][A-Za-z0-9]*( [A-Z][A-Za-z0-9]*)*$"
|
|
209
|
+
},
|
|
210
|
+
"source": "README: Naming Conventions - Test Folders"
|
|
211
|
+
},
|
|
212
|
+
{
|
|
213
|
+
"id": "NC-TESTCASE-001",
|
|
214
|
+
"category": "NamingConventions",
|
|
215
|
+
"name": "Test case names use consistent naming convention",
|
|
216
|
+
"description": "Test case names must follow a consistent naming pattern. NOTE: This rule only applies to test cases with an explicit 'name' attribute (typically callable test cases). The 'id' attribute can be any format.",
|
|
217
|
+
"appliesTo": [
|
|
218
|
+
"TestCase"
|
|
219
|
+
],
|
|
220
|
+
"severity": "minor",
|
|
221
|
+
"weight": 2,
|
|
222
|
+
"recommendation": "Use a consistent naming style for callable test case names. Examples: 'CreateAndConvertLead', 'createAndConvertLead', 'Create And Convert Lead'.",
|
|
223
|
+
"check": {
|
|
224
|
+
"type": "disabled",
|
|
225
|
+
"comment": "Disabled - testCase.id can be any format (numeric, etc). Only explicit 'name' attributes need validation.",
|
|
226
|
+
"target": "testCase.name",
|
|
227
|
+
"pattern": "^([A-Z][a-z]+(?:[A-Z][a-z]+)*|[a-z]+(?:[A-Z][a-z]+)*|[A-Z][a-z]+(?: [A-Z][a-z]+)*|[a-z]+(?: [a-z]+)*|[a-z]+(?:_[a-z]+)*)$"
|
|
228
|
+
},
|
|
229
|
+
"source": "README: Naming Conventions - Test Cases"
|
|
230
|
+
},
|
|
231
|
+
{
|
|
232
|
+
"id": "NC-PARAM-001",
|
|
233
|
+
"category": "NamingConventions",
|
|
234
|
+
"name": "Parameters and variables use camelCase",
|
|
235
|
+
"description": "Test parameters, variables, and return values must be camelCase and unique within their scope.",
|
|
236
|
+
"appliesTo": [
|
|
237
|
+
"Parameter"
|
|
238
|
+
],
|
|
239
|
+
"severity": "major",
|
|
240
|
+
"weight": 5,
|
|
241
|
+
"recommendation": "Rename parameters and variables to camelCase and ensure no duplicates in the same scope.",
|
|
242
|
+
"check": {
|
|
243
|
+
"type": "regex",
|
|
244
|
+
"target": "parameter.name",
|
|
245
|
+
"pattern": "^[a-z][A-Za-z0-9]*$"
|
|
246
|
+
},
|
|
247
|
+
"source": "README: Naming Conventions - Test Parameters, Variables, and Return Values"
|
|
248
|
+
},
|
|
249
|
+
{
|
|
250
|
+
"id": "NC-PO-001",
|
|
251
|
+
"category": "NamingConventions",
|
|
252
|
+
"name": "Page Objects use PascalCase",
|
|
253
|
+
"description": "Page Object names must follow PascalCase and represent a single logical page or screen.",
|
|
254
|
+
"appliesTo": [
|
|
255
|
+
"PageObject",
|
|
256
|
+
"TestCase"
|
|
257
|
+
],
|
|
258
|
+
"severity": "major",
|
|
259
|
+
"weight": 5,
|
|
260
|
+
"recommendation": "Rename Page Object to PascalCase (e.g. 'AccountDetailsPage').",
|
|
261
|
+
"check": {
|
|
262
|
+
"type": "regex",
|
|
263
|
+
"target": "pageObject.name",
|
|
264
|
+
"pattern": "^[A-Z][A-Za-z0-9]*$"
|
|
265
|
+
},
|
|
266
|
+
"source": "README: Naming Conventions - Page Objects"
|
|
267
|
+
},
|
|
268
|
+
{
|
|
269
|
+
"id": "NC-FIELD-001",
|
|
270
|
+
"category": "NamingConventions",
|
|
271
|
+
"name": "Field names use camelCase",
|
|
272
|
+
"description": "Field mapping names must be camelCase for consistency and readability.",
|
|
273
|
+
"appliesTo": [
|
|
274
|
+
"Field",
|
|
275
|
+
"TestCase"
|
|
276
|
+
],
|
|
277
|
+
"severity": "minor",
|
|
278
|
+
"weight": 2,
|
|
279
|
+
"recommendation": "Rename field mapping to camelCase (e.g. 'billingStreet').",
|
|
280
|
+
"check": {
|
|
281
|
+
"type": "regex",
|
|
282
|
+
"target": "field.name",
|
|
283
|
+
"pattern": "^[a-z][A-Za-z0-9]*$"
|
|
284
|
+
},
|
|
285
|
+
"source": "README: Naming Conventions - Field Names"
|
|
286
|
+
},
|
|
287
|
+
{
|
|
288
|
+
"id": "STRUCT-GROUP-001",
|
|
289
|
+
"category": "StructureAndGrouping",
|
|
290
|
+
"name": "All steps are inside Group steps or BDD structure",
|
|
291
|
+
"description": "Every test step must be contained within a Group Step, BDD design step (Given/When/Then/And/But), or Finally block to improve readability and maintenance.",
|
|
292
|
+
"appliesTo": [
|
|
293
|
+
"TestCase"
|
|
294
|
+
],
|
|
295
|
+
"severity": "minor",
|
|
296
|
+
"weight": 2,
|
|
297
|
+
"recommendation": "Wrap root-level steps into logically named Group Steps, BDD steps (Given/When/Then), or Finally blocks.",
|
|
298
|
+
"check": {
|
|
299
|
+
"type": "mustAllBeInGroups",
|
|
300
|
+
"target": "testCase.steps",
|
|
301
|
+
"acceptBddSteps": true,
|
|
302
|
+
"acceptFinallyBlocks": true
|
|
303
|
+
},
|
|
304
|
+
"source": "README: Naming Conventions - Group Steps"
|
|
305
|
+
},
|
|
306
|
+
{
|
|
307
|
+
"id": "STRUCT-SUMMARY-001",
|
|
308
|
+
"category": "StructureAndGrouping",
|
|
309
|
+
"name": "Test case has top-level summary",
|
|
310
|
+
"description": "Each test case must have a summary describing its purpose and usage.",
|
|
311
|
+
"appliesTo": [
|
|
312
|
+
"TestCase"
|
|
313
|
+
],
|
|
314
|
+
"severity": "info",
|
|
315
|
+
"weight": 1,
|
|
316
|
+
"recommendation": "Add a concise summary describing scenario, preconditions, and outcome.",
|
|
317
|
+
"check": {
|
|
318
|
+
"type": "mustExistAndNotEmpty",
|
|
319
|
+
"target": "testCase.summary"
|
|
320
|
+
},
|
|
321
|
+
"source": "README: Usage - Each test has its own summary"
|
|
322
|
+
},
|
|
323
|
+
{
|
|
324
|
+
"id": "STEP-NAMES-001",
|
|
325
|
+
"category": "StructureAndGrouping",
|
|
326
|
+
"name": "Custom step names for UI asserts and sets",
|
|
327
|
+
"description": "UI Assert, Set Values, and long SOQL query steps must be renamed to be descriptive.",
|
|
328
|
+
"appliesTo": [
|
|
329
|
+
"Step",
|
|
330
|
+
"TestCase"
|
|
331
|
+
],
|
|
332
|
+
"severity": "minor",
|
|
333
|
+
"weight": 2,
|
|
334
|
+
"recommendation": "Rename the step to describe what is being asserted or set (e.g. 'assertAccountStatusIsActive').",
|
|
335
|
+
"check": {
|
|
336
|
+
"type": "semanticPattern",
|
|
337
|
+
"target": "step",
|
|
338
|
+
"appliesWhen": [
|
|
339
|
+
"step.type in ['UIAssert','SetValues','SOQL']"
|
|
340
|
+
],
|
|
341
|
+
"mustHaveCustomName": true
|
|
342
|
+
},
|
|
343
|
+
"source": "README: Naming Conventions - Test Step Names"
|
|
344
|
+
},
|
|
345
|
+
{
|
|
346
|
+
"id": "DDT-EXCEL-001",
|
|
347
|
+
"category": "DataDrivenTesting",
|
|
348
|
+
"name": "Excel headers match field label or API name",
|
|
349
|
+
"description": "Excel/data source column headers must match either the field label or API name.",
|
|
350
|
+
"appliesTo": [
|
|
351
|
+
"DataSource"
|
|
352
|
+
],
|
|
353
|
+
"severity": "major",
|
|
354
|
+
"weight": 5,
|
|
355
|
+
"recommendation": "Rename Excel column headers to match Salesforce field labels or API names.",
|
|
356
|
+
"check": {
|
|
357
|
+
"type": "semantic",
|
|
358
|
+
"target": "dataSource.columns[]",
|
|
359
|
+
"requireMappingTo": "SalesforceField"
|
|
360
|
+
},
|
|
361
|
+
"source": "README: Naming Conventions - Excel/Data Source"
|
|
362
|
+
},
|
|
363
|
+
{
|
|
364
|
+
"id": "DDT-NO-FUNC-001",
|
|
365
|
+
"category": "DataDrivenTesting",
|
|
366
|
+
"name": "No Excel functions in data",
|
|
367
|
+
"description": "Excel test data must be static; functions are not allowed in cells.",
|
|
368
|
+
"appliesTo": [
|
|
369
|
+
"DataSource",
|
|
370
|
+
"TestCase"
|
|
371
|
+
],
|
|
372
|
+
"severity": "major",
|
|
373
|
+
"weight": 5,
|
|
374
|
+
"recommendation": "Replace Excel formulas with static values to keep tests deterministic.",
|
|
375
|
+
"check": {
|
|
376
|
+
"type": "mustNotMatchRegex",
|
|
377
|
+
"target": "dataSource.cells[]",
|
|
378
|
+
"pattern": "^="
|
|
379
|
+
},
|
|
380
|
+
"source": "README: Naming Conventions - Excel/Data Source"
|
|
381
|
+
},
|
|
382
|
+
{
|
|
383
|
+
"id": "VAR-NAMING-001",
|
|
384
|
+
"category": "DataDrivenTesting",
|
|
385
|
+
"name": "Variable names must use valid identifiers",
|
|
386
|
+
"description": "Variable names, field references, and result names must contain only letters, digits, and underscores. Spaces, hyphens, and special characters cause RUNTIME FAILURES in Provar. Examples: {Account.Name} ✅ | {Account.Account Name} ❌ (space will fail)",
|
|
387
|
+
"appliesTo": [
|
|
388
|
+
"Step"
|
|
389
|
+
],
|
|
390
|
+
"severity": "critical",
|
|
391
|
+
"weight": 8,
|
|
392
|
+
"recommendation": "Replace spaces and special characters with underscores. Use API field names (e.g., Account.Name) instead of field labels (e.g., Account.Account Name).",
|
|
393
|
+
"check": {
|
|
394
|
+
"type": "variableNaming",
|
|
395
|
+
"target": "step.variable",
|
|
396
|
+
"pattern": "^[A-Za-z_][A-Za-z0-9_]*$"
|
|
397
|
+
},
|
|
398
|
+
"source": "Provar Runtime Requirements - Variable identifier rules",
|
|
399
|
+
"notes": "CRITICAL severity - spaces/hyphens in variable names cause test step failures at runtime. Affects SetValues, variable references, result names (resultName, resultIdName, sfUiTargetResultName), and array notation."
|
|
400
|
+
},
|
|
401
|
+
{
|
|
402
|
+
"id": "DDT-VAR-001",
|
|
403
|
+
"category": "DataDrivenTesting",
|
|
404
|
+
"name": "No hardcoded values in steps",
|
|
405
|
+
"description": "Fields and values used more than once in a test must be parameterized or stored in variables.",
|
|
406
|
+
"appliesTo": [
|
|
407
|
+
"Step",
|
|
408
|
+
"TestCase"
|
|
409
|
+
],
|
|
410
|
+
"severity": "minor",
|
|
411
|
+
"weight": 3,
|
|
412
|
+
"recommendation": "Replace repeated literal values with variables or parameters defined in the test case.",
|
|
413
|
+
"check": {
|
|
414
|
+
"type": "detectDuplicatesLiterals",
|
|
415
|
+
"target": "step.value",
|
|
416
|
+
"minOccurrences": 2
|
|
417
|
+
},
|
|
418
|
+
"source": "Best Practices: Data-Driven Testing & Additional Best Practices",
|
|
419
|
+
"notes": "Severity downgraded from major to minor in Session 4 - style/cosmetic issue, not functional"
|
|
420
|
+
},
|
|
421
|
+
{
|
|
422
|
+
"id": "REUSE-CALL-001",
|
|
423
|
+
"category": "ReusabilityAndCallables",
|
|
424
|
+
"name": "Callable tests reside in Callables folder",
|
|
425
|
+
"description": "Callable tests must be stored under a designated Callables folder for discoverability.",
|
|
426
|
+
"appliesTo": [
|
|
427
|
+
"TestCase"
|
|
428
|
+
],
|
|
429
|
+
"severity": "minor",
|
|
430
|
+
"weight": 2,
|
|
431
|
+
"recommendation": "Move callable test into 'tests/Callables' (or equivalent) folder.",
|
|
432
|
+
"check": {
|
|
433
|
+
"type": "callableFolder",
|
|
434
|
+
"target": "testCase.path"
|
|
435
|
+
},
|
|
436
|
+
"source": "README: Test Case Reusability & Callables"
|
|
437
|
+
},
|
|
438
|
+
{
|
|
439
|
+
"id": "REUSE-CALL-002",
|
|
440
|
+
"category": "ReusabilityAndCallables",
|
|
441
|
+
"name": "Callable tests are parameterized",
|
|
442
|
+
"description": "Callable tests must define input parameters instead of using internal hardcoded values.",
|
|
443
|
+
"appliesTo": [
|
|
444
|
+
"TestCase"
|
|
445
|
+
],
|
|
446
|
+
"severity": "minor",
|
|
447
|
+
"weight": 2,
|
|
448
|
+
"recommendation": "Add input parameters to the callable test and reference them in steps instead of literals.",
|
|
449
|
+
"check": {
|
|
450
|
+
"type": "mustHaveParameters",
|
|
451
|
+
"target": "testCase",
|
|
452
|
+
"minParameters": 1
|
|
453
|
+
},
|
|
454
|
+
"source": "README: Callables & Best Practices text"
|
|
455
|
+
},
|
|
456
|
+
{
|
|
457
|
+
"id": "REUSE-CALL-003",
|
|
458
|
+
"category": "ReusabilityAndCallables",
|
|
459
|
+
"name": "Callable tests executable in isolation",
|
|
460
|
+
"description": "Callable tests must be self-contained and executable independently for debugging.",
|
|
461
|
+
"appliesTo": [
|
|
462
|
+
"TestCase"
|
|
463
|
+
],
|
|
464
|
+
"severity": "minor",
|
|
465
|
+
"weight": 2,
|
|
466
|
+
"recommendation": "Ensure callable tests create their own preconditions (records, connections) or accept them as parameters.",
|
|
467
|
+
"check": {
|
|
468
|
+
"type": "semantic",
|
|
469
|
+
"target": "testCase",
|
|
470
|
+
"mustNotDependOnExternalState": true
|
|
471
|
+
},
|
|
472
|
+
"source": "README: Callables & Best Practices text - Additional Best Practices"
|
|
473
|
+
},
|
|
474
|
+
{
|
|
475
|
+
"id": "ENV-CONN-001",
|
|
476
|
+
"category": "ConnectionsAndEnvironments",
|
|
477
|
+
"name": "Admin connection supports Login-As",
|
|
478
|
+
"description": "There must be an Admin connection configured to perform Login-As for other profiles.",
|
|
479
|
+
"appliesTo": [
|
|
480
|
+
"Project"
|
|
481
|
+
],
|
|
482
|
+
"severity": "major",
|
|
483
|
+
"weight": 5,
|
|
484
|
+
"recommendation": "Create an Admin connection with 'login as' permissions for target profiles.",
|
|
485
|
+
"check": {
|
|
486
|
+
"type": "mustContain",
|
|
487
|
+
"target": "project.connections",
|
|
488
|
+
"connectionRole": "AdminLoginAs"
|
|
489
|
+
},
|
|
490
|
+
"source": "Best Practices: Environment Overrides"
|
|
491
|
+
},
|
|
492
|
+
{
|
|
493
|
+
"id": "ENV-CONN-002",
|
|
494
|
+
"category": "ConnectionsAndEnvironments",
|
|
495
|
+
"name": "Connection names should not contain environment specifiers",
|
|
496
|
+
"description": "Connection names should be environment-agnostic. Avoid embedding UAT, QA, Sandbox, Prod, Production, or Scratch in connection names. Use environment overrides instead.",
|
|
497
|
+
"appliesTo": [
|
|
498
|
+
"TestCase"
|
|
499
|
+
],
|
|
500
|
+
"severity": "minor",
|
|
501
|
+
"weight": 2,
|
|
502
|
+
"recommendation": "Remove environment specifiers from connection names. Create environment-specific connections via Provar environment overrides.",
|
|
503
|
+
"check": {
|
|
504
|
+
"type": "environmentConnectionName",
|
|
505
|
+
"target": "testCase"
|
|
506
|
+
},
|
|
507
|
+
"source": "Best Practices: Environment Overrides"
|
|
508
|
+
},
|
|
509
|
+
{
|
|
510
|
+
"id": "DESIGN-APIUI-001",
|
|
511
|
+
"category": "TestCaseDesign",
|
|
512
|
+
"name": "Prefer API for setup where possible",
|
|
513
|
+
"description": "Test setup steps that only prepare data should use API/SOQL instead of UI flows when feasible.",
|
|
514
|
+
"appliesTo": [
|
|
515
|
+
"Step"
|
|
516
|
+
],
|
|
517
|
+
"severity": "minor",
|
|
518
|
+
"weight": 3,
|
|
519
|
+
"recommendation": "Replace UI-based record creation for setup with API or SOQL-based data creation where appropriate.",
|
|
520
|
+
"check": {
|
|
521
|
+
"type": "semantic",
|
|
522
|
+
"target": "step",
|
|
523
|
+
"detectSetupViaUI": true,
|
|
524
|
+
"suggestAPIAlternative": true
|
|
525
|
+
},
|
|
526
|
+
"source": "Best Practices: Test Case Design - API vs UI"
|
|
527
|
+
},
|
|
528
|
+
{
|
|
529
|
+
"id": "DESIGN-GROUP-001",
|
|
530
|
+
"category": "TestCaseDesign",
|
|
531
|
+
"name": "Use Group Steps or BDD structure for logical phases",
|
|
532
|
+
"description": "Long tests should be organized using Group Steps, BDD design steps (Given/When/Then/And/But), or Finally blocks that represent logical phases of the flow.",
|
|
533
|
+
"appliesTo": [
|
|
534
|
+
"TestCase"
|
|
535
|
+
],
|
|
536
|
+
"severity": "minor",
|
|
537
|
+
"weight": 2,
|
|
538
|
+
"recommendation": "Organize test with Group Steps (e.g. 'Setup', 'Execute Scenario', 'Validate Results'), BDD steps (Given/When/Then), or Finally blocks around related actions.",
|
|
539
|
+
"check": {
|
|
540
|
+
"type": "mustHaveMinGroups",
|
|
541
|
+
"target": "testCase.groups",
|
|
542
|
+
"minGroups": 2,
|
|
543
|
+
"acceptBddSteps": true,
|
|
544
|
+
"acceptFinallyBlocks": true
|
|
545
|
+
},
|
|
546
|
+
"source": "Best Practices: Test Case Design"
|
|
547
|
+
},
|
|
548
|
+
{
|
|
549
|
+
"id": "STEP-DISABLED-001",
|
|
550
|
+
"category": "TestCaseDesign",
|
|
551
|
+
"name": "Disabled test steps should be removed",
|
|
552
|
+
"description": "Test steps marked with <tags><string>disabled</string></tags> should be removed from the test case to maintain clean, maintainable code. Disabled steps create technical debt and confusion.",
|
|
553
|
+
"appliesTo": [
|
|
554
|
+
"Step"
|
|
555
|
+
],
|
|
556
|
+
"severity": "minor",
|
|
557
|
+
"weight": 2,
|
|
558
|
+
"recommendation": "Remove disabled test steps entirely. If the step is needed for future use, consider creating a separate test case or adding it when actually required.",
|
|
559
|
+
"check": {
|
|
560
|
+
"type": "disabledStep",
|
|
561
|
+
"target": "step"
|
|
562
|
+
},
|
|
563
|
+
"source": "Best Practices: Test Case Maintenance"
|
|
564
|
+
},
|
|
565
|
+
{
|
|
566
|
+
"id": "MAINT-FOLDER-001",
|
|
567
|
+
"category": "MaintenanceAndFolders",
|
|
568
|
+
"name": "Folder-level setup test per application segment",
|
|
569
|
+
"description": "Each folder representing an application segment must include a setup test case for connections.",
|
|
570
|
+
"appliesTo": [
|
|
571
|
+
"Folder"
|
|
572
|
+
],
|
|
573
|
+
"severity": "minor",
|
|
574
|
+
"weight": 3,
|
|
575
|
+
"recommendation": "Add a setup test case that establishes folder-wide connections (even if folder scope bug exists).",
|
|
576
|
+
"check": {
|
|
577
|
+
"type": "mustContainSetupTest",
|
|
578
|
+
"target": "folder.tests",
|
|
579
|
+
"setupNamePattern": "(?i)setup"
|
|
580
|
+
},
|
|
581
|
+
"source": "README: Test Case Maintenance"
|
|
582
|
+
},
|
|
583
|
+
{
|
|
584
|
+
"id": "MAINT-VERSION-001",
|
|
585
|
+
"category": "MaintenanceAndFolders",
|
|
586
|
+
"name": "Consistent Provar/OS/browser versions",
|
|
587
|
+
"description": "Execution environments and authors should align on Provar, OS, and browser versions.",
|
|
588
|
+
"appliesTo": [
|
|
589
|
+
"Project"
|
|
590
|
+
],
|
|
591
|
+
"severity": "info",
|
|
592
|
+
"weight": 1,
|
|
593
|
+
"recommendation": "Document and standardize supported Provar, OS, and browser versions in project configuration.",
|
|
594
|
+
"check": {
|
|
595
|
+
"type": "informational",
|
|
596
|
+
"target": "project.config"
|
|
597
|
+
},
|
|
598
|
+
"source": "Best Practices: Additional Best Practices"
|
|
599
|
+
},
|
|
600
|
+
{
|
|
601
|
+
"id": "BUILD-PLAN-001",
|
|
602
|
+
"category": "BuildAndCI",
|
|
603
|
+
"name": "Regression Test Plan exists for CI",
|
|
604
|
+
"description": "A Regression Test Plan must exist and be referenced by CI builds.",
|
|
605
|
+
"appliesTo": [
|
|
606
|
+
"Project"
|
|
607
|
+
],
|
|
608
|
+
"severity": "major",
|
|
609
|
+
"weight": 5,
|
|
610
|
+
"recommendation": "Ensure there is a 'Regression' plan and that the CI workflow references it.",
|
|
611
|
+
"check": {
|
|
612
|
+
"type": "mustContain",
|
|
613
|
+
"target": "project.plans",
|
|
614
|
+
"planName": "Regression"
|
|
615
|
+
},
|
|
616
|
+
"source": "README: Builds"
|
|
617
|
+
},
|
|
618
|
+
{
|
|
619
|
+
"id": "VALID-GUID-001",
|
|
620
|
+
"category": "TestCaseDesign",
|
|
621
|
+
"name": "Test case has valid identifier",
|
|
622
|
+
"description": "Test case must have a valid identifier: 'guid' (V3), 'id', or 'registryId' (legacy) attribute.",
|
|
623
|
+
"appliesTo": [
|
|
624
|
+
"TestCase"
|
|
625
|
+
],
|
|
626
|
+
"severity": "critical",
|
|
627
|
+
"weight": 8,
|
|
628
|
+
"recommendation": "Add a valid identifier to the <testCase> element: guid (preferred), id, or registryId attribute.",
|
|
629
|
+
"check": {
|
|
630
|
+
"type": "validIdentifier",
|
|
631
|
+
"target": "testCase"
|
|
632
|
+
},
|
|
633
|
+
"source": "INSTRUCTIONS.md: Top-Level Format & Structure"
|
|
634
|
+
},
|
|
635
|
+
{
|
|
636
|
+
"id": "VALID-STEPS-001",
|
|
637
|
+
"category": "TestCaseDesign",
|
|
638
|
+
"name": "Test case has steps element",
|
|
639
|
+
"description": "Test case must contain a <steps> element with at least one step.",
|
|
640
|
+
"appliesTo": [
|
|
641
|
+
"TestCase"
|
|
642
|
+
],
|
|
643
|
+
"severity": "critical",
|
|
644
|
+
"weight": 8,
|
|
645
|
+
"recommendation": "Add a <steps> element containing apiCall steps.",
|
|
646
|
+
"check": {
|
|
647
|
+
"type": "mustExist",
|
|
648
|
+
"target": "testCase.steps"
|
|
649
|
+
},
|
|
650
|
+
"source": "INSTRUCTIONS.md: Top-Level Format & Structure"
|
|
651
|
+
},
|
|
652
|
+
{
|
|
653
|
+
"id": "UI-CONNECT-ARGS-001",
|
|
654
|
+
"category": "ConnectionsAndEnvironments",
|
|
655
|
+
"name": "UiConnect has invalid arguments (ApexConnect arguments used)",
|
|
656
|
+
"description": "UiConnect steps have different arguments than ApexConnect. UiConnect does NOT support: autoCleanup, enableObjectIdLogging, quickUiLogin, closeAllPrimaryTabs, alreadyOpenBehaviour, lightningMode, uiApplicationName, cleanupConnectionName. Valid UiConnect arguments are: connectionName, connectionId, resultName, resultScope, reuseConnectionName, privateBrowsingMode, webBrowser.",
|
|
657
|
+
"appliesTo": [
|
|
658
|
+
"Step"
|
|
659
|
+
],
|
|
660
|
+
"severity": "critical",
|
|
661
|
+
"weight": 10,
|
|
662
|
+
"recommendation": "Remove invalid arguments from UiConnect. For Salesforce connections requiring autoCleanup, use ApexConnect instead of UiConnect. UiConnect is for non-Salesforce UI connections only.",
|
|
663
|
+
"check": {
|
|
664
|
+
"type": "uiConnectInvalidArgs",
|
|
665
|
+
"target": "step"
|
|
666
|
+
},
|
|
667
|
+
"source": "Provar API Reference: UiConnect vs ApexConnect"
|
|
668
|
+
},
|
|
669
|
+
{
|
|
670
|
+
"id": "APEX-AUTOCLEANUP-001",
|
|
671
|
+
"category": "ConnectionsAndEnvironments",
|
|
672
|
+
"name": "Prefer autoCleanup over manual ApexDeleteObject steps",
|
|
673
|
+
"description": "ApexConnect should use autoCleanup=true instead of manual ApexDeleteObject steps. autoCleanup is preferred because it requires fewer steps, is more reliable (cleanup happens even if test fails), and automatically handles deletion order. Only use manual deletes when records are created indirectly and IDs aren't captured. Note: With multiple ApexConnect steps, autoCleanup only deletes records created using that specific connection's resultName.",
|
|
674
|
+
"appliesTo": [
|
|
675
|
+
"Step"
|
|
676
|
+
],
|
|
677
|
+
"severity": "minor",
|
|
678
|
+
"weight": 2,
|
|
679
|
+
"recommendation": "Set autoCleanup=true in ApexConnect and remove manual ApexDeleteObject steps. autoCleanup automatically deletes records created via ApexCreateObject and UiWithScreen action=New (when sfUiTargetResultName captures the ID). With multiple connections, each autoCleanup only handles its own connection's records.",
|
|
680
|
+
"check": {
|
|
681
|
+
"type": "apexConnectAutoCleanup",
|
|
682
|
+
"target": "step"
|
|
683
|
+
},
|
|
684
|
+
"source": "INSTRUCTIONS.md: Salesforce Connect Steps"
|
|
685
|
+
},
|
|
686
|
+
{
|
|
687
|
+
"id": "UI-SCREEN-NAV-001",
|
|
688
|
+
"category": "TestCaseDesign",
|
|
689
|
+
"name": "First UiWithScreen must use navigate=Always or IfNeccessary",
|
|
690
|
+
"description": "The first UiWithScreen step must use navigate='Always' or 'IfNeccessary' to ensure proper navigation. Using 'Dont' will cause failures in CLI/debug mode. This rule is skipped for callable tests (visibility='Internal').",
|
|
691
|
+
"appliesTo": [
|
|
692
|
+
"Step"
|
|
693
|
+
],
|
|
694
|
+
"severity": "major",
|
|
695
|
+
"weight": 5,
|
|
696
|
+
"recommendation": "Set navigate='Always' (preferred) or 'IfNeccessary' for the first UiWithScreen step. Never use 'Dont' on the first UiWithScreen unless in a callable test.",
|
|
697
|
+
"check": {
|
|
698
|
+
"type": "uiScreenNavigation",
|
|
699
|
+
"target": "step"
|
|
700
|
+
},
|
|
701
|
+
"source": "Provar UI Navigation Best Practices - CLI/Pipeline Compatibility"
|
|
702
|
+
},
|
|
703
|
+
{
|
|
704
|
+
"id": "UI-SCREEN-NAV-002",
|
|
705
|
+
"category": "TestCaseDesign",
|
|
706
|
+
"name": "First UiWithScreen should prefer navigate=Always over IfNeccessary",
|
|
707
|
+
"description": "The first UiWithScreen step should use navigate='Always' rather than 'IfNeccessary' for more reliable navigation. While 'IfNeccessary' works, 'Always' provides more consistent behavior. This rule is skipped for callable tests (visibility='Internal').",
|
|
708
|
+
"appliesTo": [
|
|
709
|
+
"Step"
|
|
710
|
+
],
|
|
711
|
+
"severity": "minor",
|
|
712
|
+
"weight": 2,
|
|
713
|
+
"recommendation": "Change navigate='IfNeccessary' to navigate='Always' for the first UiWithScreen step for more reliable navigation.",
|
|
714
|
+
"check": {
|
|
715
|
+
"type": "uiScreenNavigationIfNeccessary",
|
|
716
|
+
"target": "step"
|
|
717
|
+
},
|
|
718
|
+
"source": "Provar UI Navigation Best Practices"
|
|
719
|
+
},
|
|
720
|
+
{
|
|
721
|
+
"id": "UI-SCREEN-OBJID-001",
|
|
722
|
+
"category": "TestCaseDesign",
|
|
723
|
+
"name": "UiWithScreen with navigate=Always for Edit/View must have sfUiTargetObjectId",
|
|
724
|
+
"description": "When UiWithScreen navigates to an Edit or View screen with navigate='Always', the sfUiTargetObjectId argument must be populated to specify which record to edit or view. Without this, Provar cannot determine which record to navigate to. This does not apply to New screens (which create new records) or List/Home screens.",
|
|
725
|
+
"appliesTo": [
|
|
726
|
+
"Step"
|
|
727
|
+
],
|
|
728
|
+
"severity": "major",
|
|
729
|
+
"weight": 5,
|
|
730
|
+
"recommendation": "Add sfUiTargetObjectId argument referencing a previously captured record ID variable (e.g., AccountId from a prior UiWithScreen with action=New or ApexCreateObject).",
|
|
731
|
+
"check": {
|
|
732
|
+
"type": "uiScreenObjectId",
|
|
733
|
+
"target": "step"
|
|
734
|
+
},
|
|
735
|
+
"source": "Provar UI Navigation Best Practices - Record Context"
|
|
736
|
+
},
|
|
737
|
+
{
|
|
738
|
+
"id": "APEX-RESULTNAME-001",
|
|
739
|
+
"category": "NamingConventions",
|
|
740
|
+
"name": "ApexConnect resultName is unique",
|
|
741
|
+
"description": "Each ApexConnect step must have a unique resultName for proper connection tracking.",
|
|
742
|
+
"appliesTo": [
|
|
743
|
+
"Step"
|
|
744
|
+
],
|
|
745
|
+
"severity": "minor",
|
|
746
|
+
"weight": 2,
|
|
747
|
+
"recommendation": "Use unique resultName values like 'ApexConnection', 'ApexConnection2', etc.",
|
|
748
|
+
"check": {
|
|
749
|
+
"type": "uniqueResultNames",
|
|
750
|
+
"target": "step"
|
|
751
|
+
},
|
|
752
|
+
"source": "INSTRUCTIONS.md: Salesforce Connect Steps",
|
|
753
|
+
"notes": "Severity downgraded from major to minor in Session 4 - naming convention, not functional issue"
|
|
754
|
+
},
|
|
755
|
+
{
|
|
756
|
+
"id": "STEP-ITEMID-001",
|
|
757
|
+
"category": "TestCaseDesign",
|
|
758
|
+
"name": "testItemId values are whole numbers",
|
|
759
|
+
"description": "All testItemId attributes must be whole numbers, not decimals.",
|
|
760
|
+
"appliesTo": [
|
|
761
|
+
"Step"
|
|
762
|
+
],
|
|
763
|
+
"severity": "critical",
|
|
764
|
+
"weight": 8,
|
|
765
|
+
"recommendation": "Use whole numbers for testItemId (1, 2, 3...) not decimals (1.0, 2.5).",
|
|
766
|
+
"check": {
|
|
767
|
+
"type": "wholeNumberTestItemId",
|
|
768
|
+
"target": "step"
|
|
769
|
+
},
|
|
770
|
+
"source": "INSTRUCTIONS.md: Top-Level Format & Structure"
|
|
771
|
+
},
|
|
772
|
+
{
|
|
773
|
+
"id": "VAR-USAGE-001",
|
|
774
|
+
"category": "DataDrivenTesting",
|
|
775
|
+
"name": "Variable references use correct syntax",
|
|
776
|
+
"description": "Variable references must use {VarName[1].Field} syntax, not {VarName.1.Field}.",
|
|
777
|
+
"appliesTo": [
|
|
778
|
+
"Step"
|
|
779
|
+
],
|
|
780
|
+
"severity": "minor",
|
|
781
|
+
"weight": 2,
|
|
782
|
+
"recommendation": "Use {ResultName[index].FieldName} where index starts at 1, not 0.",
|
|
783
|
+
"check": {
|
|
784
|
+
"type": "variableSyntax",
|
|
785
|
+
"target": "step"
|
|
786
|
+
},
|
|
787
|
+
"source": "INSTRUCTIONS.md: Apex CRUD Steps",
|
|
788
|
+
"notes": "Severity downgraded from major to minor in Session 4 - code cleanliness, similar to unused imports"
|
|
789
|
+
},
|
|
790
|
+
{
|
|
791
|
+
"id": "CUSTOM-FIELD-001",
|
|
792
|
+
"category": "NamingConventions",
|
|
793
|
+
"name": "Custom fields end with __c",
|
|
794
|
+
"description": "Custom field references must end with '__c' suffix.",
|
|
795
|
+
"appliesTo": [
|
|
796
|
+
"Step"
|
|
797
|
+
],
|
|
798
|
+
"severity": "major",
|
|
799
|
+
"weight": 5,
|
|
800
|
+
"recommendation": "Add '__c' suffix to custom field names (e.g. 'CustomField__c').",
|
|
801
|
+
"check": {
|
|
802
|
+
"type": "customFieldSuffix",
|
|
803
|
+
"target": "step"
|
|
804
|
+
},
|
|
805
|
+
"source": "INSTRUCTIONS.md: Final Reminders"
|
|
806
|
+
},
|
|
807
|
+
{
|
|
808
|
+
"id": "SOQL-SELECT-ID-001",
|
|
809
|
+
"category": "TestCaseDesign",
|
|
810
|
+
"name": "SOQL queries include Id and Name",
|
|
811
|
+
"description": "SOQL queries should retrieve Id and Name fields by default.",
|
|
812
|
+
"appliesTo": [
|
|
813
|
+
"Step"
|
|
814
|
+
],
|
|
815
|
+
"severity": "minor",
|
|
816
|
+
"weight": 2,
|
|
817
|
+
"recommendation": "Add 'Id, Name' to SELECT clause for object queries.",
|
|
818
|
+
"check": {
|
|
819
|
+
"type": "disabled",
|
|
820
|
+
"subType": "soqlSelectFields",
|
|
821
|
+
"target": "step"
|
|
822
|
+
},
|
|
823
|
+
"source": "INSTRUCTIONS.md: Apex CRUD Steps"
|
|
824
|
+
},
|
|
825
|
+
{
|
|
826
|
+
"id": "CONTROL-FOREACH-001",
|
|
827
|
+
"category": "TestCaseDesign",
|
|
828
|
+
"name": "ForEach loops have valid source collection",
|
|
829
|
+
"description": "ForEach steps must specify a valid source collection to iterate over.",
|
|
830
|
+
"appliesTo": [
|
|
831
|
+
"Step"
|
|
832
|
+
],
|
|
833
|
+
"severity": "major",
|
|
834
|
+
"weight": 4,
|
|
835
|
+
"recommendation": "Add 'list' argument to ForEach step referencing a valid collection variable.",
|
|
836
|
+
"check": {
|
|
837
|
+
"type": "mustContainArgument",
|
|
838
|
+
"apiId": "com.provar.plugins.bundled.apis.control.ForEach",
|
|
839
|
+
"argument": "list"
|
|
840
|
+
},
|
|
841
|
+
"source": "Provar Test Step Schema: Control Flow Steps"
|
|
842
|
+
},
|
|
843
|
+
{
|
|
844
|
+
"id": "CONTROL-IF-001",
|
|
845
|
+
"category": "TestCaseDesign",
|
|
846
|
+
"name": "If statements have conditions",
|
|
847
|
+
"description": "If steps must have a boolean condition expression.",
|
|
848
|
+
"appliesTo": [
|
|
849
|
+
"Step"
|
|
850
|
+
],
|
|
851
|
+
"severity": "critical",
|
|
852
|
+
"weight": 8,
|
|
853
|
+
"recommendation": "Add 'condition' argument to If step with valid boolean expression.",
|
|
854
|
+
"check": {
|
|
855
|
+
"type": "mustContainArgument",
|
|
856
|
+
"apiId": "com.provar.plugins.bundled.apis.If",
|
|
857
|
+
"argument": "condition"
|
|
858
|
+
},
|
|
859
|
+
"source": "Provar Test Step Schema: Control Flow Steps"
|
|
860
|
+
},
|
|
861
|
+
{
|
|
862
|
+
"id": "CONTROL-WHILE-001",
|
|
863
|
+
"category": "TestCaseDesign",
|
|
864
|
+
"name": "While loops have exit conditions",
|
|
865
|
+
"description": "While loops must have a condition to prevent infinite loops.",
|
|
866
|
+
"appliesTo": [
|
|
867
|
+
"Step"
|
|
868
|
+
],
|
|
869
|
+
"severity": "critical",
|
|
870
|
+
"weight": 8,
|
|
871
|
+
"recommendation": "Add 'condition' argument to While step that will eventually evaluate to false.",
|
|
872
|
+
"check": {
|
|
873
|
+
"type": "mustContainArgument",
|
|
874
|
+
"apiId": "com.provar.plugins.bundled.apis.control.DoWhile",
|
|
875
|
+
"argument": "condition"
|
|
876
|
+
},
|
|
877
|
+
"source": "Provar Test Step Schema: Control Flow Steps"
|
|
878
|
+
},
|
|
879
|
+
{
|
|
880
|
+
"id": "CONTROL-FOREACH-002",
|
|
881
|
+
"category": "TestCaseDesign",
|
|
882
|
+
"name": "ForEach loops have valueName to store current item",
|
|
883
|
+
"description": "ForEach steps must specify a valueName argument to store the current iteration value.",
|
|
884
|
+
"appliesTo": [
|
|
885
|
+
"Step"
|
|
886
|
+
],
|
|
887
|
+
"severity": "critical",
|
|
888
|
+
"weight": 8,
|
|
889
|
+
"recommendation": "Add 'valueName' argument to ForEach step to name the variable that will hold each item during iteration.",
|
|
890
|
+
"check": {
|
|
891
|
+
"type": "mustContainArgument",
|
|
892
|
+
"apiId": "com.provar.plugins.bundled.apis.control.ForEach",
|
|
893
|
+
"argument": "valueName"
|
|
894
|
+
},
|
|
895
|
+
"source": "Provar Test Step Schema: Control Flow Steps"
|
|
896
|
+
},
|
|
897
|
+
{
|
|
898
|
+
"id": "CONTROL-SWITCH-001",
|
|
899
|
+
"category": "TestCaseDesign",
|
|
900
|
+
"name": "Switch statements have value expression",
|
|
901
|
+
"description": "Switch steps must specify a value argument to evaluate against cases.",
|
|
902
|
+
"appliesTo": [
|
|
903
|
+
"Step"
|
|
904
|
+
],
|
|
905
|
+
"severity": "critical",
|
|
906
|
+
"weight": 8,
|
|
907
|
+
"recommendation": "Add 'value' argument to Switch step with the expression to evaluate.",
|
|
908
|
+
"check": {
|
|
909
|
+
"type": "mustContainArgument",
|
|
910
|
+
"apiId": "com.provar.plugins.bundled.apis.control.Switch",
|
|
911
|
+
"argument": "value"
|
|
912
|
+
},
|
|
913
|
+
"source": "Provar Test Step Schema: Control Flow Steps"
|
|
914
|
+
},
|
|
915
|
+
{
|
|
916
|
+
"id": "CONTROL-SLEEP-002",
|
|
917
|
+
"category": "TestCaseDesign",
|
|
918
|
+
"name": "Sleep steps have duration specified",
|
|
919
|
+
"description": "Sleep steps must specify sleepSecs argument with the number of seconds to wait.",
|
|
920
|
+
"appliesTo": [
|
|
921
|
+
"Step"
|
|
922
|
+
],
|
|
923
|
+
"severity": "critical",
|
|
924
|
+
"weight": 8,
|
|
925
|
+
"recommendation": "Add 'sleepSecs' argument to Sleep step with a positive decimal value.",
|
|
926
|
+
"check": {
|
|
927
|
+
"type": "mustContainArgument",
|
|
928
|
+
"apiId": "com.provar.plugins.bundled.apis.control.Sleep",
|
|
929
|
+
"argument": "sleepSecs"
|
|
930
|
+
},
|
|
931
|
+
"source": "Provar Test Step Schema: Control Flow Steps"
|
|
932
|
+
},
|
|
933
|
+
{
|
|
934
|
+
"id": "CONTROL-WAITFOR-001",
|
|
935
|
+
"category": "TestCaseDesign",
|
|
936
|
+
"name": "WaitFor steps have condition",
|
|
937
|
+
"description": "WaitFor steps must specify a condition argument to evaluate each iteration.",
|
|
938
|
+
"appliesTo": [
|
|
939
|
+
"Step"
|
|
940
|
+
],
|
|
941
|
+
"severity": "critical",
|
|
942
|
+
"weight": 8,
|
|
943
|
+
"recommendation": "Add 'condition' argument to WaitFor step with a boolean expression to wait for.",
|
|
944
|
+
"check": {
|
|
945
|
+
"type": "mustContainArgument",
|
|
946
|
+
"apiId": "com.provar.plugins.bundled.apis.control.WaitFor",
|
|
947
|
+
"argument": "condition"
|
|
948
|
+
},
|
|
949
|
+
"source": "Provar Test Step Schema: Control Flow Steps"
|
|
950
|
+
},
|
|
951
|
+
{
|
|
952
|
+
"id": "CONTROL-WAITFOR-002",
|
|
953
|
+
"category": "TestCaseDesign",
|
|
954
|
+
"name": "WaitFor steps have max iterations limit",
|
|
955
|
+
"description": "WaitFor steps must specify maxIterations to prevent infinite waiting.",
|
|
956
|
+
"appliesTo": [
|
|
957
|
+
"Step"
|
|
958
|
+
],
|
|
959
|
+
"severity": "major",
|
|
960
|
+
"weight": 5,
|
|
961
|
+
"recommendation": "Add 'maxIterations' argument to WaitFor step to limit wait cycles.",
|
|
962
|
+
"check": {
|
|
963
|
+
"type": "mustContainArgument",
|
|
964
|
+
"apiId": "com.provar.plugins.bundled.apis.control.WaitFor",
|
|
965
|
+
"argument": "maxIterations"
|
|
966
|
+
},
|
|
967
|
+
"source": "Provar Test Step Schema: Control Flow Steps"
|
|
968
|
+
},
|
|
969
|
+
{
|
|
970
|
+
"id": "ASSERT-COMPARISON-001",
|
|
971
|
+
"category": "TestCaseDesign",
|
|
972
|
+
"name": "AssertValues has comparisonType",
|
|
973
|
+
"description": "AssertValues steps must specify a comparisonType argument (EqualTo, Contains, etc.).",
|
|
974
|
+
"appliesTo": [
|
|
975
|
+
"Step"
|
|
976
|
+
],
|
|
977
|
+
"severity": "critical",
|
|
978
|
+
"weight": 8,
|
|
979
|
+
"recommendation": "Add 'comparisonType' argument with valid value: EqualTo, NotEqualTo, Contains, NotContains, StartsWith, EndsWith, GreaterThan, LessThan, Matches, IsNull, NotNull.",
|
|
980
|
+
"check": {
|
|
981
|
+
"type": "mustContainArgument",
|
|
982
|
+
"apiId": "com.provar.plugins.bundled.apis.AssertValues",
|
|
983
|
+
"argument": "comparisonType"
|
|
984
|
+
},
|
|
985
|
+
"source": "Provar Test Step Schema: Assertion Steps"
|
|
986
|
+
},
|
|
987
|
+
{
|
|
988
|
+
"id": "ASSERT-EXPECTED-001",
|
|
989
|
+
"category": "TestCaseDesign",
|
|
990
|
+
"name": "AssertValues has expectedValue",
|
|
991
|
+
"description": "AssertValues steps must specify an expectedValue argument containing the value to test.",
|
|
992
|
+
"appliesTo": [
|
|
993
|
+
"Step"
|
|
994
|
+
],
|
|
995
|
+
"severity": "critical",
|
|
996
|
+
"weight": 8,
|
|
997
|
+
"recommendation": "Add 'expectedValue' argument with the variable or value being verified.",
|
|
998
|
+
"check": {
|
|
999
|
+
"type": "mustContainArgument",
|
|
1000
|
+
"apiId": "com.provar.plugins.bundled.apis.AssertValues",
|
|
1001
|
+
"argument": "expectedValue"
|
|
1002
|
+
},
|
|
1003
|
+
"source": "Provar Test Step Schema: Assertion Steps"
|
|
1004
|
+
},
|
|
1005
|
+
{
|
|
1006
|
+
"id": "ASSERT-ACTUAL-001",
|
|
1007
|
+
"category": "TestCaseDesign",
|
|
1008
|
+
"name": "AssertValues has actualValue",
|
|
1009
|
+
"description": "AssertValues steps must specify an actualValue argument containing the expected result. NOTE: actualValue CAN be empty when using NotEqualTo to check if a string is NOT blank (common pattern for null/blank checks).",
|
|
1010
|
+
"appliesTo": [
|
|
1011
|
+
"Step"
|
|
1012
|
+
],
|
|
1013
|
+
"severity": "critical",
|
|
1014
|
+
"weight": 8,
|
|
1015
|
+
"recommendation": "Add 'actualValue' argument with the expected result to compare against. For blank string checks, actualValue can be empty with NotEqualTo comparison.",
|
|
1016
|
+
"check": {
|
|
1017
|
+
"type": "disabled",
|
|
1018
|
+
"comment": "DISABLED: False positive - actualValue CAN be empty for NotEqualTo blank string checks. Valid pattern: <argument id='actualValue'/> with NotEqualTo checks if string is not blank. Need to reimplement as conditional check that allows empty actualValue when comparisonType=NotEqualTo.",
|
|
1019
|
+
"subtype": "assertActualValue"
|
|
1020
|
+
},
|
|
1021
|
+
"source": "Provar Test Step Schema: Assertion Steps"
|
|
1022
|
+
},
|
|
1023
|
+
{
|
|
1024
|
+
"id": "BDD-GIVEN-001",
|
|
1025
|
+
"category": "TestCaseDesign",
|
|
1026
|
+
"name": "Given steps have description",
|
|
1027
|
+
"description": "BDD Given steps must specify a description argument explaining the precondition.",
|
|
1028
|
+
"appliesTo": [
|
|
1029
|
+
"Step"
|
|
1030
|
+
],
|
|
1031
|
+
"severity": "major",
|
|
1032
|
+
"weight": 5,
|
|
1033
|
+
"recommendation": "Add 'description' argument to Given step with a clear precondition statement.",
|
|
1034
|
+
"check": {
|
|
1035
|
+
"type": "mustContainArgument",
|
|
1036
|
+
"apiId": "com.provar.plugins.bundled.apis.bdd.Given",
|
|
1037
|
+
"argument": "description"
|
|
1038
|
+
},
|
|
1039
|
+
"source": "Provar Test Step Schema: BDD Steps"
|
|
1040
|
+
},
|
|
1041
|
+
{
|
|
1042
|
+
"id": "BDD-WHEN-001",
|
|
1043
|
+
"category": "TestCaseDesign",
|
|
1044
|
+
"name": "When steps have description",
|
|
1045
|
+
"description": "BDD When steps must specify a description argument explaining the action.",
|
|
1046
|
+
"appliesTo": [
|
|
1047
|
+
"Step"
|
|
1048
|
+
],
|
|
1049
|
+
"severity": "major",
|
|
1050
|
+
"weight": 5,
|
|
1051
|
+
"recommendation": "Add 'description' argument to When step with a clear action statement.",
|
|
1052
|
+
"check": {
|
|
1053
|
+
"type": "mustContainArgument",
|
|
1054
|
+
"apiId": "com.provar.plugins.bundled.apis.bdd.When",
|
|
1055
|
+
"argument": "description"
|
|
1056
|
+
},
|
|
1057
|
+
"source": "Provar Test Step Schema: BDD Steps"
|
|
1058
|
+
},
|
|
1059
|
+
{
|
|
1060
|
+
"id": "BDD-THEN-001",
|
|
1061
|
+
"category": "TestCaseDesign",
|
|
1062
|
+
"name": "Then steps have description",
|
|
1063
|
+
"description": "BDD Then steps must specify a description argument explaining the expected outcome.",
|
|
1064
|
+
"appliesTo": [
|
|
1065
|
+
"Step"
|
|
1066
|
+
],
|
|
1067
|
+
"severity": "major",
|
|
1068
|
+
"weight": 5,
|
|
1069
|
+
"recommendation": "Add 'description' argument to Then step with a clear expected outcome statement.",
|
|
1070
|
+
"check": {
|
|
1071
|
+
"type": "mustContainArgument",
|
|
1072
|
+
"apiId": "com.provar.plugins.bundled.apis.bdd.Then",
|
|
1073
|
+
"argument": "description"
|
|
1074
|
+
},
|
|
1075
|
+
"source": "Provar Test Step Schema: BDD Steps"
|
|
1076
|
+
},
|
|
1077
|
+
{
|
|
1078
|
+
"id": "SOQL-QUERY-001",
|
|
1079
|
+
"category": "TestCaseDesign",
|
|
1080
|
+
"name": "ApexSoqlQuery has soqlQuery argument",
|
|
1081
|
+
"description": "ApexSoqlQuery steps must specify a soqlQuery argument with the SOQL statement.",
|
|
1082
|
+
"appliesTo": [
|
|
1083
|
+
"Step"
|
|
1084
|
+
],
|
|
1085
|
+
"severity": "critical",
|
|
1086
|
+
"weight": 8,
|
|
1087
|
+
"recommendation": "Add 'soqlQuery' argument with valid SOQL statement (e.g., SELECT Id, Name FROM Account).",
|
|
1088
|
+
"check": {
|
|
1089
|
+
"type": "mustContainArgument",
|
|
1090
|
+
"apiId": "com.provar.plugins.forcedotcom.core.testapis.ApexSoqlQuery",
|
|
1091
|
+
"argument": "soqlQuery"
|
|
1092
|
+
},
|
|
1093
|
+
"source": "Provar Test Step Schema: Apex Steps"
|
|
1094
|
+
},
|
|
1095
|
+
{
|
|
1096
|
+
"id": "DB-CONNECT-001",
|
|
1097
|
+
"category": "ConnectionsAndEnvironments",
|
|
1098
|
+
"name": "DbConnect has connectionName",
|
|
1099
|
+
"description": "DbConnect steps must specify a connectionName argument identifying the database connection.",
|
|
1100
|
+
"appliesTo": [
|
|
1101
|
+
"Step"
|
|
1102
|
+
],
|
|
1103
|
+
"severity": "critical",
|
|
1104
|
+
"weight": 8,
|
|
1105
|
+
"recommendation": "Add 'connectionName' argument with the database connection identifier from environment settings.",
|
|
1106
|
+
"check": {
|
|
1107
|
+
"type": "mustContainArgument",
|
|
1108
|
+
"apiId": "com.provar.plugins.bundled.apis.db.DbConnect",
|
|
1109
|
+
"argument": "connectionName"
|
|
1110
|
+
},
|
|
1111
|
+
"source": "Provar Test Step Schema: Database Steps"
|
|
1112
|
+
},
|
|
1113
|
+
{
|
|
1114
|
+
"id": "DB-CONNECT-002",
|
|
1115
|
+
"category": "ConnectionsAndEnvironments",
|
|
1116
|
+
"name": "DbConnect has resultName",
|
|
1117
|
+
"description": "DbConnect steps must specify a resultName argument to store the connection reference.",
|
|
1118
|
+
"appliesTo": [
|
|
1119
|
+
"Step"
|
|
1120
|
+
],
|
|
1121
|
+
"severity": "critical",
|
|
1122
|
+
"weight": 8,
|
|
1123
|
+
"recommendation": "Add 'resultName' argument with the variable name to store the connection reference.",
|
|
1124
|
+
"check": {
|
|
1125
|
+
"type": "mustContainArgument",
|
|
1126
|
+
"apiId": "com.provar.plugins.bundled.apis.db.DbConnect",
|
|
1127
|
+
"argument": "resultName"
|
|
1128
|
+
},
|
|
1129
|
+
"source": "Provar Test Step Schema: Database Steps"
|
|
1130
|
+
},
|
|
1131
|
+
{
|
|
1132
|
+
"id": "SQL-QUERY-001",
|
|
1133
|
+
"category": "TestCaseDesign",
|
|
1134
|
+
"name": "SqlQuery has query argument",
|
|
1135
|
+
"description": "SqlQuery steps must specify a query argument with the SQL statement.",
|
|
1136
|
+
"appliesTo": [
|
|
1137
|
+
"Step"
|
|
1138
|
+
],
|
|
1139
|
+
"severity": "critical",
|
|
1140
|
+
"weight": 8,
|
|
1141
|
+
"recommendation": "Add 'query' argument with valid SQL statement.",
|
|
1142
|
+
"check": {
|
|
1143
|
+
"type": "mustContainArgument",
|
|
1144
|
+
"apiId": "com.provar.plugins.bundled.apis.db.SqlQuery",
|
|
1145
|
+
"argument": "query"
|
|
1146
|
+
},
|
|
1147
|
+
"source": "Provar Test Step Schema: Database Steps"
|
|
1148
|
+
},
|
|
1149
|
+
{
|
|
1150
|
+
"id": "SQL-QUERY-002",
|
|
1151
|
+
"category": "TestCaseDesign",
|
|
1152
|
+
"name": "SqlQuery has dbConnectionName",
|
|
1153
|
+
"description": "SqlQuery steps must specify a dbConnectionName argument referencing the database connection.",
|
|
1154
|
+
"appliesTo": [
|
|
1155
|
+
"Step"
|
|
1156
|
+
],
|
|
1157
|
+
"severity": "critical",
|
|
1158
|
+
"weight": 8,
|
|
1159
|
+
"recommendation": "Add 'dbConnectionName' argument with the resultName from a DbConnect step.",
|
|
1160
|
+
"check": {
|
|
1161
|
+
"type": "mustContainArgument",
|
|
1162
|
+
"apiId": "com.provar.plugins.bundled.apis.db.SqlQuery",
|
|
1163
|
+
"argument": "dbConnectionName"
|
|
1164
|
+
},
|
|
1165
|
+
"source": "Provar Test Step Schema: Database Steps"
|
|
1166
|
+
},
|
|
1167
|
+
{
|
|
1168
|
+
"id": "REST-CONN-001",
|
|
1169
|
+
"category": "ConnectionsAndEnvironments",
|
|
1170
|
+
"name": "WebConnect has connectionName",
|
|
1171
|
+
"description": "WebConnect steps must specify a connectionName argument identifying the web service connection.",
|
|
1172
|
+
"appliesTo": [
|
|
1173
|
+
"Step"
|
|
1174
|
+
],
|
|
1175
|
+
"severity": "critical",
|
|
1176
|
+
"weight": 8,
|
|
1177
|
+
"recommendation": "Add 'connectionName' argument with the web service connection identifier.",
|
|
1178
|
+
"check": {
|
|
1179
|
+
"type": "mustContainArgument",
|
|
1180
|
+
"apiId": "com.provar.plugins.bundled.apis.restservice.WebConnect",
|
|
1181
|
+
"argument": "connectionName"
|
|
1182
|
+
},
|
|
1183
|
+
"source": "Provar Test Step Schema: Web Service Steps"
|
|
1184
|
+
},
|
|
1185
|
+
{
|
|
1186
|
+
"id": "REST-CONN-002",
|
|
1187
|
+
"category": "ConnectionsAndEnvironments",
|
|
1188
|
+
"name": "WebConnect has resultName",
|
|
1189
|
+
"description": "WebConnect steps must specify a resultName argument to store the connection reference.",
|
|
1190
|
+
"appliesTo": [
|
|
1191
|
+
"Step"
|
|
1192
|
+
],
|
|
1193
|
+
"severity": "critical",
|
|
1194
|
+
"weight": 8,
|
|
1195
|
+
"recommendation": "Add 'resultName' argument with the variable name to store the connection reference.",
|
|
1196
|
+
"check": {
|
|
1197
|
+
"type": "mustContainArgument",
|
|
1198
|
+
"apiId": "com.provar.plugins.bundled.apis.restservice.WebConnect",
|
|
1199
|
+
"argument": "resultName"
|
|
1200
|
+
},
|
|
1201
|
+
"source": "Provar Test Step Schema: Web Service Steps"
|
|
1202
|
+
},
|
|
1203
|
+
{
|
|
1204
|
+
"id": "REST-REQUEST-001",
|
|
1205
|
+
"category": "TestCaseDesign",
|
|
1206
|
+
"name": "RestRequest has connectionName",
|
|
1207
|
+
"description": "RestRequest steps must specify a connectionName argument referencing the web service connection.",
|
|
1208
|
+
"appliesTo": [
|
|
1209
|
+
"Step"
|
|
1210
|
+
],
|
|
1211
|
+
"severity": "critical",
|
|
1212
|
+
"weight": 8,
|
|
1213
|
+
"recommendation": "Add 'connectionName' argument with the resultName from a WebConnect step.",
|
|
1214
|
+
"check": {
|
|
1215
|
+
"type": "mustContainArgument",
|
|
1216
|
+
"apiId": "com.provar.plugins.bundled.apis.restservice.RestRequest",
|
|
1217
|
+
"argument": "connectionName"
|
|
1218
|
+
},
|
|
1219
|
+
"source": "Provar Test Step Schema: Web Service Steps"
|
|
1220
|
+
},
|
|
1221
|
+
{
|
|
1222
|
+
"id": "VAR-REFERENCE-001",
|
|
1223
|
+
"category": "TestCaseDesign",
|
|
1224
|
+
"name": "Variables are defined before use",
|
|
1225
|
+
"description": "Variables referenced in test should be defined via SetValues or result from previous steps.",
|
|
1226
|
+
"appliesTo": [
|
|
1227
|
+
"Step",
|
|
1228
|
+
"TestCase"
|
|
1229
|
+
],
|
|
1230
|
+
"severity": "major",
|
|
1231
|
+
"weight": 5,
|
|
1232
|
+
"recommendation": "Define variable with SetValues step or ensure it's populated by previous step (SOQL, CreateObject, etc.).",
|
|
1233
|
+
"check": {
|
|
1234
|
+
"type": "variableDefinition",
|
|
1235
|
+
"target": "testCase"
|
|
1236
|
+
},
|
|
1237
|
+
"source": "Provar Test Step Schema: Variable Validation"
|
|
1238
|
+
},
|
|
1239
|
+
{
|
|
1240
|
+
"id": "CONN-APEX-001",
|
|
1241
|
+
"category": "ConnectionsAndEnvironments",
|
|
1242
|
+
"name": "Apex API calls reference valid connections",
|
|
1243
|
+
"description": "Apex API operations must reference an existing ApexConnect connection.",
|
|
1244
|
+
"appliesTo": [
|
|
1245
|
+
"Step"
|
|
1246
|
+
],
|
|
1247
|
+
"severity": "critical",
|
|
1248
|
+
"weight": 8,
|
|
1249
|
+
"recommendation": "Ensure ApexConnect step exists before Apex API calls and connection names match.",
|
|
1250
|
+
"check": {
|
|
1251
|
+
"type": "apexConnectionReference",
|
|
1252
|
+
"target": "testCase"
|
|
1253
|
+
},
|
|
1254
|
+
"source": "Provar Test Step Schema: Connection Validation"
|
|
1255
|
+
},
|
|
1256
|
+
{
|
|
1257
|
+
"id": "CONN-DB-001",
|
|
1258
|
+
"category": "ConnectionsAndEnvironments",
|
|
1259
|
+
"name": "Database operations reference valid connections",
|
|
1260
|
+
"description": "Database API operations must reference an existing DbConnect connection.",
|
|
1261
|
+
"appliesTo": [
|
|
1262
|
+
"Step"
|
|
1263
|
+
],
|
|
1264
|
+
"severity": "critical",
|
|
1265
|
+
"weight": 8,
|
|
1266
|
+
"recommendation": "Ensure DbConnect step exists before database operations and connection names match.",
|
|
1267
|
+
"check": {
|
|
1268
|
+
"type": "dbConnectionReference",
|
|
1269
|
+
"target": "testCase"
|
|
1270
|
+
},
|
|
1271
|
+
"source": "Provar Test Step Schema: Connection Validation"
|
|
1272
|
+
},
|
|
1273
|
+
{
|
|
1274
|
+
"id": "CONN-UI-001",
|
|
1275
|
+
"category": "ConnectionsAndEnvironments",
|
|
1276
|
+
"name": "UI operations reference valid connections",
|
|
1277
|
+
"description": "UI API operations must reference an existing UiConnect connection.",
|
|
1278
|
+
"appliesTo": [
|
|
1279
|
+
"Step"
|
|
1280
|
+
],
|
|
1281
|
+
"severity": "critical",
|
|
1282
|
+
"weight": 8,
|
|
1283
|
+
"recommendation": "Ensure UiConnect step exists before UI operations and connection names match.",
|
|
1284
|
+
"check": {
|
|
1285
|
+
"type": "uiConnectionReference",
|
|
1286
|
+
"target": "testCase"
|
|
1287
|
+
},
|
|
1288
|
+
"source": "Provar Test Step Schema: Connection Validation"
|
|
1289
|
+
},
|
|
1290
|
+
{
|
|
1291
|
+
"id": "UI-CONN-LITERAL-001",
|
|
1292
|
+
"category": "ConnectionsAndEnvironments",
|
|
1293
|
+
"name": "uiConnectionName must be a literal string",
|
|
1294
|
+
"description": "UiWithScreen/UiDoAction/UiAssert steps must set uiConnectionName to a literal string (the resultName from UiConnect/ApexConnect). Using <value class=\"variable\"> for uiConnectionName causes Provar to error.",
|
|
1295
|
+
"appliesTo": [
|
|
1296
|
+
"Step"
|
|
1297
|
+
],
|
|
1298
|
+
"severity": "critical",
|
|
1299
|
+
"weight": 8,
|
|
1300
|
+
"recommendation": "Set uiConnectionName using <value class=\"value\" valueClass=\"string\">AdminConnection</value> (not a variable reference).",
|
|
1301
|
+
"check": {
|
|
1302
|
+
"type": "uiConnectionNameLiteral",
|
|
1303
|
+
"target": "step"
|
|
1304
|
+
},
|
|
1305
|
+
"source": "Provar Runtime Requirement - UI connection name must be literal"
|
|
1306
|
+
},
|
|
1307
|
+
{
|
|
1308
|
+
"id": "SOQL-STRUCTURE-001",
|
|
1309
|
+
"category": "TestCaseDesign",
|
|
1310
|
+
"name": "SOQL queries have SELECT and FROM clauses",
|
|
1311
|
+
"description": "Valid SOQL queries must contain both SELECT and FROM clauses.",
|
|
1312
|
+
"appliesTo": [
|
|
1313
|
+
"Step"
|
|
1314
|
+
],
|
|
1315
|
+
"severity": "critical",
|
|
1316
|
+
"weight": 8,
|
|
1317
|
+
"recommendation": "Ensure SOQL query has format: SELECT <fields> FROM <object>",
|
|
1318
|
+
"check": {
|
|
1319
|
+
"type": "soqlStructure",
|
|
1320
|
+
"target": "step",
|
|
1321
|
+
"apiId": "com.provar.plugins.forcedotcom.core.testapis.ApexSoqlQuery"
|
|
1322
|
+
},
|
|
1323
|
+
"source": "Provar Test Step Schema: SOQL Validation"
|
|
1324
|
+
},
|
|
1325
|
+
{
|
|
1326
|
+
"id": "SOQL-WHERE-001",
|
|
1327
|
+
"category": "TestCaseDesign",
|
|
1328
|
+
"name": "SOQL queries include WHERE or LIMIT clause",
|
|
1329
|
+
"description": "SOQL queries must have WHERE clause or LIMIT to prevent unreasonably large result sets and potential heap size errors. This prevents Apex governance violations and performance issues.",
|
|
1330
|
+
"appliesTo": [
|
|
1331
|
+
"Step"
|
|
1332
|
+
],
|
|
1333
|
+
"severity": "major",
|
|
1334
|
+
"weight": 5,
|
|
1335
|
+
"recommendation": "Add WHERE clause to filter results (e.g. WHERE Id = :recordId) or LIMIT clause to cap result size (e.g. LIMIT 100). For large data volumes, always include both WHERE and LIMIT.",
|
|
1336
|
+
"check": {
|
|
1337
|
+
"type": "soqlWhereOrLimit",
|
|
1338
|
+
"target": "step",
|
|
1339
|
+
"apiId": "com.provar.plugins.forcedotcom.core.testapis.ApexSoqlQuery"
|
|
1340
|
+
},
|
|
1341
|
+
"source": "SOQL Best Practices: Building Efficient & Selective Queries"
|
|
1342
|
+
},
|
|
1343
|
+
{
|
|
1344
|
+
"id": "SOQL-IN-LOOP-001",
|
|
1345
|
+
"category": "TestCaseDesign",
|
|
1346
|
+
"name": "SOQL queries must not be inside loops",
|
|
1347
|
+
"description": "SOQL queries inside For/ForEach/While/DoWhile loops violate Apex governor limits. Each SOQL query execution consumes from the 100 SOQL query limit per transaction. Queries in loops can quickly exceed this limit causing 'Too many SOQL queries: 101' errors.",
|
|
1348
|
+
"appliesTo": [
|
|
1349
|
+
"Step"
|
|
1350
|
+
],
|
|
1351
|
+
"severity": "major",
|
|
1352
|
+
"weight": 5,
|
|
1353
|
+
"recommendation": "Move SOQL query outside the loop and store results in a collection. Then iterate over the collection. Use SOQL WHERE clauses to filter in the query rather than filtering in a loop. Consider using aggregate queries or bulk processing patterns.",
|
|
1354
|
+
"check": {
|
|
1355
|
+
"type": "soqlInLoop",
|
|
1356
|
+
"target": "step",
|
|
1357
|
+
"apiId": "com.provar.plugins.forcedotcom.core.testapis.ApexSoqlQuery"
|
|
1358
|
+
},
|
|
1359
|
+
"source": "SOQL Best Practices: Avoid SOQL inside FOR Loops - Apex Governance"
|
|
1360
|
+
},
|
|
1361
|
+
{
|
|
1362
|
+
"id": "APEX-CONNECTION-REF-001",
|
|
1363
|
+
"category": "ApexAPI",
|
|
1364
|
+
"name": "Apex API steps must reference a valid connection",
|
|
1365
|
+
"description": "All Apex API calls (ApexCreate, ApexUpdate, ApexDelete, ApexRead, ApexSoqlQuery) must reference a valid Salesforce connection defined earlier in the test using ApexConnect. Tests will fail if connection is missing, empty, or references an undefined connection name.",
|
|
1366
|
+
"appliesTo": [
|
|
1367
|
+
"Step"
|
|
1368
|
+
],
|
|
1369
|
+
"severity": "major",
|
|
1370
|
+
"weight": 5,
|
|
1371
|
+
"recommendation": "Add an ApexConnect step before any Apex API calls to establish a Salesforce connection. Reference the connection name in all subsequent Apex API calls. Use variables for connection names if dynamically determined.",
|
|
1372
|
+
"check": {
|
|
1373
|
+
"type": "apexConnectionReference",
|
|
1374
|
+
"target": "step"
|
|
1375
|
+
},
|
|
1376
|
+
"source": "Provar API Best Practices - Connection Management"
|
|
1377
|
+
},
|
|
1378
|
+
{
|
|
1379
|
+
"id": "APEX-OBJECT-TYPE-001",
|
|
1380
|
+
"category": "ApexAPI",
|
|
1381
|
+
"name": "Apex CRUD operations must have valid object types",
|
|
1382
|
+
"description": "ApexCreate, ApexUpdate, ApexDelete, and ApexRead steps must specify a valid Salesforce object type (e.g., Account, Contact, Broker__c). Object type must start with capital letter and contain only alphanumeric characters and underscores. Invalid or missing object types cause test failures.",
|
|
1383
|
+
"appliesTo": [
|
|
1384
|
+
"Step"
|
|
1385
|
+
],
|
|
1386
|
+
"severity": "major",
|
|
1387
|
+
"weight": 5,
|
|
1388
|
+
"recommendation": "Specify the correct Salesforce object API name (e.g., Account, Contact, Custom__c). Use standard object names or custom object API names ending with __c. Verify object exists in the target Salesforce org.",
|
|
1389
|
+
"check": {
|
|
1390
|
+
"type": "apexObjectType",
|
|
1391
|
+
"target": "step"
|
|
1392
|
+
},
|
|
1393
|
+
"source": "Provar API Best Practices - Object Validation"
|
|
1394
|
+
},
|
|
1395
|
+
{
|
|
1396
|
+
"id": "APEX-UPDATE-ID-001",
|
|
1397
|
+
"category": "ApexAPI",
|
|
1398
|
+
"name": "ApexUpdateObject must have valid record ID",
|
|
1399
|
+
"description": "ApexUpdateObject requires a valid Salesforce record ID (15 or 18 characters) or a variable reference containing the ID. Missing, empty, or invalid record IDs cause test failures. Record IDs must be alphanumeric Salesforce IDs.",
|
|
1400
|
+
"appliesTo": [
|
|
1401
|
+
"Step"
|
|
1402
|
+
],
|
|
1403
|
+
"severity": "major",
|
|
1404
|
+
"weight": 5,
|
|
1405
|
+
"recommendation": "Provide a valid 15 or 18 character Salesforce record ID, or use a variable like {AccountId} that contains a valid ID from a prior Create or Read operation. Verify the record exists before attempting update.",
|
|
1406
|
+
"check": {
|
|
1407
|
+
"type": "apexRecordId",
|
|
1408
|
+
"target": "step"
|
|
1409
|
+
},
|
|
1410
|
+
"source": "Provar API Best Practices - Record ID Validation"
|
|
1411
|
+
},
|
|
1412
|
+
{
|
|
1413
|
+
"id": "APEX-UPDATE-FIELDS-001",
|
|
1414
|
+
"category": "ApexAPI",
|
|
1415
|
+
"name": "ApexUpdateObject must specify fields to update",
|
|
1416
|
+
"description": "ApexUpdateObject steps should specify at least one field to update in the field definitions. Update operations without field specifications have no effect and represent incomplete test logic. System arguments (objectType, recordId, objectId, apexConnectionName, resultScope, assignmentRuleId, resultIdName) are excluded from this check.",
|
|
1417
|
+
"appliesTo": [
|
|
1418
|
+
"Step"
|
|
1419
|
+
],
|
|
1420
|
+
"severity": "major",
|
|
1421
|
+
"weight": 5,
|
|
1422
|
+
"recommendation": "Add field definitions to the ApexUpdateObject step and provide values for at least one field to update. For example, specify Name, Status, or other relevant fields with test data or variable references.",
|
|
1423
|
+
"check": {
|
|
1424
|
+
"type": "apexUpdateFields",
|
|
1425
|
+
"target": "step"
|
|
1426
|
+
},
|
|
1427
|
+
"source": "Provar API Best Practices - Field Validation"
|
|
1428
|
+
},
|
|
1429
|
+
{
|
|
1430
|
+
"id": "APEX-DELETE-ID-001",
|
|
1431
|
+
"category": "ApexAPI",
|
|
1432
|
+
"name": "ApexDeleteObject must have valid record ID",
|
|
1433
|
+
"description": "ApexDeleteObject requires a valid Salesforce record ID (15 or 18 characters) or a variable reference containing the ID. Missing, empty, or invalid record IDs cause test failures. Record IDs must be alphanumeric Salesforce IDs.",
|
|
1434
|
+
"appliesTo": [
|
|
1435
|
+
"Step"
|
|
1436
|
+
],
|
|
1437
|
+
"severity": "major",
|
|
1438
|
+
"weight": 5,
|
|
1439
|
+
"recommendation": "Provide a valid 15 or 18 character Salesforce record ID, or use a variable like {AccountId} that contains a valid ID from a prior Create or Read operation. Verify the record exists before attempting delete.",
|
|
1440
|
+
"check": {
|
|
1441
|
+
"type": "apexRecordId",
|
|
1442
|
+
"target": "step"
|
|
1443
|
+
},
|
|
1444
|
+
"source": "Provar API Best Practices - Record ID Validation"
|
|
1445
|
+
},
|
|
1446
|
+
{
|
|
1447
|
+
"id": "APEX-READ-ID-001",
|
|
1448
|
+
"category": "ApexAPI",
|
|
1449
|
+
"name": "ApexReadObject must have valid record ID",
|
|
1450
|
+
"description": "ApexReadObject requires a valid Salesforce record ID (15 or 18 characters) or a variable reference containing the ID. Missing, empty, or invalid record IDs cause test failures. Record IDs must be alphanumeric Salesforce IDs.",
|
|
1451
|
+
"appliesTo": [
|
|
1452
|
+
"Step"
|
|
1453
|
+
],
|
|
1454
|
+
"severity": "major",
|
|
1455
|
+
"weight": 5,
|
|
1456
|
+
"recommendation": "Provide a valid 15 or 18 character Salesforce record ID, or use a variable like {AccountId} that contains a valid ID from a prior Create operation. Verify the record exists before attempting read.",
|
|
1457
|
+
"check": {
|
|
1458
|
+
"type": "apexRecordId",
|
|
1459
|
+
"target": "step"
|
|
1460
|
+
},
|
|
1461
|
+
"source": "Provar API Best Practices - Record ID Validation"
|
|
1462
|
+
},
|
|
1463
|
+
{
|
|
1464
|
+
"id": "APEX-CREATE-FIELDS-001",
|
|
1465
|
+
"category": "ApexAPI",
|
|
1466
|
+
"name": "ApexCreateObject with fields must populate at least one field",
|
|
1467
|
+
"description": "When ApexCreateObject includes a fields structure, at least one field must be populated with a value. Empty field structures indicate incomplete test implementation and may cause test failures or create records with default values only.",
|
|
1468
|
+
"appliesTo": [
|
|
1469
|
+
"Step"
|
|
1470
|
+
],
|
|
1471
|
+
"severity": "major",
|
|
1472
|
+
"weight": 5,
|
|
1473
|
+
"recommendation": "Populate at least one field with a value or variable reference in the ApexCreateObject fields structure. Remove the fields structure if not needed. Verify required fields for the object are populated.",
|
|
1474
|
+
"check": {
|
|
1475
|
+
"type": "apexCreateFields",
|
|
1476
|
+
"target": "step"
|
|
1477
|
+
},
|
|
1478
|
+
"source": "Provar API Best Practices - Field Validation"
|
|
1479
|
+
},
|
|
1480
|
+
{
|
|
1481
|
+
"id": "APEX-CREATE-METADATA-001",
|
|
1482
|
+
"category": "ApexAPI",
|
|
1483
|
+
"name": "ApexCreateObject and ApexUpdateObject must include parameter metadata",
|
|
1484
|
+
"description": "ApexCreateObject and ApexUpdateObject steps MUST include <parameterGeneratorProperties> and <generatedParameters> sections after </arguments>. These metadata sections enable Provar IDE features like field auto-complete, field suggestions, and proper object binding. Missing metadata causes degraded IDE experience and prevents field discovery.",
|
|
1485
|
+
"appliesTo": [
|
|
1486
|
+
"Step"
|
|
1487
|
+
],
|
|
1488
|
+
"severity": "major",
|
|
1489
|
+
"weight": 5,
|
|
1490
|
+
"recommendation": "Add <parameterGeneratorProperties> with ConnectionName and CustomObjectName properties, and <generatedParameters> with apiParam elements for each field used in arguments. Study existing tier4 examples to see the exact structure required.",
|
|
1491
|
+
"check": {
|
|
1492
|
+
"type": "apexCreateMetadata",
|
|
1493
|
+
"target": "step"
|
|
1494
|
+
},
|
|
1495
|
+
"source": "Provar IDE Requirements - Parameter Metadata for Object Operations"
|
|
1496
|
+
},
|
|
1497
|
+
{
|
|
1498
|
+
"id": "APEX-CREATE-UPDATE-STRUCTURE-001",
|
|
1499
|
+
"category": "ApexAPI",
|
|
1500
|
+
"name": "ApexUpdateObject/ApexCreateObject field arguments must be direct, not nested in uiObjectFieldValue",
|
|
1501
|
+
"description": "ApexUpdateObject and ApexCreateObject steps MUST use direct field arguments (e.g., <argument id='Status'><value.../></argument>) instead of nesting fields within uiObjectFieldValue elements inside the values argument. The values argument should contain only an empty valueList. Nested uiObjectFieldValue structures cause runtime failures and prevent proper field serialization.",
|
|
1502
|
+
"appliesTo": [
|
|
1503
|
+
"Step"
|
|
1504
|
+
],
|
|
1505
|
+
"severity": "major",
|
|
1506
|
+
"weight": 5,
|
|
1507
|
+
"recommendation": "Use direct field arguments: <argument id='FieldName'><value class='value' valueClass='string'>FieldValue</value></argument> with a separate empty <argument id='values'><value class='valueList' mutable='Mutable'/></argument>. Do NOT nest fields inside uiObjectFieldValue within the values argument.",
|
|
1508
|
+
"check": {
|
|
1509
|
+
"type": "apexUpdateFieldStructure",
|
|
1510
|
+
"target": "step"
|
|
1511
|
+
},
|
|
1512
|
+
"source": "Provar Runtime Requirements - API Object Field Structure"
|
|
1513
|
+
},
|
|
1514
|
+
{
|
|
1515
|
+
"id": "APEX-READ-FIELDS-STRUCTURE-001",
|
|
1516
|
+
"category": "ApexAPI",
|
|
1517
|
+
"name": "ApexReadObject must use generatedParameters for field references, not fields argument with textType",
|
|
1518
|
+
"description": "ApexReadObject steps should NOT use a 'fields' argument containing textType elements for field names. Instead, fields must be properly referenced via generatedParameters with modelBinding attributes. The incorrect pattern uses <argument id='fields'><value class='valueList'><textType>FieldName</textType></value></argument> which prevents proper field metadata and IDE integration. The correct pattern includes proper <generatedParameters> with <apiParam group='fields' modelBinding='sf:ui:binding:object?field=FieldName&object=ObjectType'> elements.",
|
|
1519
|
+
"appliesTo": [
|
|
1520
|
+
"Step"
|
|
1521
|
+
],
|
|
1522
|
+
"severity": "major",
|
|
1523
|
+
"weight": 5,
|
|
1524
|
+
"recommendation": "Remove the 'fields' argument with textType elements. Ensure each field has a corresponding <apiParam> entry in <generatedParameters> with proper modelBinding. Fields should be declared through generatedParameters, not through a fields argument.",
|
|
1525
|
+
"check": {
|
|
1526
|
+
"type": "apexReadFieldsStructure",
|
|
1527
|
+
"target": "step"
|
|
1528
|
+
},
|
|
1529
|
+
"source": "Provar IDE Requirements - ApexReadObject Field Metadata"
|
|
1530
|
+
},
|
|
1531
|
+
{
|
|
1532
|
+
"id": "APEX-APIPARAM-HALLUCINATION-001",
|
|
1533
|
+
"category": "XMLSchema",
|
|
1534
|
+
"name": "Apex CRUD apiParam elements must be self-closing without summary/type children",
|
|
1535
|
+
"description": "In ApexCreateObject, ApexUpdateObject, and ApexReadObject steps, <apiParam> elements within <generatedParameters> MUST be self-closing tags without child elements. LLMs commonly hallucinate <summary> and <type> elements inside apiParam, which causes Provar to reject the test case as invalid XML. The correct pattern is: <apiParam group=\"fields\" modelBinding=\"sf:ui:binding:object?object=Account&field=Name\" name=\"Name\" title=\"Name\"/>. The hallucinated pattern includes <summary>description</summary> and <type><textType>...</textType></type> children which are ONLY valid in other contexts (RestRequest, UiDoAction) but NEVER in Apex CRUD operations.",
|
|
1536
|
+
"appliesTo": [
|
|
1537
|
+
"Step"
|
|
1538
|
+
],
|
|
1539
|
+
"severity": "critical",
|
|
1540
|
+
"weight": 10,
|
|
1541
|
+
"recommendation": "Remove all <summary> and <type> child elements from apiParam elements in ApexCreateObject, ApexUpdateObject, and ApexReadObject steps. Use self-closing <apiParam .../> tags. For example: <apiParam group=\"fields\" modelBinding=\"sf:ui:binding:object?object=Account&field=Name\" name=\"Name\" title=\"Name\"/>",
|
|
1542
|
+
"check": {
|
|
1543
|
+
"type": "apexApiParamHallucination",
|
|
1544
|
+
"target": "step"
|
|
1545
|
+
},
|
|
1546
|
+
"notes": "CRITICAL severity - hallucinated summary/type elements in Apex CRUD apiParam cause Provar to mark test as invalid XML. Based on corpus analysis: 100% of Apex CRUD apiParam elements use self-closing tags without children. LLMs confuse this with RestRequest/UiDoAction patterns which DO allow summary/type children.",
|
|
1547
|
+
"source": "Corpus analysis: InternalProjects - Apex CRUD apiParam structure validation"
|
|
1548
|
+
},
|
|
1549
|
+
{
|
|
1550
|
+
"id": "APEX-READ-ASSERTIONS-001",
|
|
1551
|
+
"category": "ApexAPI",
|
|
1552
|
+
"name": "ApexReadObject should use resultAssertions instead of separate AssertValues",
|
|
1553
|
+
"description": "When asserting field values after ApexReadObject, use <resultAssertions> within the ApexReadObject step rather than separate AssertValues API calls. This pattern is more efficient, provides better error messages, and maintains field metadata associations. The resultAssertions element should contain <resultAssertion> children with comparisonType, resultName, title, and optional <expected> elements.",
|
|
1554
|
+
"appliesTo": [
|
|
1555
|
+
"Step"
|
|
1556
|
+
],
|
|
1557
|
+
"severity": "minor",
|
|
1558
|
+
"weight": 3,
|
|
1559
|
+
"recommendation": "Add <resultAssertions> block within ApexReadObject after the <arguments> section. Move AssertValues logic into <resultAssertion> elements with appropriate comparisonType (EqualTo, IsPresent, NotEqualTo, etc.) and expected values where applicable.",
|
|
1560
|
+
"check": {
|
|
1561
|
+
"type": "apexReadAssertions",
|
|
1562
|
+
"target": "step"
|
|
1563
|
+
},
|
|
1564
|
+
"source": "Provar Best Practices - Efficient Read and Assert Pattern"
|
|
1565
|
+
},
|
|
1566
|
+
{
|
|
1567
|
+
"id": "APEX-EXTRACT-LAYOUT-001",
|
|
1568
|
+
"category": "ApexAPI",
|
|
1569
|
+
"name": "ApexExtractLayout must have object, file type, and path",
|
|
1570
|
+
"description": "ApexExtractLayout requires three parameters: object (Salesforce object API name), fileType (.xlsx only supported), and filePath (where to save the layout). Missing any parameter causes test failure. Only .xlsx file format is supported for layout extraction.",
|
|
1571
|
+
"appliesTo": [
|
|
1572
|
+
"Step"
|
|
1573
|
+
],
|
|
1574
|
+
"severity": "major",
|
|
1575
|
+
"weight": 5,
|
|
1576
|
+
"recommendation": "Specify the Salesforce object API name, set fileType to .xlsx, and provide a valid file path for the extracted layout. Ensure the directory path exists and is writable. Use relative paths for portability.",
|
|
1577
|
+
"check": {
|
|
1578
|
+
"type": "apexExtractLayout",
|
|
1579
|
+
"target": "step"
|
|
1580
|
+
},
|
|
1581
|
+
"source": "Provar API Best Practices - Layout Extraction"
|
|
1582
|
+
},
|
|
1583
|
+
{
|
|
1584
|
+
"id": "APEX-ASSERT-LAYOUT-001",
|
|
1585
|
+
"category": "ApexAPI",
|
|
1586
|
+
"name": "ApexAssertLayout must have object and expected file",
|
|
1587
|
+
"description": "ApexAssertLayout requires two parameters: object (Salesforce object API name) and expectedFile (path to expected .xlsx layout file). Missing either parameter causes test failure. Only .xlsx file format is supported for layout assertions.",
|
|
1588
|
+
"appliesTo": [
|
|
1589
|
+
"Step"
|
|
1590
|
+
],
|
|
1591
|
+
"severity": "major",
|
|
1592
|
+
"weight": 5,
|
|
1593
|
+
"recommendation": "Specify the Salesforce object API name and provide path to expected .xlsx layout file. Ensure the expected file exists and is in .xlsx format. Use ApexExtractLayout to generate baseline layouts.",
|
|
1594
|
+
"check": {
|
|
1595
|
+
"type": "apexAssertLayout",
|
|
1596
|
+
"target": "step"
|
|
1597
|
+
},
|
|
1598
|
+
"source": "Provar API Best Practices - Layout Assertion"
|
|
1599
|
+
},
|
|
1600
|
+
{
|
|
1601
|
+
"id": "CONN-ARG-001",
|
|
1602
|
+
"category": "ApexAPI",
|
|
1603
|
+
"name": "Connection arguments must use correct naming convention",
|
|
1604
|
+
"description": "Different step types require specific connection argument names: 'connectionName' for Connect steps (ApexConnect, UiConnect, DbConnect, WebConnect) and web requests (RestRequest, SoapRequest); 'apexConnectionName' for Apex CRUD/SOQL operations (ApexCreateObject, ApexReadObject, ApexUpdateObject, ApexDeleteObject, ApexSoqlQuery, ApexBulk, etc.); 'uiConnectionName' for UI steps (UiWithScreen, UiDoAction, UiAssert); 'dbConnectionName' for database operations (DbDelete, DbInsert, DbRead, DbUpdate, SqlQuery). Using incorrect connection argument names causes connection lookup failures.",
|
|
1605
|
+
"appliesTo": [
|
|
1606
|
+
"Step"
|
|
1607
|
+
],
|
|
1608
|
+
"severity": "minor",
|
|
1609
|
+
"weight": 2,
|
|
1610
|
+
"recommendation": "Use the correct connection argument name for each step type. Connect steps and web requests use 'connectionName'. Apex CRUD/SOQL operations use 'apexConnectionName'. UI steps use 'uiConnectionName'. Database operations use 'dbConnectionName'. Review Provar API documentation for the specific step type being used.",
|
|
1611
|
+
"check": {
|
|
1612
|
+
"type": "connectionArgumentNaming",
|
|
1613
|
+
"target": "step"
|
|
1614
|
+
},
|
|
1615
|
+
"source": "Provar API Best Practices - Connection Argument Conventions"
|
|
1616
|
+
},
|
|
1617
|
+
{
|
|
1618
|
+
"id": "APEX-PARAM-GEN-URI-001",
|
|
1619
|
+
"category": "ApexAPI",
|
|
1620
|
+
"name": "Apex CRUD operations should include parameterGeneratorUri",
|
|
1621
|
+
"description": "ApexCreateObject, ApexReadObject, and ApexUpdateObject steps should include the parameterGeneratorUri attribute for proper IDE support. This attribute enables field auto-complete, field suggestions, and proper object binding in Provar IDE. Missing this attribute degrades IDE experience and may cause issues in some Provar versions.",
|
|
1622
|
+
"appliesTo": [
|
|
1623
|
+
"Step"
|
|
1624
|
+
],
|
|
1625
|
+
"severity": "minor",
|
|
1626
|
+
"weight": 2,
|
|
1627
|
+
"recommendation": "Add parameterGeneratorUri attribute to Apex CRUD steps: ApexCreateObject uses 'command:com.provar.plugins.forcedotcom.ui.commands.CreateCustomObjectTestStepCommand', ApexReadObject uses 'command:com.provar.plugins.forcedotcom.ui.commands.ReadCustomObjectTestStepCommand', ApexUpdateObject uses 'command:com.provar.plugins.forcedotcom.ui.commands.UpdateCustomObjectTestStepCommand'.",
|
|
1628
|
+
"check": {
|
|
1629
|
+
"type": "apexParameterGeneratorUri",
|
|
1630
|
+
"target": "step"
|
|
1631
|
+
},
|
|
1632
|
+
"source": "Provar IDE Requirements - Parameter Generator Metadata"
|
|
1633
|
+
},
|
|
1634
|
+
{
|
|
1635
|
+
"id": "ASSERT-ARG-ORDER-001",
|
|
1636
|
+
"category": "TestCaseDesign",
|
|
1637
|
+
"name": "AssertValues arguments must be in correct order",
|
|
1638
|
+
"description": "AssertValues steps should contain the variable being tested (from Read/Query operations) and actualValue containing the expected literal value. Reversed arguments cause assertion logic to be backwards - comparing literals against variables instead of variables against expected values. This can result in unclear test step outcomes, although results are the same.",
|
|
1639
|
+
"appliesTo": [
|
|
1640
|
+
"Step"
|
|
1641
|
+
],
|
|
1642
|
+
"severity": "info",
|
|
1643
|
+
"weight": 1,
|
|
1644
|
+
"recommendation": "Correct the argument order: expectedValue should contain the variable reference (e.g., {OpportunityRead.Name}), actualValue should contain the expected literal value (e.g., 'Demo Opportunity'). The assertion logic is: 'Assert that expectedValue equals actualValue', which reads as 'Assert that {variable} equals literal'.",
|
|
1645
|
+
"check": {
|
|
1646
|
+
"type": "assertValuesArgumentOrder",
|
|
1647
|
+
"target": "step"
|
|
1648
|
+
},
|
|
1649
|
+
"source": "Provar API Best Practices - Assertion Patterns"
|
|
1650
|
+
},
|
|
1651
|
+
{
|
|
1652
|
+
"id": "ASSERT-API-001",
|
|
1653
|
+
"category": "TestCaseDesign",
|
|
1654
|
+
"name": "Must use AssertValues API, not deprecated Assert API",
|
|
1655
|
+
"description": "Test cases must use the supported com.provar.plugins.bundled.apis.AssertValues API with expectedValue/actualValue/comparisonType arguments. The com.provar.plugins.bundled.core.testapis.Assert API is not supported and will not load properly in Provar. AssertValues provides more flexible comparison operators and clearer assertion semantics.",
|
|
1656
|
+
"appliesTo": [
|
|
1657
|
+
"Step"
|
|
1658
|
+
],
|
|
1659
|
+
"severity": "critical",
|
|
1660
|
+
"weight": 8,
|
|
1661
|
+
"recommendation": "Replace com.provar.plugins.bundled.core.testapis.Assert steps with com.provar.plugins.bundled.apis.AssertValues. Use expectedValue (the variable being tested), comparisonType (EqualTo, NotEqualTo, etc.), actualValue (the expected result), and failureMessage (explanation of what failed).",
|
|
1662
|
+
"check": {
|
|
1663
|
+
"type": "assertApi",
|
|
1664
|
+
"target": "step"
|
|
1665
|
+
},
|
|
1666
|
+
"source": "Provar API Reference - Assertion Steps"
|
|
1667
|
+
},
|
|
1668
|
+
{
|
|
1669
|
+
"id": "ASSERT-VALUES-COMPARISON-001",
|
|
1670
|
+
"category": "TestCaseDesign",
|
|
1671
|
+
"name": "AssertValues should have meaningful expected values",
|
|
1672
|
+
"description": "AssertValues steps using comparison operators (EqualTo, Contains, GreaterThan, etc.) should specify meaningful expectedValue arguments. Empty or missing expected values defeat the purpose of assertions and provide no validation. Without expected values, assertions cannot verify correct behavior. Comparison types requiring values: EqualTo, NotEqualTo, Contains, NotContains, StartsWith, NotStartsWith, EndsWith, NotEndsWith, Matches, NotMatches, GreaterThan, LessThan, GreaterThanOrEqualTo, LessThanOrEqualTo.",
|
|
1673
|
+
"appliesTo": [
|
|
1674
|
+
"Step"
|
|
1675
|
+
],
|
|
1676
|
+
"severity": "major",
|
|
1677
|
+
"weight": 5,
|
|
1678
|
+
"recommendation": "Specify a clear expectedValue for the assertion based on requirements. Use test data variables or literals that represent the expected state. Empty expected values provide no validation. For example: expectedValue='{Account.Name}', comparisonType='EqualTo', actualValue='ACME Corporation'.",
|
|
1679
|
+
"check": {
|
|
1680
|
+
"type": "assertValuesComparison",
|
|
1681
|
+
"target": "step",
|
|
1682
|
+
"apiId": "com.provar.plugins.bundled.apis.AssertValues"
|
|
1683
|
+
},
|
|
1684
|
+
"source": "Provar Test Step Schema: Meaningful Assertions (Phase 2 API Usage Analysis)"
|
|
1685
|
+
},
|
|
1686
|
+
{
|
|
1687
|
+
"id": "CONTROL-SLEEP-001",
|
|
1688
|
+
"category": "TestCaseDesign",
|
|
1689
|
+
"name": "Sleep step duration and frequency issues",
|
|
1690
|
+
"description": "Sleep steps with duration > 10 seconds cause unnecessary test execution time (major). Sleep inside loops multiplies wait time and indicates polling anti-pattern (minor). More than 5 sleeps per 50 steps indicates excessive waiting rather than proper synchronization (minor).",
|
|
1691
|
+
"appliesTo": [
|
|
1692
|
+
"Step"
|
|
1693
|
+
],
|
|
1694
|
+
"severity": "major",
|
|
1695
|
+
"weight": 5,
|
|
1696
|
+
"recommendation": "Replace long sleeps with explicit waits for conditions. Move sleeps outside loops and use proper wait-until-ready patterns. Replace multiple sleeps with single targeted waits. Use UI wait conditions, API polling with timeout, or event-driven synchronization.",
|
|
1697
|
+
"check": {
|
|
1698
|
+
"type": "controlSleep",
|
|
1699
|
+
"target": "step"
|
|
1700
|
+
},
|
|
1701
|
+
"source": "Provar Best Practices - Synchronization and Waits"
|
|
1702
|
+
},
|
|
1703
|
+
{
|
|
1704
|
+
"id": "CONTROL-WHILE-MAX-001",
|
|
1705
|
+
"category": "TestCaseDesign",
|
|
1706
|
+
"name": "While loop must have termination condition",
|
|
1707
|
+
"description": "While loops without counterEnd and without a terminating condition (<=, <, >=, >, ==, !=) can run infinitely if no termination is defined (major). While loops with condition='true' or '{true}' are infinite loops that rely solely on break statements, making them difficult to debug (minor). A loop is valid if it has either counterEnd OR a comparison condition.",
|
|
1708
|
+
"appliesTo": [
|
|
1709
|
+
"Step"
|
|
1710
|
+
],
|
|
1711
|
+
"severity": "major",
|
|
1712
|
+
"weight": 5,
|
|
1713
|
+
"recommendation": "Either set counterEnd to a reasonable limit (e.g., 10, 50, 100) OR use a terminating condition with a comparison operator (e.g., {Counter <= MaxIterations}). Avoid while{true} - use a meaningful condition instead. Add timeout handling and clear exit conditions. Consider using For loop with fixed iteration count if possible.",
|
|
1714
|
+
"check": {
|
|
1715
|
+
"type": "controlWhileMaxIterations",
|
|
1716
|
+
"target": "step"
|
|
1717
|
+
},
|
|
1718
|
+
"source": "Provar Best Practices - Loop Control and Safety"
|
|
1719
|
+
},
|
|
1720
|
+
{
|
|
1721
|
+
"id": "CONTROL-FINALLY-001",
|
|
1722
|
+
"category": "TestCaseDesign",
|
|
1723
|
+
"name": "Finally block must have description and be at end",
|
|
1724
|
+
"description": "Finally blocks without descriptions make it unclear what cleanup actions are performed (major). Finally blocks not at the end of the test may not execute if test stops before reaching them (minor). Finally is meant for cleanup that always runs regardless of test outcome.",
|
|
1725
|
+
"appliesTo": [
|
|
1726
|
+
"Step"
|
|
1727
|
+
],
|
|
1728
|
+
"severity": "major",
|
|
1729
|
+
"weight": 5,
|
|
1730
|
+
"recommendation": "Add clear description to Finally block explaining what cleanup actions are performed (e.g., 'Delete test data', 'Logout'). Move Finally block to the end of the test to ensure cleanup runs. Use Finally for critical cleanup like deleting test records or closing connections.",
|
|
1731
|
+
"check": {
|
|
1732
|
+
"type": "controlFinallyEnhanced",
|
|
1733
|
+
"target": "step"
|
|
1734
|
+
},
|
|
1735
|
+
"source": "Provar Best Practices - Exception Handling and Cleanup"
|
|
1736
|
+
},
|
|
1737
|
+
{
|
|
1738
|
+
"id": "SOQL-RESULT-001",
|
|
1739
|
+
"category": "TestCaseDesign",
|
|
1740
|
+
"name": "SOQL queries specify resultListName",
|
|
1741
|
+
"description": "SOQL queries must specify resultListName to capture query results.",
|
|
1742
|
+
"appliesTo": [
|
|
1743
|
+
"Step"
|
|
1744
|
+
],
|
|
1745
|
+
"severity": "critical",
|
|
1746
|
+
"weight": 8,
|
|
1747
|
+
"recommendation": "Add 'resultListName' argument to store query results.",
|
|
1748
|
+
"check": {
|
|
1749
|
+
"type": "soqlResultName",
|
|
1750
|
+
"target": "step",
|
|
1751
|
+
"apiId": "com.provar.plugins.forcedotcom.core.testapis.ApexSoqlQuery"
|
|
1752
|
+
},
|
|
1753
|
+
"source": "Provar Test Step Schema: SOQL Validation"
|
|
1754
|
+
},
|
|
1755
|
+
{
|
|
1756
|
+
"id": "DATA-TYPE-BOOL-001",
|
|
1757
|
+
"category": "TestCaseDesign",
|
|
1758
|
+
"name": "Boolean values are 'true' or 'false'",
|
|
1759
|
+
"description": "Boolean value elements must contain exactly 'true' or 'false' (lowercase).",
|
|
1760
|
+
"appliesTo": [
|
|
1761
|
+
"Step"
|
|
1762
|
+
],
|
|
1763
|
+
"severity": "critical",
|
|
1764
|
+
"weight": 8,
|
|
1765
|
+
"recommendation": "Set boolean value to 'true' or 'false' (not 'True', 'FALSE', '1', '0', etc.).",
|
|
1766
|
+
"check": {
|
|
1767
|
+
"type": "booleanValue",
|
|
1768
|
+
"target": "step"
|
|
1769
|
+
},
|
|
1770
|
+
"source": "Provar Test Step Schema: Data Type Validation"
|
|
1771
|
+
},
|
|
1772
|
+
{
|
|
1773
|
+
"id": "DATA-TYPE-NUMBER-001",
|
|
1774
|
+
"category": "TestCaseDesign",
|
|
1775
|
+
"name": "Numeric values are valid numbers",
|
|
1776
|
+
"description": "DISABLED: Rule generates false positives. Flags numeric-looking strings in SetValues without checking Salesforce field types. Many text fields (ZIP codes, IDs with leading zeros, year fields) legitimately store numeric strings.",
|
|
1777
|
+
"appliesTo": [
|
|
1778
|
+
"Step"
|
|
1779
|
+
],
|
|
1780
|
+
"severity": "info",
|
|
1781
|
+
"weight": 0,
|
|
1782
|
+
"recommendation": "Review if this value should be valueClass='decimal' for calculations, or valueClass='string' for text fields like IDs, ZIP codes, phone numbers. Consider field type in Salesforce.",
|
|
1783
|
+
"check": {
|
|
1784
|
+
"type": "numericValue",
|
|
1785
|
+
"target": "step"
|
|
1786
|
+
},
|
|
1787
|
+
"source": "Provar Test Step Schema: Data Type Validation (DISABLED due to false positives - needs field-type awareness)"
|
|
1788
|
+
},
|
|
1789
|
+
{
|
|
1790
|
+
"id": "SETVALUES-STRUCTURE-001",
|
|
1791
|
+
"category": "TestCaseDesign",
|
|
1792
|
+
"name": "SetValues steps have namedValues container",
|
|
1793
|
+
"description": "SetValues steps must have <namedValues> container element.",
|
|
1794
|
+
"appliesTo": [
|
|
1795
|
+
"Step"
|
|
1796
|
+
],
|
|
1797
|
+
"severity": "critical",
|
|
1798
|
+
"weight": 8,
|
|
1799
|
+
"recommendation": "Add <namedValues> container inside values argument.",
|
|
1800
|
+
"check": {
|
|
1801
|
+
"type": "setValuesStructure",
|
|
1802
|
+
"target": "step",
|
|
1803
|
+
"apiId": "com.provar.plugins.bundled.apis.control.SetValues"
|
|
1804
|
+
},
|
|
1805
|
+
"source": "Provar Test Step Schema: SetValues Validation"
|
|
1806
|
+
},
|
|
1807
|
+
{
|
|
1808
|
+
"id": "SETVALUES-NAME-001",
|
|
1809
|
+
"category": "NamingConventions",
|
|
1810
|
+
"name": "SetValues namedValue elements have name attribute",
|
|
1811
|
+
"description": "Each namedValue element must have a 'name' attribute defining the variable name.",
|
|
1812
|
+
"appliesTo": [
|
|
1813
|
+
"Step"
|
|
1814
|
+
],
|
|
1815
|
+
"severity": "critical",
|
|
1816
|
+
"weight": 8,
|
|
1817
|
+
"recommendation": "Add 'name' attribute to each <namedValue> element.",
|
|
1818
|
+
"check": {
|
|
1819
|
+
"type": "namedValueName",
|
|
1820
|
+
"target": "step",
|
|
1821
|
+
"apiId": "com.provar.plugins.bundled.apis.control.SetValues"
|
|
1822
|
+
},
|
|
1823
|
+
"source": "Provar Test Step Schema: SetValues Validation"
|
|
1824
|
+
},
|
|
1825
|
+
{
|
|
1826
|
+
"id": "SETVALUES-VALUE-001",
|
|
1827
|
+
"category": "TestCaseDesign",
|
|
1828
|
+
"name": "SetValues namedValue elements have value element",
|
|
1829
|
+
"description": "Each namedValue must contain a <value> element defining the variable value.",
|
|
1830
|
+
"appliesTo": [
|
|
1831
|
+
"Step"
|
|
1832
|
+
],
|
|
1833
|
+
"severity": "critical",
|
|
1834
|
+
"weight": 8,
|
|
1835
|
+
"recommendation": "Add <value> element inside each <namedValue>.",
|
|
1836
|
+
"check": {
|
|
1837
|
+
"type": "namedValueValue",
|
|
1838
|
+
"target": "step",
|
|
1839
|
+
"apiId": "com.provar.plugins.bundled.apis.control.SetValues"
|
|
1840
|
+
},
|
|
1841
|
+
"source": "Provar Test Step Schema: SetValues Validation"
|
|
1842
|
+
},
|
|
1843
|
+
{
|
|
1844
|
+
"id": "SETVALUES-INVALID-ELEMENT-001",
|
|
1845
|
+
"category": "StructureAndGrouping",
|
|
1846
|
+
"name": "SetValues must not contain invalid child elements",
|
|
1847
|
+
"description": "SetValues steps must use the correct structure with <namedValues> containing <namedValue> elements. AI models sometimes hallucinate invalid elements like <namedValueSet>, <name>, or nested <namedValues> incorrectly. The correct structure is: <namedValues mutable=\"Mutable\"><namedValue name=\"valuePath\">...</namedValue><namedValue name=\"value\">...</namedValue><namedValue name=\"valueScope\">...</namedValue></namedValues>",
|
|
1848
|
+
"appliesTo": [
|
|
1849
|
+
"Step"
|
|
1850
|
+
],
|
|
1851
|
+
"severity": "critical",
|
|
1852
|
+
"weight": 10,
|
|
1853
|
+
"recommendation": "Use correct SetValues structure: <value class=\"valueList\" mutable=\"Mutable\"><namedValues mutable=\"Mutable\"><namedValue name=\"valuePath\"><value class=\"value\" valueClass=\"string\">VariableName</value></namedValue><namedValue name=\"value\"><value>...</value></namedValue><namedValue name=\"valueScope\"><value class=\"value\" valueClass=\"string\">Test</value></namedValue></namedValues></value>. Each variable requires one <namedValues> block with three <namedValue> children: valuePath, value, and valueScope.",
|
|
1854
|
+
"check": {
|
|
1855
|
+
"type": "setValuesInvalidElements",
|
|
1856
|
+
"target": "step",
|
|
1857
|
+
"apiId": "com.provar.plugins.bundled.apis.control.SetValues",
|
|
1858
|
+
"invalidElements": ["namedValueSet", "name"]
|
|
1859
|
+
},
|
|
1860
|
+
"notes": "AI models hallucinate <namedValueSet> and <name> elements which are not valid in Provar SetValues. The correct child elements are: <namedValues> (container) and <namedValue> (with name attribute).",
|
|
1861
|
+
"source": "Field observation: SetValues structure hallucination pattern"
|
|
1862
|
+
},
|
|
1863
|
+
{
|
|
1864
|
+
"id": "VAR-PROPERTY-001",
|
|
1865
|
+
"category": "TestCaseDesign",
|
|
1866
|
+
"name": "Variable property references must be valid",
|
|
1867
|
+
"description": "Variable property references (<path element>) must reference valid properties. For SOQL query results, only fields in the SELECT clause are valid. Common hallucinations include: .size, .length, .count (use Count() function), duplicate path elements, and referencing fields not in the query.",
|
|
1868
|
+
"appliesTo": [
|
|
1869
|
+
"Step"
|
|
1870
|
+
],
|
|
1871
|
+
"severity": "major",
|
|
1872
|
+
"weight": 6,
|
|
1873
|
+
"recommendation": "For list size, use Count(variable) function. For properties, ensure the field is included in the SOQL SELECT clause or ApexReadObject generatedParameters. Remove duplicate path elements.",
|
|
1874
|
+
"check": {
|
|
1875
|
+
"type": "invalidVariableProperty",
|
|
1876
|
+
"target": "step",
|
|
1877
|
+
"invalidProperties": ["size", "length", "count", "empty", "isEmpty", "first", "last", "keys", "values"]
|
|
1878
|
+
},
|
|
1879
|
+
"notes": "AI models hallucinate properties that don't exist on Provar variables: .size/.length/.count (use Count() function), .empty (use Count()=0), .first/.last (use index notation). Also catches: duplicate path elements (VarName.VarName.Field), undefined properties not in SOQL SELECT or generatedParameters.",
|
|
1880
|
+
"source": "Field observation: Variable property hallucination patterns"
|
|
1881
|
+
},
|
|
1882
|
+
{
|
|
1883
|
+
"id": "CLEANUP-CONSISTENCY-001",
|
|
1884
|
+
"category": "TestCaseDesign",
|
|
1885
|
+
"name": "Manual cleanup matches object creation",
|
|
1886
|
+
"description": "When autoCleanup=false, test should have ApexDeleteObject steps matching ApexCreateObject steps.",
|
|
1887
|
+
"appliesTo": [
|
|
1888
|
+
"TestCase"
|
|
1889
|
+
],
|
|
1890
|
+
"severity": "major",
|
|
1891
|
+
"weight": 5,
|
|
1892
|
+
"recommendation": "Add ApexDeleteObject steps for created objects or set autoCleanup=true.",
|
|
1893
|
+
"check": {
|
|
1894
|
+
"type": "cleanupConsistency",
|
|
1895
|
+
"target": "testCase"
|
|
1896
|
+
},
|
|
1897
|
+
"source": "Provar Test Step Schema: AutoCleanup Validation"
|
|
1898
|
+
},
|
|
1899
|
+
{
|
|
1900
|
+
"id": "CLEANUP-ORDER-001",
|
|
1901
|
+
"category": "TestCaseDesign",
|
|
1902
|
+
"name": "Cleanup deletes objects in reverse order",
|
|
1903
|
+
"description": "When manually deleting objects, delete child objects before parent objects.",
|
|
1904
|
+
"appliesTo": [
|
|
1905
|
+
"TestCase"
|
|
1906
|
+
],
|
|
1907
|
+
"severity": "minor",
|
|
1908
|
+
"weight": 2,
|
|
1909
|
+
"recommendation": "Reorder ApexDeleteObject steps: delete child records before parent records.",
|
|
1910
|
+
"check": {
|
|
1911
|
+
"type": "cleanupOrder",
|
|
1912
|
+
"target": "testCase"
|
|
1913
|
+
},
|
|
1914
|
+
"source": "Provar Test Step Schema: AutoCleanup Best Practices"
|
|
1915
|
+
},
|
|
1916
|
+
{
|
|
1917
|
+
"id": "CREATE-RESULT-001",
|
|
1918
|
+
"category": "TestCaseDesign",
|
|
1919
|
+
"name": "ApexCreateObject steps specify resultIdName",
|
|
1920
|
+
"description": "ApexCreateObject should specify resultIdName to capture created record ID for later reference or cleanup.",
|
|
1921
|
+
"appliesTo": [
|
|
1922
|
+
"Step"
|
|
1923
|
+
],
|
|
1924
|
+
"severity": "major",
|
|
1925
|
+
"weight": 5,
|
|
1926
|
+
"recommendation": "Add 'resultIdName' argument to store created record ID (e.g. 'CreatedAccountId', 'OpportunityId').",
|
|
1927
|
+
"check": {
|
|
1928
|
+
"type": "createResultName",
|
|
1929
|
+
"target": "step",
|
|
1930
|
+
"apiId": "com.provar.plugins.forcedotcom.core.testapis.ApexCreateObject"
|
|
1931
|
+
},
|
|
1932
|
+
"source": "Provar Test Step Schema: Apex CRUD Best Practices"
|
|
1933
|
+
},
|
|
1934
|
+
{
|
|
1935
|
+
"id": "CALLABLE-VISIBILITY-001",
|
|
1936
|
+
"category": "ReusabilityAndCallables",
|
|
1937
|
+
"name": "Called test cases are marked as Callable",
|
|
1938
|
+
"description": "Test cases invoked via TestCaseCall must have visibility='Callable' attribute.",
|
|
1939
|
+
"appliesTo": [
|
|
1940
|
+
"Step"
|
|
1941
|
+
],
|
|
1942
|
+
"severity": "critical",
|
|
1943
|
+
"weight": 8,
|
|
1944
|
+
"recommendation": "Set visibility='Callable' on called test case or reference a callable test.",
|
|
1945
|
+
"check": {
|
|
1946
|
+
"type": "callableVisibility",
|
|
1947
|
+
"target": "step",
|
|
1948
|
+
"apiId": "com.provar.core.testapis.TestCaseCall"
|
|
1949
|
+
},
|
|
1950
|
+
"source": "Provar Test Step Schema: Callable Validation"
|
|
1951
|
+
},
|
|
1952
|
+
{
|
|
1953
|
+
"id": "LOG-LEVEL-001",
|
|
1954
|
+
"category": "TestCaseDesign",
|
|
1955
|
+
"name": "Log messages use appropriate log levels",
|
|
1956
|
+
"description": "Log steps should use appropriate log levels: INFO for general info, WARN for warnings, ERROR for errors.",
|
|
1957
|
+
"appliesTo": [
|
|
1958
|
+
"Step"
|
|
1959
|
+
],
|
|
1960
|
+
"severity": "info",
|
|
1961
|
+
"weight": 1,
|
|
1962
|
+
"recommendation": "Set logLevel argument to appropriate value: INFO, WARN, ERROR, or DEBUG.",
|
|
1963
|
+
"check": {
|
|
1964
|
+
"type": "logLevel",
|
|
1965
|
+
"target": "step",
|
|
1966
|
+
"apiId": "com.provar.plugins.bundled.apis.control.Log"
|
|
1967
|
+
},
|
|
1968
|
+
"source": "Provar Test Step Schema: Logging Best Practices"
|
|
1969
|
+
},
|
|
1970
|
+
{
|
|
1971
|
+
"id": "UI-ASSERT-TYPE-001",
|
|
1972
|
+
"category": "TestCaseDesign",
|
|
1973
|
+
"name": "UiAssert steps specify assertion type",
|
|
1974
|
+
"description": "UiAssert steps should specify assertionType (Visible, Value, Enabled, etc.) for clarity.",
|
|
1975
|
+
"appliesTo": [
|
|
1976
|
+
"Step"
|
|
1977
|
+
],
|
|
1978
|
+
"severity": "minor",
|
|
1979
|
+
"weight": 2,
|
|
1980
|
+
"recommendation": "Add 'assertionType' argument specifying the type of assertion.",
|
|
1981
|
+
"check": {
|
|
1982
|
+
"type": "uiAssertType",
|
|
1983
|
+
"target": "step",
|
|
1984
|
+
"apiId": "com.provar.plugins.forcedotcom.core.ui.UiAssert"
|
|
1985
|
+
},
|
|
1986
|
+
"source": "Provar Test Step Schema: UI Validation Best Practices"
|
|
1987
|
+
},
|
|
1988
|
+
{
|
|
1989
|
+
"id": "UI-DOACTION-VALUE-001",
|
|
1990
|
+
"category": "TestCaseDesign",
|
|
1991
|
+
"name": "UiDoAction Set requires value argument",
|
|
1992
|
+
"description": "UiDoAction steps with interactionType='Set' must have a value argument.",
|
|
1993
|
+
"appliesTo": [
|
|
1994
|
+
"Step"
|
|
1995
|
+
],
|
|
1996
|
+
"severity": "critical",
|
|
1997
|
+
"weight": 8,
|
|
1998
|
+
"recommendation": "Add 'value' argument when using Set interaction type.",
|
|
1999
|
+
"check": {
|
|
2000
|
+
"type": "uiDoActionValue",
|
|
2001
|
+
"target": "step",
|
|
2002
|
+
"apiId": "com.provar.plugins.forcedotcom.core.ui.UiDoAction"
|
|
2003
|
+
},
|
|
2004
|
+
"source": "Provar Test Step Schema: UI Interaction Validation"
|
|
2005
|
+
},
|
|
2006
|
+
{
|
|
2007
|
+
"id": "UI-LOCATOR-BINDING-001",
|
|
2008
|
+
"category": "TestCaseDesign",
|
|
2009
|
+
"name": "Ui locator built-in actions use object binding",
|
|
2010
|
+
"description": "For Salesforce built-in actions (New, Edit, Save, Convert, etc.), uiLocator URIs must use an object binding (sf:ui:binding:object?object=...&action=...). Using the action binding form (sf:ui:binding:action?actionName=...) is known to cause runtime execution failures in generated tests.",
|
|
2011
|
+
"appliesTo": [
|
|
2012
|
+
"Step"
|
|
2013
|
+
],
|
|
2014
|
+
"severity": "major",
|
|
2015
|
+
"weight": 5,
|
|
2016
|
+
"recommendation": "Use an object-binding locator like: ui:locator?name=<ActionName>&binding=sf%3Aui%3Abinding%3Aobject%3Fobject%3D<ObjectApiName>%26action%3D<ActionName> (replace object/action accordingly).",
|
|
2017
|
+
"check": {
|
|
2018
|
+
"type": "uiLocatorBinding",
|
|
2019
|
+
"target": "step",
|
|
2020
|
+
"apiId": "com.provar.plugins.forcedotcom.core.ui.UiDoAction"
|
|
2021
|
+
},
|
|
2022
|
+
"source": "Field observation: uiLocator bindings using action?actionName are unreliable; prefer object binding for built-in actions"
|
|
2023
|
+
},
|
|
2024
|
+
{
|
|
2025
|
+
"id": "UI-ASSERT-FIELDLOCATOR-001",
|
|
2026
|
+
"category": "TestCaseDesign",
|
|
2027
|
+
"name": "UiAssert fieldLocator uses object+field binding",
|
|
2028
|
+
"description": "UiAssert embedded field assertions (uiFieldAssertion) must use a fieldLocator with an object binding (sf:ui:binding:object?field=<Field>&object=<Object>) and a locator name matching the field (name=<Field>). Incorrect fieldLocator bindings can cause runtime failures or assertions targeting the wrong element.",
|
|
2029
|
+
"appliesTo": [
|
|
2030
|
+
"Step"
|
|
2031
|
+
],
|
|
2032
|
+
"severity": "major",
|
|
2033
|
+
"weight": 5,
|
|
2034
|
+
"recommendation": "Use one of: (1) sf:ui:locator?name=<FieldApiName>&binding=sf%3Aui%3Abinding%3Aobject%3Ffield%3D<FieldApiName>%26object%3D<ObjectApiName> (ensure name and field match), (2) ui:pageobject:field?field=<elementName>&pageId=pageobjects.<PageObjectClass>, or (3) ui:locator?name=<fieldName> for Page Object locator references.",
|
|
2035
|
+
"check": {
|
|
2036
|
+
"type": "uiAssertFieldLocatorBinding",
|
|
2037
|
+
"target": "step",
|
|
2038
|
+
"apiId": "com.provar.plugins.forcedotcom.core.ui.UiAssert"
|
|
2039
|
+
},
|
|
2040
|
+
"source": "Field observation: UiAssert fieldLocator must use valid binding pattern (SF metadata, Page Object, or locator reference)"
|
|
2041
|
+
},
|
|
2042
|
+
{
|
|
2043
|
+
"id": "UI-ASSERT-FIELDLOCATOR-002",
|
|
2044
|
+
"category": "TestCaseDesign",
|
|
2045
|
+
"name": "UiAssert fieldAssertion must not wrap fieldLocator in uiLocator",
|
|
2046
|
+
"description": "UiAssert embedded field assertions (uiFieldAssertion) must use fieldLocator with a uri attribute directly. DO NOT wrap the locator in a nested <uiLocator/> element. The correct format is <fieldLocator uri=\"...\"/> not <fieldLocator><uiLocator uri=\"...\"/></fieldLocator>. This incorrect nesting causes runtime failures.",
|
|
2047
|
+
"appliesTo": [
|
|
2048
|
+
"Step"
|
|
2049
|
+
],
|
|
2050
|
+
"severity": "major",
|
|
2051
|
+
"weight": 5,
|
|
2052
|
+
"recommendation": "Use <fieldLocator uri=\"ui:locator?name=FieldApiName&binding=sf%3Aui%3Abinding%3Aobject%3Ffield%3DFieldApiName%26object%3DObjectApiName\"/> directly without wrapping in <uiLocator/>.",
|
|
2053
|
+
"check": {
|
|
2054
|
+
"type": "uiAssertFieldLocatorStructure",
|
|
2055
|
+
"target": "step",
|
|
2056
|
+
"apiId": "com.provar.plugins.forcedotcom.core.ui.UiAssert"
|
|
2057
|
+
},
|
|
2058
|
+
"source": "Field observation: UiAssert fieldLocator must be direct uri attribute, not nested uiLocator element"
|
|
2059
|
+
},
|
|
2060
|
+
{
|
|
2061
|
+
"id": "UI-ASSERT-FIELDLOCATOR-003",
|
|
2062
|
+
"category": "TestCaseDesign",
|
|
2063
|
+
"name": "UiAssert bare locator in Salesforce metadata context causes render failure",
|
|
2064
|
+
"description": "UiAssert uiFieldAssertion using a bare locator reference (ui:locator?name=X without a binding parameter) inside a UiWithScreen whose target is a Salesforce metadata context (sf:ui:target?object=...) will fail to render in Provar Automation. Bare locators are only valid inside Page Object contexts. In Salesforce metadata contexts, the fieldLocator must include a full binding (ui:locator?name=X&binding=sf%3Aui%3Abinding%3Aobject%3Fobject%3D...%26field%3D...). This commonly occurs when asserting button visibility (e.g., Run, Edit, Delete buttons) as uiFieldAssertions — buttons are not Salesforce fields and cannot be asserted this way.",
|
|
2065
|
+
"appliesTo": [
|
|
2066
|
+
"Step"
|
|
2067
|
+
],
|
|
2068
|
+
"severity": "critical",
|
|
2069
|
+
"weight": 10,
|
|
2070
|
+
"recommendation": "Remove the bare-locator uiFieldAssertion for non-field elements (buttons, actions). If you need to verify a button exists, use a UiDoAction click interaction instead of a UiAssert fieldAssertion. For Salesforce field assertions, always include the full binding: <fieldLocator uri=\"ui:locator?name=FieldApiName&binding=sf%3Aui%3Abinding%3Aobject%3Fobject%3DObjectApiName%26field%3DFieldApiName\"/>. NEVER use bare ui:locator?name=X without a binding inside a Salesforce metadata UiWithScreen.",
|
|
2071
|
+
"check": {
|
|
2072
|
+
"type": "uiAssertBareLocatorInSfContext",
|
|
2073
|
+
"target": "step",
|
|
2074
|
+
"apiId": "com.provar.plugins.forcedotcom.core.ui.UiAssert"
|
|
2075
|
+
},
|
|
2076
|
+
"source": "Field observation: Bare ui:locator?name=X in Salesforce metadata UiWithScreen causes Provar Automation render failure (CRITICAL)"
|
|
2077
|
+
},
|
|
2078
|
+
{
|
|
2079
|
+
"id": "UI-LOCATOR-ACTION-001",
|
|
2080
|
+
"category": "TestCaseDesign",
|
|
2081
|
+
"name": "UiDoAction locator URIs must use valid patterns",
|
|
2082
|
+
"description": "Validates all UiDoAction locator URI patterns including: (1) SF locators: ui:locator?name=...&binding=sf:ui:binding:object?... with required name and binding params, (2) Page Object Fields: ui:pageobject:field?field=...&pageId=pageobjects... with required field and pageId params, (3) Page Object Methods: ui:pageobject:method?pageId=...&operation=... with required pageId and operation params, (4) Lead Conversion context-aware patterns (Convert on View vs submitConvert in Dialog). Detects malformed URIs like '%3Action' which should be '%3Faction'.",
|
|
2083
|
+
"appliesTo": [
|
|
2084
|
+
"Step"
|
|
2085
|
+
],
|
|
2086
|
+
"severity": "major",
|
|
2087
|
+
"weight": 5,
|
|
2088
|
+
"recommendation": "SF locators: ui:locator?name=<Name>&binding=sf%3Aui%3Abinding%3Aobject%3F<params>. Page Object Fields: ui:pageobject:field?field=<elementName>&pageId=pageobjects.<Class>. Page Object Methods: ui:pageobject:method?pageId=pageobjects.<Class>&operation=<methodName>. Ensure '%3F' (?) precedes query params in bindings, not '%3A' (colon).",
|
|
2089
|
+
"check": {
|
|
2090
|
+
"type": "uiLocatorActionBinding",
|
|
2091
|
+
"target": "step",
|
|
2092
|
+
"apiId": "com.provar.plugins.forcedotcom.core.ui.UiDoAction"
|
|
2093
|
+
},
|
|
2094
|
+
"notes": "Validates: (1) SF locator structure (name, binding params), (2) Page Object Field locators (field, pageId params), (3) Page Object Method locators (pageId, operation params), (4) Lead Conversion context patterns, (5) Malformed URI detection (%3Action→%3Faction).",
|
|
2095
|
+
"source": "Field observation: UiDoAction locators require specific URI formats for proper Provar UI rendering"
|
|
2096
|
+
},
|
|
2097
|
+
{
|
|
2098
|
+
"id": "UI-SCREEN-TARGET-001",
|
|
2099
|
+
"category": "TestCaseDesign",
|
|
2100
|
+
"name": "UiWithScreen target URIs must use valid patterns",
|
|
2101
|
+
"description": "Validates all UiWithScreen target URI patterns including: (1) SF targets with valid parameters (object/action, lightningComponent, lightningWebComponent, auraComponent, application/tab, lookup, fieldService, action-only), (2) Page Object targets: ui:pageobject:target?pageId=pageobjects... with required pageId param starting with 'pageobjects.' prefix.",
|
|
2102
|
+
"appliesTo": [
|
|
2103
|
+
"Step"
|
|
2104
|
+
],
|
|
2105
|
+
"severity": "major",
|
|
2106
|
+
"weight": 5,
|
|
2107
|
+
"recommendation": "SF Object-Action targets: sf:ui:target?object=<Object>&action=<Action>. SF Lightning targets: sf:ui:target?lightningComponent=<Component>. Page Object targets: ui:pageobject:target?pageId=pageobjects.<PageObjectClass>.",
|
|
2108
|
+
"check": {
|
|
2109
|
+
"type": "uiWithScreenTarget",
|
|
2110
|
+
"target": "step",
|
|
2111
|
+
"apiId": "com.provar.plugins.forcedotcom.core.ui.UiWithScreen"
|
|
2112
|
+
},
|
|
2113
|
+
"notes": "Validates: (1) SF target has at least one valid param (object, action, lightningComponent, lightningWebComponent, auraComponent, application, lookup, fieldService), (2) Page Object target has pageId with 'pageobjects.' prefix.",
|
|
2114
|
+
"source": "Field observation: UiWithScreen targets require specific URI formats for proper Provar UI rendering"
|
|
2115
|
+
},
|
|
2116
|
+
{
|
|
2117
|
+
"id": "APEX-REUSE-CONN-001",
|
|
2118
|
+
"category": "TestCaseDesign",
|
|
2119
|
+
"name": "ApexConnect reuseConnectionName should be left blank",
|
|
2120
|
+
"description": "The reuseConnectionName argument in ApexConnect steps should be left empty/blank. AI models may hallucinate invalid values like 'Reuse' which cause runtime errors.",
|
|
2121
|
+
"appliesTo": [
|
|
2122
|
+
"Step"
|
|
2123
|
+
],
|
|
2124
|
+
"severity": "major",
|
|
2125
|
+
"weight": 5,
|
|
2126
|
+
"recommendation": "Leave reuseConnectionName blank: <argument id=\"reuseConnectionName\"/> or <argument id=\"reuseConnectionName\"><value class=\"value\"/></argument>. Do not use string values.",
|
|
2127
|
+
"check": {
|
|
2128
|
+
"type": "apexConnectReuseConnection",
|
|
2129
|
+
"target": "step",
|
|
2130
|
+
"apiId": "com.provar.plugins.forcedotcom.core.testapis.ApexConnect"
|
|
2131
|
+
},
|
|
2132
|
+
"notes": "Corpus analysis shows reuseConnectionName is always empty in valid test cases. Non-empty values are AI hallucinations.",
|
|
2133
|
+
"source": "Field observation: ApexConnect reuseConnectionName hallucination pattern"
|
|
2134
|
+
},
|
|
2135
|
+
{
|
|
2136
|
+
"id": "APEX-CONNECT-ARGS-001",
|
|
2137
|
+
"category": "XMLSchema",
|
|
2138
|
+
"name": "ApexConnect - Only valid argument IDs allowed",
|
|
2139
|
+
"description": "ApexConnect steps must use ONLY the 21 valid argument IDs defined in Provar schema. AI models commonly hallucinate plausible-sounding but invalid arguments like autoPopulateRequiredFields, assertObjectFieldsPopulated, commandTimeout which cause Provar to reject the test case.",
|
|
2140
|
+
"appliesTo": [
|
|
2141
|
+
"Step"
|
|
2142
|
+
],
|
|
2143
|
+
"severity": "critical",
|
|
2144
|
+
"weight": 10,
|
|
2145
|
+
"recommendation": "Only use these 21 valid ApexConnect argument IDs: connectionName, resultName, resultScope, uiApplicationName, quickUiLogin, closeAllPrimaryTabs, reuseConnectionName, alreadyOpenBehaviour, autoCleanup, cleanupConnectionName, logFileLocation, connectionId, enableObjectIdLogging, privateBrowsingMode, lightningMode, username, password, securityToken, environment, webBrowser. If you don't have a value, leave the argument empty: <argument id=\"username\"/> rather than inventing values.",
|
|
2146
|
+
"check": {
|
|
2147
|
+
"type": "apexConnectValidArguments",
|
|
2148
|
+
"target": "step",
|
|
2149
|
+
"apiId": "com.provar.plugins.forcedotcom.core.testapis.ApexConnect",
|
|
2150
|
+
"validArgumentIds": [
|
|
2151
|
+
"connectionName",
|
|
2152
|
+
"resultName",
|
|
2153
|
+
"resultScope",
|
|
2154
|
+
"uiApplicationName",
|
|
2155
|
+
"quickUiLogin",
|
|
2156
|
+
"closeAllPrimaryTabs",
|
|
2157
|
+
"reuseConnectionName",
|
|
2158
|
+
"alreadyOpenBehaviour",
|
|
2159
|
+
"autoCleanup",
|
|
2160
|
+
"cleanupConnectionName",
|
|
2161
|
+
"logFileLocation",
|
|
2162
|
+
"connectionId",
|
|
2163
|
+
"enableObjectIdLogging",
|
|
2164
|
+
"privateBrowsingMode",
|
|
2165
|
+
"lightningMode",
|
|
2166
|
+
"username",
|
|
2167
|
+
"password",
|
|
2168
|
+
"securityToken",
|
|
2169
|
+
"environment",
|
|
2170
|
+
"webBrowser"
|
|
2171
|
+
]
|
|
2172
|
+
},
|
|
2173
|
+
"notes": "Provar performs strict schema validation and will reject test cases with unknown argument IDs. Common hallucinated arguments: autoPopulateRequiredFields, assertObjectFieldsPopulated, commandTimeout, autoRetry, timeout, connectionTimeout.",
|
|
2174
|
+
"source": "Provar XML Schema: ApexConnect Argument Whitelist"
|
|
2175
|
+
},
|
|
2176
|
+
{
|
|
2177
|
+
"id": "APEX-CONNECT-CONNID-001",
|
|
2178
|
+
"category": "XMLSchema",
|
|
2179
|
+
"name": "ApexConnect connectionId must use valueClass='id'",
|
|
2180
|
+
"description": "The connectionId argument in ApexConnect steps MUST use valueClass='id' with a GUID format value (xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx), NOT valueClass='string'. Using valueClass='string' breaks connection references in Provar.",
|
|
2181
|
+
"appliesTo": [
|
|
2182
|
+
"Step"
|
|
2183
|
+
],
|
|
2184
|
+
"severity": "critical",
|
|
2185
|
+
"weight": 10,
|
|
2186
|
+
"recommendation": "Use correct format: <argument id=\"connectionId\"><value class=\"value\" valueClass=\"id\">bce7fd3f-0f81-4c5c-ab68-c3edd44b5d1e</value></argument>. NEVER use valueClass=\"string\" or value=\"default\". If you don't have a specific GUID, leave the argument empty: <argument id=\"connectionId\"/>",
|
|
2187
|
+
"check": {
|
|
2188
|
+
"type": "apexConnectConnectionIdValueClass",
|
|
2189
|
+
"target": "step",
|
|
2190
|
+
"apiId": "com.provar.plugins.forcedotcom.core.testapis.ApexConnect"
|
|
2191
|
+
},
|
|
2192
|
+
"notes": "AI models commonly use valueClass='string' with value='default' which is invalid. connectionId must reference a valid Provar connection using its GUID identifier with valueClass='id'.",
|
|
2193
|
+
"source": "Provar XML Schema: connectionId Type Constraint"
|
|
2194
|
+
},
|
|
2195
|
+
{
|
|
2196
|
+
"id": "UI-WAIT-VALUECLASS-001",
|
|
2197
|
+
"category": "TestCaseDesign",
|
|
2198
|
+
"name": "Wait arguments must use uiWait value class",
|
|
2199
|
+
"description": "The beforeWait, afterWait, and autoRetry arguments in UiDoAction and UiAssert steps must use class=\"uiWait\" with a uri attribute. Using valueClass=\"boolean\" is incorrect and causes runtime errors.",
|
|
2200
|
+
"appliesTo": [
|
|
2201
|
+
"Step"
|
|
2202
|
+
],
|
|
2203
|
+
"severity": "major",
|
|
2204
|
+
"weight": 5,
|
|
2205
|
+
"recommendation": "Use uiWait class with uri: <value class=\"uiWait\" uri=\"default\"/> or <value class=\"uiWait\" uri=\"ui:wait:timed?seconds=5\"/>. Valid uri patterns: default, NoWait, ui:wait:timed?seconds=N, ui:wait:autoRetry:timeout=N, ui:wait:auraBusy?timeout=N, ui:wait:pageload?timeout=N.",
|
|
2206
|
+
"check": {
|
|
2207
|
+
"type": "uiWaitValueClass",
|
|
2208
|
+
"target": "step",
|
|
2209
|
+
"apiId": "com.provar.plugins.forcedotcom.core.ui.UiDoAction"
|
|
2210
|
+
},
|
|
2211
|
+
"notes": "AI models sometimes hallucinate boolean values for wait arguments. The correct pattern is class=\"uiWait\" with uri attribute.",
|
|
2212
|
+
"source": "Field observation: Wait argument valueClass hallucination pattern"
|
|
2213
|
+
},
|
|
2214
|
+
{
|
|
2215
|
+
"id": "UI-LOCATOR-SAVE-001",
|
|
2216
|
+
"category": "TestCaseDesign",
|
|
2217
|
+
"name": "Save button locator must use correct pattern",
|
|
2218
|
+
"description": "Save button locators in UiDoAction steps must use lowercase 'save' and correct URI parameter order. Common error: using 'Save' (capital S) with action before object, or using %3A instead of %3F after object?",
|
|
2219
|
+
"appliesTo": [
|
|
2220
|
+
"Step"
|
|
2221
|
+
],
|
|
2222
|
+
"severity": "major",
|
|
2223
|
+
"weight": 5,
|
|
2224
|
+
"recommendation": "Use correct pattern: name=save (lowercase), binding=sf%3Aui%3Abinding%3Aobject%3Fobject%3D<Object>%26action%3Dsave. Common mistakes: (1) Capital S in 'Save', (2) action before object in binding, (3) using %3Action instead of %3Faction, (4) using %3A instead of %3F after 'object?'",
|
|
2225
|
+
"check": {
|
|
2226
|
+
"type": "uiLocatorSaveButton",
|
|
2227
|
+
"target": "step",
|
|
2228
|
+
"apiId": "com.provar.plugins.forcedotcom.core.ui.UiDoAction"
|
|
2229
|
+
},
|
|
2230
|
+
"notes": "Corpus analysis shows Save button should use lowercase 'save' with object parameter before action parameter. Pattern: object?object=<Type>&action=save",
|
|
2231
|
+
"source": "Field observation: Save button locator pattern"
|
|
2232
|
+
},
|
|
2233
|
+
{
|
|
2234
|
+
"id": "UI-TARGET-ACTION-001",
|
|
2235
|
+
"category": "TestCaseDesign",
|
|
2236
|
+
"name": "UiWithScreen target uses invalid action value",
|
|
2237
|
+
"description": "The action parameter in sf:ui:target URIs must use valid Provar action names. AI models commonly hallucinate 'action=Home' when the correct value is 'action=ObjectHome'.",
|
|
2238
|
+
"appliesTo": [
|
|
2239
|
+
"Step"
|
|
2240
|
+
],
|
|
2241
|
+
"severity": "major",
|
|
2242
|
+
"weight": 5,
|
|
2243
|
+
"recommendation": "Use valid action values: ObjectHome (for object list/home), New, View, Edit, List, Convert, GlobalSearch, QuickAction, etc. Common fixes: Home→ObjectHome, Create→New, Update→Edit, Details→View.",
|
|
2244
|
+
"check": {
|
|
2245
|
+
"type": "uiTargetActionValue",
|
|
2246
|
+
"target": "step",
|
|
2247
|
+
"apiId": "com.provar.plugins.forcedotcom.core.ui.UiWithScreen"
|
|
2248
|
+
},
|
|
2249
|
+
"notes": "Corpus analysis shows valid actions: ObjectHome, New, View, Edit, List, Convert, GlobalSearch, QuickAction, ChangeStatus, Search, recordTypeNew, attachFile, addProd, ownershipTransfer, PathAssistant, piSubmit, OppContactRoleMultiAdd, ActivityTimeline. 'Home' alone is never valid - always use 'ObjectHome'.",
|
|
2250
|
+
"source": "Field observation: action=Home hallucination pattern"
|
|
2251
|
+
},
|
|
2252
|
+
{
|
|
2253
|
+
"id": "CONTROL-SLEEP-001",
|
|
2254
|
+
"category": "TestCaseDesign",
|
|
2255
|
+
"name": "Sleep duration should be under 5 seconds",
|
|
2256
|
+
"description": "Fixed sleep durations should be kept short (<5 seconds). Prefer WaitFor with condition over fixed Sleep.",
|
|
2257
|
+
"appliesTo": [
|
|
2258
|
+
"Step"
|
|
2259
|
+
],
|
|
2260
|
+
"severity": "info",
|
|
2261
|
+
"weight": 1,
|
|
2262
|
+
"recommendation": "Use WaitFor with condition instead of Sleep, or reduce sleep duration to <5 seconds.",
|
|
2263
|
+
"check": {
|
|
2264
|
+
"type": "sleepDuration",
|
|
2265
|
+
"target": "step",
|
|
2266
|
+
"apiId": "com.provar.plugins.bundled.apis.control.Sleep",
|
|
2267
|
+
"maxDuration": 5
|
|
2268
|
+
},
|
|
2269
|
+
"source": "Provar Best Practices: Prefer conditional waits over fixed sleeps"
|
|
2270
|
+
},
|
|
2271
|
+
{
|
|
2272
|
+
"id": "CONTROL-FINALLY-001",
|
|
2273
|
+
"category": "StructureAndGrouping",
|
|
2274
|
+
"name": "Finally block should be at end of test",
|
|
2275
|
+
"description": "Finally blocks should be placed at the end of the test to ensure cleanup always executes.",
|
|
2276
|
+
"appliesTo": [
|
|
2277
|
+
"Step"
|
|
2278
|
+
],
|
|
2279
|
+
"severity": "major",
|
|
2280
|
+
"weight": 5,
|
|
2281
|
+
"recommendation": "Move Finally block to end of test case.",
|
|
2282
|
+
"check": {
|
|
2283
|
+
"type": "finallyPlacement",
|
|
2284
|
+
"target": "step",
|
|
2285
|
+
"apiId": "com.provar.plugins.bundled.apis.control.Finally"
|
|
2286
|
+
},
|
|
2287
|
+
"source": "Provar Best Practices: Finally blocks for reliable cleanup"
|
|
2288
|
+
},
|
|
2289
|
+
{
|
|
2290
|
+
"id": "BDD-GIVEN-FIRST-001",
|
|
2291
|
+
"category": "StructureAndGrouping",
|
|
2292
|
+
"name": "BDD scenario should start with Given",
|
|
2293
|
+
"description": "BDD test cases should start with Given step to establish preconditions.",
|
|
2294
|
+
"appliesTo": [
|
|
2295
|
+
"TestCase"
|
|
2296
|
+
],
|
|
2297
|
+
"severity": "info",
|
|
2298
|
+
"weight": 1,
|
|
2299
|
+
"recommendation": "Start BDD scenario with Given step.",
|
|
2300
|
+
"check": {
|
|
2301
|
+
"type": "bddGivenFirst",
|
|
2302
|
+
"target": "testCase"
|
|
2303
|
+
},
|
|
2304
|
+
"source": "BDD Best Practices: Given-When-Then structure"
|
|
2305
|
+
},
|
|
2306
|
+
{
|
|
2307
|
+
"id": "BDD-ORDER-001",
|
|
2308
|
+
"category": "StructureAndGrouping",
|
|
2309
|
+
"name": "BDD steps should follow logical order",
|
|
2310
|
+
"description": "BDD steps should follow Given->When->Then order (with And/But as connectors).",
|
|
2311
|
+
"appliesTo": [
|
|
2312
|
+
"TestCase"
|
|
2313
|
+
],
|
|
2314
|
+
"severity": "info",
|
|
2315
|
+
"weight": 1,
|
|
2316
|
+
"recommendation": "Reorder BDD steps to follow Given->When->Then pattern.",
|
|
2317
|
+
"check": {
|
|
2318
|
+
"type": "bddOrder",
|
|
2319
|
+
"target": "testCase"
|
|
2320
|
+
},
|
|
2321
|
+
"source": "BDD Best Practices: Scenario structure"
|
|
2322
|
+
},
|
|
2323
|
+
{
|
|
2324
|
+
"id": "BDD-AND-LIMIT-001",
|
|
2325
|
+
"category": "StructureAndGrouping",
|
|
2326
|
+
"name": "Limit And/But chain length",
|
|
2327
|
+
"description": "Chains of And/But steps should be limited to 3-5 for readability.",
|
|
2328
|
+
"appliesTo": [
|
|
2329
|
+
"TestCase"
|
|
2330
|
+
],
|
|
2331
|
+
"severity": "info",
|
|
2332
|
+
"weight": 1,
|
|
2333
|
+
"recommendation": "Break long And/But chains into separate scenarios.",
|
|
2334
|
+
"check": {
|
|
2335
|
+
"type": "bddAndChainLength",
|
|
2336
|
+
"target": "testCase",
|
|
2337
|
+
"maxChainLength": 5
|
|
2338
|
+
},
|
|
2339
|
+
"source": "BDD Best Practices: Readability"
|
|
2340
|
+
},
|
|
2341
|
+
{
|
|
2342
|
+
"id": "APEX-EXECUTE-SYNTAX-001",
|
|
2343
|
+
"category": "TestCaseDesign",
|
|
2344
|
+
"name": "ApexExecute code should be valid Apex syntax",
|
|
2345
|
+
"description": "Anonymous Apex code blocks should be syntactically valid.",
|
|
2346
|
+
"appliesTo": [
|
|
2347
|
+
"Step"
|
|
2348
|
+
],
|
|
2349
|
+
"severity": "critical",
|
|
2350
|
+
"weight": 8,
|
|
2351
|
+
"recommendation": "Validate Apex syntax before execution. Test in Developer Console first.",
|
|
2352
|
+
"check": {
|
|
2353
|
+
"type": "apexSyntax",
|
|
2354
|
+
"target": "step",
|
|
2355
|
+
"apiId": "com.provar.plugins.forcedotcom.core.testapis.ApexExecute"
|
|
2356
|
+
},
|
|
2357
|
+
"source": "Salesforce Development Best Practices"
|
|
2358
|
+
},
|
|
2359
|
+
{
|
|
2360
|
+
"id": "APEX-BULK-LIMIT-001",
|
|
2361
|
+
"category": "TestCaseDesign",
|
|
2362
|
+
"name": "ApexBulk should be used for large data volumes",
|
|
2363
|
+
"description": "Use ApexBulk for operations with more than 200 records.",
|
|
2364
|
+
"appliesTo": [
|
|
2365
|
+
"Step"
|
|
2366
|
+
],
|
|
2367
|
+
"severity": "info",
|
|
2368
|
+
"weight": 1,
|
|
2369
|
+
"recommendation": "Use ApexBulk instead of individual CRUD operations for >200 records.",
|
|
2370
|
+
"check": {
|
|
2371
|
+
"type": "apexBulkUsage",
|
|
2372
|
+
"target": "testCase"
|
|
2373
|
+
},
|
|
2374
|
+
"source": "Salesforce API Best Practices: Bulk operations"
|
|
2375
|
+
},
|
|
2376
|
+
{
|
|
2377
|
+
"id": "UI-NAVIGATE-PREFER-SCREEN-001",
|
|
2378
|
+
"category": "TestCaseDesign",
|
|
2379
|
+
"name": "Prefer UiWithScreen over UiNavigate for Salesforce",
|
|
2380
|
+
"description": "For Salesforce navigation, prefer UiWithScreen over direct URL navigation with UiNavigate.",
|
|
2381
|
+
"appliesTo": [
|
|
2382
|
+
"Step"
|
|
2383
|
+
],
|
|
2384
|
+
"severity": "info",
|
|
2385
|
+
"weight": 1,
|
|
2386
|
+
"recommendation": "Use UiWithScreen for Salesforce screens instead of UiNavigate.",
|
|
2387
|
+
"check": {
|
|
2388
|
+
"type": "uiNavigateUsage",
|
|
2389
|
+
"target": "step",
|
|
2390
|
+
"apiId": "com.provar.plugins.forcedotcom.core.ui.UiNavigate"
|
|
2391
|
+
},
|
|
2392
|
+
"source": "Provar UI Testing Best Practices"
|
|
2393
|
+
},
|
|
2394
|
+
{
|
|
2395
|
+
"id": "UI-FILL-VERIFY-001",
|
|
2396
|
+
"category": "TestCaseDesign",
|
|
2397
|
+
"name": "Verify fields after UiFill",
|
|
2398
|
+
"description": "UiFill should be followed by UiAssert to verify all fields were filled correctly.",
|
|
2399
|
+
"appliesTo": [
|
|
2400
|
+
"Step"
|
|
2401
|
+
],
|
|
2402
|
+
"severity": "info",
|
|
2403
|
+
"weight": 1,
|
|
2404
|
+
"recommendation": "Add UiAssert steps after UiFill to verify field values.",
|
|
2405
|
+
"check": {
|
|
2406
|
+
"type": "uiFillVerification",
|
|
2407
|
+
"target": "step",
|
|
2408
|
+
"apiId": "com.provar.plugins.forcedotcom.core.ui.UiFill"
|
|
2409
|
+
},
|
|
2410
|
+
"source": "Provar UI Testing Best Practices"
|
|
2411
|
+
},
|
|
2412
|
+
{
|
|
2413
|
+
"id": "UI-ALERT-HANDLE-001",
|
|
2414
|
+
"category": "TestCaseDesign",
|
|
2415
|
+
"name": "UiHandleAlert should capture alert text",
|
|
2416
|
+
"description": "When handling alerts with GetText action, store the text in a result variable.",
|
|
2417
|
+
"appliesTo": [
|
|
2418
|
+
"Step"
|
|
2419
|
+
],
|
|
2420
|
+
"severity": "info",
|
|
2421
|
+
"weight": 1,
|
|
2422
|
+
"recommendation": "Set resultName when using GetText action on UiHandleAlert.",
|
|
2423
|
+
"check": {
|
|
2424
|
+
"type": "uiHandleAlertResult",
|
|
2425
|
+
"target": "step",
|
|
2426
|
+
"apiId": "com.provar.plugins.forcedotcom.core.ui.UiHandleAlert"
|
|
2427
|
+
},
|
|
2428
|
+
"source": "Provar UI Testing Best Practices"
|
|
2429
|
+
},
|
|
2430
|
+
{
|
|
2431
|
+
"id": "UTIL-REPLACE-EMPTY-001",
|
|
2432
|
+
"category": "TestCaseDesign",
|
|
2433
|
+
"name": "Replace searchString should not be empty",
|
|
2434
|
+
"description": "Replace step searchString parameter should not be empty string.",
|
|
2435
|
+
"appliesTo": [
|
|
2436
|
+
"Step"
|
|
2437
|
+
],
|
|
2438
|
+
"severity": "major",
|
|
2439
|
+
"weight": 5,
|
|
2440
|
+
"recommendation": "Provide non-empty searchString for Replace operation.",
|
|
2441
|
+
"check": {
|
|
2442
|
+
"type": "replaceSearchString",
|
|
2443
|
+
"target": "step",
|
|
2444
|
+
"apiId": "com.provar.plugins.bundled.apis.string.Replace"
|
|
2445
|
+
},
|
|
2446
|
+
"source": "Utility Operations Best Practices"
|
|
2447
|
+
},
|
|
2448
|
+
{
|
|
2449
|
+
"id": "UTIL-SPLIT-DELIMITER-001",
|
|
2450
|
+
"category": "TestCaseDesign",
|
|
2451
|
+
"name": "Split delimiter should not be empty",
|
|
2452
|
+
"description": "Split step delimiter parameter should not be empty string.",
|
|
2453
|
+
"appliesTo": [
|
|
2454
|
+
"Step"
|
|
2455
|
+
],
|
|
2456
|
+
"severity": "major",
|
|
2457
|
+
"weight": 5,
|
|
2458
|
+
"recommendation": "Provide non-empty delimiter for Split operation.",
|
|
2459
|
+
"check": {
|
|
2460
|
+
"type": "splitDelimiter",
|
|
2461
|
+
"target": "step",
|
|
2462
|
+
"apiId": "com.provar.plugins.bundled.apis.string.Split"
|
|
2463
|
+
},
|
|
2464
|
+
"source": "Utility Operations Best Practices"
|
|
2465
|
+
},
|
|
2466
|
+
{
|
|
2467
|
+
"id": "UTIL-MATCH-REGEX-001",
|
|
2468
|
+
"category": "TestCaseDesign",
|
|
2469
|
+
"name": "Match regex pattern should be valid",
|
|
2470
|
+
"description": "When using isRegex=true, pattern must be valid regular expression.",
|
|
2471
|
+
"appliesTo": [
|
|
2472
|
+
"Step"
|
|
2473
|
+
],
|
|
2474
|
+
"severity": "critical",
|
|
2475
|
+
"weight": 8,
|
|
2476
|
+
"recommendation": "Validate regex pattern syntax before using in Match step.",
|
|
2477
|
+
"check": {
|
|
2478
|
+
"type": "matchRegexValid",
|
|
2479
|
+
"target": "step",
|
|
2480
|
+
"apiId": "com.provar.plugins.bundled.apis.string.Match"
|
|
2481
|
+
},
|
|
2482
|
+
"source": "Utility Operations Best Practices"
|
|
2483
|
+
},
|
|
2484
|
+
{
|
|
2485
|
+
"id": "DATA-DB-WHERE-001",
|
|
2486
|
+
"category": "TestCaseDesign",
|
|
2487
|
+
"name": "DbDelete and DbUpdate should have WHERE clause",
|
|
2488
|
+
"description": "Database delete and update operations should include WHERE clause to avoid affecting all records.",
|
|
2489
|
+
"appliesTo": [
|
|
2490
|
+
"Step"
|
|
2491
|
+
],
|
|
2492
|
+
"severity": "critical",
|
|
2493
|
+
"weight": 8,
|
|
2494
|
+
"recommendation": "Add WHERE clause to DbDelete/DbUpdate to limit affected records.",
|
|
2495
|
+
"check": {
|
|
2496
|
+
"type": "dbWhereClause",
|
|
2497
|
+
"target": "step",
|
|
2498
|
+
"apiIds": [
|
|
2499
|
+
"com.provar.plugins.bundled.apis.db.DbDelete",
|
|
2500
|
+
"com.provar.plugins.bundled.apis.db.DbUpdate"
|
|
2501
|
+
]
|
|
2502
|
+
},
|
|
2503
|
+
"source": "Database Testing Best Practices"
|
|
2504
|
+
},
|
|
2505
|
+
{
|
|
2506
|
+
"id": "DATA-REST-METHOD-001",
|
|
2507
|
+
"category": "TestCaseDesign",
|
|
2508
|
+
"name": "RestRequest method should be valid HTTP method",
|
|
2509
|
+
"description": "RestRequest method must be one of: GET, POST, PUT, DELETE, PATCH, HEAD, OPTIONS.",
|
|
2510
|
+
"appliesTo": [
|
|
2511
|
+
"Step"
|
|
2512
|
+
],
|
|
2513
|
+
"severity": "critical",
|
|
2514
|
+
"weight": 8,
|
|
2515
|
+
"recommendation": "Use valid HTTP method (GET, POST, PUT, DELETE, PATCH).",
|
|
2516
|
+
"check": {
|
|
2517
|
+
"type": "restHttpMethod",
|
|
2518
|
+
"target": "step",
|
|
2519
|
+
"apiId": "com.provar.plugins.bundled.apis.restservice.RestRequest",
|
|
2520
|
+
"validMethods": [
|
|
2521
|
+
"GET",
|
|
2522
|
+
"POST",
|
|
2523
|
+
"PUT",
|
|
2524
|
+
"DELETE",
|
|
2525
|
+
"PATCH",
|
|
2526
|
+
"HEAD",
|
|
2527
|
+
"OPTIONS"
|
|
2528
|
+
]
|
|
2529
|
+
},
|
|
2530
|
+
"source": "REST API Testing Best Practices"
|
|
2531
|
+
},
|
|
2532
|
+
{
|
|
2533
|
+
"id": "DATA-REST-BODY-001",
|
|
2534
|
+
"category": "TestCaseDesign",
|
|
2535
|
+
"name": "POST/PUT/PATCH should have request body",
|
|
2536
|
+
"description": "RestRequest with POST, PUT, or PATCH method should include request body.",
|
|
2537
|
+
"appliesTo": [
|
|
2538
|
+
"Step"
|
|
2539
|
+
],
|
|
2540
|
+
"severity": "major",
|
|
2541
|
+
"weight": 5,
|
|
2542
|
+
"recommendation": "Add request body for POST/PUT/PATCH operations.",
|
|
2543
|
+
"check": {
|
|
2544
|
+
"type": "restRequestBody",
|
|
2545
|
+
"target": "step",
|
|
2546
|
+
"apiId": "com.provar.plugins.bundled.apis.restservice.RestRequest"
|
|
2547
|
+
},
|
|
2548
|
+
"source": "REST API Testing Best Practices"
|
|
2549
|
+
},
|
|
2550
|
+
{
|
|
2551
|
+
"id": "DATA-REST-STATUS-001",
|
|
2552
|
+
"category": "TestCaseDesign",
|
|
2553
|
+
"name": "Validate REST response status",
|
|
2554
|
+
"description": "RestRequest should be followed by assertion to validate response status code.",
|
|
2555
|
+
"appliesTo": [
|
|
2556
|
+
"Step"
|
|
2557
|
+
],
|
|
2558
|
+
"severity": "info",
|
|
2559
|
+
"weight": 1,
|
|
2560
|
+
"recommendation": "Add Assert step to validate REST response status code.",
|
|
2561
|
+
"check": {
|
|
2562
|
+
"type": "restStatusValidation",
|
|
2563
|
+
"target": "step",
|
|
2564
|
+
"apiId": "com.provar.plugins.bundled.apis.restservice.RestRequest"
|
|
2565
|
+
},
|
|
2566
|
+
"source": "REST API Testing Best Practices"
|
|
2567
|
+
},
|
|
2568
|
+
{
|
|
2569
|
+
"id": "DATA-SOAP-XML-001",
|
|
2570
|
+
"category": "TestCaseDesign",
|
|
2571
|
+
"name": "SOAP request body should be well-formed XML",
|
|
2572
|
+
"description": "WebServiceRequest requestBody must be valid XML for SOAP operations.",
|
|
2573
|
+
"appliesTo": [
|
|
2574
|
+
"Step"
|
|
2575
|
+
],
|
|
2576
|
+
"severity": "critical",
|
|
2577
|
+
"weight": 8,
|
|
2578
|
+
"recommendation": "Validate SOAP XML syntax before execution.",
|
|
2579
|
+
"check": {
|
|
2580
|
+
"type": "soapXmlValid",
|
|
2581
|
+
"target": "step",
|
|
2582
|
+
"apiId": "com.provar.plugins.bundled.apis.web.WebServiceRequest"
|
|
2583
|
+
},
|
|
2584
|
+
"source": "SOAP Web Service Testing Best Practices"
|
|
2585
|
+
},
|
|
2586
|
+
{
|
|
2587
|
+
"id": "AI-SESSION-WEBCONNECT-001",
|
|
2588
|
+
"category": "TestCaseDesign",
|
|
2589
|
+
"name": "AIAgentSession requires WebConnect first",
|
|
2590
|
+
"description": "AIAgentSession step must be preceded by WebConnect step.",
|
|
2591
|
+
"appliesTo": [
|
|
2592
|
+
"Step"
|
|
2593
|
+
],
|
|
2594
|
+
"severity": "critical",
|
|
2595
|
+
"weight": 8,
|
|
2596
|
+
"recommendation": "Add WebConnect step before AIAgentSession.",
|
|
2597
|
+
"check": {
|
|
2598
|
+
"type": "aiSessionWebConnect",
|
|
2599
|
+
"target": "step",
|
|
2600
|
+
"apiId": "com.provar.plugins.forcedotcom.core.testapis.ai.AIAgentSession"
|
|
2601
|
+
},
|
|
2602
|
+
"source": "Agentforce Testing Best Practices"
|
|
2603
|
+
},
|
|
2604
|
+
{
|
|
2605
|
+
"id": "AI-CONVERSATION-SESSION-001",
|
|
2606
|
+
"category": "TestCaseDesign",
|
|
2607
|
+
"name": "AIAgentConversation requires valid session",
|
|
2608
|
+
"description": "AIAgentConversation must reference sessionID from AIAgentSession.",
|
|
2609
|
+
"appliesTo": [
|
|
2610
|
+
"Step"
|
|
2611
|
+
],
|
|
2612
|
+
"severity": "critical",
|
|
2613
|
+
"weight": 8,
|
|
2614
|
+
"recommendation": "Ensure AIAgentSession is called first and sessionID is stored.",
|
|
2615
|
+
"check": {
|
|
2616
|
+
"type": "aiConversationSession",
|
|
2617
|
+
"target": "step",
|
|
2618
|
+
"apiId": "com.provar.plugins.forcedotcom.core.testapis.ai.AIAgentConversation"
|
|
2619
|
+
},
|
|
2620
|
+
"source": "Agentforce Testing Best Practices"
|
|
2621
|
+
},
|
|
2622
|
+
{
|
|
2623
|
+
"id": "AI-UTTERANCE-COUNT-001",
|
|
2624
|
+
"category": "TestCaseDesign",
|
|
2625
|
+
"name": "GenerateUtterance count should be reasonable",
|
|
2626
|
+
"description": "GenerateUtterance count parameter should be between 1-100 for performance.",
|
|
2627
|
+
"appliesTo": [
|
|
2628
|
+
"Step"
|
|
2629
|
+
],
|
|
2630
|
+
"severity": "info",
|
|
2631
|
+
"weight": 1,
|
|
2632
|
+
"recommendation": "Set GenerateUtterance count between 1-100.",
|
|
2633
|
+
"check": {
|
|
2634
|
+
"type": "aiUtteranceCount",
|
|
2635
|
+
"target": "step",
|
|
2636
|
+
"apiId": "com.provar.plugins.forcedotcom.core.testapis.ai.GenerateUtterance",
|
|
2637
|
+
"minCount": 1,
|
|
2638
|
+
"maxCount": 100
|
|
2639
|
+
},
|
|
2640
|
+
"source": "AI Testing Best Practices"
|
|
2641
|
+
},
|
|
2642
|
+
{
|
|
2643
|
+
"id": "AI-IMAGE-CONFIDENCE-001",
|
|
2644
|
+
"category": "TestCaseDesign",
|
|
2645
|
+
"name": "ImageValidator confidence should be 0.0-1.0",
|
|
2646
|
+
"description": "ImageValidator confidenceThreshold must be between 0.0 and 1.0.",
|
|
2647
|
+
"appliesTo": [
|
|
2648
|
+
"Step"
|
|
2649
|
+
],
|
|
2650
|
+
"severity": "major",
|
|
2651
|
+
"weight": 5,
|
|
2652
|
+
"recommendation": "Set confidenceThreshold between 0.0 and 1.0.",
|
|
2653
|
+
"check": {
|
|
2654
|
+
"type": "imageConfidence",
|
|
2655
|
+
"target": "step",
|
|
2656
|
+
"apiId": "com.provar.plugins.forcedotcom.core.testapis.ai.ImageValidator",
|
|
2657
|
+
"minConfidence": 0.0,
|
|
2658
|
+
"maxConfidence": 1.0
|
|
2659
|
+
},
|
|
2660
|
+
"source": "AI Image Testing Best Practices"
|
|
2661
|
+
},
|
|
2662
|
+
{
|
|
2663
|
+
"id": "SF-LAYOUT-EXTRACT-BEFORE-ASSERT-001",
|
|
2664
|
+
"category": "TestCaseDesign",
|
|
2665
|
+
"name": "ExtractSalesforceLayout before AssertSalesforceLayout",
|
|
2666
|
+
"description": "AssertSalesforceLayout should be preceded by ExtractSalesforceLayout to create baseline. Otherwise ensure you have a compatible data source to compare to.",
|
|
2667
|
+
"appliesTo": [
|
|
2668
|
+
"Step"
|
|
2669
|
+
],
|
|
2670
|
+
"severity": "minor",
|
|
2671
|
+
"weight": 2,
|
|
2672
|
+
"recommendation": "Run ExtractSalesforceLayout first to create baseline layout file.",
|
|
2673
|
+
"check": {
|
|
2674
|
+
"type": "layoutExtractBeforeAssert",
|
|
2675
|
+
"target": "step",
|
|
2676
|
+
"apiId": "com.provar.plugins.forcedotcom.core.testapis.ApexAssertLayout"
|
|
2677
|
+
},
|
|
2678
|
+
"source": "Salesforce Layout Testing Best Practices"
|
|
2679
|
+
},
|
|
2680
|
+
{
|
|
2681
|
+
"id": "SF-CONVERT-LEAD-STATUS-001",
|
|
2682
|
+
"category": "TestCaseDesign",
|
|
2683
|
+
"name": "ConvertLead status must be valid",
|
|
2684
|
+
"description": "ConvertLead convertedStatus must be a valid lead conversion status.",
|
|
2685
|
+
"appliesTo": [
|
|
2686
|
+
"Step"
|
|
2687
|
+
],
|
|
2688
|
+
"severity": "critical",
|
|
2689
|
+
"weight": 8,
|
|
2690
|
+
"recommendation": "Use valid lead conversion status (Converted, Qualified, etc.).",
|
|
2691
|
+
"check": {
|
|
2692
|
+
"type": "convertLeadStatus",
|
|
2693
|
+
"target": "step",
|
|
2694
|
+
"apiId": "com.provar.plugins.forcedotcom.core.testapis.ApexConvertLead"
|
|
2695
|
+
},
|
|
2696
|
+
"source": "Salesforce Lead Management Best Practices"
|
|
2697
|
+
},
|
|
2698
|
+
{
|
|
2699
|
+
"id": "FILE-READ-PATH-001",
|
|
2700
|
+
"category": "TestCaseDesign",
|
|
2701
|
+
"name": "Read dataUrl should be valid file path",
|
|
2702
|
+
"description": "Read step dataUrl must point to existing Excel/CSV file.",
|
|
2703
|
+
"appliesTo": [
|
|
2704
|
+
"Step"
|
|
2705
|
+
],
|
|
2706
|
+
"severity": "critical",
|
|
2707
|
+
"weight": 8,
|
|
2708
|
+
"recommendation": "Verify file exists before Read operation.",
|
|
2709
|
+
"check": {
|
|
2710
|
+
"type": "fileReadPath",
|
|
2711
|
+
"target": "step",
|
|
2712
|
+
"apiId": "com.provar.plugins.bundled.apis.io.Read"
|
|
2713
|
+
},
|
|
2714
|
+
"source": "File I/O Best Practices"
|
|
2715
|
+
},
|
|
2716
|
+
{
|
|
2717
|
+
"id": "FILE-WRITE-PATH-001",
|
|
2718
|
+
"category": "TestCaseDesign",
|
|
2719
|
+
"name": "Write dataUrl should be writable",
|
|
2720
|
+
"description": "Write step dataUrl directory must be writable.",
|
|
2721
|
+
"appliesTo": [
|
|
2722
|
+
"Step"
|
|
2723
|
+
],
|
|
2724
|
+
"severity": "critical",
|
|
2725
|
+
"weight": 8,
|
|
2726
|
+
"recommendation": "Ensure output directory exists and is writable.",
|
|
2727
|
+
"check": {
|
|
2728
|
+
"type": "fileWritePath",
|
|
2729
|
+
"target": "step",
|
|
2730
|
+
"apiId": "com.provar.plugins.bundled.apis.io.Write"
|
|
2731
|
+
},
|
|
2732
|
+
"source": "File I/O Best Practices"
|
|
2733
|
+
},
|
|
2734
|
+
{
|
|
2735
|
+
"id": "MESSAGING-SUBSCRIBE-BEFORE-RECEIVE-001",
|
|
2736
|
+
"category": "TestCaseDesign",
|
|
2737
|
+
"name": "Subscribe before ReceiveMessage",
|
|
2738
|
+
"description": "ReceiveMessage should be preceded by Subscribe to topic/channel.",
|
|
2739
|
+
"appliesTo": [
|
|
2740
|
+
"Step"
|
|
2741
|
+
],
|
|
2742
|
+
"severity": "critical",
|
|
2743
|
+
"weight": 8,
|
|
2744
|
+
"recommendation": "Add Subscribe step before ReceiveMessage.",
|
|
2745
|
+
"check": {
|
|
2746
|
+
"type": "subscribeBeforeReceive",
|
|
2747
|
+
"target": "step",
|
|
2748
|
+
"apiId": "com.provar.plugins.bundled.apis.messaging.ReceiveMessage"
|
|
2749
|
+
},
|
|
2750
|
+
"source": "Messaging Best Practices"
|
|
2751
|
+
},
|
|
2752
|
+
{
|
|
2753
|
+
"id": "MESSAGING-TIMEOUT-001",
|
|
2754
|
+
"category": "TestCaseDesign",
|
|
2755
|
+
"name": "ReceiveMessage timeout should be reasonable",
|
|
2756
|
+
"description": "ReceiveMessage timeout should be between 5-60 seconds.",
|
|
2757
|
+
"appliesTo": [
|
|
2758
|
+
"Step"
|
|
2759
|
+
],
|
|
2760
|
+
"severity": "info",
|
|
2761
|
+
"weight": 1,
|
|
2762
|
+
"recommendation": "Set timeout between 5-60 seconds.",
|
|
2763
|
+
"check": {
|
|
2764
|
+
"type": "messageTimeout",
|
|
2765
|
+
"target": "step",
|
|
2766
|
+
"apiId": "com.provar.plugins.bundled.apis.messaging.ReceiveMessage",
|
|
2767
|
+
"minTimeout": 5,
|
|
2768
|
+
"maxTimeout": 60
|
|
2769
|
+
},
|
|
2770
|
+
"source": "Messaging Best Practices"
|
|
2771
|
+
},
|
|
2772
|
+
{
|
|
2773
|
+
"id": "RENDER-ARG-001",
|
|
2774
|
+
"category": "StructureAndGrouping",
|
|
2775
|
+
"name": "All arguments must have value elements",
|
|
2776
|
+
"description": "Every <argument> element must contain a <value> child element, even if empty. Missing <value> elements prevent test cases from loading in Provar.",
|
|
2777
|
+
"appliesTo": [
|
|
2778
|
+
"TestCase",
|
|
2779
|
+
"Step"
|
|
2780
|
+
],
|
|
2781
|
+
"severity": "critical",
|
|
2782
|
+
"weight": 10,
|
|
2783
|
+
"recommendation": "Add <value class=\"value\" valueClass=\"string\"></value> (or appropriate type) to all <argument> elements.",
|
|
2784
|
+
"check": {
|
|
2785
|
+
"type": "disabled",
|
|
2786
|
+
"comment": "DISABLED: Too many false positives (1378/1454 = 95%). Valid Provar test cases have many optional arguments without values. Need to reimplement as targeted check for specific critical arguments only (apexConnectionName, objectType, etc.).",
|
|
2787
|
+
"subtype": "argumentValue"
|
|
2788
|
+
},
|
|
2789
|
+
"source": "Provar XML Schema Requirements"
|
|
2790
|
+
},
|
|
2791
|
+
{
|
|
2792
|
+
"id": "RENDER-CASE-001",
|
|
2793
|
+
"category": "StructureAndGrouping",
|
|
2794
|
+
"name": "valueClass attributes must use lowercase",
|
|
2795
|
+
"description": "All valueClass attributes must use lowercase values (e.g., 'boolean' not 'Boolean', 'string' not 'String'). Incorrect casing prevents test cases from loading in Provar.",
|
|
2796
|
+
"appliesTo": [
|
|
2797
|
+
"TestCase",
|
|
2798
|
+
"Step"
|
|
2799
|
+
],
|
|
2800
|
+
"severity": "critical",
|
|
2801
|
+
"weight": 10,
|
|
2802
|
+
"recommendation": "Change all valueClass attributes to lowercase: boolean, string, decimal, integer, date, datetime, variable, compound, funcCall, value, valueList.",
|
|
2803
|
+
"check": {
|
|
2804
|
+
"type": "valueClassCasing",
|
|
2805
|
+
"subtype": "valueClassCasing"
|
|
2806
|
+
},
|
|
2807
|
+
"source": "Provar XML Schema Requirements"
|
|
2808
|
+
},
|
|
2809
|
+
{
|
|
2810
|
+
"id": "RENDER-BOOL-001",
|
|
2811
|
+
"category": "StructureAndGrouping",
|
|
2812
|
+
"name": "Boolean values must use lowercase",
|
|
2813
|
+
"description": "Boolean values in <value> elements must be lowercase 'true' or 'false', not 'True', 'False', 'TRUE', or 'FALSE'. Incorrect casing prevents test cases from loading in Provar.",
|
|
2814
|
+
"appliesTo": [
|
|
2815
|
+
"TestCase",
|
|
2816
|
+
"Step"
|
|
2817
|
+
],
|
|
2818
|
+
"severity": "critical",
|
|
2819
|
+
"weight": 10,
|
|
2820
|
+
"recommendation": "Change all boolean values to lowercase: true, false.",
|
|
2821
|
+
"check": {
|
|
2822
|
+
"type": "booleanCasing",
|
|
2823
|
+
"subtype": "booleanCasing"
|
|
2824
|
+
},
|
|
2825
|
+
"source": "Provar XML Schema Requirements"
|
|
2826
|
+
},
|
|
2827
|
+
{
|
|
2828
|
+
"id": "RENDER-ROOT-001",
|
|
2829
|
+
"category": "StructureAndGrouping",
|
|
2830
|
+
"name": "Test case root element should not have unknown attributes",
|
|
2831
|
+
"description": "The root <testCase> element should only contain known attributes: guid, id, name, visibility, registryId, failureBehaviour. Unknown attributes may prevent test cases from loading in Provar.",
|
|
2832
|
+
"appliesTo": [
|
|
2833
|
+
"TestCase"
|
|
2834
|
+
],
|
|
2835
|
+
"severity": "minor",
|
|
2836
|
+
"weight": 3,
|
|
2837
|
+
"recommendation": "Remove unknown attributes from the root <testCase> element.",
|
|
2838
|
+
"check": {
|
|
2839
|
+
"type": "rootAttributes",
|
|
2840
|
+
"subtype": "rootAttributes"
|
|
2841
|
+
},
|
|
2842
|
+
"source": "Provar XML Schema Requirements"
|
|
2843
|
+
},
|
|
2844
|
+
{
|
|
2845
|
+
"id": "TEST-LENGTH-001",
|
|
2846
|
+
"category": "TestCaseDesign",
|
|
2847
|
+
"name": "Test case should not be excessively long",
|
|
2848
|
+
"description": "Test cases with more than 150 steps become difficult to maintain, debug, and understand. Consider breaking long tests into smaller, focused test cases or using callable test cases.",
|
|
2849
|
+
"appliesTo": [
|
|
2850
|
+
"TestCase"
|
|
2851
|
+
],
|
|
2852
|
+
"severity": "minor",
|
|
2853
|
+
"weight": 3,
|
|
2854
|
+
"recommendation": "Break test into smaller test cases or extract common logic into callable test cases.",
|
|
2855
|
+
"check": {
|
|
2856
|
+
"type": "testLength",
|
|
2857
|
+
"threshold": 150
|
|
2858
|
+
},
|
|
2859
|
+
"source": "Provar Best Practices: Test Design and Maintainability"
|
|
2860
|
+
},
|
|
2861
|
+
{
|
|
2862
|
+
"id": "PICKLIST-001",
|
|
2863
|
+
"category": "TestCaseDesign",
|
|
2864
|
+
"name": "Picklist values should match Salesforce metadata",
|
|
2865
|
+
"description": "Picklist values used in test data should match the valid values defined in Salesforce. Common hallucinated values like 'Active' for Campaign.Status or 'Open' for Opportunity.StageName are often invalid and will cause test failures.",
|
|
2866
|
+
"appliesTo": [
|
|
2867
|
+
"Step",
|
|
2868
|
+
"TestCase"
|
|
2869
|
+
],
|
|
2870
|
+
"severity": "major",
|
|
2871
|
+
"weight": 7,
|
|
2872
|
+
"recommendation": "Use only picklist values provided in the metadata context. For Campaign.Status, valid values are typically: Planned, In Progress, Completed, Aborted. For Opportunity.StageName, check your org's sales process stages. For Lead.Status, check your org's lead process. Never assume generic values like 'Active', 'Inactive', 'Open', 'Closed' unless explicitly confirmed.",
|
|
2873
|
+
"check": {
|
|
2874
|
+
"type": "hallucinatedPicklistValue",
|
|
2875
|
+
"target": "step",
|
|
2876
|
+
"knownHallucinations": {
|
|
2877
|
+
"Campaign.Status": ["Active", "Inactive", "Running", "Paused", "Pending", "Draft"],
|
|
2878
|
+
"Opportunity.StageName": ["Active", "Open", "In Progress", "Pending", "Draft", "New"],
|
|
2879
|
+
"Lead.Status": ["Active", "Inactive", "Pending", "Converted", "Rejected"],
|
|
2880
|
+
"Case.Status": ["Active", "Inactive", "Pending", "Draft", "Resolved"],
|
|
2881
|
+
"Task.Status": ["Active", "Pending", "Done", "Cancelled"],
|
|
2882
|
+
"Event.Status": ["Active", "Pending", "Cancelled"]
|
|
2883
|
+
}
|
|
2884
|
+
},
|
|
2885
|
+
"notes": "Common AI hallucination: Using 'Active' as a status value when it doesn't exist in the org's picklist. Campaign.Status typically uses: Planned, In Progress, Completed, Aborted.",
|
|
2886
|
+
"source": "Standard Salesforce object picklist values and common hallucination patterns"
|
|
2887
|
+
},
|
|
2888
|
+
{
|
|
2889
|
+
"id": "VALUE-CLASS-001",
|
|
2890
|
+
"category": "StructureAndGrouping",
|
|
2891
|
+
"name": "Value elements must use valid class attribute",
|
|
2892
|
+
"description": "The 'class' attribute on <value> elements must be one of the valid Provar value classes. Invalid class values like 'null' cause runtime errors. Valid classes are: value, variable, compound, funcCall, valueList, uiWait, uiLocator, uiTarget, uiInteraction, restTarget, excelTarget, csvTarget, namedValues, url, template, and expression operators (add, sub, mult, div, eq, ne, gt, lt, ge, le, and, or, match).",
|
|
2893
|
+
"appliesTo": [
|
|
2894
|
+
"Step",
|
|
2895
|
+
"TestCase"
|
|
2896
|
+
],
|
|
2897
|
+
"severity": "critical",
|
|
2898
|
+
"weight": 10,
|
|
2899
|
+
"recommendation": "Use a valid class attribute. For literal values use class='value' with appropriate valueClass (string, boolean, decimal, id, date, dateTime). For variables use class='variable'. For empty/optional arguments, omit the <value> element entirely: <argument id=\"argId\"/>",
|
|
2900
|
+
"check": {
|
|
2901
|
+
"type": "invalidValueClass",
|
|
2902
|
+
"target": "step",
|
|
2903
|
+
"validClasses": ["value", "variable", "compound", "funcCall", "valueList", "uiWait", "uiLocator", "uiTarget", "uiInteraction", "restTarget", "excelTarget", "csvTarget", "namedValues", "url", "template", "add", "sub", "mult", "div", "eq", "ne", "gt", "lt", "ge", "le", "and", "or", "match"],
|
|
2904
|
+
"validValueClasses": ["string", "boolean", "decimal", "id", "date", "dateTime"]
|
|
2905
|
+
},
|
|
2906
|
+
"notes": "Based on corpus analysis of 1451 test files with 329,424 <value> elements. Common AI hallucination: class='null' (never valid). Empty arguments should have no <value> child.",
|
|
2907
|
+
"source": "Corpus analysis: All valid value class patterns from InternalProjects"
|
|
2908
|
+
},
|
|
2909
|
+
{
|
|
2910
|
+
"id": "UI-ASSERT-STRUCT-001",
|
|
2911
|
+
"category": "StructureAndGrouping",
|
|
2912
|
+
"name": "UiAssert steps must include all required arguments",
|
|
2913
|
+
"description": "UiAssert steps must include columnAssertions, pageAssertions, resultScope, captureAfter, beforeWait, and autoRetry arguments. Based on corpus analysis of 4,577 UiAssert instances, 100% include these arguments (even if empty). Missing these required arguments causes Provar validation failures at runtime.",
|
|
2914
|
+
"appliesTo": [
|
|
2915
|
+
"Step"
|
|
2916
|
+
],
|
|
2917
|
+
"severity": "critical",
|
|
2918
|
+
"weight": 8,
|
|
2919
|
+
"recommendation": "Include all required arguments in UiAssert steps. Use empty value lists for unused assertions: <argument id=\"columnAssertions\"><value class=\"valueList\" mutable=\"Mutable\"/></argument>. For resultScope use 'Test'. For captureAfter use 'false'. For beforeWait and autoRetry, include empty argument tags: <argument id=\"beforeWait\"/><argument id=\"autoRetry\"/>",
|
|
2920
|
+
"check": {
|
|
2921
|
+
"type": "uiAssertMissingArguments",
|
|
2922
|
+
"target": "step",
|
|
2923
|
+
"requiredArguments": ["fieldAssertions", "columnAssertions", "pageAssertions", "resultScope", "captureAfter", "beforeWait", "autoRetry"],
|
|
2924
|
+
"recommendedArguments": ["resultName"]
|
|
2925
|
+
},
|
|
2926
|
+
"notes": "Based on corpus analysis of 4,577 UiAssert instances across 694 test files. All instances include these arguments. AI commonly omits columnAssertions, pageAssertions, resultScope, beforeWait, and autoRetry.",
|
|
2927
|
+
"source": "Corpus analysis: InternalProjects - 100% of UiAssert steps have these arguments"
|
|
2928
|
+
},
|
|
2929
|
+
{
|
|
2930
|
+
"id": "UI-ASSERT-STRUCT-002",
|
|
2931
|
+
"category": "StructureAndGrouping",
|
|
2932
|
+
"name": "UiAssert steps must NOT contain generatedParameters",
|
|
2933
|
+
"description": "UiAssert steps should NOT contain <generatedParameters> element. Based on corpus analysis of 4,577 UiAssert instances, 0% contain generatedParameters. This element is 100% hallucinated by AI and causes Provar validation failures.",
|
|
2934
|
+
"appliesTo": [
|
|
2935
|
+
"Step"
|
|
2936
|
+
],
|
|
2937
|
+
"severity": "critical",
|
|
2938
|
+
"weight": 10,
|
|
2939
|
+
"recommendation": "Remove the entire <generatedParameters> section from UiAssert steps. Unlike UiDoAction steps which use generatedParameters for field metadata, UiAssert steps never use this element. The assertion configuration is entirely within the arguments element.",
|
|
2940
|
+
"check": {
|
|
2941
|
+
"type": "uiAssertHallucinatedGeneratedParameters",
|
|
2942
|
+
"target": "step"
|
|
2943
|
+
},
|
|
2944
|
+
"notes": "Based on corpus analysis of 4,577 UiAssert instances. 0 instances have generatedParameters - this is a 100% AI hallucination. UiDoAction correctly uses generatedParameters, but UiAssert does not.",
|
|
2945
|
+
"source": "Corpus analysis: InternalProjects - 0% of UiAssert steps have generatedParameters"
|
|
2946
|
+
},
|
|
2947
|
+
{
|
|
2948
|
+
"id": "UI-BINDING-ORDER-001",
|
|
2949
|
+
"category": "LocatorPatterns",
|
|
2950
|
+
"name": "UI binding parameter order must have object= first",
|
|
2951
|
+
"description": "In UI binding URIs, the object= parameter MUST come FIRST, followed by field= or action=. Wrong order causes 'Unknown control' errors in Provar at runtime.",
|
|
2952
|
+
"appliesTo": [
|
|
2953
|
+
"Step"
|
|
2954
|
+
],
|
|
2955
|
+
"severity": "critical",
|
|
2956
|
+
"weight": 10,
|
|
2957
|
+
"recommendation": "Correct the binding parameter order. Use 'object?object=ObjectName&action=ActionName' or 'object?object=ObjectName&field=FieldName'. Never put action= or field= before object=.",
|
|
2958
|
+
"check": {
|
|
2959
|
+
"type": "bindingParameterOrder",
|
|
2960
|
+
"target": "step"
|
|
2961
|
+
},
|
|
2962
|
+
"examples": {
|
|
2963
|
+
"correct": [
|
|
2964
|
+
"sf:ui:binding:object?object=Lead&action=New",
|
|
2965
|
+
"sf:ui:binding:object?object=Account&field=Name",
|
|
2966
|
+
"sf%3Aui%3Abinding%3Aobject%3Fobject%3DLead%26action%3DNew"
|
|
2967
|
+
],
|
|
2968
|
+
"incorrect": [
|
|
2969
|
+
"sf:ui:binding:object?action=New&object=Lead",
|
|
2970
|
+
"sf:ui:binding:object?field=Name&object=Account",
|
|
2971
|
+
"sf%3Aui%3Abinding%3Aobject%3Faction%3DNew%26object%3DLead"
|
|
2972
|
+
]
|
|
2973
|
+
},
|
|
2974
|
+
"notes": "Based on corpus analysis: Field bindings 94.4% have object= first (6,017 vs 354). Action bindings 94.7% have object= first (1,554 vs 87). The 5% with wrong order are edge cases that often cause issues.",
|
|
2975
|
+
"source": "Corpus analysis: InternalProjects - 95% of bindings have object= parameter first"
|
|
2976
|
+
},
|
|
2977
|
+
{
|
|
2978
|
+
"id": "UI-FIELD-METADATA-001",
|
|
2979
|
+
"category": "TestCaseDesign",
|
|
2980
|
+
"name": "UiDoAction/UiAssert fields should exist in Salesforce metadata",
|
|
2981
|
+
"description": "When Salesforce object metadata is provided in the context, UI operations (UiDoAction field sets, UiAssert field assertions) should only reference fields that exist in the metadata. Fields mentioned in user story but not present in Salesforce metadata may cause 'Unknown control' errors at runtime. This validation catches hallucinated or non-existent fields before test execution.",
|
|
2982
|
+
"appliesTo": [
|
|
2983
|
+
"Step"
|
|
2984
|
+
],
|
|
2985
|
+
"severity": "major",
|
|
2986
|
+
"weight": 5,
|
|
2987
|
+
"recommendation": "Only use fields that exist in the Salesforce org metadata. If a field from the user story shows 'Unknown control' in Provar, either: (1) the field doesn't exist in this org, (2) the field API name is different, or (3) the field isn't on the page layout. Check the org's metadata or remove the field operation from the test.",
|
|
2988
|
+
"check": {
|
|
2989
|
+
"type": "disabled",
|
|
2990
|
+
"target": "step",
|
|
2991
|
+
"apiIds": [
|
|
2992
|
+
"com.provar.plugins.forcedotcom.core.ui.UiDoAction",
|
|
2993
|
+
"com.provar.plugins.forcedotcom.core.ui.UiAssert"
|
|
2994
|
+
]
|
|
2995
|
+
},
|
|
2996
|
+
"notes": "This validation requires metadata context (objects with field information) to be passed to the validator. When metadata is not available, this validation is skipped. Common issue: user story mentions 'Account Number' but org only has 'AccountNumber' or field doesn't exist at all.",
|
|
2997
|
+
"source": "Field observation: AI generates steps for fields mentioned in user story that don't exist in org metadata"
|
|
2998
|
+
},
|
|
2999
|
+
{
|
|
3000
|
+
"id": "PO-FIELD-EXISTS-001",
|
|
3001
|
+
"category": "TestCaseDesign",
|
|
3002
|
+
"name": "Page Object locator references non-existent field",
|
|
3003
|
+
"description": "When Page Objects are provided, ui:pageobject:field locators must reference fields (WebElement variables) that actually exist in the corresponding Page Object Java class. AI models commonly hallucinate field names that are not defined in the provided Page Objects, causing 'Unknown control' errors at runtime.",
|
|
3004
|
+
"appliesTo": [
|
|
3005
|
+
"Step"
|
|
3006
|
+
],
|
|
3007
|
+
"severity": "major",
|
|
3008
|
+
"weight": 5,
|
|
3009
|
+
"recommendation": "Only reference field names that exist in the provided Page Object definitions. Check the Page Object class for available WebElement variables. If the field you need is not in the Page Object, either: (1) use SF metadata binding (sf:ui:binding:object) instead, or (2) ensure the Page Object includes the required field. Available fields are listed in the Page Object section of the generation context.",
|
|
3010
|
+
"check": {
|
|
3011
|
+
"type": "pageObjectFieldExists",
|
|
3012
|
+
"target": "step",
|
|
3013
|
+
"apiIds": [
|
|
3014
|
+
"com.provar.plugins.forcedotcom.core.ui.UiDoAction",
|
|
3015
|
+
"com.provar.plugins.forcedotcom.core.ui.UiAssert"
|
|
3016
|
+
]
|
|
3017
|
+
},
|
|
3018
|
+
"notes": "This validation requires page_objects context to be passed to the validator via additional_context. When page_objects are not available, validation is skipped. Common issue: AI generates field names based on the user story text rather than the actual Page Object WebElement variable names.",
|
|
3019
|
+
"source": "Field observation: AI generates PO field references for fields not defined in provided Page Objects"
|
|
3020
|
+
},
|
|
3021
|
+
{
|
|
3022
|
+
"id": "FUNCCALL-VALID-001",
|
|
3023
|
+
"category": "XMLSchema",
|
|
3024
|
+
"name": "funcCall id must be a valid Provar function",
|
|
3025
|
+
"description": "When using <value class=\"funcCall\" id=\"FunctionName\">, the function name must be one of Provar's known built-in functions. Unknown function names cause runtime errors as Provar cannot evaluate them. Common hallucinations include JavaScript-style functions like 'concat', 'toString', 'parseInt' instead of Provar's actual functions.",
|
|
3026
|
+
"appliesTo": [
|
|
3027
|
+
"TestCase"
|
|
3028
|
+
],
|
|
3029
|
+
"severity": "major",
|
|
3030
|
+
"weight": 6,
|
|
3031
|
+
"recommendation": "Use only valid Provar function names: Count, DateAdd, DateFormat, DateParse, GetEnvironmentVariable, GetSelectedEnvironment, IsSorted, Not, NumberFormat, Round, StringNormalize, StringReplace, StringTrim, TestCaseErrors, TestCaseName, TestCaseOutcome, TestCasePath, TestCaseSuccessful, TestRunErrors, UniqueId. NOTE: Concatenate, PadLeft, PadRight, Substring are NOT valid - use valueList for string building.",
|
|
3032
|
+
"check": {
|
|
3033
|
+
"type": "validFuncCallId",
|
|
3034
|
+
"target": "testCase"
|
|
3035
|
+
},
|
|
3036
|
+
"examples": {
|
|
3037
|
+
"correct": [
|
|
3038
|
+
"<value class=\"funcCall\" id=\"Count\">",
|
|
3039
|
+
"<value class=\"funcCall\" id=\"DateAdd\">",
|
|
3040
|
+
"<value class=\"funcCall\" id=\"UniqueId\">",
|
|
3041
|
+
"<value class=\"funcCall\" id=\"StringReplace\">",
|
|
3042
|
+
"<value class=\"funcCall\" id=\"Round\">"
|
|
3043
|
+
],
|
|
3044
|
+
"incorrect": [
|
|
3045
|
+
"<value class=\"funcCall\" id=\"Concatenate\">",
|
|
3046
|
+
"<value class=\"funcCall\" id=\"PadLeft\">",
|
|
3047
|
+
"<value class=\"funcCall\" id=\"concat\">",
|
|
3048
|
+
"<value class=\"funcCall\" id=\"toString\">",
|
|
3049
|
+
"<value class=\"funcCall\" id=\"Format\">"
|
|
3050
|
+
]
|
|
3051
|
+
},
|
|
3052
|
+
"notes": "Provar functions use PascalCase naming. Concatenate, PadLeft, PadRight, Substring are NOT valid Provar funcCalls - for string building use valueList or compound. Common hallucinations: 'concat' and 'Concatenate' should use valueList instead.",
|
|
3053
|
+
"source": "Corpus analysis: PROVAR_BUILTIN_VARIABLES.md and BUILTIN_FUNCTIONS_ANALYSIS.md document all valid Provar functions"
|
|
3054
|
+
},
|
|
3055
|
+
{
|
|
3056
|
+
"id": "UI-ASSERT-COMPOUND-001",
|
|
3057
|
+
"category": "TestCaseDesign",
|
|
3058
|
+
"name": "UiAssert must use compound fields for component field assertions",
|
|
3059
|
+
"description": "Salesforce compound fields (Name=FirstName+LastName, BillingAddress=BillingStreet+BillingCity+BillingState+BillingPostalCode+BillingCountry, ShippingAddress, MailingAddress) are displayed as single fields in the UI View screen but set as individual component fields in UiDoAction. UiAssert steps MUST assert the compound field (e.g., 'Name', 'BillingAddress') using a compound value that combines the individual component variables, NOT assert the individual component fields directly which don't exist in the View UI.",
|
|
3060
|
+
"appliesTo": [
|
|
3061
|
+
"Step"
|
|
3062
|
+
],
|
|
3063
|
+
"severity": "major",
|
|
3064
|
+
"weight": 6,
|
|
3065
|
+
"recommendation": "For Name field: Assert 'Name' using <value class=\"compound\"><parts><variable><path element=\"FirstName\"/></variable><value class=\"value\" valueClass=\"string\"> </value><variable><path element=\"LastName\"/></variable></parts></value>. For Address fields: Assert 'BillingAddress' or 'ShippingAddress' as the compound field, not individual BillingStreet/BillingCity etc. The fieldLocator should reference the compound field name (e.g., field=Name, field=BillingAddress).",
|
|
3066
|
+
"check": {
|
|
3067
|
+
"type": "uiAssertCompoundField",
|
|
3068
|
+
"target": "step",
|
|
3069
|
+
"apiId": "com.provar.plugins.forcedotcom.core.ui.UiAssert",
|
|
3070
|
+
"compoundFields": {
|
|
3071
|
+
"Name": ["FirstName", "LastName", "Salutation", "MiddleName", "Suffix"],
|
|
3072
|
+
"BillingAddress": ["BillingStreet", "BillingCity", "BillingState", "BillingStateCode", "BillingPostalCode", "BillingCountry", "BillingCountryCode", "BillingLatitude", "BillingLongitude"],
|
|
3073
|
+
"ShippingAddress": ["ShippingStreet", "ShippingCity", "ShippingState", "ShippingStateCode", "ShippingPostalCode", "ShippingCountry", "ShippingCountryCode", "ShippingLatitude", "ShippingLongitude"],
|
|
3074
|
+
"MailingAddress": ["MailingStreet", "MailingCity", "MailingState", "MailingStateCode", "MailingPostalCode", "MailingCountry", "MailingCountryCode", "MailingLatitude", "MailingLongitude"]
|
|
3075
|
+
}
|
|
3076
|
+
},
|
|
3077
|
+
"examples": {
|
|
3078
|
+
"correct": [
|
|
3079
|
+
"<uiFieldAssertion resultName=\"Name\"><fieldLocator uri=\"ui:locator?name=Name&binding=sf%3Aui%3Abinding%3Aobject%3Ffield%3DName%26object%3DLead\"/><attributeAssertions><uiAttributeAssertion attributeName=\"value\" comparisonType=\"EqualTo\"><value class=\"compound\"><parts><variable><path element=\"LeadFirstName\"/></variable><value class=\"value\" valueClass=\"string\"> </value><variable><path element=\"LeadLastName\"/></variable></parts></value></uiAttributeAssertion></attributeAssertions></uiFieldAssertion>",
|
|
3080
|
+
"<uiFieldAssertion resultName=\"BillingAddress\"><fieldLocator uri=\"ui:locator?name=BillingAddress&binding=sf%3Aui%3Abinding%3Aobject%3Ffield%3DBillingAddress%26object%3DAccount\"/>...</uiFieldAssertion>"
|
|
3081
|
+
],
|
|
3082
|
+
"incorrect": [
|
|
3083
|
+
"<uiFieldAssertion resultName=\"FirstName\"><fieldLocator uri=\"...field=FirstName...\"/>... (FirstName doesn't exist as separate field in View UI)",
|
|
3084
|
+
"<uiFieldAssertion resultName=\"BillingCity\"><fieldLocator uri=\"...field=BillingCity...\"/>... (individual address components not visible separately)"
|
|
3085
|
+
]
|
|
3086
|
+
},
|
|
3087
|
+
"notes": "UiDoAction sets individual fields (FirstName, LastName, BillingStreet, etc.) because the New/Edit form has separate inputs. UiAssert on View screen must use compound field names (Name, BillingAddress) because that's how Salesforce displays them. The assertion value should use <value class=\"compound\"> with <parts> containing variables for each component separated by appropriate delimiters (space for names, newlines for addresses).",
|
|
3088
|
+
"source": "Field observation: Salesforce UI View screens display compound fields, not individual components"
|
|
3089
|
+
},
|
|
3090
|
+
{
|
|
3091
|
+
"id": "UI-LOOKUP-ID-001",
|
|
3092
|
+
"category": "TestCaseDesign",
|
|
3093
|
+
"name": "UiDoAction lookup fields should use Name values, not IDs",
|
|
3094
|
+
"description": "When setting lookup/reference fields via UiDoAction (UI automation), use the record Name (text value the user sees) instead of the record Id. The Salesforce UI expects display values like 'ABC Corp' for Account lookups, not 18-character IDs like '001xx000003DGWaAAO'. Using IDs in UI fields causes lookup failures. Note: Apex CRUD operations (ApexCreateObject, ApexUpdateObject) correctly use IDs for lookup fields.",
|
|
3095
|
+
"appliesTo": [
|
|
3096
|
+
"Step"
|
|
3097
|
+
],
|
|
3098
|
+
"severity": "major",
|
|
3099
|
+
"weight": 6,
|
|
3100
|
+
"recommendation": "For UiDoAction on lookup fields (AccountId, ContactId, OwnerId, etc.): 1) Create a variable for the record Name BEFORE creating the record (e.g., AccountName = 'ABC Corp'), 2) Use that Name variable in UiDoAction for lookups, 3) Only use ID variables for Apex CRUD operations or sfUiTargetObjectId navigation. Example: Set AccountName='ABC Corp' via SetValues, then use {AccountName} for UiDoAction on AccountId field.",
|
|
3101
|
+
"check": {
|
|
3102
|
+
"type": "uiLookupIdUsage",
|
|
3103
|
+
"target": "step",
|
|
3104
|
+
"apiId": "com.provar.plugins.forcedotcom.core.ui.UiDoAction",
|
|
3105
|
+
"lookupFieldSuffixes": ["Id", "ID"],
|
|
3106
|
+
"idVariableSuffixes": ["Id", "ID", "RecordId"]
|
|
3107
|
+
},
|
|
3108
|
+
"examples": {
|
|
3109
|
+
"correct": [
|
|
3110
|
+
"SetValues: AccountName = 'ABC Corp'",
|
|
3111
|
+
"UiDoAction: Set AccountId field using {AccountName} variable",
|
|
3112
|
+
"ApexCreateObject: Set AccountId using {AccountId} variable (IDs are correct for Apex)"
|
|
3113
|
+
],
|
|
3114
|
+
"incorrect": [
|
|
3115
|
+
"UiDoAction: Set AccountId field using {AccountId} variable (ID won't work in UI lookup)",
|
|
3116
|
+
"UiDoAction: Set OwnerId field using {UserId} variable"
|
|
3117
|
+
]
|
|
3118
|
+
},
|
|
3119
|
+
"notes": "Salesforce UI lookup fields expect the display Name (what the user types/sees), not the record ID. The UI performs a search on the Name and selects the matching record. Apex CRUD operations directly accept IDs because they bypass the UI. This is a common AI generation mistake.",
|
|
3120
|
+
"source": "Field observation: UI lookup fields require Name values, not record IDs"
|
|
3121
|
+
},
|
|
3122
|
+
{
|
|
3123
|
+
"id": "ASSERT-DATE-FORMAT-001",
|
|
3124
|
+
"category": "TestCaseDesign",
|
|
3125
|
+
"name": "Date/DateTime assertions should use proper format functions",
|
|
3126
|
+
"description": "Salesforce Date and DateTime fields are stored in specific formats: DateTime uses ISO 8601 format (2026-01-14T05:36:12.000+0000) and Date uses yyyy-MM-dd format (2026-01-14). When asserting these fields in UI or API tests, raw date strings may cause assertion failures due to format mismatches, timezone differences, or precision issues. Use Provar's date functions (DateFormat, DateParse, DateAdd, TODAY, NOW) for reliable date comparisons.",
|
|
3127
|
+
"appliesTo": [
|
|
3128
|
+
"Step"
|
|
3129
|
+
],
|
|
3130
|
+
"severity": "minor",
|
|
3131
|
+
"weight": 4,
|
|
3132
|
+
"recommendation": "For date assertions: 1) Use DateFormat() to format dates consistently: DateFormat(variable, 'yyyy-MM-dd', 'GMT'), 2) Use DateParse() to parse date strings with known formats, 3) Use TODAY for current date comparisons (yyyy-MM-dd format), 4) Use NOW for current datetime (yyyy-MM-dd HH:mm:ss.SSS format), 5) Use DateAdd() for date calculations. Always specify timezone for DateTime comparisons. Example: Assert CloseDate equals DateFormat(ExpectedDate, 'yyyy-MM-dd').",
|
|
3133
|
+
"check": {
|
|
3134
|
+
"type": "dateFormatAssertion",
|
|
3135
|
+
"target": "step",
|
|
3136
|
+
"apiIds": [
|
|
3137
|
+
"com.provar.plugins.forcedotcom.core.ui.UiAssert",
|
|
3138
|
+
"com.provar.plugins.bundled.apis.AssertValues"
|
|
3139
|
+
],
|
|
3140
|
+
"dateFieldPatterns": ["Date$", "DateTime$", "^Close", "^Start", "^End", "^Created", "^LastModified", "^Birth", "^Expir"],
|
|
3141
|
+
"sfDateTimeFormat": "yyyy-MM-dd'T'HH:mm:ss.SSSZ",
|
|
3142
|
+
"sfDateFormat": "yyyy-MM-dd"
|
|
3143
|
+
},
|
|
3144
|
+
"examples": {
|
|
3145
|
+
"correct": [
|
|
3146
|
+
"AssertValues: Assert CloseDate equals DateFormat(ExpectedCloseDate, 'yyyy-MM-dd')",
|
|
3147
|
+
"UiAssert: Assert CreatedDate using DateFormat(expectedDate, 'yyyy-MM-dd', 'GMT')",
|
|
3148
|
+
"AssertValues: Assert field using TODAY for today's date",
|
|
3149
|
+
"AssertValues: Assert datetime using DateParse(value, 'yyyy-MM-dd\\'T\\'HH:mm:ss.SSSZ')"
|
|
3150
|
+
],
|
|
3151
|
+
"incorrect": [
|
|
3152
|
+
"AssertValues: Assert CloseDate equals '2026-01-14' (hardcoded date string)",
|
|
3153
|
+
"UiAssert: Assert CreatedDate equals {CreatedDateVariable} (raw datetime may have timezone/format issues)",
|
|
3154
|
+
"AssertValues: Assert DateTime field using string comparison (precision mismatch)"
|
|
3155
|
+
]
|
|
3156
|
+
},
|
|
3157
|
+
"notes": "Salesforce DateTime format: 2026-01-14T05:36:12.000+0000 (ISO 8601). Salesforce Date format: 2026-01-14. NOW returns: 2026-01-15 15:03:16.690. TODAY returns: 2026-01-15. DateFormat output format defaults to yyyy-MM-dd. DateParse input format defaults to international format. Always consider timezone when comparing DateTime values across different contexts.",
|
|
3158
|
+
"source": "Field observation: Date/DateTime format inconsistencies cause assertion failures"
|
|
3159
|
+
},
|
|
3160
|
+
{
|
|
3161
|
+
"id": "RENDER-DATE-VALUECLASS-001",
|
|
3162
|
+
"category": "XMLSchema",
|
|
3163
|
+
"name": "valueClass='date' requires epoch timestamp, not date string",
|
|
3164
|
+
"description": "When using valueClass='date' or valueClass='dateTime' in Provar test cases, the value MUST be an epoch timestamp in milliseconds (a numeric value like 1736899200000), NOT an ISO date string like '2025-01-15' or '2025-01-15T00:00:00.000Z'. Using string date formats with valueClass='date' causes the test case to fail loading in Provar IDE. This is a critical XML schema violation that completely breaks the test case.",
|
|
3165
|
+
"appliesTo": [
|
|
3166
|
+
"Step",
|
|
3167
|
+
"TestCase"
|
|
3168
|
+
],
|
|
3169
|
+
"severity": "critical",
|
|
3170
|
+
"weight": 10,
|
|
3171
|
+
"recommendation": "Either: 1) Use valueClass='string' if you want to specify dates as strings like '2025-01-15', OR 2) Use valueClass='date' with epoch timestamp in milliseconds (e.g., 1736899200000 for 2025-01-15). For ApexCreateObject/ApexUpdateObject date fields, use valueClass='string' with the date in 'YYYY-MM-DD' format. To convert: new Date('2025-01-15').getTime() = 1736899200000.",
|
|
3172
|
+
"check": {
|
|
3173
|
+
"type": "dateValueClassFormat",
|
|
3174
|
+
"target": "step"
|
|
3175
|
+
},
|
|
3176
|
+
"examples": {
|
|
3177
|
+
"correct": [
|
|
3178
|
+
"<value class=\"value\" valueClass=\"date\">1736899200000</value>",
|
|
3179
|
+
"<value class=\"value\" valueClass=\"string\">2025-01-15</value>",
|
|
3180
|
+
"<value class=\"value\" valueClass=\"dateTime\">1736899200000</value>"
|
|
3181
|
+
],
|
|
3182
|
+
"incorrect": [
|
|
3183
|
+
"<value class=\"value\" valueClass=\"date\">2025-01-15</value>",
|
|
3184
|
+
"<value class=\"value\" valueClass=\"date\">2025-01-15T00:00:00.000Z</value>",
|
|
3185
|
+
"<value class=\"value\" valueClass=\"dateTime\">2025-01-15T12:30:00</value>"
|
|
3186
|
+
]
|
|
3187
|
+
},
|
|
3188
|
+
"notes": "Provar internally stores dates as epoch timestamps (milliseconds since 1970-01-01). When valueClass='date' is used, Provar expects a numeric epoch value. String dates cause XML parsing/loading failures. All valid corpus examples use epoch timestamps with valueClass='date': 1637798400000, 1513209600000, etc.",
|
|
3189
|
+
"source": "Corpus analysis: All valueClass='date' values are epoch timestamps; string dates cause Provar IDE load failures"
|
|
3190
|
+
}
|
|
3191
|
+
]
|
|
3192
|
+
}
|