@provartesting/provardx-cli 1.5.3 → 1.6.1

package/lib/mcp/rules/provar_best_practices_rules.json

@@ -338,10 +338,10 @@
       "id": "VAR-NAMING-001",
       "category": "DataDrivenTesting",
       "name": "Variable names must use valid identifiers",
-      "description": "Variable names, field references, and result names must contain only letters, digits, and underscores. Spaces, hyphens, and special characters cause RUNTIME FAILURES in Provar. Examples: {Account.Name} ✅ | {Account.Account Name} ❌ (space will fail)",
+      "description": "Variable names, field references, and result names must contain only letters, digits, and underscores. Spaces, hyphens, and special characters cause RUNTIME FAILURES in Provar (the test case still loads, so this is a major runtime defect, not a load-blocking critical). Examples: {Account.Name} ✅ | {Account.Account Name} ❌ (space will fail)",
       "appliesTo": ["Step"],
-      "severity": "critical",
-      "weight": 8,
+      "severity": "major",
+      "weight": 5,
       "recommendation": "Replace spaces and special characters with underscores. Use API field names (e.g., Account.Name) instead of field labels (e.g., Account.Account Name).",
       "check": {
         "type": "variableNaming",
@@ -892,7 +892,7 @@
       "appliesTo": ["Step"],
       "severity": "critical",
       "weight": 8,
-      "recommendation": "Add 'comparisonType' argument with valid value: EqualTo, NotEqualTo, Contains, NotContains, StartsWith, EndsWith, GreaterThan, LessThan, Matches, IsNull, NotNull.",
+      "recommendation": "Add 'comparisonType' argument with a value from the AssertValues subset of ComparisonType: EqualTo, NotEqualTo, GreaterThan, GreaterThanOrEqualTo, LessThan, LessThanOrEqualTo, IsPresent, IsEmpty, Matches, NotMatches, Contains, NotContains, StartsWith, NotStartsWith, EndsWith, NotEndsWith. Note: UI Assert (uiAttributeAssertion) steps accept only the narrower subset EqualTo, Contains, StartsWith, EndsWith, Matches, None; a value used outside its step's subset is load-blocking.",
       "check": {
         "type": "mustContainArgument",
         "apiId": "com.provar.plugins.bundled.apis.AssertValues",
@@ -1506,7 +1506,7 @@
       "id": "ASSERT-VALUES-COMPARISON-001",
       "category": "TestCaseDesign",
       "name": "AssertValues should have meaningful expected values",
-      "description": "AssertValues steps using comparison operators (EqualTo, Contains, GreaterThan, etc.) should specify meaningful expectedValue arguments. Empty or missing expected values defeat the purpose of assertions and provide no validation. Without expected values, assertions cannot verify correct behavior. Comparison types requiring values: EqualTo, NotEqualTo, Contains, NotContains, StartsWith, NotStartsWith, EndsWith, NotEndsWith, Matches, NotMatches, GreaterThan, LessThan, GreaterThanOrEqualTo, LessThanOrEqualTo.",
+      "description": "AssertValues steps using comparison operators (EqualTo, Contains, GreaterThan, etc.) should specify meaningful expectedValue arguments. Empty or missing expected values defeat the purpose of assertions and provide no validation. Without expected values, assertions cannot verify correct behavior. The AssertValues subset of ComparisonType is: EqualTo, NotEqualTo, GreaterThan, GreaterThanOrEqualTo, LessThan, LessThanOrEqualTo, IsPresent, IsEmpty, Matches, NotMatches, Contains, NotContains, StartsWith, NotStartsWith, EndsWith, NotEndsWith. Of these, the value-requiring comparators are EqualTo, NotEqualTo, Contains, NotContains, StartsWith, NotStartsWith, EndsWith, NotEndsWith, Matches, NotMatches, GreaterThan, LessThan, GreaterThanOrEqualTo, LessThanOrEqualTo (the presence checks IsPresent and IsEmpty do not need an expectedValue). UI Assert (uiAttributeAssertion) steps use a narrower subset: EqualTo, Contains, StartsWith, EndsWith, Matches, None.",
       "appliesTo": ["Step"],
       "severity": "major",
       "weight": 5,
@@ -2670,10 +2670,10 @@
       "id": "UI-BINDING-ORDER-001",
       "category": "LocatorPatterns",
       "name": "UI binding parameter order must have object= first",
-      "description": "In UI binding URIs, the object= parameter MUST come FIRST, followed by field= or action=. Wrong order causes 'Unknown control' errors in Provar at runtime.",
+      "description": "In UI binding URIs, the object= parameter MUST come FIRST, followed by field= or action=. Wrong order causes 'Unknown control' errors in Provar at runtime (the test case still loads, so this is a major runtime defect, not a load-blocking critical).",
       "appliesTo": ["Step"],
-      "severity": "critical",
-      "weight": 10,
+      "severity": "major",
+      "weight": 5,
       "recommendation": "Correct the binding parameter order. Use 'object?object=ObjectName&action=ActionName' or 'object?object=ObjectName&field=FieldName'. Never put action= or field= before object=.",
       "check": {
         "type": "bindingParameterOrder",
@@ -2958,6 +2958,117 @@
       },
       "notes": "Mirrors QH UiActionNestingStructureValidator (lambda/src/validator/best_practices_engine.py). Emits one violation per offending step so that (rule_id, test_item_id) de-dups against the API. UiWithRow plays a dual role: itself a UI action that must be nested, and a container whose substeps clause satisfies the rule for its descendants.",
       "source": "Field observation: flat-emitted UI actions (sibling of UiWithScreen) fail to render in Provar IDE despite scoring 100/94 in earlier local validation"
+    },
+    {
+      "id": "VAR-STRING-LITERAL-001",
+      "category": "TestCaseDesign",
+      "name": "Variable reference stored as plain string",
+      "description": "An argument value contains a {VarName} or {Obj.Field} pattern stored as class=\"value\" valueClass=\"string\". Provar passes the literal string to the API instead of resolving the variable. In data arguments the step silently receives the wrong input; in UI target arguments (sfUiTargetObjectId, sfUiTargetResultName) the literal {VarName} lands in the URL as %7BVarName%7D and the step fails at runtime (record 'no longer available'). The test case still loads, so this is a runtime execution error (major), not a load failure (critical) or a style warning (minor).",
+      "appliesTo": ["Step"],
+      "severity": "major",
+      "weight": 5,
+      "recommendation": "Replace with <value class=\"variable\"><path element=\"VarName\"/></value>. Use dotted paths for nested references: <path element=\"Obj\"/><path element=\"Field\"/>. If using provar.testcase.generate, the {VarName} syntax in attributes is converted automatically.",
+      "check": {
+        "type": "varStringLiteral"
+      },
+      "source": "Provar Test Step Schema: variable references must use class=\"variable\", not string literals"
+    },
+    {
+      "id": "CONN-DB-002",
+      "category": "ConnectionsAndEnvironments",
+      "name": "DbConnect resultName must match dbConnectionName in DB operations",
+      "description": "The resultName set on a DbConnect step is the connection handle variable that all subsequent DB operations (SqlQuery, DbRead, DbInsert, DbUpdate, DbDelete) must reference via their dbConnectionName argument. A mismatch between DbConnect resultName and dbConnectionName causes a runtime failure because the operation cannot find the open connection. This is the leading cause of silent Database step failures in Provar test suites.",
+      "appliesTo": ["Step"],
+      "severity": "major",
+      "weight": 5,
+      "recommendation": "Ensure every DB operation step's dbConnectionName argument exactly matches the resultName set on the preceding DbConnect step. For example, if DbConnect sets resultName='SQLServer', every SqlQuery/DbRead/DbInsert/DbUpdate/DbDelete in the same test must set dbConnectionName='SQLServer'.",
+      "check": {
+        "type": "dbConnectResultNameMismatch",
+        "target": "testCase"
+      },
+      "source": "Provar DB Connection Best Practices - DbConnect resultName / dbConnectionName alignment"
+    },
+    {
+      "id": "SETVALUES-FUNC-STR-001",
+      "category": "StructureAndGrouping",
+      "name": "SetValues must not use string interpolation for function calls",
+      "description": "SetValues steps must use the correct XML structure for function calls (<value class=\"funcCall\">) rather than embedding Provar's curly-brace template syntax as a plain string literal (e.g. {Count(AccountList)}). At runtime Provar reads the string literally and does not evaluate it, so the target variable is set to the text \"{Count(AccountList)}\" instead of the computed count. This pattern is generated by AI models that reproduce the Provar UI tooltip syntax verbatim.",
+      "appliesTo": ["Step"],
+      "severity": "major",
+      "weight": 5,
+      "recommendation": "Replace the string literal with the proper funcCall XML: <value class=\"funcCall\" id=\"Count\"><argument id=\"value\"><value class=\"variable\"><path element=\"ListVariableName\"/></value></argument></value>. Supported functions include Count, Sum, Avg, Min, Max, Concat, StringLength, etc.",
+      "check": {
+        "type": "setValuesFuncCallString",
+        "target": "step"
+      },
+      "notes": "AI models output {Count(var)} in a valueClass=string element because they mirror Provar's UI tooltip. The correct form is a funcCall value element. This causes silent test failures where the variable receives the literal string rather than the computed value.",
+      "source": "Field observation: SetValues function-call string interpolation anti-pattern"
+    },
+    {
+      "id": "SETVALUES-ZERO-IDX-001",
+      "category": "StructureAndGrouping",
+      "name": "SetValues string expression must not use [0] index",
+      "description": "Provar's string-template engine is 1-indexed: {List[1]} accesses the first element. Using {List[0].Field} in a valueClass=string expression causes an out-of-bounds exception at runtime because index 0 is invalid in the template system. Note: the XML variable-path structure (<filter class=\"index\"><index valueClass=\"decimal\">0</index>) correctly uses 0-based indexing via a different code path — this rule targets only the broken string-template form.",
+      "appliesTo": ["Step"],
+      "severity": "major",
+      "weight": 5,
+      "recommendation": "Either switch to the proper XML variable path structure (<value class=\"variable\"><path element=\"List\"><filter class=\"index\"><index valueClass=\"decimal\">0</index></filter></path><path element=\"FieldName\"/></value>) or, if you must use string interpolation, change [0] to [1] to access the first element.",
+      "check": {
+        "type": "setValuesZeroIndexString",
+        "target": "step"
+      },
+      "notes": "AI models that generate string-template expressions derive index numbers from the literal test intent (e.g. 'first item = index 0') but Provar string templates are 1-indexed. This causes runtime out-of-bounds errors.",
+      "source": "Field observation: SetValues zero-index string interpolation anti-pattern"
+    },
+    {
+      "id": "ASSERT-STR-VAR-001",
+      "category": "StructureAndGrouping",
+      "name": "AssertValues must not use string literal to reference a variable",
+      "description": "AssertValues steps must use a proper XML variable reference (<value class=\"variable\">) when comparing against a stored variable. Using a string literal that wraps a variable name — e.g. <value class=\"value\" valueClass=\"string\">{RowCount}</value> — causes Provar to compare the literal text \"{RowCount}\" against the actual value, which always produces a false failure. The same applies to function-call expressions such as {Count(Results)}.",
+      "appliesTo": ["Step"],
+      "severity": "major",
+      "weight": 5,
+      "recommendation": "Replace the string literal with a proper variable reference: <value class=\"variable\"><path element=\"RowCount\"/></value>. For function calls, use a preceding SetValues step to compute the value into a variable, then reference that variable in AssertValues.",
+      "check": {
+        "type": "assertValuesStringExpr",
+        "target": "step"
+      },
+      "notes": "AI models generate {VarName} in a valueClass=string element on the expectedValue side because they reproduce the Provar UI display format. The correct form is a class=variable element. This causes every assertion to fail with a string-vs-value mismatch.",
+      "source": "Field observation: AssertValues string-variable reference anti-pattern"
+    },
+    {
+      "id": "UI-LOCATOR-BUTTON-CASING-001",
+      "category": "TestCaseDesign",
+      "name": "Standard Salesforce flow buttons must use correct locator pattern",
+      "description": "Standard Salesforce flow buttons must use the correct locator name. Cancel must use lowercase 'name=cancel'. The Continue button on the Record Type selection screen uses a special path-based locator 'name=save&path=selectRecordType' — using 'name=continue' or 'name=Continue' will not resolve and causes Provar to show 'Not Available' at runtime. AI models commonly generate these incorrectly.",
+      "appliesTo": ["Step"],
+      "severity": "major",
+      "weight": 5,
+      "recommendation": "Cancel button: use name=cancel (lowercase). Continue button on record type selection: use name=save&path=selectRecordType (NOT name=continue or name=Continue). Corpus-validated patterns from 2000+ real test cases.",
+      "check": {
+        "type": "uiLocatorButtonCasing",
+        "target": "step",
+        "apiId": "com.provar.plugins.forcedotcom.core.ui.UiDoAction"
+      },
+      "notes": "Corpus analysis: Cancel uses name=cancel (10 occurrences) vs name=Cancel (4). Continue on record type selection consistently uses name=save&path=selectRecordType across all corpus examples.",
+      "source": "Corpus analysis: AllPOCProjects + InternalProjects (2108 test cases)"
+    },
+    {
+      "id": "UI-LOCATOR-RECORDTYPE-001",
+      "category": "TestCaseDesign",
+      "name": "Record Type field locator must use name=RecordType not name=recordTypeId",
+      "description": "On the Record Type selection screen (action=recordTypeNew), the Record Type picker field must use 'name=RecordType' in the locator URI with 'field=RecordTypeId' in the binding. AI models commonly generate 'name=recordTypeId' (the camelCase API name), which causes Provar to show 'Not Available' and the step fails at runtime.",
+      "appliesTo": ["Step"],
+      "severity": "major",
+      "weight": 5,
+      "recommendation": "Use name=RecordType (not name=recordTypeId or name=recordType) with field=RecordTypeId in the binding. Correct example: ui:locator?name=RecordType&binding=sf%3Aui%3Abinding%3Aobject%3Fobject%3D%7BtargetUrl%3Aobject%7D%26field%3DRecordTypeId",
+      "check": {
+        "type": "uiLocatorRecordTypeField",
+        "target": "step",
+        "apiId": "com.provar.plugins.forcedotcom.core.ui.UiDoAction"
+      },
+      "notes": "Corpus analysis: 258 occurrences of name=RecordType+field=RecordTypeId vs 0 occurrences of name=recordTypeId. The {targetUrl:object} binding resolves dynamically to the current screen's object.",
+      "source": "Corpus analysis: AllPOCProjects + InternalProjects (2108 test cases)"
     }
   ]
 }

package/lib/mcp/rules/provar_layer1_rules.json

@@ -0,0 +1,151 @@
+{
+  "schemaVersion": "1.0",
+  "name": "Provar Test Case Layer-1 Structural Validity Rules",
+  "description": "Single source of truth for the Layer-1 (structural validity) rule catalog. The DETECTION logic stays imperative in src/mcp/tools/testCaseValidate.ts; only the rule metadata (id, severity, applies_to, description) and the best-practice ownership/suppression mapping are centralized here. Consumers: testCaseValidate.ts (derives the bridge-suppression owned-set), scripts/build-validation-rule-registry.cjs (renders the Layer-1 registry rows), and test/unit/mcp/validationRuleRegistry.test.ts (drift guard). The `owns_bp_rules` field lists the Layer-2 `critical` best-practice rule ids whose concept this Layer-1 check already owns — those criticals are NOT bridged into is_valid (avoids double-reporting). Order is display order for the registry; keep it stable. severity is ERROR or WARNING (INFO reserved).",
+  "rules": [
+    {
+      "id": "TC_001",
+      "severity": "ERROR",
+      "applies_to": "document",
+      "description": "XML declaration present (<?xml …?> first line)."
+    },
+    {
+      "id": "TC_002",
+      "severity": "ERROR",
+      "applies_to": "document",
+      "description": "XML is well-formed (parses without error)."
+    },
+    {
+      "id": "TC_003",
+      "severity": "ERROR",
+      "applies_to": "document",
+      "description": "Root element is <testCase>.",
+      "owns_bp_rules": ["SCHEMA-ROOT-001"]
+    },
+    {
+      "id": "TC_010",
+      "severity": "ERROR",
+      "applies_to": "testCase",
+      "description": "testCase id, when present, is a non-negative integer (id is optional; guid is the identifier).",
+      "owns_bp_rules": ["SCHEMA-ID-001"]
+    },
+    {
+      "id": "TC_011",
+      "severity": "ERROR",
+      "applies_to": "testCase",
+      "description": "testCase has a guid attribute.",
+      "owns_bp_rules": ["VALID-GUID-001"]
+    },
+    {
+      "id": "TC_012",
+      "severity": "ERROR",
+      "applies_to": "testCase",
+      "description": "testCase guid is a valid UUID v4."
+    },
+    {
+      "id": "TC_020",
+      "severity": "ERROR",
+      "applies_to": "testCase",
+      "description": "testCase has a <steps> element.",
+      "owns_bp_rules": ["SCHEMA-STEPS-001", "VALID-STEPS-001"]
+    },
+    {
+      "id": "TC_030",
+      "severity": "ERROR",
+      "applies_to": "apiCall",
+      "description": "Each apiCall has a guid attribute."
+    },
+    {
+      "id": "TC_031",
+      "severity": "ERROR",
+      "applies_to": "apiCall",
+      "description": "Each apiCall guid is a valid UUID v4."
+    },
+    {
+      "id": "TC_032",
+      "severity": "ERROR",
+      "applies_to": "apiCall",
+      "description": "Each apiCall has an apiId attribute."
+    },
+    {
+      "id": "TC_033",
+      "severity": "WARNING",
+      "applies_to": "apiCall",
+      "description": "Each apiCall has a descriptive name attribute."
+    },
+    {
+      "id": "TC_034",
+      "severity": "ERROR",
+      "applies_to": "apiCall",
+      "description": "Each apiCall has a testItemId attribute.",
+      "owns_bp_rules": ["STEP-ITEMID-001"]
+    },
+    {
+      "id": "TC_035",
+      "severity": "ERROR",
+      "applies_to": "apiCall",
+      "description": "apiCall testItemId is a whole number."
+    },
+    {
+      "id": "DATA-001",
+      "severity": "WARNING",
+      "applies_to": "testCase",
+      "description": "<dataTable> only iterates under a test plan; flags direct testCase-mode execution."
+    },
+    {
+      "id": "VAR-REF-001",
+      "severity": "WARNING",
+      "applies_to": "argument",
+      "description": "A whole-token {Var} stored as valueClass=\"string\" (use class=\"variable\")."
+    },
+    {
+      "id": "VAR-REF-002",
+      "severity": "WARNING",
+      "applies_to": "argument",
+      "description": "{Var} tokens embedded in a plain string (use class=\"compound\")."
+    },
+    {
+      "id": "UI-TARGET-001",
+      "severity": "ERROR",
+      "applies_to": "apiCall",
+      "description": "UiWithScreen/UiWithRow target uses class=\"uiTarget\"."
+    },
+    {
+      "id": "UI-LOCATOR-001",
+      "severity": "ERROR",
+      "applies_to": "apiCall",
+      "description": "UI action locator uses class=\"uiLocator\"."
+    },
+    {
+      "id": "UI-INTERACTION-001",
+      "severity": "ERROR",
+      "applies_to": "apiCall",
+      "description": "UiDoAction interaction uses class=\"uiInteraction\"."
+    },
+    {
+      "id": "UI-ASSERT-STRUCTURE-001",
+      "severity": "ERROR",
+      "applies_to": "apiCall",
+      "description": "UiAssert uses nested field/column/page assertion containers, not a flat argument."
+    },
+    {
+      "id": "SETVALUES-STRUCTURE-001",
+      "severity": "ERROR",
+      "applies_to": "apiCall",
+      "description": "SetValues values argument uses class=\"valueList\" with <namedValues>."
+    },
+    {
+      "id": "ASSERT-001",
+      "severity": "WARNING",
+      "applies_to": "apiCall",
+      "description": "AssertValues namedValues format flagged for variable/Apex comparisons."
+    },
+    {
+      "id": "COMPARISON-TYPE-001",
+      "severity": "ERROR",
+      "applies_to": "apiCall",
+      "description": "comparisonType is within the step-scoped enum subset (load-blocking otherwise).",
+      "owns_bp_rules": ["COMPARISON-TYPE-ENUM-001"]
+    }
+  ]
+}