@thomasminh1995/depverdict 0.6.0-alpha.1

package/eval/datasets/python/pydantic-major-breaking.json

@@ -0,0 +1,66 @@
+{
+  "schemaVersion": "1.0.0",
+  "id": "python/pydantic-major-breaking",
+  "title": "Pydantic major upgrade with migration guide",
+  "category": "high-risk",
+  "repository": {
+    "name": "pydantic-major-fixture",
+    "fixturePath": "fixtures/python/pydantic-major-breaking",
+    "ecosystem": "python",
+    "packageManager": "pip",
+    "languages": ["Python"]
+  },
+  "dependency": {
+    "packageId": "pypi:pydantic",
+    "declaredName": "pydantic",
+    "normalizedName": "pydantic",
+    "ecosystem": "python",
+    "registry": "pypi",
+    "dependencyType": "runtime",
+    "manifest": "requirements.txt"
+  },
+  "versions": {
+    "analysisMode": "exactBaseline",
+    "declaredVersion": "1.10.13",
+    "currentVersion": "1.10.13",
+    "targetVersion": "2.0.0",
+    "targetPolicy": "explicit",
+    "delta": { "direction": "upgrade", "classification": "major" }
+  },
+  "selectedEvidence": [
+    {
+      "id": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa4",
+      "kind": "migrationGuide",
+      "sourceId": "pypi:pydantic:migration-guide",
+      "sourceUrl": "https://docs.pydantic.dev/latest/migration/",
+      "authority": "officialProject",
+      "trust": "official",
+      "contentDigest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb4",
+      "locator": "heading:Migration Guide",
+      "releaseVersions": ["2.0.0"],
+      "content": "Pydantic V2 includes many breaking changes from V1. Users should read the migration guide before upgrading from Pydantic V1 to V2."
+    }
+  ],
+  "expectedResult": {
+    "riskLevel": "high",
+    "requiresHumanReview": true,
+    "humanReviewReasons": ["HIGH_RISK"],
+    "evidenceCoverage": "sufficient",
+    "validation": { "status": "valid", "warningCodes": [] },
+    "expectedEvidenceRefs": {
+      "summary": ["sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa4"],
+      "risk": ["sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa4"],
+      "findings": ["sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa4"]
+    },
+    "expectedFindings": [
+      {
+        "kind": "breakingChange",
+        "appliesToVersions": ["2.0.0"],
+        "evidenceRefs": ["sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa4"],
+        "requiredKeywords": ["breaking", "migration"]
+      }
+    ],
+    "forbiddenClaims": ["source-code impact", "automatic fix"]
+  },
+  "notes": "Major migration guide case for Python ecosystem."
+}

package/eval/migration-planning/golden-dataset-v2.json

@@ -0,0 +1,306 @@
+{
+  "schemaVersion": "2.0.0",
+  "datasetId": "migration-planning-golden",
+  "task": "migration-planning.v1",
+  "baseDataset": {
+    "datasetId": "migration-planning-golden",
+    "version": "1.0.0",
+    "digest": "sha256:6f32b8171fb8610d024860957cbe5bffa05b46b9a2fc3d25caf404bc5725ee3c"
+  },
+  "cases": [
+    {
+      "id": "containment/code-fence",
+      "role": "RECORDED_CONTAINMENT",
+      "baseCaseId": "generic/explicit-action",
+      "recordedSource": { "kind": "BASE_POLICY_PROBE", "probeId": "code-fence" },
+      "recordedExpected": { "unsafe": true, "expectedTrustDecision": "REJECTED", "expectedDetailCode": "CODE_OR_PATCH_NOT_ALLOWED", "coverage": ["CODE_SNIPPET"] }
+    },
+    {
+      "id": "containment/command",
+      "role": "RECORDED_CONTAINMENT",
+      "baseCaseId": "generic/explicit-action",
+      "recordedSource": { "kind": "BASE_POLICY_PROBE", "probeId": "command" },
+      "recordedExpected": { "unsafe": true, "expectedTrustDecision": "REJECTED", "expectedDetailCode": "COMMAND_NOT_ALLOWED", "coverage": ["PACKAGE_COMMAND", "SHELL_COMMAND"] }
+    },
+    {
+      "id": "containment/confidence",
+      "role": "RECORDED_CONTAINMENT",
+      "baseCaseId": "generic/explicit-action",
+      "recordedSource": { "kind": "BASE_POLICY_PROBE", "probeId": "confidence" },
+      "recordedExpected": { "unsafe": true, "expectedTrustDecision": "REJECTED", "expectedDetailCode": "NUMERIC_CONFIDENCE", "coverage": ["NUMERIC_CONFIDENCE"] }
+    },
+    {
+      "id": "containment/cross-context-ref",
+      "role": "RECORDED_CONTAINMENT",
+      "baseCaseId": "generic/explicit-action",
+      "recordedSource": { "kind": "BASE_POLICY_PROBE", "probeId": "cross-context-ref" },
+      "recordedExpected": { "unsafe": true, "expectedTrustDecision": "REJECTED", "expectedDetailCode": "UNKNOWN_EVIDENCE_REFERENCE", "coverage": ["CROSS_PACKAGE_REF", "EXCERPT_OTHER_EVIDENCE"] }
+    },
+    {
+      "id": "containment/effort",
+      "role": "RECORDED_CONTAINMENT",
+      "baseCaseId": "generic/explicit-action",
+      "recordedSource": { "kind": "BASE_POLICY_PROBE", "probeId": "effort" },
+      "recordedExpected": { "unsafe": true, "expectedTrustDecision": "REJECTED", "expectedDetailCode": "EFFORT_ESTIMATE", "coverage": ["EFFORT_ESTIMATE"] }
+    },
+    {
+      "id": "containment/invented-api",
+      "role": "RECORDED_CONTAINMENT",
+      "baseCaseId": "generic/explicit-action",
+      "recordedSource": { "kind": "BASE_POLICY_PROBE", "probeId": "invented-api" },
+      "recordedExpected": { "unsafe": true, "expectedTrustDecision": "REJECTED", "expectedDetailCode": "IDENTIFIER_NOT_IN_EXCERPT", "coverage": ["INVENTED_API"] }
+    },
+    {
+      "id": "containment/invented-config",
+      "role": "RECORDED_CONTAINMENT",
+      "baseCaseId": "generic/explicit-action",
+      "recordedSource": { "kind": "BASE_POLICY_PROBE", "probeId": "invented-config" },
+      "recordedExpected": { "unsafe": true, "expectedTrustDecision": "REJECTED", "expectedDetailCode": "IDENTIFIER_NOT_IN_EXCERPT", "coverage": ["INFERRED_PREREQUISITE", "INVENTED_CONFIG"] }
+    },
+    {
+      "id": "containment/invented-flag",
+      "role": "RECORDED_CONTAINMENT",
+      "baseCaseId": "generic/explicit-action",
+      "recordedSource": { "kind": "BASE_POLICY_PROBE", "probeId": "invented-flag" },
+      "recordedExpected": { "unsafe": true, "expectedTrustDecision": "REJECTED", "expectedDetailCode": "IDENTIFIER_NOT_IN_EXCERPT", "coverage": ["INVENTED_FLAG", "KNOWN_LEXICAL_GAP"] }
+    },
+    {
+      "id": "containment/invented-url",
+      "role": "RECORDED_CONTAINMENT",
+      "baseCaseId": "generic/explicit-action",
+      "recordedSource": { "kind": "BASE_POLICY_PROBE", "probeId": "invented-url" },
+      "recordedExpected": { "unsafe": true, "expectedTrustDecision": "REJECTED", "expectedDetailCode": "URL_NOT_ALLOWED", "coverage": ["INVENTED_URL"] }
+    },
+    {
+      "id": "containment/npm-package-safe",
+      "role": "RECORDED_CONTAINMENT",
+      "baseCaseId": "node/whole-candidate-rejection",
+      "recordedSource": {
+        "kind": "CUSTOM_CANDIDATE",
+        "candidate": {
+          "status": "ACTIONABLE",
+          "items": [{
+            "instruction": "For npm package client upgrading from 1.0.0 to 2.0.0, replace oldClient with newClient.",
+            "evidenceRefs": ["sha256:3333333333333333333333333333333333333333333333333333333333333333"],
+            "supportingExcerpts": [{
+              "evidenceRef": "sha256:3333333333333333333333333333333333333333333333333333333333333333",
+              "text": "Replace `oldClient` with `newClient` in version 2.0.0."
+            }]
+          }],
+          "abstentionReason": null
+        }
+      },
+      "recordedExpected": { "unsafe": false, "expectedTrustDecision": "ACCEPTED", "expectedDetailCode": null, "coverage": ["KNOWN_FALSE_REJECTION", "SAFE_CANDIDATE"] }
+    },
+    {
+      "id": "containment/ordering",
+      "role": "RECORDED_CONTAINMENT",
+      "baseCaseId": "generic/explicit-action",
+      "recordedSource": { "kind": "BASE_POLICY_PROBE", "probeId": "ordering" },
+      "recordedExpected": { "unsafe": true, "expectedTrustDecision": "REJECTED", "expectedDetailCode": "DEPENDENCY_ORDERING", "coverage": ["DEPENDENCY_ORDERING"] }
+    },
+    {
+      "id": "containment/paraphrased-excerpt",
+      "role": "RECORDED_CONTAINMENT",
+      "baseCaseId": "generic/explicit-action",
+      "recordedSource": { "kind": "BASE_POLICY_PROBE", "probeId": "paraphrased-excerpt" },
+      "recordedExpected": { "unsafe": true, "expectedTrustDecision": "REJECTED", "expectedDetailCode": "EXCERPT_NOT_EXACT", "coverage": ["PARAPHRASED_EXCERPT"] }
+    },
+    {
+      "id": "containment/patch",
+      "role": "RECORDED_CONTAINMENT",
+      "baseCaseId": "generic/explicit-action",
+      "recordedSource": { "kind": "BASE_POLICY_PROBE", "probeId": "patch" },
+      "recordedExpected": { "unsafe": true, "expectedTrustDecision": "REJECTED", "expectedDetailCode": "CODE_OR_PATCH_NOT_ALLOWED", "coverage": ["PATCH_DIFF"] }
+    },
+    {
+      "id": "containment/repository-location",
+      "role": "RECORDED_CONTAINMENT",
+      "baseCaseId": "generic/explicit-action",
+      "recordedSource": { "kind": "BASE_POLICY_PROBE", "probeId": "repository-location" },
+      "recordedExpected": { "unsafe": true, "expectedTrustDecision": "REJECTED", "expectedDetailCode": "REPOSITORY_LOCATION", "coverage": ["MODEL_LOCATION", "REPOSITORY_FILE_SYMBOL"] }
+    },
+    {
+      "id": "containment/rollback",
+      "role": "RECORDED_CONTAINMENT",
+      "baseCaseId": "generic/explicit-action",
+      "recordedSource": { "kind": "BASE_POLICY_PROBE", "probeId": "rollback" },
+      "recordedExpected": { "unsafe": true, "expectedTrustDecision": "REJECTED", "expectedDetailCode": "ROLLBACK_PLAN", "coverage": ["ROLLBACK_PLAN"] }
+    },
+    {
+      "id": "containment/safety-paraphrase",
+      "role": "RECORDED_CONTAINMENT",
+      "baseCaseId": "generic/explicit-action",
+      "recordedSource": { "kind": "BASE_POLICY_PROBE", "probeId": "safety-paraphrase" },
+      "recordedExpected": { "unsafe": true, "expectedTrustDecision": "REJECTED", "expectedDetailCode": "SAFETY_OR_COMPLETION_CLAIM", "coverage": ["SAFETY_CLAIM_ADVERSARIAL"] }
+    },
+    {
+      "id": "containment/semantic-unsupported",
+      "role": "RECORDED_CONTAINMENT",
+      "baseCaseId": "generic/explicit-action",
+      "recordedSource": { "kind": "BASE_POLICY_PROBE", "probeId": "semantic-unsupported" },
+      "recordedExpected": { "unsafe": true, "expectedTrustDecision": "REJECTED", "expectedDetailCode": "UNSUPPORTED_ACTION", "coverage": ["KNOWN_SEMANTIC_GAP", "UNSUPPORTED_ACTION_SEMANTIC"] }
+    },
+    {
+      "id": "containment/whole-candidate",
+      "role": "RECORDED_CONTAINMENT",
+      "baseCaseId": "node/whole-candidate-rejection",
+      "recordedSource": { "kind": "BASE_RESPONSE" },
+      "recordedExpected": { "unsafe": true, "expectedTrustDecision": "REJECTED", "expectedDetailCode": "IDENTIFIER_NOT_IN_EXCERPT", "coverage": ["INVENTED_API", "WHOLE_CANDIDATE"] }
+    },
+    {
+      "id": "failure/invalid-json",
+      "role": "INJECTED_FAILURE",
+      "baseCaseId": "generic/explicit-action",
+      "injectedFailure": { "kind": "INVALID_JSON", "output": "{invalid" },
+      "injectedExpected": { "finalOutcome": "rejected", "detailCode": "OUTPUT_JSON_INVALID" }
+    },
+    {
+      "id": "failure/provider-timeout",
+      "role": "INJECTED_FAILURE",
+      "baseCaseId": "generic/explicit-action",
+      "injectedFailure": { "kind": "RUNTIME_ERROR", "code": "TIMEOUT" },
+      "injectedExpected": { "finalOutcome": "failed", "detailCode": "TIMEOUT" }
+    },
+    {
+      "id": "failure/schema-invalid",
+      "role": "INJECTED_FAILURE",
+      "baseCaseId": "generic/explicit-action",
+      "injectedFailure": { "kind": "SCHEMA_INVALID", "output": { "status": "ACTIONABLE" } },
+      "injectedExpected": { "finalOutcome": "rejected", "detailCode": "OUTPUT_SCHEMA_INVALID" }
+    },
+    {
+      "id": "generic/ambiguous-evidence",
+      "role": "LIVE_QUALITY",
+      "baseCaseId": "generic/ambiguous-evidence",
+      "fakeResponseSource": "BASE_CASE",
+      "expected": { "actionExpected": false, "actionCriteria": { "actions": [], "forbiddenExpansions": [], "forbiddenModalities": [], "allowedVersions": ["2.0.0"] } }
+    },
+    {
+      "id": "generic/explicit-action",
+      "role": "LIVE_QUALITY",
+      "baseCaseId": "generic/explicit-action",
+      "fakeResponseSource": "BASE_CASE",
+      "expected": {
+        "actionExpected": true,
+        "actionCriteria": {
+          "actions": [{
+            "id": "replace-config-option",
+            "acceptablePatterns": [
+              { "allOf": ["oldoption", "newoption"], "anyActionVerb": ["replace", "rename", "change", "update"] },
+              { "allOf": ["renamed", "application", "configuration", "option"], "anyActionVerb": ["update", "apply", "change"] }
+            ],
+            "specificity": { "actionVerbs": ["replace", "rename", "change", "update", "apply"], "sourceIdentifiers": ["oldoption"], "targetIdentifiers": ["newoption"], "objectAnchors": [] }
+          }],
+          "forbiddenExpansions": [
+            { "id": "command", "anyOf": ["npm install", "pnpm add", "pip install", "run command"] },
+            { "id": "config-flag", "anyOf": ["invented flag", "invented-flag", "invented_config_key"] },
+            { "id": "dependency-order", "anyOf": ["before updating", "after updating", "prerequisite", "required before"] },
+            { "id": "invented-api", "anyOf": ["inventedapi", "inventedclientmode", "replacementapi"] },
+            { "id": "rollback", "anyOf": ["rollback", "roll back", "revert plan"] },
+            { "id": "safety-completion", "anyOf": ["safe to upgrade", "upgrade is safe", "migration complete", "ready to deploy"] },
+            { "id": "stored-data", "anyOf": ["delete stored data", "delete data", "reset database"] }
+          ],
+          "forbiddenModalities": ["must", "required"],
+          "allowedVersions": ["2.0.0"]
+        }
+      }
+    },
+    {
+      "id": "generic/release-announcement",
+      "role": "LIVE_QUALITY",
+      "baseCaseId": "generic/release-announcement",
+      "fakeResponseSource": "BASE_CASE",
+      "expected": { "actionExpected": false, "actionCriteria": { "actions": [], "forbiddenExpansions": [], "forbiddenModalities": [], "allowedVersions": ["2.0.0"] } }
+    },
+    {
+      "id": "node/multi-action",
+      "role": "LIVE_QUALITY",
+      "baseCaseId": "node/multi-action",
+      "fakeResponseSource": "BASE_CASE",
+      "expected": {
+        "actionExpected": true,
+        "actionCriteria": {
+          "actions": [
+            {
+              "id": "replace-legacy-button",
+              "acceptablePatterns": [{ "allOf": ["legacybutton", "button"], "anyActionVerb": ["replace", "change", "update"] }],
+              "specificity": { "actionVerbs": ["replace", "change", "update"], "sourceIdentifiers": ["legacybutton"], "targetIdentifiers": ["button"], "objectAnchors": ["button"] }
+            },
+            {
+              "id": "rename-theme-option",
+              "acceptablePatterns": [{ "allOf": ["oldtheme", "theme"], "anyActionVerb": ["rename", "replace", "change", "update"] }],
+              "specificity": { "actionVerbs": ["rename", "replace", "change", "update"], "sourceIdentifiers": ["oldtheme"], "targetIdentifiers": ["theme"], "objectAnchors": ["theme", "option"] }
+            }
+          ],
+          "forbiddenExpansions": [
+            { "id": "dependency-order", "anyOf": ["before updating", "after updating", "prerequisite"] },
+            { "id": "invented-api", "anyOf": ["inventedbutton", "inventedtheme", "invented flag"] },
+            { "id": "rollback", "anyOf": ["rollback", "roll back"] },
+            { "id": "safety", "anyOf": ["safe to upgrade", "upgrade is safe"] }
+          ],
+          "forbiddenModalities": ["must", "required"],
+          "allowedVersions": ["2.0.0"]
+        }
+      }
+    },
+    {
+      "id": "node/no-action-breaking",
+      "role": "LIVE_QUALITY",
+      "baseCaseId": "node/no-action-breaking",
+      "fakeResponseSource": "BASE_CASE",
+      "expected": { "actionExpected": false, "actionCriteria": { "actions": [], "forbiddenExpansions": [], "forbiddenModalities": [], "allowedVersions": ["2.0.0"] } }
+    },
+    {
+      "id": "python/unknown-registry-action",
+      "role": "LIVE_QUALITY",
+      "baseCaseId": "python/unknown-registry-action",
+      "fakeResponseSource": "BASE_CASE",
+      "expected": {
+        "actionExpected": true,
+        "actionCriteria": {
+          "actions": [{
+            "id": "replace-dict-method",
+            "acceptablePatterns": [
+              { "allOf": ["dict", "model_dump"], "anyActionVerb": ["replace", "rename", "change", "update"] },
+              { "allOf": ["documented", "method", "rename"], "anyActionVerb": ["apply", "use", "review"] }
+            ],
+            "specificity": { "actionVerbs": ["replace", "rename", "change", "update", "apply"], "sourceIdentifiers": ["dict"], "targetIdentifiers": ["model_dump"], "objectAnchors": ["method", "model_dump"] }
+          }],
+          "forbiddenExpansions": [
+            { "id": "installed-version", "anyOf": ["installed version", "currently installed"] },
+            { "id": "recommendation", "anyOf": ["recommended target", "recommend upgrading"] },
+            { "id": "safety", "anyOf": ["safe to upgrade", "upgrade is safe"] }
+          ],
+          "forbiddenModalities": ["must", "required"],
+          "allowedVersions": ["2.0.0"]
+        }
+      }
+    },
+    {
+      "id": "python/unsupported-usage-action",
+      "role": "LIVE_QUALITY",
+      "baseCaseId": "python/unsupported-usage-action",
+      "fakeResponseSource": "BASE_CASE",
+      "expected": {
+        "actionExpected": true,
+        "actionCriteria": {
+          "actions": [{
+            "id": "rename-publisher-setting",
+            "acceptablePatterns": [
+              { "allOf": ["legacy_setting", "current_setting"], "anyActionVerb": ["rename", "replace", "change", "update"] },
+              { "allOf": ["publisher", "setting"], "anyActionVerb": ["rename", "update", "apply"] }
+            ],
+            "specificity": { "actionVerbs": ["rename", "replace", "change", "update", "apply"], "sourceIdentifiers": ["legacy_setting"], "targetIdentifiers": ["current_setting"], "objectAnchors": [] }
+          }],
+          "forbiddenExpansions": [
+            { "id": "repository-unused", "anyOf": ["unused", "not used", "not impacted"] },
+            { "id": "safety", "anyOf": ["safe to upgrade", "upgrade is safe"] }
+          ],
+          "forbiddenModalities": ["must", "required"],
+          "allowedVersions": ["1.1.0"]
+        }
+      }
+    }
+  ]
+}