@thomasminh1995/depverdict 0.6.0-alpha.1

package/eval/README.md

@@ -0,0 +1,86 @@
+# DepVerdict Golden Evaluation Dataset
+
+This directory contains provider-independent golden cases for AI Version Analysis quality evaluation.
+
+Historical fixture, prompt, schema, and qualification identifiers may retain the
+UpgradeLens protocol name under the `0.6.x` compatibility contract. They are not
+current product branding and must not be renamed without a versioned evaluation and
+qualification migration.
+
+AE-01 only defines dataset contracts and sample cases. It does not add an evaluation runner, metrics, telemetry, prompt optimization, or model comparison.
+
+## Structure
+
+```text
+eval/
+  schemas/
+    golden-case.schema.json
+    expected-result.schema.json
+  datasets/
+    node/
+    python/
+    generic/
+```
+
+Each file under `eval/datasets/**` is one JSON golden case. The first path segment groups cases by ecosystem or cross-ecosystem behavior. Future ecosystems such as Maven, NuGet, Cargo, Go, or Ruby should add sibling folders without changing the core case schema.
+
+## Golden Case Contract
+
+A case describes:
+
+- repository fixture identity;
+- ecosystem and dependency identity;
+- current and target version facts;
+- selected evidence that the AI is allowed to use;
+- expected result assertions.
+
+Cases are portable and provider-neutral. They must not store prompts, raw model outputs, chain-of-thought, provider names, model names, latency, token usage, or telemetry.
+
+## Expected Result Contract
+
+Expected results store only machine-checkable outcomes:
+
+- expected `riskLevel`;
+- expected human review boolean and reason codes;
+- expected evidence coverage and validation state;
+- expected evidence references;
+- expected finding kinds, applicable versions, evidence refs, and optional required keywords;
+- forbidden claim categories such as `source-code impact`.
+
+The expected result is not a transcript of what a model should say. Summaries and prose are intentionally not golden-string matched.
+
+## Naming
+
+Use lowercase kebab-case:
+
+```text
+eval/datasets/<group>/<dependency>-<scenario>.json
+```
+
+Examples:
+
+```text
+eval/datasets/node/react-major-breaking.json
+eval/datasets/python/pydantic-major-breaking.json
+eval/datasets/generic/missing-evidence.json
+```
+
+The case `id` must match the dataset grouping style, such as `node/react-major-breaking`.
+
+## Contributor Guidelines
+
+When adding a case:
+
+1. Keep evidence bounded and directly relevant to the current → target analysis.
+2. Use stable fake digests that match the `sha256:<64 hex>` shape.
+3. Do not include provider-specific fields or prompts.
+4. Do not encode chain-of-thought or hidden reasoning.
+5. Prefer one behavioral focus per case.
+6. Use `generic/` for cross-ecosystem behavior such as missing evidence, declared constraints, conflicts, and validation edge cases.
+7. Add new ecosystem folders only when the case needs ecosystem-specific version behavior.
+
+## Validation
+
+Dataset validation is schema-only in AE-01. The test suite loads every `eval/datasets/**/*.json` file and validates it against `eval/schemas/golden-case.schema.json`, which references `expected-result.schema.json`.
+
+AE-02 can build on this by adding an evaluation runner, metrics, fake-model replay, and model/prompt comparison. AE-01 deliberately stops before those steps.

package/eval/datasets/generic/declared-constraint.json

@@ -0,0 +1,59 @@
+{
+  "schemaVersion": "1.0.0",
+  "id": "generic/declared-constraint",
+  "title": "Declared constraint without resolved current version",
+  "category": "declared-constraint",
+  "repository": {
+    "name": "declared-constraint-fixture",
+    "fixturePath": "fixtures/generic/declared-constraint",
+    "ecosystem": "generic",
+    "packageManager": null,
+    "languages": ["Generic"]
+  },
+  "dependency": {
+    "packageId": "generic:example",
+    "declaredName": "example",
+    "normalizedName": "example",
+    "ecosystem": "generic",
+    "registry": "generic",
+    "dependencyType": "dependency",
+    "manifest": "manifest.json"
+  },
+  "versions": {
+    "analysisMode": "declaredConstraint",
+    "declaredVersion": "^1.0.0",
+    "currentVersion": null,
+    "targetVersion": "2.0.0",
+    "targetPolicy": "explicit",
+    "delta": { "direction": "unknown", "classification": "unknown" }
+  },
+  "selectedEvidence": [
+    {
+      "id": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa5",
+      "kind": "releaseNotes",
+      "sourceId": "generic:example:release-notes",
+      "sourceUrl": "https://example.org/releases/2.0.0",
+      "authority": "officialProject",
+      "trust": "official",
+      "contentDigest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb5",
+      "locator": "heading:2.0.0",
+      "releaseVersions": ["2.0.0"],
+      "content": "Version 2.0.0 documents target release behavior. The installed current version is not known from the declared constraint."
+    }
+  ],
+  "expectedResult": {
+    "riskLevel": "unknown",
+    "requiresHumanReview": true,
+    "humanReviewReasons": ["UNKNOWN_RISK", "VERSION_UNCERTAIN"],
+    "evidenceCoverage": "sufficient",
+    "validation": { "status": "valid", "warningCodes": [] },
+    "expectedEvidenceRefs": {
+      "summary": ["sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa5"],
+      "risk": [],
+      "findings": []
+    },
+    "expectedFindings": [],
+    "forbiddenClaims": ["installed version", "exact delta", "low risk"]
+  },
+  "notes": "Constraint mode can use target-scoped evidence but must not pretend to know the installed baseline."
+}

package/eval/datasets/generic/evidence-conflict.json

@@ -0,0 +1,73 @@
+{
+  "schemaVersion": "1.0.0",
+  "id": "generic/evidence-conflict",
+  "title": "Conflicting evidence forces review",
+  "category": "conflict",
+  "repository": {
+    "name": "evidence-conflict-fixture",
+    "fixturePath": "fixtures/generic/evidence-conflict",
+    "ecosystem": "generic",
+    "packageManager": null,
+    "languages": ["Generic"]
+  },
+  "dependency": {
+    "packageId": "generic:example",
+    "declaredName": "example",
+    "normalizedName": "example",
+    "ecosystem": "generic",
+    "registry": "generic",
+    "dependencyType": "dependency",
+    "manifest": "manifest.json"
+  },
+  "versions": {
+    "analysisMode": "exactBaseline",
+    "declaredVersion": "1.0.0",
+    "currentVersion": "1.0.0",
+    "targetVersion": "2.0.0",
+    "targetPolicy": "explicit",
+    "delta": { "direction": "upgrade", "classification": "major" }
+  },
+  "selectedEvidence": [
+    {
+      "id": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa6",
+      "kind": "breakingChanges",
+      "sourceId": "generic:example:official-notes",
+      "sourceUrl": "https://example.org/releases/2.0.0",
+      "authority": "officialProject",
+      "trust": "official",
+      "contentDigest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb6",
+      "locator": "heading:Breaking changes",
+      "releaseVersions": ["2.0.0"],
+      "content": "Version 2.0.0 documents a breaking configuration change.",
+      "conflictsWith": ["sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa7"]
+    },
+    {
+      "id": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa7",
+      "kind": "releaseNotes",
+      "sourceId": "generic:example:registry",
+      "sourceUrl": "https://registry.example.org/example/2.0.0",
+      "authority": "registryAuthoritative",
+      "trust": "publisher",
+      "contentDigest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb7",
+      "locator": "release:2.0.0",
+      "releaseVersions": ["2.0.0"],
+      "content": "Version 2.0.0 is listed as a routine release with no migration note.",
+      "conflictsWith": ["sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa6"]
+    }
+  ],
+  "expectedResult": {
+    "riskLevel": "unknown",
+    "requiresHumanReview": true,
+    "humanReviewReasons": ["UNKNOWN_RISK", "EVIDENCE_PARTIAL", "SOURCE_CONFLICT"],
+    "evidenceCoverage": "partial",
+    "validation": { "status": "validWithWarnings", "warningCodes": ["SOURCE_CONFLICT"] },
+    "expectedEvidenceRefs": {
+      "summary": ["sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa6", "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa7"],
+      "risk": [],
+      "findings": []
+    },
+    "expectedFindings": [],
+    "forbiddenClaims": ["low risk", "conflict resolved", "source-code impact"]
+  },
+  "notes": "Conflicting evidence should stay visible instead of silently choosing a winner."
+}

package/eval/datasets/generic/major-breaking-release.json

@@ -0,0 +1,66 @@
+{
+  "schemaVersion": "1.0.0",
+  "id": "generic/major-breaking-release",
+  "title": "Generic major release with mandatory breaking change",
+  "category": "high-risk",
+  "repository": {
+    "name": "major-breaking-fixture",
+    "fixturePath": "fixtures/generic/major-breaking-release",
+    "ecosystem": "java",
+    "packageManager": "maven",
+    "languages": ["Java"]
+  },
+  "dependency": {
+    "packageId": "maven:org.example:example-core",
+    "declaredName": "org.example:example-core",
+    "normalizedName": "org.example:example-core",
+    "ecosystem": "java",
+    "registry": "maven",
+    "dependencyType": "dependency",
+    "manifest": "pom.xml"
+  },
+  "versions": {
+    "analysisMode": "exactBaseline",
+    "declaredVersion": "1.9.0",
+    "currentVersion": "1.9.0",
+    "targetVersion": "2.0.0",
+    "targetPolicy": "explicit",
+    "delta": { "direction": "upgrade", "classification": "major" }
+  },
+  "selectedEvidence": [
+    {
+      "id": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa8",
+      "kind": "migrationGuide",
+      "sourceId": "maven:org.example:example-core:migration",
+      "sourceUrl": "https://example.org/example-core/2.0/migration",
+      "authority": "officialProject",
+      "trust": "official",
+      "contentDigest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb8",
+      "locator": "heading:2.0 Migration",
+      "releaseVersions": ["2.0.0"],
+      "content": "Version 2.0.0 removes deprecated configuration keys. Projects upgrading from 1.x must review the migration notes before adopting 2.0.0."
+    }
+  ],
+  "expectedResult": {
+    "riskLevel": "high",
+    "requiresHumanReview": true,
+    "humanReviewReasons": ["HIGH_RISK"],
+    "evidenceCoverage": "sufficient",
+    "validation": { "status": "valid", "warningCodes": [] },
+    "expectedEvidenceRefs": {
+      "summary": ["sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa8"],
+      "risk": ["sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa8"],
+      "findings": ["sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa8"]
+    },
+    "expectedFindings": [
+      {
+        "kind": "breakingChange",
+        "appliesToVersions": ["2.0.0"],
+        "evidenceRefs": ["sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa8"],
+        "requiredKeywords": ["removes", "migration"]
+      }
+    ],
+    "forbiddenClaims": ["source-code impact", "automatic migration"]
+  },
+  "notes": "Cross-ecosystem high-risk case to prove the dataset is not npm-specific."
+}

package/eval/datasets/generic/missing-evidence.json

@@ -0,0 +1,46 @@
+{
+  "schemaVersion": "1.0.0",
+  "id": "generic/missing-evidence",
+  "title": "Missing evidence yields unknown risk",
+  "category": "missing-evidence",
+  "repository": {
+    "name": "missing-evidence-fixture",
+    "fixturePath": "fixtures/generic/missing-evidence",
+    "ecosystem": "generic",
+    "packageManager": null,
+    "languages": ["Generic"]
+  },
+  "dependency": {
+    "packageId": "generic:example",
+    "declaredName": "example",
+    "normalizedName": "example",
+    "ecosystem": "generic",
+    "registry": "generic",
+    "dependencyType": "dependency",
+    "manifest": "manifest.json"
+  },
+  "versions": {
+    "analysisMode": "exactBaseline",
+    "declaredVersion": "1.0.0",
+    "currentVersion": "1.0.0",
+    "targetVersion": "2.0.0",
+    "targetPolicy": "explicit",
+    "delta": { "direction": "upgrade", "classification": "major" }
+  },
+  "selectedEvidence": [],
+  "expectedResult": {
+    "riskLevel": "unknown",
+    "requiresHumanReview": true,
+    "humanReviewReasons": ["UNKNOWN_RISK", "EVIDENCE_NONE", "ANALYSIS_FAILED"],
+    "evidenceCoverage": "none",
+    "validation": { "status": "validWithWarnings", "warningCodes": ["EVIDENCE_MISSING"] },
+    "expectedEvidenceRefs": {
+      "summary": [],
+      "risk": [],
+      "findings": []
+    },
+    "expectedFindings": [],
+    "forbiddenClaims": ["low risk", "breaking change", "source-code impact"]
+  },
+  "notes": "No model call should be required when selected evidence is empty."
+}

package/eval/datasets/generic/patch-release-low.json

@@ -0,0 +1,59 @@
+{
+  "schemaVersion": "1.0.0",
+  "id": "generic/patch-release-low",
+  "title": "Generic patch release with explicit no-breaking evidence",
+  "category": "low-risk",
+  "repository": {
+    "name": "patch-low-fixture",
+    "fixturePath": "fixtures/generic/patch-release-low",
+    "ecosystem": "ruby",
+    "packageManager": "bundler",
+    "languages": ["Ruby"]
+  },
+  "dependency": {
+    "packageId": "rubygems:example",
+    "declaredName": "example",
+    "normalizedName": "example",
+    "ecosystem": "ruby",
+    "registry": "rubygems",
+    "dependencyType": "dependency",
+    "manifest": "Gemfile"
+  },
+  "versions": {
+    "analysisMode": "exactBaseline",
+    "declaredVersion": "3.4.1",
+    "currentVersion": "3.4.1",
+    "targetVersion": "3.4.2",
+    "targetPolicy": "explicit",
+    "delta": { "direction": "upgrade", "classification": "patch" }
+  },
+  "selectedEvidence": [
+    {
+      "id": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa9",
+      "kind": "releaseNotes",
+      "sourceId": "rubygems:example:release-notes",
+      "sourceUrl": "https://example.org/ruby/example/releases/3.4.2",
+      "authority": "officialProject",
+      "trust": "official",
+      "contentDigest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb9",
+      "locator": "release:3.4.2",
+      "releaseVersions": ["3.4.2"],
+      "content": "Version 3.4.2 fixes a documentation typo and a small internal bug. No breaking changes, deprecations, or compatibility actions are documented."
+    }
+  ],
+  "expectedResult": {
+    "riskLevel": "low",
+    "requiresHumanReview": false,
+    "humanReviewReasons": [],
+    "evidenceCoverage": "sufficient",
+    "validation": { "status": "valid", "warningCodes": [] },
+    "expectedEvidenceRefs": {
+      "summary": ["sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa9"],
+      "risk": ["sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa9"],
+      "findings": []
+    },
+    "expectedFindings": [],
+    "forbiddenClaims": ["breaking change", "migration plan", "source-code impact"]
+  },
+  "notes": "Cross-ecosystem low-risk patch case."
+}

package/eval/datasets/node/axios-patch-low.json

@@ -0,0 +1,59 @@
+{
+  "schemaVersion": "1.0.0",
+  "id": "node/axios-patch-low",
+  "title": "Axios patch release with bug fixes only",
+  "category": "low-risk",
+  "repository": {
+    "name": "axios-patch-fixture",
+    "fixturePath": "fixtures/node/axios-patch-low",
+    "ecosystem": "node",
+    "packageManager": "npm",
+    "languages": ["JavaScript"]
+  },
+  "dependency": {
+    "packageId": "npm:axios",
+    "declaredName": "axios",
+    "normalizedName": "axios",
+    "ecosystem": "node",
+    "registry": "npm",
+    "dependencyType": "dependency",
+    "manifest": "package.json"
+  },
+  "versions": {
+    "analysisMode": "exactBaseline",
+    "declaredVersion": "1.6.7",
+    "currentVersion": "1.6.7",
+    "targetVersion": "1.6.8",
+    "targetPolicy": "explicit",
+    "delta": { "direction": "upgrade", "classification": "patch" }
+  },
+  "selectedEvidence": [
+    {
+      "id": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa2",
+      "kind": "releaseNotes",
+      "sourceId": "npm:axios:release-notes",
+      "sourceUrl": "https://github.com/axios/axios/releases/tag/v1.6.8",
+      "authority": "officialProject",
+      "trust": "official",
+      "contentDigest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb2",
+      "locator": "release:v1.6.8",
+      "releaseVersions": ["1.6.8"],
+      "content": "Axios 1.6.8 contains bug fixes and maintenance updates. No breaking changes or migration actions are documented for this patch release."
+    }
+  ],
+  "expectedResult": {
+    "riskLevel": "low",
+    "requiresHumanReview": false,
+    "humanReviewReasons": [],
+    "evidenceCoverage": "sufficient",
+    "validation": { "status": "valid", "warningCodes": [] },
+    "expectedEvidenceRefs": {
+      "summary": ["sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa2"],
+      "risk": ["sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa2"],
+      "findings": []
+    },
+    "expectedFindings": [],
+    "forbiddenClaims": ["breaking change", "migration plan", "source-code impact"]
+  },
+  "notes": "Low risk requires explicit evidence that the patch has no documented breaking or migration content."
+}

package/eval/datasets/node/react-major-breaking.json

@@ -0,0 +1,66 @@
+{
+  "schemaVersion": "1.0.0",
+  "id": "node/react-major-breaking",
+  "title": "React major release with documented breaking change",
+  "category": "high-risk",
+  "repository": {
+    "name": "react-major-fixture",
+    "fixturePath": "fixtures/node/react-major-breaking",
+    "ecosystem": "node",
+    "packageManager": "npm",
+    "languages": ["JavaScript"]
+  },
+  "dependency": {
+    "packageId": "npm:react",
+    "declaredName": "react",
+    "normalizedName": "react",
+    "ecosystem": "node",
+    "registry": "npm",
+    "dependencyType": "dependency",
+    "manifest": "package.json"
+  },
+  "versions": {
+    "analysisMode": "exactBaseline",
+    "declaredVersion": "18.2.0",
+    "currentVersion": "18.2.0",
+    "targetVersion": "19.0.0",
+    "targetPolicy": "explicit",
+    "delta": { "direction": "upgrade", "classification": "major" }
+  },
+  "selectedEvidence": [
+    {
+      "id": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "kind": "breakingChanges",
+      "sourceId": "npm:react:official-release-notes",
+      "sourceUrl": "https://react.dev/blog/2024/04/25/react-19",
+      "authority": "officialProject",
+      "trust": "official",
+      "contentDigest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
+      "locator": "heading:Breaking changes",
+      "releaseVersions": ["19.0.0"],
+      "content": "React 19 includes documented breaking changes for deprecated APIs and requires applications to review compatibility before upgrading."
+    }
+  ],
+  "expectedResult": {
+    "riskLevel": "high",
+    "requiresHumanReview": true,
+    "humanReviewReasons": ["HIGH_RISK"],
+    "evidenceCoverage": "sufficient",
+    "validation": { "status": "valid", "warningCodes": [] },
+    "expectedEvidenceRefs": {
+      "summary": ["sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"],
+      "risk": ["sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"],
+      "findings": ["sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"]
+    },
+    "expectedFindings": [
+      {
+        "kind": "breakingChange",
+        "appliesToVersions": ["19.0.0"],
+        "evidenceRefs": ["sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"],
+        "requiredKeywords": ["breaking", "deprecated"]
+      }
+    ],
+    "forbiddenClaims": ["source-code impact", "migration plan"]
+  },
+  "notes": "Major release evidence explicitly documents breaking changes."
+}

package/eval/datasets/node/react-minor-compatibility.json

@@ -0,0 +1,66 @@
+{
+  "schemaVersion": "1.0.0",
+  "id": "node/react-minor-compatibility",
+  "title": "React minor release with compatibility note",
+  "category": "medium-risk",
+  "repository": {
+    "name": "react-minor-fixture",
+    "fixturePath": "fixtures/node/react-minor-compatibility",
+    "ecosystem": "node",
+    "packageManager": "npm",
+    "languages": ["JavaScript"]
+  },
+  "dependency": {
+    "packageId": "npm:react",
+    "declaredName": "react",
+    "normalizedName": "react",
+    "ecosystem": "node",
+    "registry": "npm",
+    "dependencyType": "dependency",
+    "manifest": "package.json"
+  },
+  "versions": {
+    "analysisMode": "exactBaseline",
+    "declaredVersion": "18.2.0",
+    "currentVersion": "18.2.0",
+    "targetVersion": "18.3.0",
+    "targetPolicy": "explicit",
+    "delta": { "direction": "upgrade", "classification": "minor" }
+  },
+  "selectedEvidence": [
+    {
+      "id": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1",
+      "kind": "compatibility",
+      "sourceId": "npm:react:official-release-notes",
+      "sourceUrl": "https://react.dev/blog/2024/04/25/react-19-upgrade-guide",
+      "authority": "officialProject",
+      "trust": "official",
+      "contentDigest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb1",
+      "locator": "heading:18.3 warnings",
+      "releaseVersions": ["18.3.0"],
+      "content": "React 18.3 adds warnings that help prepare for React 19. The release is intended to surface compatibility issues before the next major upgrade."
+    }
+  ],
+  "expectedResult": {
+    "riskLevel": "medium",
+    "requiresHumanReview": false,
+    "humanReviewReasons": [],
+    "evidenceCoverage": "sufficient",
+    "validation": { "status": "valid", "warningCodes": [] },
+    "expectedEvidenceRefs": {
+      "summary": ["sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1"],
+      "risk": ["sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1"],
+      "findings": ["sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1"]
+    },
+    "expectedFindings": [
+      {
+        "kind": "compatibility",
+        "appliesToVersions": ["18.3.0"],
+        "evidenceRefs": ["sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1"],
+        "requiredKeywords": ["warnings", "compatibility"]
+      }
+    ],
+    "forbiddenClaims": ["source-code impact", "mandatory migration"]
+  },
+  "notes": "Minor release has compatibility warnings but no documented hard break."
+}

package/eval/datasets/python/fastapi-deprecation.json

@@ -0,0 +1,66 @@
+{
+  "schemaVersion": "1.0.0",
+  "id": "python/fastapi-deprecation",
+  "title": "FastAPI release with documented deprecation",
+  "category": "medium-risk",
+  "repository": {
+    "name": "fastapi-deprecation-fixture",
+    "fixturePath": "fixtures/python/fastapi-deprecation",
+    "ecosystem": "python",
+    "packageManager": "pip",
+    "languages": ["Python"]
+  },
+  "dependency": {
+    "packageId": "pypi:fastapi",
+    "declaredName": "fastapi",
+    "normalizedName": "fastapi",
+    "ecosystem": "python",
+    "registry": "pypi",
+    "dependencyType": "runtime",
+    "manifest": "requirements.txt"
+  },
+  "versions": {
+    "analysisMode": "exactBaseline",
+    "declaredVersion": "0.109.0",
+    "currentVersion": "0.109.0",
+    "targetVersion": "0.110.0",
+    "targetPolicy": "explicit",
+    "delta": { "direction": "upgrade", "classification": "minor" }
+  },
+  "selectedEvidence": [
+    {
+      "id": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa3",
+      "kind": "deprecations",
+      "sourceId": "pypi:fastapi:release-notes",
+      "sourceUrl": "https://fastapi.tiangolo.com/release-notes/",
+      "authority": "officialProject",
+      "trust": "official",
+      "contentDigest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb3",
+      "locator": "heading:0.110.0",
+      "releaseVersions": ["0.110.0"],
+      "content": "FastAPI 0.110.0 documents a deprecation for an older parameter behavior. Existing projects should review usage before relying on the old behavior."
+    }
+  ],
+  "expectedResult": {
+    "riskLevel": "medium",
+    "requiresHumanReview": false,
+    "humanReviewReasons": [],
+    "evidenceCoverage": "sufficient",
+    "validation": { "status": "valid", "warningCodes": [] },
+    "expectedEvidenceRefs": {
+      "summary": ["sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa3"],
+      "risk": ["sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa3"],
+      "findings": ["sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa3"]
+    },
+    "expectedFindings": [
+      {
+        "kind": "deprecation",
+        "appliesToVersions": ["0.110.0"],
+        "evidenceRefs": ["sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa3"],
+        "requiredKeywords": ["deprecation"]
+      }
+    ],
+    "forbiddenClaims": ["source-code impact", "major breaking change"]
+  },
+  "notes": "Deprecation evidence should not be escalated to high risk without a documented hard break."
+}