@thomasminh1995/depverdict 0.6.0-alpha.1

package/schemas/ai-scorecard.schema.json

@@ -0,0 +1,132 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://raw.githubusercontent.com/thomasMinh1995/UpgradeLens/main/schemas/ai-scorecard.schema.json",
+  "title": "UpgradeLens AI Engineering Scorecard",
+  "description": "Deterministic scorecard derived from AI quality metrics.",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "schemaVersion",
+    "generatedAt",
+    "generator",
+    "input",
+    "overallScore",
+    "categoryScores",
+    "metricDetails",
+    "recommendations"
+  ],
+  "properties": {
+    "schemaVersion": { "const": "1.0.0" },
+    "generatedAt": { "type": "string", "format": "date-time" },
+    "generator": { "$ref": "#/$defs/generator" },
+    "input": { "$ref": "#/$defs/input" },
+    "overallScore": { "$ref": "#/$defs/score" },
+    "categoryScores": { "$ref": "#/$defs/categoryScores" },
+    "metricDetails": { "$ref": "#/$defs/metricDetails" },
+    "recommendations": {
+      "type": "array",
+      "items": { "$ref": "#/$defs/recommendation" }
+    }
+  },
+  "$defs": {
+    "score": { "type": "integer", "minimum": 0, "maximum": 100 },
+    "ratio": { "type": "number", "minimum": 0, "maximum": 1 },
+    "nonNegativeInteger": { "type": "integer", "minimum": 0 },
+    "generator": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["name", "version"],
+      "properties": {
+        "name": { "const": "UpgradeLens" },
+        "version": { "type": "string", "minLength": 1 }
+      }
+    },
+    "input": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["metrics"],
+      "properties": {
+        "metrics": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["schemaVersion", "generatedAt", "datasetVersion", "promptVersion", "model", "caseCount"],
+          "properties": {
+            "schemaVersion": { "const": "1.0.0" },
+            "generatedAt": { "type": "string", "format": "date-time" },
+            "datasetVersion": { "type": "string", "minLength": 1 },
+            "promptVersion": { "type": "string", "minLength": 1 },
+            "model": {
+              "type": "object",
+              "additionalProperties": false,
+              "required": ["provider", "name"],
+              "properties": {
+                "provider": { "type": "string", "minLength": 1 },
+                "name": { "type": "string", "minLength": 1 }
+              }
+            },
+            "caseCount": { "$ref": "#/$defs/nonNegativeInteger" }
+          }
+        }
+      }
+    },
+    "categoryScores": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "riskAnalysis",
+        "humanReview",
+        "evidenceQuality",
+        "trustLayer",
+        "deterministicQuality"
+      ],
+      "properties": {
+        "riskAnalysis": { "$ref": "#/$defs/score" },
+        "humanReview": { "$ref": "#/$defs/score" },
+        "evidenceQuality": { "$ref": "#/$defs/score" },
+        "trustLayer": { "$ref": "#/$defs/score" },
+        "deterministicQuality": { "$ref": "#/$defs/score" }
+      }
+    },
+    "metricDetails": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "casePassRate",
+        "riskClassificationAccuracy",
+        "humanReviewAccuracy",
+        "humanReviewReasonAccuracy",
+        "evidenceReferenceAccuracy",
+        "evidenceCoverageAccuracy",
+        "evidenceReferenceCoverage",
+        "unsupportedClaimRate",
+        "validationPassRate",
+        "deterministicPassRate"
+      ],
+      "properties": {
+        "casePassRate": { "$ref": "#/$defs/ratio" },
+        "riskClassificationAccuracy": { "$ref": "#/$defs/ratio" },
+        "humanReviewAccuracy": { "$ref": "#/$defs/ratio" },
+        "humanReviewReasonAccuracy": { "$ref": "#/$defs/ratio" },
+        "evidenceReferenceAccuracy": { "$ref": "#/$defs/ratio" },
+        "evidenceCoverageAccuracy": { "$ref": "#/$defs/ratio" },
+        "evidenceReferenceCoverage": { "$ref": "#/$defs/ratio" },
+        "unsupportedClaimRate": { "$ref": "#/$defs/ratio" },
+        "validationPassRate": { "$ref": "#/$defs/ratio" },
+        "deterministicPassRate": { "$ref": "#/$defs/ratio" }
+      }
+    },
+    "recommendation": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["severity", "category", "code", "message"],
+      "properties": {
+        "severity": { "enum": ["info", "medium", "high"] },
+        "category": {
+          "enum": ["Risk Analysis", "Human Review", "Evidence Quality", "Trust Layer", "Deterministic Quality", "General"]
+        },
+        "code": { "type": "string", "minLength": 1 },
+        "message": { "type": "string", "minLength": 1 }
+      }
+    }
+  }
+}

package/schemas/benchmark-report.schema.json

@@ -0,0 +1,189 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://raw.githubusercontent.com/thomasMinh1995/UpgradeLens/main/schemas/benchmark-report.schema.json",
+  "title": "UpgradeLens AI Benchmark Report",
+  "description": "Machine-readable report for prompt and model benchmark runs.",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "schemaVersion",
+    "generatedAt",
+    "generator",
+    "benchmark",
+    "runs",
+    "ranking"
+  ],
+  "properties": {
+    "schemaVersion": { "const": "1.0.0" },
+    "generatedAt": { "type": "string", "format": "date-time" },
+    "generator": { "$ref": "#/$defs/generator" },
+    "benchmark": { "$ref": "#/$defs/benchmark" },
+    "runs": {
+      "type": "array",
+      "items": { "$ref": "#/$defs/runResult" }
+    },
+    "ranking": {
+      "type": "array",
+      "items": { "$ref": "#/$defs/rankingEntry" }
+    }
+  },
+  "$defs": {
+    "score": { "type": "integer", "minimum": 0, "maximum": 100 },
+    "ratio": { "type": "number", "minimum": 0, "maximum": 1 },
+    "nullableNumber": {
+      "type": ["number", "null"],
+      "minimum": 0
+    },
+    "generator": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["name", "version"],
+      "properties": {
+        "name": { "const": "UpgradeLens" },
+        "version": { "type": "string", "minLength": 1 }
+      }
+    },
+    "model": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["provider", "name"],
+      "properties": {
+        "provider": { "type": "string", "minLength": 1 },
+        "name": { "type": "string", "minLength": 1 }
+      }
+    },
+    "runtime": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["type"],
+      "properties": {
+        "type": { "type": "string", "minLength": 1 },
+        "settings": {
+          "type": "object",
+          "additionalProperties": true
+        }
+      }
+    },
+    "benchmark": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["name", "configPath", "datasetPath", "runCount"],
+      "properties": {
+        "name": { "type": "string", "minLength": 1 },
+        "configPath": { "type": ["string", "null"] },
+        "datasetPath": { "type": "string", "minLength": 1 },
+        "runCount": { "type": "integer", "minimum": 0 }
+      }
+    },
+    "metrics": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "overallScore",
+        "riskAccuracy",
+        "humanReviewAccuracy",
+        "evidenceQuality",
+        "unsupportedClaimRate",
+        "deterministicPassRate"
+      ],
+      "properties": {
+        "overallScore": { "$ref": "#/$defs/score" },
+        "riskAccuracy": { "$ref": "#/$defs/ratio" },
+        "humanReviewAccuracy": { "$ref": "#/$defs/ratio" },
+        "evidenceQuality": { "$ref": "#/$defs/score" },
+        "unsupportedClaimRate": { "$ref": "#/$defs/ratio" },
+        "deterministicPassRate": { "$ref": "#/$defs/ratio" }
+      }
+    },
+    "performance": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["latencyMs", "tokenUsage", "estimatedCost"],
+      "properties": {
+        "latencyMs": { "$ref": "#/$defs/nullableNumber" },
+        "tokenUsage": { "$ref": "#/$defs/nullableNumber" },
+        "estimatedCost": { "$ref": "#/$defs/nullableNumber" }
+      }
+    },
+    "scorecardSummary": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["overallScore", "categoryScores", "recommendationCodes"],
+      "properties": {
+        "overallScore": { "$ref": "#/$defs/score" },
+        "categoryScores": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "riskAnalysis",
+            "humanReview",
+            "evidenceQuality",
+            "trustLayer",
+            "deterministicQuality"
+          ],
+          "properties": {
+            "riskAnalysis": { "$ref": "#/$defs/score" },
+            "humanReview": { "$ref": "#/$defs/score" },
+            "evidenceQuality": { "$ref": "#/$defs/score" },
+            "trustLayer": { "$ref": "#/$defs/score" },
+            "deterministicQuality": { "$ref": "#/$defs/score" }
+          }
+        },
+        "recommendationCodes": {
+          "type": "array",
+          "uniqueItems": true,
+          "items": { "type": "string", "minLength": 1 }
+        }
+      }
+    },
+    "runResult": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "id",
+        "promptVersion",
+        "model",
+        "runtime",
+        "datasetPath",
+        "evaluation",
+        "metrics",
+        "scorecard",
+        "performance"
+      ],
+      "properties": {
+        "id": { "type": "string", "minLength": 1 },
+        "promptVersion": { "type": "string", "minLength": 1 },
+        "model": { "$ref": "#/$defs/model" },
+        "runtime": { "$ref": "#/$defs/runtime" },
+        "datasetPath": { "type": "string", "minLength": 1 },
+        "evaluation": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["totalCases", "passed", "failed"],
+          "properties": {
+            "totalCases": { "type": "integer", "minimum": 0 },
+            "passed": { "type": "integer", "minimum": 0 },
+            "failed": { "type": "integer", "minimum": 0 }
+          }
+        },
+        "metrics": { "$ref": "#/$defs/metrics" },
+        "scorecard": { "$ref": "#/$defs/scorecardSummary" },
+        "performance": { "$ref": "#/$defs/performance" }
+      }
+    },
+    "rankingEntry": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["rank", "runId", "overallScore", "riskAccuracy", "evidenceQuality", "latencyMs", "estimatedCost"],
+      "properties": {
+        "rank": { "type": "integer", "minimum": 1 },
+        "runId": { "type": "string", "minLength": 1 },
+        "overallScore": { "$ref": "#/$defs/score" },
+        "riskAccuracy": { "$ref": "#/$defs/ratio" },
+        "evidenceQuality": { "$ref": "#/$defs/score" },
+        "latencyMs": { "$ref": "#/$defs/nullableNumber" },
+        "estimatedCost": { "$ref": "#/$defs/nullableNumber" }
+      }
+    }
+  }
+}

package/schemas/benchmark.schema.json

@@ -0,0 +1,55 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://raw.githubusercontent.com/thomasMinh1995/UpgradeLens/main/schemas/benchmark.schema.json",
+  "title": "UpgradeLens AI Benchmark Configuration",
+  "description": "Provider-neutral benchmark matrix for comparing prompt versions, models, and runtimes.",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["schemaVersion", "name", "datasetPath", "runs"],
+  "properties": {
+    "schemaVersion": { "const": "1.0.0" },
+    "name": { "type": "string", "minLength": 1 },
+    "datasetPath": { "type": "string", "minLength": 1 },
+    "runs": {
+      "type": "array",
+      "minItems": 1,
+      "items": { "$ref": "#/$defs/run" }
+    }
+  },
+  "$defs": {
+    "run": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["id", "promptVersion", "model", "runtime"],
+      "properties": {
+        "id": {
+          "type": "string",
+          "pattern": "^[a-zA-Z0-9][a-zA-Z0-9._-]*$"
+        },
+        "promptVersion": { "type": "string", "minLength": 1 },
+        "datasetPath": { "type": "string", "minLength": 1 },
+        "model": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["provider", "name"],
+          "properties": {
+            "provider": { "type": "string", "minLength": 1 },
+            "name": { "type": "string", "minLength": 1 }
+          }
+        },
+        "runtime": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["type"],
+          "properties": {
+            "type": { "type": "string", "minLength": 1 },
+            "settings": {
+              "type": "object",
+              "additionalProperties": true
+            }
+          }
+        }
+      }
+    }
+  }
+}

package/schemas/capability-profile.schema.json

@@ -0,0 +1,44 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://raw.githubusercontent.com/thomasMinh1995/UpgradeLens/main/schemas/capability-profile.schema.json",
+  "title": "UpgradeLens AI Runtime Capability Profile",
+  "description": "Portable descriptive metadata for one declared runtime capability set.",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "schemaVersion",
+    "capabilityId",
+    "protocol",
+    "structuredOutput",
+    "jsonMode",
+    "streaming",
+    "toolCalling",
+    "responsesApi",
+    "usageMetadata",
+    "identityVerification",
+    "timeoutSupported"
+  ],
+  "properties": {
+    "schemaVersion": { "const": "1.0.0" },
+    "capabilityId": { "$ref": "#/$defs/id" },
+    "protocol": { "$ref": "#/$defs/id" },
+    "structuredOutput": {
+      "enum": ["jsonSchema", "jsonMode", "promptOnly", "none"]
+    },
+    "jsonMode": { "type": "boolean" },
+    "streaming": { "type": "boolean" },
+    "toolCalling": { "type": "boolean" },
+    "responsesApi": { "type": "boolean" },
+    "usageMetadata": { "type": "boolean" },
+    "identityVerification": { "type": "boolean" },
+    "timeoutSupported": { "type": "boolean" }
+  },
+  "$defs": {
+    "id": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 128,
+      "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$"
+    }
+  }
+}

package/schemas/conformance-report.schema.json

@@ -0,0 +1,150 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://upgradelens.dev/schemas/conformance-report.schema.json",
+  "title": "UpgradeLens Offline Conformance Report",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "schemaVersion",
+    "generatedAt",
+    "generator",
+    "suite",
+    "runtime",
+    "deploymentProfileDigest",
+    "capabilityProfileDigest",
+    "conformanceScope",
+    "summary",
+    "capabilities",
+    "cases",
+    "failures",
+    "recommendation"
+  ],
+  "properties": {
+    "schemaVersion": { "const": "1.0.0" },
+    "generatedAt": { "type": "string", "format": "date-time" },
+    "generator": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["name", "version"],
+      "properties": {
+        "name": { "type": "string", "minLength": 1 },
+        "version": { "type": "string", "minLength": 1 }
+      }
+    },
+    "suite": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["mode", "protocol", "scope", "caseCount"],
+      "properties": {
+        "mode": { "const": "offlineFixture" },
+        "protocol": { "const": "openai-compatible-chat-completions" },
+        "scope": { "const": "offline-runtime-protocol" },
+        "caseCount": { "type": "integer", "minimum": 1 }
+      }
+    },
+    "runtime": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["provider", "model"],
+      "properties": {
+        "provider": { "type": "string", "minLength": 1 },
+        "model": { "type": "string", "minLength": 1 }
+      }
+    },
+    "deploymentProfileDigest": { "$ref": "#/$defs/digest" },
+    "capabilityProfileDigest": { "$ref": "#/$defs/digest" },
+    "conformanceScope": { "const": "offline-runtime-protocol" },
+    "summary": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["total", "passed", "failed", "skipped"],
+      "properties": {
+        "total": { "type": "integer", "minimum": 0 },
+        "passed": { "type": "integer", "minimum": 0 },
+        "failed": { "type": "integer", "minimum": 0 },
+        "skipped": { "type": "integer", "minimum": 0 }
+      }
+    },
+    "capabilities": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "connectivity",
+        "authentication",
+        "timeout",
+        "cancellation",
+        "structuredOutput",
+        "jsonSchema",
+        "jsonMode",
+        "responseValidation",
+        "usage",
+        "identity",
+        "contentType",
+        "httpErrors",
+        "oversizedResponse",
+        "redirectProtection",
+        "providerErrors"
+      ],
+      "properties": {
+        "connectivity": { "$ref": "#/$defs/capabilityStatus" },
+        "authentication": { "$ref": "#/$defs/capabilityStatus" },
+        "timeout": { "$ref": "#/$defs/capabilityStatus" },
+        "cancellation": { "$ref": "#/$defs/capabilityStatus" },
+        "structuredOutput": { "$ref": "#/$defs/capabilityStatus" },
+        "jsonSchema": { "$ref": "#/$defs/capabilityStatus" },
+        "jsonMode": { "$ref": "#/$defs/capabilityStatus" },
+        "responseValidation": { "$ref": "#/$defs/capabilityStatus" },
+        "usage": { "$ref": "#/$defs/capabilityStatus" },
+        "identity": { "$ref": "#/$defs/capabilityStatus" },
+        "contentType": { "$ref": "#/$defs/capabilityStatus" },
+        "httpErrors": { "$ref": "#/$defs/capabilityStatus" },
+        "oversizedResponse": { "$ref": "#/$defs/capabilityStatus" },
+        "redirectProtection": { "$ref": "#/$defs/capabilityStatus" },
+        "providerErrors": { "$ref": "#/$defs/capabilityStatus" }
+      }
+    },
+    "cases": {
+      "type": "array",
+      "items": { "$ref": "#/$defs/caseResult" }
+    },
+    "failures": {
+      "type": "array",
+      "items": { "$ref": "#/$defs/failure" }
+    },
+    "recommendation": {
+      "enum": ["CONFORMANT", "PARTIALLY_CONFORMANT", "NON_CONFORMANT"]
+    }
+  },
+  "$defs": {
+    "digest": {
+      "type": "string",
+      "pattern": "^sha256:[a-f0-9]{64}$"
+    },
+    "capabilityStatus": {
+      "enum": ["PASS", "FAIL", "NOT_SUPPORTED"]
+    },
+    "caseResult": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["id", "capability", "required", "status", "expected", "observed"],
+      "properties": {
+        "id": { "type": "string", "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$" },
+        "capability": { "type": "string", "minLength": 1 },
+        "required": { "type": "boolean" },
+        "status": { "enum": ["PASS", "FAIL", "SKIP"] },
+        "expected": { "type": "string", "minLength": 1 },
+        "observed": { "type": "string", "minLength": 1 }
+      }
+    },
+    "failure": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["caseId", "code", "message"],
+      "properties": {
+        "caseId": { "type": "string", "minLength": 1 },
+        "code": { "type": "string", "minLength": 1 },
+        "message": { "type": "string", "minLength": 1 }
+      }
+    }
+  }
+}

package/schemas/deployment-profile.schema.json

@@ -0,0 +1,64 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://raw.githubusercontent.com/thomasMinh1995/UpgradeLens/main/schemas/deployment-profile.schema.json",
+  "title": "UpgradeLens AI Runtime Deployment Profile",
+  "description": "Portable, secret-free metadata for one exact runtime deployment.",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "schemaVersion",
+    "deploymentId",
+    "provider",
+    "endpoint",
+    "model",
+    "capabilityProfile",
+    "capabilityProfileDigest",
+    "timeoutSeconds",
+    "maxResponseBytes"
+  ],
+  "properties": {
+    "schemaVersion": { "const": "1.0.0" },
+    "deploymentId": { "$ref": "#/$defs/id" },
+    "provider": { "$ref": "#/$defs/id" },
+    "endpoint": {
+      "type": "string",
+      "format": "uri",
+      "minLength": 1,
+      "maxLength": 2048
+    },
+    "model": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 256
+    },
+    "capabilityProfile": { "$ref": "#/$defs/id" },
+    "capabilityProfileDigest": { "$ref": "#/$defs/digest" },
+    "modelRevision": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 256
+    },
+    "timeoutSeconds": {
+      "type": "integer",
+      "minimum": 1,
+      "maximum": 86400
+    },
+    "maxResponseBytes": {
+      "type": "integer",
+      "minimum": 1,
+      "maximum": 1073741824
+    }
+  },
+  "$defs": {
+    "digest": {
+      "type": "string",
+      "pattern": "^sha256:[a-f0-9]{64}$"
+    },
+    "id": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 128,
+      "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$"
+    }
+  }
+}