@reviewsensei/cli-win32-x64 0.1.0

package/bin/_internal/review_sensei/schemas/evaluation-corpus.schema.json

@@ -0,0 +1,316 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://reviewsensei.dev/schemas/v1/evaluation-corpus.schema.json",
+  "title": "ReviewSensei evaluation corpus v1",
+  "description": "Closed structural contract for one privacy-safe evaluation corpus.",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "schema_version",
+    "corpus_id",
+    "version",
+    "description",
+    "license",
+    "provenance",
+    "files",
+    "sample_repository",
+    "review_configuration",
+    "deterministic_thresholds",
+    "quality_thresholds",
+    "cases"
+  ],
+  "properties": {
+    "schema_version": {
+      "const": "1.0"
+    },
+    "corpus_id": {
+      "type": "string",
+      "minLength": 1,
+      "pattern": "^[a-z0-9][a-z0-9._-]*$"
+    },
+    "version": {
+      "type": "string",
+      "minLength": 1
+    },
+    "description": {
+      "type": "string",
+      "minLength": 1
+    },
+    "license": {
+      "type": "string",
+      "minLength": 1
+    },
+    "provenance": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["synthetic", "statement"],
+      "properties": {
+        "synthetic": {
+          "const": true
+        },
+        "statement": {
+          "type": "string",
+          "minLength": 1
+        }
+      }
+    },
+    "files": {
+      "type": "array",
+      "minItems": 1,
+      "maxItems": 512,
+      "uniqueItems": true,
+      "items": {
+        "$ref": "#/$defs/fileEntry"
+      }
+    },
+    "sample_repository": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["base_root", "head_root"],
+      "properties": {
+        "base_root": {
+          "type": "string",
+          "minLength": 1
+        },
+        "head_root": {
+          "type": "string",
+          "minLength": 1
+        }
+      }
+    },
+    "review_configuration": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["id"],
+      "properties": {
+        "id": {
+          "type": "string",
+          "minLength": 1
+        }
+      }
+    },
+    "deterministic_thresholds": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["exact_fixture_result_rate", "expected_contract_rejection_rate"],
+      "properties": {
+        "exact_fixture_result_rate": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 1
+        },
+        "expected_contract_rejection_rate": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 1
+        }
+      }
+    },
+    "quality_thresholds": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "actionable_precision_minimum",
+        "false_positive_rate_maximum",
+        "expected_finding_recall_minimum",
+        "location_validity_minimum",
+        "category_coverage_minimum"
+      ],
+      "properties": {
+        "actionable_precision_minimum": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 1
+        },
+        "false_positive_rate_maximum": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 1
+        },
+        "expected_finding_recall_minimum": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 1
+        },
+        "location_validity_minimum": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 1
+        },
+        "category_coverage_minimum": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 1
+        }
+      }
+    },
+    "cases": {
+      "type": "array",
+      "minItems": 1,
+      "maxItems": 64,
+      "uniqueItems": true,
+      "items": {
+        "$ref": "#/$defs/case"
+      }
+    }
+  },
+  "$defs": {
+    "fileEntry": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["path", "sha256", "purpose"],
+      "properties": {
+        "path": {
+          "type": "string",
+          "minLength": 1
+        },
+        "sha256": {
+          "type": "string",
+          "pattern": "^[a-f0-9]{64}$"
+        },
+        "size": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "purpose": {
+          "type": "string",
+          "minLength": 1
+        }
+      }
+    },
+    "expectedFinding": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["path", "line", "category", "body_terms"],
+      "properties": {
+        "path": {
+          "type": "string",
+          "minLength": 1
+        },
+        "line": {
+          "type": "integer",
+          "minimum": 1
+        },
+        "category": {
+          "type": "string",
+          "minLength": 1
+        },
+        "body_terms": {
+          "type": "array",
+          "minItems": 1,
+          "maxItems": 32,
+          "uniqueItems": true,
+          "items": {
+            "type": "string",
+            "minLength": 1
+          }
+        }
+      }
+    },
+    "case": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "id",
+        "title",
+        "kind",
+        "category",
+        "diff_path",
+        "response_path",
+        "expected_error_category"
+      ],
+      "properties": {
+        "id": {
+          "type": "string",
+          "pattern": "^[a-z0-9][a-z0-9._-]*$"
+        },
+        "title": {
+          "type": "string",
+          "minLength": 1
+        },
+        "kind": {
+          "enum": ["quality", "contract-rejection"]
+        },
+        "category": {
+          "type": "string",
+          "minLength": 1
+        },
+        "diff_path": {
+          "type": "string",
+          "minLength": 1
+        },
+        "response_path": {
+          "type": "string",
+          "minLength": 1
+        },
+        "expected_result_path": {
+          "type": "string",
+          "minLength": 1
+        },
+        "expected_findings": {
+          "type": "array",
+          "maxItems": 16,
+          "items": {
+            "$ref": "#/$defs/expectedFinding"
+          }
+        },
+        "acceptable_no_finding": {
+          "type": "boolean"
+        },
+        "expected_error_category": {
+          "type": "string",
+          "minLength": 1
+        }
+      }
+    }
+  },
+  "examples": [
+    {
+      "schema_version": "1.0",
+      "corpus_id": "review-sensei-synthetic-v1",
+      "version": "1.0",
+      "description": "Synthetic regression corpus.",
+      "license": "CC0-1.0",
+      "provenance": {
+        "synthetic": true,
+        "statement": "All contents are synthetic."
+      },
+      "files": [
+        {
+          "path": "corpus.json",
+          "sha256": "0000000000000000000000000000000000000000000000000000000000000000",
+          "purpose": "manifest"
+        }
+      ],
+      "sample_repository": {
+        "base_root": "sample-repository/base",
+        "head_root": "sample-repository/head"
+      },
+      "review_configuration": {
+        "id": "packaged-defaults-v1"
+      },
+      "deterministic_thresholds": {
+        "exact_fixture_result_rate": 1.0,
+        "expected_contract_rejection_rate": 1.0
+      },
+      "quality_thresholds": {
+        "actionable_precision_minimum": 0.75,
+        "false_positive_rate_maximum": 0.25,
+        "expected_finding_recall_minimum": 0.75,
+        "location_validity_minimum": 1.0,
+        "category_coverage_minimum": 0.75
+      },
+      "cases": [
+        {
+          "id": "example",
+          "title": "Example case",
+          "kind": "quality",
+          "category": "correctness",
+          "diff_path": "diffs/example.patch",
+          "response_path": "responses/example.json",
+          "expected_error_category": "none"
+        }
+      ]
+    }
+  ]
+}

package/bin/_internal/review_sensei/schemas/evaluation-report.schema.json

@@ -0,0 +1,371 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://reviewsensei.dev/schemas/v1/evaluation-report.schema.json",
+  "title": "ReviewSensei evaluation report v1",
+  "description": "Closed structural contract for one non-secret evaluation report.",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "schema_version",
+    "corpus",
+    "run",
+    "configuration",
+    "privacy",
+    "cases",
+    "metrics",
+    "deterministic",
+    "quality",
+    "threshold_failures",
+    "passed"
+  ],
+  "properties": {
+    "schema_version": {
+      "const": "1.0"
+    },
+    "corpus": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["id", "version", "sha256"],
+      "properties": {
+        "id": {
+          "type": "string",
+          "minLength": 1
+        },
+        "version": {
+          "type": "string",
+          "minLength": 1
+        },
+        "sha256": {
+          "type": "string",
+          "pattern": "^[a-f0-9]{64}$"
+        }
+      }
+    },
+    "run": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "mode",
+        "review_sensei_version",
+        "provider",
+        "provider_version",
+        "model",
+        "endpoint_scope",
+        "package_stage_digest",
+        "configuration_digest"
+      ],
+      "properties": {
+        "mode": {
+          "enum": ["fixture", "live"]
+        },
+        "review_sensei_version": {
+          "type": "string",
+          "minLength": 1
+        },
+        "provider": {
+          "type": "string",
+          "minLength": 1
+        },
+        "provider_version": {
+          "type": ["string", "null"]
+        },
+        "model": {
+          "type": ["string", "null"]
+        },
+        "endpoint_scope": {
+          "enum": ["none", "loopback", "remote"]
+        },
+        "package_stage_digest": {
+          "type": "string",
+          "pattern": "^[a-f0-9]{64}$"
+        },
+        "configuration_digest": {
+          "type": "string",
+          "pattern": "^[a-f0-9]{64}$"
+        }
+      }
+    },
+    "configuration": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["review_configuration_id"],
+      "properties": {
+        "review_configuration_id": {
+          "type": "string",
+          "minLength": 1
+        }
+      }
+    },
+    "privacy": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["status", "scanned_inventory_count"],
+      "properties": {
+        "status": {
+          "enum": ["clean"]
+        },
+        "scanned_inventory_count": {
+          "type": "integer",
+          "minimum": 0
+        }
+      }
+    },
+    "cases": {
+      "type": "array",
+      "minItems": 1,
+      "maxItems": 64,
+      "items": {
+        "$ref": "#/$defs/caseResult"
+      }
+    },
+    "metrics": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "provider_calls",
+        "prompt_bytes",
+        "response_bytes",
+        "token_proxy_4_bytes",
+        "elapsed_total_ms",
+        "elapsed_mean_ms",
+        "elapsed_p95_ms"
+      ],
+      "properties": {
+        "provider_calls": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "prompt_bytes": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "response_bytes": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "token_proxy_4_bytes": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "elapsed_total_ms": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "elapsed_mean_ms": {
+          "type": "number",
+          "minimum": 0
+        },
+        "elapsed_p95_ms": {
+          "type": "number",
+          "minimum": 0
+        }
+      }
+    },
+    "deterministic": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "exact_fixture_result_rate",
+        "expected_contract_rejection_rate"
+      ],
+      "properties": {
+        "exact_fixture_result_rate": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 1
+        },
+        "expected_contract_rejection_rate": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 1
+        }
+      }
+    },
+    "quality": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "actionable_precision",
+        "false_positive_rate",
+        "expected_finding_recall",
+        "location_validity",
+        "category_coverage"
+      ],
+      "properties": {
+        "actionable_precision": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 1
+        },
+        "false_positive_rate": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 1
+        },
+        "expected_finding_recall": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 1
+        },
+        "location_validity": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 1
+        },
+        "category_coverage": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 1
+        }
+      }
+    },
+    "threshold_failures": {
+      "type": "array",
+      "uniqueItems": true,
+      "items": {
+        "type": "string",
+        "minLength": 1
+      }
+    },
+    "passed": {
+      "type": "boolean"
+    }
+  },
+  "$defs": {
+    "caseResult": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "id",
+        "kind",
+        "category",
+        "status",
+        "expected_matches",
+        "actual_matches",
+        "false_positives",
+        "location_valid",
+        "category_valid",
+        "elapsed_ms",
+        "provider_calls",
+        "prompt_bytes",
+        "response_bytes"
+      ],
+      "properties": {
+        "id": {
+          "type": "string",
+          "minLength": 1
+        },
+        "kind": {
+          "enum": ["quality", "contract-rejection"]
+        },
+        "category": {
+          "type": "string",
+          "minLength": 1
+        },
+        "status": {
+          "enum": ["passed", "failed", "rejected-expected", "rejected-unexpected"]
+        },
+        "expected_matches": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "actual_matches": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "false_positives": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "location_valid": {
+          "type": "boolean"
+        },
+        "category_valid": {
+          "type": "boolean"
+        },
+        "elapsed_ms": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "provider_calls": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "prompt_bytes": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "response_bytes": {
+          "type": "integer",
+          "minimum": 0
+        }
+      }
+    }
+  },
+  "examples": [
+    {
+      "schema_version": "1.0",
+      "corpus": {
+        "id": "review-sensei-synthetic-v1",
+        "version": "1.0",
+        "sha256": "0000000000000000000000000000000000000000000000000000000000000000"
+      },
+      "run": {
+        "mode": "fixture",
+        "review_sensei_version": "0.1.0",
+        "provider": "fixture",
+        "provider_version": null,
+        "model": "fixture-v1",
+        "endpoint_scope": "none",
+        "package_stage_digest": "0000000000000000000000000000000000000000000000000000000000000000",
+        "configuration_digest": "0000000000000000000000000000000000000000000000000000000000000000"
+      },
+      "configuration": {
+        "review_configuration_id": "packaged-defaults-v1"
+      },
+      "privacy": {
+        "status": "clean",
+        "scanned_inventory_count": 1
+      },
+      "cases": [
+        {
+          "id": "example",
+          "kind": "quality",
+          "category": "correctness",
+          "status": "passed",
+          "expected_matches": 0,
+          "actual_matches": 0,
+          "false_positives": 0,
+          "location_valid": true,
+          "category_valid": true,
+          "elapsed_ms": 0,
+          "provider_calls": 1,
+          "prompt_bytes": 0,
+          "response_bytes": 0
+        }
+      ],
+      "metrics": {
+        "provider_calls": 1,
+        "prompt_bytes": 0,
+        "response_bytes": 0,
+        "token_proxy_4_bytes": 0,
+        "elapsed_total_ms": 0,
+        "elapsed_mean_ms": 0,
+        "elapsed_p95_ms": 0
+      },
+      "deterministic": {
+        "exact_fixture_result_rate": 1.0,
+        "expected_contract_rejection_rate": 1.0
+      },
+      "quality": {
+        "actionable_precision": 0.0,
+        "false_positive_rate": 0.0,
+        "expected_finding_recall": 1.0,
+        "location_validity": 1.0,
+        "category_coverage": 1.0
+      },
+      "threshold_failures": [],
+      "passed": true
+    }
+  ]
+}