@oracle-agent/oracle 0.35.41 → 0.35.43

package/public/oracle-splash/eval/benchmarks/evm-escrow-20260827.json

@@ -0,0 +1,170 @@
+{
+  "schemaVersion": 1,
+  "benchmark": "evm-escrow-20260827",
+  "measuredAt": "2026-08-27",
+  "revalidatedAt": "2026-09-06T09:39:45.527927+00:00",
+  "sampleSize": 1,
+  "scope": "Historical custom EVM milestone escrow task. Local compile and tests only, no deployment. Existing generated artifacts revalidated offline, not fresh model generations. Oracle means MCP plus protocol-builder skill, not a native-harness head-to-head.",
+  "valid_arms": [
+    {
+      "id": "codex-clean",
+      "label": "Codex",
+      "executor": "Codex CLI 0.150.1",
+      "model": "GPT-5.6-Sol",
+      "oracle": false,
+      "own_tests_passed": 8,
+      "own_tests_total": 8,
+      "hidden_tests_passed": 28,
+      "hidden_tests_total": 28,
+      "targeted_audit": "One literal checks-effects-interactions ordering miss in fund()",
+      "sourceHashes": {
+        "src/MilestoneEscrow.sol": "047f85e8aa9fd9b66605587d2169700079b32a9ec50c63869b5fd25ae2c009f0",
+        "src/MilestoneEscrowFactory.sol": "90d39dbdedc7225682eadfb3763a215d9bbfbd528d45a8f6d1aaac7bb287a8c5"
+      },
+      "revalidated": {
+        "own": {
+          "pass": 8,
+          "fail": 0,
+          "total": 8,
+          "exitCode": 0,
+          "sha256": "ce7beb69dae574d41ea364858ac6604efa4d2541a5043ad07a583b55ff9761f0"
+        },
+        "hidden": {
+          "pass": 28,
+          "fail": 0,
+          "total": 28,
+          "exitCode": 0,
+          "sha256": "2bf446735fb4f96139dbc3dfa956293137cb545db32316be6b3c533411d0a07d"
+        }
+      }
+    },
+    {
+      "id": "codex-oracle-clean",
+      "label": "Codex + Oracle",
+      "executor": "Codex CLI 0.150.1",
+      "model": "GPT-5.6-Sol",
+      "oracle": true,
+      "own_tests_passed": 9,
+      "own_tests_total": 9,
+      "hidden_tests_passed": 28,
+      "hidden_tests_total": 28,
+      "targeted_audit": "No targeted specification gaps found",
+      "sourceHashes": {
+        "src/MilestoneEscrow.sol": "ea3c81c318a115d06653e96878e0f21324ca069add6250dfb6805576e84a577b",
+        "src/MilestoneEscrowFactory.sol": "b223d7ad6ed84710914b8c1f69eca3d2931867c3b6330d831af5cd55eabf0f78"
+      },
+      "revalidated": {
+        "own": {
+          "pass": 9,
+          "fail": 0,
+          "total": 9,
+          "exitCode": 0,
+          "sha256": "fa61737fe69a9faba7b17948c1642511f40334ca9a891654bc4e251f6beddf6f"
+        },
+        "hidden": {
+          "pass": 28,
+          "fail": 0,
+          "total": 28,
+          "exitCode": 0,
+          "sha256": "280324e4bb106861ddb24e113da9b75ef3cd0e236a860b882c6fb7c2d9228b72"
+        }
+      }
+    },
+    {
+      "id": "claude-clean",
+      "label": "Claude",
+      "executor": "Claude Code 2.1.247",
+      "model": "Opus 5",
+      "oracle": false,
+      "own_tests_passed": 82,
+      "own_tests_total": 82,
+      "hidden_tests_passed": 28,
+      "hidden_tests_total": 28,
+      "targeted_audit": "No targeted specification gaps found",
+      "sourceHashes": {
+        "src/MilestoneEscrow.sol": "64242699273c9964365dc2fc0993c2825f170a7eea003e41a394e678144a74f7",
+        "src/MilestoneEscrowFactory.sol": "51114938302c42bc02f6b99298f7bebfeb9aa91c48176dae23934aff37f67af2"
+      },
+      "revalidated": {
+        "own": {
+          "pass": 82,
+          "fail": 0,
+          "total": 82,
+          "exitCode": 0,
+          "sha256": "3ff353edf0860d0e78cd43c7514c56c5a896f464ac95247b697ac14ad366d4db"
+        },
+        "hidden": {
+          "pass": 28,
+          "fail": 0,
+          "total": 28,
+          "exitCode": 0,
+          "sha256": "98535ea6941193695fd52e5fb4effa4f29f59855ec080d3b7e9300c7574fa584"
+        }
+      }
+    },
+    {
+      "id": "claude-oracle-clean",
+      "label": "Claude + Oracle",
+      "executor": "Claude Code 2.1.247",
+      "model": "Opus 5",
+      "oracle": true,
+      "own_tests_passed": 91,
+      "own_tests_total": 91,
+      "hidden_tests_passed": 28,
+      "hidden_tests_total": 28,
+      "targeted_audit": "No targeted specification gaps found",
+      "sourceHashes": {
+        "src/MilestoneEscrow.sol": "d05e890d3bfddf4226bbbee158bdc5449821a8eecc91ac19b18cc47b561df484",
+        "src/MilestoneEscrowFactory.sol": "4fb845f7749288511f27404592b5310ab1a9eaecf50f079c871c9c76c24a04f6"
+      },
+      "revalidated": {
+        "own": {
+          "pass": 91,
+          "fail": 0,
+          "total": 91,
+          "exitCode": 0,
+          "sha256": "8a6f0b98dbbbc85298735d282f3ff95e8da69edee47ed24951f0e0a76dc86552"
+        },
+        "hidden": {
+          "pass": 28,
+          "fail": 0,
+          "total": 28,
+          "exitCode": 0,
+          "sha256": "7f8691a9de041fadfe57e7158fde3fefc58806abc79eee9a6631f49bd9fe3565"
+        }
+      }
+    }
+  ],
+  "controls": [
+    {
+      "name": "good",
+      "pass": 28,
+      "fail": 0,
+      "total": 28,
+      "exitCode": 0
+    },
+    {
+      "name": "bad",
+      "pass": 19,
+      "fail": 9,
+      "total": 28,
+      "exitCode": 1
+    }
+  ],
+  "limitations": [
+    "One trial per arm, no generalized superiority claim.",
+    "Self-generated test counts measure assurance effort, not comparable coverage.",
+    "Targeted source review is historical and is not a formal security audit.",
+    "Historical Claude arms used different subscription lanes, account identity is not claimed equivalent."
+  ],
+  "excluded_runs": [
+    {
+      "id": "claude-oracle-invalid-timeout-path",
+      "reason": "Benchmark wrapper was terminated before completion"
+    },
+    {
+      "id": "claude-oracle-invalid-mcp-timeout",
+      "reason": "Required Oracle protocol_eval exceeded Claude Code's MCP timeout"
+    }
+  ]
+}