harness-everything 0.47.0 → 0.47.2

package/benchmarks/issue290-pilot/evidence/group-summary.json

@@ -0,0 +1,684 @@
+[
+  {
+    "api_equivalent_cost_usd": null,
+    "api_equivalent_cost_usd_known_subtotal": null,
+    "api_equivalent_input_cost_usd": null,
+    "api_equivalent_input_cost_usd_known_subtotal": null,
+    "arm": "A",
+    "artifact_passes": 0,
+    "budget_failures": 0,
+    "engine": "claude",
+    "exact_reference_bytes_lower_bound": null,
+    "exact_reference_bytes_lower_bound_known_subtotal": null,
+    "mean_wall_seconds": null,
+    "mean_wall_seconds_known_subtotal": null,
+    "n_api_equivalent_cost_usd_excluded": 2,
+    "n_api_equivalent_cost_usd_included": 0,
+    "n_api_equivalent_input_cost_usd_excluded": 2,
+    "n_api_equivalent_input_cost_usd_included": 0,
+    "n_attempted": 2,
+    "n_exact_reference_bytes_lower_bound_excluded": 2,
+    "n_exact_reference_bytes_lower_bound_included": 0,
+    "n_mean_wall_seconds_excluded": 2,
+    "n_mean_wall_seconds_included": 0,
+    "n_quality_excluded": 2,
+    "n_quality_included": 0,
+    "n_reference_bytes_excluded": 2,
+    "n_reference_bytes_included": 0,
+    "reference_bytes": null,
+    "reference_bytes_known_subtotal": null,
+    "series": "issue290-fable-20261014",
+    "workload": "polyglot-simple-linked-list-staged"
+  },
+  {
+    "api_equivalent_cost_usd": null,
+    "api_equivalent_cost_usd_known_subtotal": null,
+    "api_equivalent_input_cost_usd": null,
+    "api_equivalent_input_cost_usd_known_subtotal": null,
+    "arm": "B",
+    "artifact_passes": 0,
+    "budget_failures": 0,
+    "engine": "claude",
+    "exact_reference_bytes_lower_bound": null,
+    "exact_reference_bytes_lower_bound_known_subtotal": null,
+    "mean_wall_seconds": null,
+    "mean_wall_seconds_known_subtotal": null,
+    "n_api_equivalent_cost_usd_excluded": 2,
+    "n_api_equivalent_cost_usd_included": 0,
+    "n_api_equivalent_input_cost_usd_excluded": 2,
+    "n_api_equivalent_input_cost_usd_included": 0,
+    "n_attempted": 2,
+    "n_exact_reference_bytes_lower_bound_excluded": 2,
+    "n_exact_reference_bytes_lower_bound_included": 0,
+    "n_mean_wall_seconds_excluded": 2,
+    "n_mean_wall_seconds_included": 0,
+    "n_quality_excluded": 2,
+    "n_quality_included": 0,
+    "n_reference_bytes_excluded": 2,
+    "n_reference_bytes_included": 0,
+    "reference_bytes": null,
+    "reference_bytes_known_subtotal": null,
+    "series": "issue290-fable-20261014",
+    "workload": "polyglot-simple-linked-list-staged"
+  },
+  {
+    "api_equivalent_cost_usd": null,
+    "api_equivalent_cost_usd_known_subtotal": null,
+    "api_equivalent_input_cost_usd": null,
+    "api_equivalent_input_cost_usd_known_subtotal": null,
+    "arm": "A",
+    "artifact_passes": 1,
+    "budget_failures": 1,
+    "engine": "codex",
+    "exact_reference_bytes_lower_bound": 52706,
+    "exact_reference_bytes_lower_bound_known_subtotal": 52706,
+    "mean_wall_seconds": 519.8735,
+    "mean_wall_seconds_known_subtotal": 1039.747,
+    "n_api_equivalent_cost_usd_excluded": 2,
+    "n_api_equivalent_cost_usd_included": 0,
+    "n_api_equivalent_input_cost_usd_excluded": 2,
+    "n_api_equivalent_input_cost_usd_included": 0,
+    "n_attempted": 2,
+    "n_exact_reference_bytes_lower_bound_excluded": 0,
+    "n_exact_reference_bytes_lower_bound_included": 2,
+    "n_mean_wall_seconds_excluded": 0,
+    "n_mean_wall_seconds_included": 2,
+    "n_quality_excluded": 0,
+    "n_quality_included": 2,
+    "n_reference_bytes_excluded": 0,
+    "n_reference_bytes_included": 2,
+    "reference_bytes": 52706,
+    "reference_bytes_known_subtotal": 52706,
+    "series": "issue290-fable-20261014",
+    "workload": "polyglot-simple-linked-list-staged"
+  },
+  {
+    "api_equivalent_cost_usd": null,
+    "api_equivalent_cost_usd_known_subtotal": null,
+    "api_equivalent_input_cost_usd": null,
+    "api_equivalent_input_cost_usd_known_subtotal": null,
+    "arm": "B",
+    "artifact_passes": 0,
+    "budget_failures": 2,
+    "engine": "codex",
+    "exact_reference_bytes_lower_bound": 18190,
+    "exact_reference_bytes_lower_bound_known_subtotal": 18190,
+    "mean_wall_seconds": 600.881,
+    "mean_wall_seconds_known_subtotal": 1201.762,
+    "n_api_equivalent_cost_usd_excluded": 2,
+    "n_api_equivalent_cost_usd_included": 0,
+    "n_api_equivalent_input_cost_usd_excluded": 2,
+    "n_api_equivalent_input_cost_usd_included": 0,
+    "n_attempted": 2,
+    "n_exact_reference_bytes_lower_bound_excluded": 0,
+    "n_exact_reference_bytes_lower_bound_included": 2,
+    "n_mean_wall_seconds_excluded": 0,
+    "n_mean_wall_seconds_included": 2,
+    "n_quality_excluded": 0,
+    "n_quality_included": 2,
+    "n_reference_bytes_excluded": 0,
+    "n_reference_bytes_included": 2,
+    "reference_bytes": 18190,
+    "reference_bytes_known_subtotal": 18190,
+    "series": "issue290-fable-20261014",
+    "workload": "polyglot-simple-linked-list-staged"
+  },
+  {
+    "api_equivalent_cost_usd": 0.06994134,
+    "api_equivalent_cost_usd_known_subtotal": 0.06994134,
+    "api_equivalent_input_cost_usd": 0.04232984000000001,
+    "api_equivalent_input_cost_usd_known_subtotal": 0.04232984000000001,
+    "arm": "A",
+    "artifact_passes": 2,
+    "budget_failures": 0,
+    "engine": "claude",
+    "exact_reference_bytes_lower_bound": 28933,
+    "exact_reference_bytes_lower_bound_known_subtotal": 28933,
+    "mean_wall_seconds": 197.2785,
+    "mean_wall_seconds_known_subtotal": 394.557,
+    "n_api_equivalent_cost_usd_excluded": 0,
+    "n_api_equivalent_cost_usd_included": 2,
+    "n_api_equivalent_input_cost_usd_excluded": 0,
+    "n_api_equivalent_input_cost_usd_included": 2,
+    "n_attempted": 2,
+    "n_exact_reference_bytes_lower_bound_excluded": 0,
+    "n_exact_reference_bytes_lower_bound_included": 2,
+    "n_mean_wall_seconds_excluded": 0,
+    "n_mean_wall_seconds_included": 2,
+    "n_quality_excluded": 0,
+    "n_quality_included": 2,
+    "n_reference_bytes_excluded": 0,
+    "n_reference_bytes_included": 2,
+    "reference_bytes": 28933,
+    "reference_bytes_known_subtotal": 28933,
+    "series": "issue290-fable-claude-retry-20261014",
+    "workload": "polyglot-simple-linked-list-staged"
+  },
+  {
+    "api_equivalent_cost_usd": 0.10466848000000001,
+    "api_equivalent_cost_usd_known_subtotal": 0.10466848000000001,
+    "api_equivalent_input_cost_usd": 0.06321098,
+    "api_equivalent_input_cost_usd_known_subtotal": 0.06321098,
+    "arm": "B",
+    "artifact_passes": 2,
+    "budget_failures": 0,
+    "engine": "claude",
+    "exact_reference_bytes_lower_bound": 30824,
+    "exact_reference_bytes_lower_bound_known_subtotal": 30824,
+    "mean_wall_seconds": 218.853,
+    "mean_wall_seconds_known_subtotal": 437.706,
+    "n_api_equivalent_cost_usd_excluded": 0,
+    "n_api_equivalent_cost_usd_included": 2,
+    "n_api_equivalent_input_cost_usd_excluded": 0,
+    "n_api_equivalent_input_cost_usd_included": 2,
+    "n_attempted": 2,
+    "n_exact_reference_bytes_lower_bound_excluded": 0,
+    "n_exact_reference_bytes_lower_bound_included": 2,
+    "n_mean_wall_seconds_excluded": 0,
+    "n_mean_wall_seconds_included": 2,
+    "n_quality_excluded": 0,
+    "n_quality_included": 2,
+    "n_reference_bytes_excluded": 0,
+    "n_reference_bytes_included": 2,
+    "reference_bytes": 30824,
+    "reference_bytes_known_subtotal": 30824,
+    "series": "issue290-fable-claude-retry-20261014",
+    "workload": "polyglot-simple-linked-list-staged"
+  },
+  {
+    "api_equivalent_cost_usd": 0.03758254,
+    "api_equivalent_cost_usd_known_subtotal": 0.03758254,
+    "api_equivalent_input_cost_usd": 0.022892040000000002,
+    "api_equivalent_input_cost_usd_known_subtotal": 0.022892040000000002,
+    "arm": "A",
+    "artifact_passes": 2,
+    "budget_failures": 0,
+    "engine": "claude",
+    "exact_reference_bytes_lower_bound": 8315,
+    "exact_reference_bytes_lower_bound_known_subtotal": 8315,
+    "mean_wall_seconds": 86.9075,
+    "mean_wall_seconds_known_subtotal": 173.815,
+    "n_api_equivalent_cost_usd_excluded": 0,
+    "n_api_equivalent_cost_usd_included": 2,
+    "n_api_equivalent_input_cost_usd_excluded": 0,
+    "n_api_equivalent_input_cost_usd_included": 2,
+    "n_attempted": 2,
+    "n_exact_reference_bytes_lower_bound_excluded": 0,
+    "n_exact_reference_bytes_lower_bound_included": 2,
+    "n_mean_wall_seconds_excluded": 0,
+    "n_mean_wall_seconds_included": 2,
+    "n_quality_excluded": 0,
+    "n_quality_included": 2,
+    "n_reference_bytes_excluded": 0,
+    "n_reference_bytes_included": 2,
+    "reference_bytes": 8315,
+    "reference_bytes_known_subtotal": 8315,
+    "series": "issue290-formal-20261014",
+    "workload": "polyglot-simple-linked-list-staged"
+  },
+  {
+    "api_equivalent_cost_usd": 0.05576189999999999,
+    "api_equivalent_cost_usd_known_subtotal": 0.05576189999999999,
+    "api_equivalent_input_cost_usd": 0.0345839,
+    "api_equivalent_input_cost_usd_known_subtotal": 0.0345839,
+    "arm": "B",
+    "artifact_passes": 2,
+    "budget_failures": 0,
+    "engine": "claude",
+    "exact_reference_bytes_lower_bound": 6130,
+    "exact_reference_bytes_lower_bound_known_subtotal": 6130,
+    "mean_wall_seconds": 132.3965,
+    "mean_wall_seconds_known_subtotal": 264.793,
+    "n_api_equivalent_cost_usd_excluded": 0,
+    "n_api_equivalent_cost_usd_included": 2,
+    "n_api_equivalent_input_cost_usd_excluded": 0,
+    "n_api_equivalent_input_cost_usd_included": 2,
+    "n_attempted": 2,
+    "n_exact_reference_bytes_lower_bound_excluded": 0,
+    "n_exact_reference_bytes_lower_bound_included": 2,
+    "n_mean_wall_seconds_excluded": 0,
+    "n_mean_wall_seconds_included": 2,
+    "n_quality_excluded": 0,
+    "n_quality_included": 2,
+    "n_reference_bytes_excluded": 0,
+    "n_reference_bytes_included": 2,
+    "reference_bytes": 6130,
+    "reference_bytes_known_subtotal": 6130,
+    "series": "issue290-formal-20261014",
+    "workload": "polyglot-simple-linked-list-staged"
+  },
+  {
+    "api_equivalent_cost_usd": 0.011655869999999999,
+    "api_equivalent_cost_usd_known_subtotal": 0.011655869999999999,
+    "api_equivalent_input_cost_usd": 0.009629370000000002,
+    "api_equivalent_input_cost_usd_known_subtotal": 0.009629370000000002,
+    "arm": "A",
+    "artifact_passes": 0,
+    "budget_failures": 0,
+    "engine": "claude",
+    "exact_reference_bytes_lower_bound": 37914,
+    "exact_reference_bytes_lower_bound_known_subtotal": 37914,
+    "mean_wall_seconds": 12.1095,
+    "mean_wall_seconds_known_subtotal": 24.219,
+    "n_api_equivalent_cost_usd_excluded": 0,
+    "n_api_equivalent_cost_usd_included": 2,
+    "n_api_equivalent_input_cost_usd_excluded": 0,
+    "n_api_equivalent_input_cost_usd_included": 2,
+    "n_attempted": 2,
+    "n_exact_reference_bytes_lower_bound_excluded": 0,
+    "n_exact_reference_bytes_lower_bound_included": 2,
+    "n_mean_wall_seconds_excluded": 0,
+    "n_mean_wall_seconds_included": 2,
+    "n_quality_excluded": 0,
+    "n_quality_included": 2,
+    "n_reference_bytes_excluded": 0,
+    "n_reference_bytes_included": 2,
+    "reference_bytes": 37914,
+    "reference_bytes_known_subtotal": 37914,
+    "series": "issue290-formal-20261014",
+    "workload": "profile-lookup"
+  },
+  {
+    "api_equivalent_cost_usd": 0.009994530000000001,
+    "api_equivalent_cost_usd_known_subtotal": 0.009994530000000001,
+    "api_equivalent_input_cost_usd": 0.00810303,
+    "api_equivalent_input_cost_usd_known_subtotal": 0.00810303,
+    "arm": "B",
+    "artifact_passes": 0,
+    "budget_failures": 0,
+    "engine": "claude",
+    "exact_reference_bytes_lower_bound": 7142,
+    "exact_reference_bytes_lower_bound_known_subtotal": 7142,
+    "mean_wall_seconds": 15.641,
+    "mean_wall_seconds_known_subtotal": 31.282,
+    "n_api_equivalent_cost_usd_excluded": 0,
+    "n_api_equivalent_cost_usd_included": 2,
+    "n_api_equivalent_input_cost_usd_excluded": 0,
+    "n_api_equivalent_input_cost_usd_included": 2,
+    "n_attempted": 2,
+    "n_exact_reference_bytes_lower_bound_excluded": 0,
+    "n_exact_reference_bytes_lower_bound_included": 2,
+    "n_mean_wall_seconds_excluded": 0,
+    "n_mean_wall_seconds_included": 2,
+    "n_quality_excluded": 0,
+    "n_quality_included": 2,
+    "n_reference_bytes_excluded": 0,
+    "n_reference_bytes_included": 2,
+    "reference_bytes": 7142,
+    "reference_bytes_known_subtotal": 7142,
+    "series": "issue290-formal-20261014",
+    "workload": "profile-lookup"
+  },
+  {
+    "api_equivalent_cost_usd": 0.022799550000000002,
+    "api_equivalent_cost_usd_known_subtotal": 0.022799550000000002,
+    "api_equivalent_input_cost_usd": 0.01640455,
+    "api_equivalent_input_cost_usd_known_subtotal": 0.01640455,
+    "arm": "A",
+    "artifact_passes": 2,
+    "budget_failures": 0,
+    "engine": "claude",
+    "exact_reference_bytes_lower_bound": 30156,
+    "exact_reference_bytes_lower_bound_known_subtotal": 30156,
+    "mean_wall_seconds": 45.414500000000004,
+    "mean_wall_seconds_known_subtotal": 90.82900000000001,
+    "n_api_equivalent_cost_usd_excluded": 0,
+    "n_api_equivalent_cost_usd_included": 2,
+    "n_api_equivalent_input_cost_usd_excluded": 0,
+    "n_api_equivalent_input_cost_usd_included": 2,
+    "n_attempted": 2,
+    "n_exact_reference_bytes_lower_bound_excluded": 0,
+    "n_exact_reference_bytes_lower_bound_included": 2,
+    "n_mean_wall_seconds_excluded": 0,
+    "n_mean_wall_seconds_included": 2,
+    "n_quality_excluded": 0,
+    "n_quality_included": 2,
+    "n_reference_bytes_excluded": 0,
+    "n_reference_bytes_included": 2,
+    "reference_bytes": 30156,
+    "reference_bytes_known_subtotal": 30156,
+    "series": "issue290-formal-20261014",
+    "workload": "quixbugs-find-in-sorted"
+  },
+  {
+    "api_equivalent_cost_usd": 0.03047896,
+    "api_equivalent_cost_usd_known_subtotal": 0.03047896,
+    "api_equivalent_input_cost_usd": 0.021595959999999997,
+    "api_equivalent_input_cost_usd_known_subtotal": 0.021595959999999997,
+    "arm": "B",
+    "artifact_passes": 2,
+    "budget_failures": 0,
+    "engine": "claude",
+    "exact_reference_bytes_lower_bound": 6247,
+    "exact_reference_bytes_lower_bound_known_subtotal": 6247,
+    "mean_wall_seconds": 77.2675,
+    "mean_wall_seconds_known_subtotal": 154.535,
+    "n_api_equivalent_cost_usd_excluded": 0,
+    "n_api_equivalent_cost_usd_included": 2,
+    "n_api_equivalent_input_cost_usd_excluded": 0,
+    "n_api_equivalent_input_cost_usd_included": 2,
+    "n_attempted": 2,
+    "n_exact_reference_bytes_lower_bound_excluded": 0,
+    "n_exact_reference_bytes_lower_bound_included": 2,
+    "n_mean_wall_seconds_excluded": 0,
+    "n_mean_wall_seconds_included": 2,
+    "n_quality_excluded": 0,
+    "n_quality_included": 2,
+    "n_reference_bytes_excluded": 0,
+    "n_reference_bytes_included": 2,
+    "reference_bytes": 6247,
+    "reference_bytes_known_subtotal": 6247,
+    "series": "issue290-formal-20261014",
+    "workload": "quixbugs-find-in-sorted"
+  },
+  {
+    "api_equivalent_cost_usd": 0.05112488,
+    "api_equivalent_cost_usd_known_subtotal": 0.05112488,
+    "api_equivalent_input_cost_usd": 0.03154488000000001,
+    "api_equivalent_input_cost_usd_known_subtotal": 0.03154488000000001,
+    "arm": "A",
+    "artifact_passes": 2,
+    "budget_failures": 0,
+    "engine": "codex",
+    "exact_reference_bytes_lower_bound": 44046,
+    "exact_reference_bytes_lower_bound_known_subtotal": 44046,
+    "mean_wall_seconds": 227.9965,
+    "mean_wall_seconds_known_subtotal": 455.993,
+    "n_api_equivalent_cost_usd_excluded": 0,
+    "n_api_equivalent_cost_usd_included": 2,
+    "n_api_equivalent_input_cost_usd_excluded": 0,
+    "n_api_equivalent_input_cost_usd_included": 2,
+    "n_attempted": 2,
+    "n_exact_reference_bytes_lower_bound_excluded": 0,
+    "n_exact_reference_bytes_lower_bound_included": 2,
+    "n_mean_wall_seconds_excluded": 0,
+    "n_mean_wall_seconds_included": 2,
+    "n_quality_excluded": 0,
+    "n_quality_included": 2,
+    "n_reference_bytes_excluded": 0,
+    "n_reference_bytes_included": 2,
+    "reference_bytes": 44046,
+    "reference_bytes_known_subtotal": 44046,
+    "series": "issue290-formal-20261014",
+    "workload": "polyglot-simple-linked-list-staged"
+  },
+  {
+    "api_equivalent_cost_usd": 0.050625540000000004,
+    "api_equivalent_cost_usd_known_subtotal": 0.050625540000000004,
+    "api_equivalent_input_cost_usd": 0.03136804,
+    "api_equivalent_input_cost_usd_known_subtotal": 0.03136804,
+    "arm": "B",
+    "artifact_passes": 1,
+    "budget_failures": 0,
+    "engine": "codex",
+    "exact_reference_bytes_lower_bound": 6132,
+    "exact_reference_bytes_lower_bound_known_subtotal": 6132,
+    "mean_wall_seconds": 213.98399999999998,
+    "mean_wall_seconds_known_subtotal": 427.96799999999996,
+    "n_api_equivalent_cost_usd_excluded": 0,
+    "n_api_equivalent_cost_usd_included": 2,
+    "n_api_equivalent_input_cost_usd_excluded": 0,
+    "n_api_equivalent_input_cost_usd_included": 2,
+    "n_attempted": 2,
+    "n_exact_reference_bytes_lower_bound_excluded": 0,
+    "n_exact_reference_bytes_lower_bound_included": 2,
+    "n_mean_wall_seconds_excluded": 0,
+    "n_mean_wall_seconds_included": 2,
+    "n_quality_excluded": 0,
+    "n_quality_included": 2,
+    "n_reference_bytes_excluded": 0,
+    "n_reference_bytes_included": 2,
+    "reference_bytes": 6132,
+    "reference_bytes_known_subtotal": 6132,
+    "series": "issue290-formal-20261014",
+    "workload": "polyglot-simple-linked-list-staged"
+  },
+  {
+    "api_equivalent_cost_usd": 0.00425154,
+    "api_equivalent_cost_usd_known_subtotal": 0.00425154,
+    "api_equivalent_input_cost_usd": 0.00349404,
+    "api_equivalent_input_cost_usd_known_subtotal": 0.00349404,
+    "arm": "A",
+    "artifact_passes": 2,
+    "budget_failures": 0,
+    "engine": "codex",
+    "exact_reference_bytes_lower_bound": 37914,
+    "exact_reference_bytes_lower_bound_known_subtotal": 37914,
+    "mean_wall_seconds": 18.1465,
+    "mean_wall_seconds_known_subtotal": 36.293,
+    "n_api_equivalent_cost_usd_excluded": 0,
+    "n_api_equivalent_cost_usd_included": 2,
+    "n_api_equivalent_input_cost_usd_excluded": 0,
+    "n_api_equivalent_input_cost_usd_included": 2,
+    "n_attempted": 2,
+    "n_exact_reference_bytes_lower_bound_excluded": 0,
+    "n_exact_reference_bytes_lower_bound_included": 2,
+    "n_mean_wall_seconds_excluded": 0,
+    "n_mean_wall_seconds_included": 2,
+    "n_quality_excluded": 0,
+    "n_quality_included": 2,
+    "n_reference_bytes_excluded": 0,
+    "n_reference_bytes_included": 2,
+    "reference_bytes": 37914,
+    "reference_bytes_known_subtotal": 37914,
+    "series": "issue290-formal-20261014",
+    "workload": "profile-lookup"
+  },
+  {
+    "api_equivalent_cost_usd": 0.00527972,
+    "api_equivalent_cost_usd_known_subtotal": 0.00527972,
+    "api_equivalent_input_cost_usd": 0.0045942199999999996,
+    "api_equivalent_input_cost_usd_known_subtotal": 0.0045942199999999996,
+    "arm": "B",
+    "artifact_passes": 2,
+    "budget_failures": 0,
+    "engine": "codex",
+    "exact_reference_bytes_lower_bound": 7142,
+    "exact_reference_bytes_lower_bound_known_subtotal": 7142,
+    "mean_wall_seconds": 16.628,
+    "mean_wall_seconds_known_subtotal": 33.256,
+    "n_api_equivalent_cost_usd_excluded": 0,
+    "n_api_equivalent_cost_usd_included": 2,
+    "n_api_equivalent_input_cost_usd_excluded": 0,
+    "n_api_equivalent_input_cost_usd_included": 2,
+    "n_attempted": 2,
+    "n_exact_reference_bytes_lower_bound_excluded": 0,
+    "n_exact_reference_bytes_lower_bound_included": 2,
+    "n_mean_wall_seconds_excluded": 0,
+    "n_mean_wall_seconds_included": 2,
+    "n_quality_excluded": 0,
+    "n_quality_included": 2,
+    "n_reference_bytes_excluded": 0,
+    "n_reference_bytes_included": 2,
+    "reference_bytes": 7142,
+    "reference_bytes_known_subtotal": 7142,
+    "series": "issue290-formal-20261014",
+    "workload": "profile-lookup"
+  },
+  {
+    "api_equivalent_cost_usd": 0.019668940000000003,
+    "api_equivalent_cost_usd_known_subtotal": 0.019668940000000003,
+    "api_equivalent_input_cost_usd": 0.01362994,
+    "api_equivalent_input_cost_usd_known_subtotal": 0.01362994,
+    "arm": "A",
+    "artifact_passes": 2,
+    "budget_failures": 0,
+    "engine": "codex",
+    "exact_reference_bytes_lower_bound": 30156,
+    "exact_reference_bytes_lower_bound_known_subtotal": 30156,
+    "mean_wall_seconds": 107.881,
+    "mean_wall_seconds_known_subtotal": 215.762,
+    "n_api_equivalent_cost_usd_excluded": 0,
+    "n_api_equivalent_cost_usd_included": 2,
+    "n_api_equivalent_input_cost_usd_excluded": 0,
+    "n_api_equivalent_input_cost_usd_included": 2,
+    "n_attempted": 2,
+    "n_exact_reference_bytes_lower_bound_excluded": 0,
+    "n_exact_reference_bytes_lower_bound_included": 2,
+    "n_mean_wall_seconds_excluded": 0,
+    "n_mean_wall_seconds_included": 2,
+    "n_quality_excluded": 0,
+    "n_quality_included": 2,
+    "n_reference_bytes_excluded": 0,
+    "n_reference_bytes_included": 2,
+    "reference_bytes": 30156,
+    "reference_bytes_known_subtotal": 30156,
+    "series": "issue290-formal-20261014",
+    "workload": "quixbugs-find-in-sorted"
+  },
+  {
+    "api_equivalent_cost_usd": 0.03405532,
+    "api_equivalent_cost_usd_known_subtotal": 0.03405532,
+    "api_equivalent_input_cost_usd": 0.02638182,
+    "api_equivalent_input_cost_usd_known_subtotal": 0.02638182,
+    "arm": "B",
+    "artifact_passes": 2,
+    "budget_failures": 0,
+    "engine": "codex",
+    "exact_reference_bytes_lower_bound": 7731,
+    "exact_reference_bytes_lower_bound_known_subtotal": 7731,
+    "mean_wall_seconds": 139.159,
+    "mean_wall_seconds_known_subtotal": 278.318,
+    "n_api_equivalent_cost_usd_excluded": 0,
+    "n_api_equivalent_cost_usd_included": 2,
+    "n_api_equivalent_input_cost_usd_excluded": 0,
+    "n_api_equivalent_input_cost_usd_included": 2,
+    "n_attempted": 2,
+    "n_exact_reference_bytes_lower_bound_excluded": 0,
+    "n_exact_reference_bytes_lower_bound_included": 2,
+    "n_mean_wall_seconds_excluded": 0,
+    "n_mean_wall_seconds_included": 2,
+    "n_quality_excluded": 0,
+    "n_quality_included": 2,
+    "n_reference_bytes_excluded": 0,
+    "n_reference_bytes_included": 2,
+    "reference_bytes": 7731,
+    "reference_bytes_known_subtotal": 7731,
+    "series": "issue290-formal-20261014",
+    "workload": "quixbugs-find-in-sorted"
+  },
+  {
+    "api_equivalent_cost_usd": null,
+    "api_equivalent_cost_usd_known_subtotal": null,
+    "api_equivalent_input_cost_usd": null,
+    "api_equivalent_input_cost_usd_known_subtotal": null,
+    "arm": "B",
+    "artifact_passes": 0,
+    "budget_failures": 0,
+    "engine": "claude",
+    "exact_reference_bytes_lower_bound": null,
+    "exact_reference_bytes_lower_bound_known_subtotal": null,
+    "mean_wall_seconds": null,
+    "mean_wall_seconds_known_subtotal": null,
+    "n_api_equivalent_cost_usd_excluded": 1,
+    "n_api_equivalent_cost_usd_included": 0,
+    "n_api_equivalent_input_cost_usd_excluded": 1,
+    "n_api_equivalent_input_cost_usd_included": 0,
+    "n_attempted": 1,
+    "n_exact_reference_bytes_lower_bound_excluded": 1,
+    "n_exact_reference_bytes_lower_bound_included": 0,
+    "n_mean_wall_seconds_excluded": 1,
+    "n_mean_wall_seconds_included": 0,
+    "n_quality_excluded": 1,
+    "n_quality_included": 0,
+    "n_reference_bytes_excluded": 1,
+    "n_reference_bytes_included": 0,
+    "reference_bytes": null,
+    "reference_bytes_known_subtotal": null,
+    "series": "issue290-host-preflight-20261014",
+    "workload": "profile-lookup"
+  },
+  {
+    "api_equivalent_cost_usd": null,
+    "api_equivalent_cost_usd_known_subtotal": null,
+    "api_equivalent_input_cost_usd": null,
+    "api_equivalent_input_cost_usd_known_subtotal": null,
+    "arm": "B",
+    "artifact_passes": 0,
+    "budget_failures": 0,
+    "engine": "codex",
+    "exact_reference_bytes_lower_bound": null,
+    "exact_reference_bytes_lower_bound_known_subtotal": null,
+    "mean_wall_seconds": null,
+    "mean_wall_seconds_known_subtotal": null,
+    "n_api_equivalent_cost_usd_excluded": 1,
+    "n_api_equivalent_cost_usd_included": 0,
+    "n_api_equivalent_input_cost_usd_excluded": 1,
+    "n_api_equivalent_input_cost_usd_included": 0,
+    "n_attempted": 1,
+    "n_exact_reference_bytes_lower_bound_excluded": 1,
+    "n_exact_reference_bytes_lower_bound_included": 0,
+    "n_mean_wall_seconds_excluded": 1,
+    "n_mean_wall_seconds_included": 0,
+    "n_quality_excluded": 1,
+    "n_quality_included": 0,
+    "n_reference_bytes_excluded": 1,
+    "n_reference_bytes_included": 0,
+    "reference_bytes": null,
+    "reference_bytes_known_subtotal": null,
+    "series": "issue290-host-preflight-20261014",
+    "workload": "profile-lookup"
+  },
+  {
+    "api_equivalent_cost_usd": null,
+    "api_equivalent_cost_usd_known_subtotal": null,
+    "api_equivalent_input_cost_usd": null,
+    "api_equivalent_input_cost_usd_known_subtotal": null,
+    "arm": "B",
+    "artifact_passes": 0,
+    "budget_failures": 0,
+    "engine": "claude",
+    "exact_reference_bytes_lower_bound": null,
+    "exact_reference_bytes_lower_bound_known_subtotal": null,
+    "mean_wall_seconds": null,
+    "mean_wall_seconds_known_subtotal": null,
+    "n_api_equivalent_cost_usd_excluded": 1,
+    "n_api_equivalent_cost_usd_included": 0,
+    "n_api_equivalent_input_cost_usd_excluded": 1,
+    "n_api_equivalent_input_cost_usd_included": 0,
+    "n_attempted": 1,
+    "n_exact_reference_bytes_lower_bound_excluded": 1,
+    "n_exact_reference_bytes_lower_bound_included": 0,
+    "n_mean_wall_seconds_excluded": 1,
+    "n_mean_wall_seconds_included": 0,
+    "n_quality_excluded": 1,
+    "n_quality_included": 0,
+    "n_reference_bytes_excluded": 1,
+    "n_reference_bytes_included": 0,
+    "reference_bytes": null,
+    "reference_bytes_known_subtotal": null,
+    "series": "issue290-host-preflight2-20261014",
+    "workload": "profile-lookup"
+  },
+  {
+    "api_equivalent_cost_usd": null,
+    "api_equivalent_cost_usd_known_subtotal": null,
+    "api_equivalent_input_cost_usd": null,
+    "api_equivalent_input_cost_usd_known_subtotal": null,
+    "arm": "B",
+    "artifact_passes": 0,
+    "budget_failures": 0,
+    "engine": "codex",
+    "exact_reference_bytes_lower_bound": null,
+    "exact_reference_bytes_lower_bound_known_subtotal": null,
+    "mean_wall_seconds": null,
+    "mean_wall_seconds_known_subtotal": null,
+    "n_api_equivalent_cost_usd_excluded": 1,
+    "n_api_equivalent_cost_usd_included": 0,
+    "n_api_equivalent_input_cost_usd_excluded": 1,
+    "n_api_equivalent_input_cost_usd_included": 0,
+    "n_attempted": 1,
+    "n_exact_reference_bytes_lower_bound_excluded": 1,
+    "n_exact_reference_bytes_lower_bound_included": 0,
+    "n_mean_wall_seconds_excluded": 1,
+    "n_mean_wall_seconds_included": 0,
+    "n_quality_excluded": 1,
+    "n_quality_included": 0,
+    "n_reference_bytes_excluded": 1,
+    "n_reference_bytes_included": 0,
+    "reference_bytes": null,
+    "reference_bytes_known_subtotal": null,
+    "series": "issue290-host-preflight2-20261014",
+    "workload": "profile-lookup"
+  }
+]