@ictechgy/context-guard 0.4.16 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +74 -0
- package/README.ko.md +91 -1
- package/README.md +95 -1
- package/docs/distribution.md +100 -0
- package/package.json +5 -1
- package/plugins/context-guard/.claude-plugin/plugin.json +1 -1
- package/plugins/context-guard/README.ko.md +34 -1
- package/plugins/context-guard/README.md +36 -1
- package/plugins/context-guard/bin/bash_reference_policy.py +967 -0
- package/plugins/context-guard/bin/context-guard-artifact +1 -1
- package/plugins/context-guard/bin/context-guard-bench +4216 -103
- package/plugins/context-guard/bin/context-guard-failed-nudge +95 -23
- package/plugins/context-guard/bin/context-guard-guard-read +6 -2
- package/plugins/context-guard/bin/context-guard-mcp +2 -1
- package/plugins/context-guard/bin/context-guard-pack +2086 -142
- package/plugins/context-guard/bin/context-guard-rewrite-bash +497 -45
- package/plugins/context-guard/bin/context-guard-sanitize-output +178 -22
- package/plugins/context-guard/bin/context-guard-setup +901 -28
- package/plugins/context-guard/bin/context-guard-statusline +33 -2
- package/plugins/context-guard/bin/context-guard-statusline-merged +71 -20
- package/plugins/context-guard/bin/context-guard-task-memory +635 -0
- package/plugins/context-guard/bin/context-guard-trim-output +706 -35
- package/plugins/context-guard/lib/context_guard_commands.py +25 -1
- package/plugins/context-guard/lib/context_pack_git_boundary.py +19 -0
- package/plugins/context-guard/lib/context_pack_identity.py +115 -0
- package/plugins/context-guard/lib/context_pack_receipts.py +9 -0
- package/plugins/context-guard/lib/context_pack_rendering.py +12 -0
- package/plugins/context-guard/lib/context_pack_scanning.py +10 -0
- package/plugins/context-guard/lib/context_pack_selection.py +6 -0
- package/plugins/context-guard/lib/credential_policy.py +10 -2
|
@@ -54,6 +54,7 @@ dry-run 모드는 실제 호출은 하지 않고 어떤 명령이 실행될지
|
|
|
54
54
|
from __future__ import annotations
|
|
55
55
|
|
|
56
56
|
import argparse
|
|
57
|
+
import base64
|
|
57
58
|
import collections
|
|
58
59
|
from contextlib import contextmanager, nullcontext
|
|
59
60
|
import csv
|
|
@@ -71,13 +72,14 @@ import stat
|
|
|
71
72
|
import struct
|
|
72
73
|
import subprocess
|
|
73
74
|
import sys
|
|
75
|
+
import tarfile
|
|
74
76
|
import tempfile
|
|
75
77
|
import time
|
|
76
78
|
import unicodedata
|
|
77
79
|
from collections.abc import Iterable
|
|
78
80
|
from dataclasses import dataclass, field, replace
|
|
79
81
|
from fractions import Fraction
|
|
80
|
-
from pathlib import Path
|
|
82
|
+
from pathlib import Path, PurePosixPath
|
|
81
83
|
from typing import Any, Callable, Mapping, Sequence
|
|
82
84
|
|
|
83
85
|
try:
|
|
@@ -400,6 +402,7 @@ MEASUREMENT_DOCUMENTED_HOOK_EVENTS = (
|
|
|
400
402
|
)
|
|
401
403
|
SUCCESS_COMMAND_OUTPUT_MAX_BYTES = 64_000
|
|
402
404
|
VERSION_OUTPUT_MAX_BYTES = 16_000
|
|
405
|
+
MEASUREMENT_CLI_PROBE_OUTPUT_MAX_BYTES = 65_536
|
|
403
406
|
PROCESS_TERMINATE_GRACE_SECONDS = 2.0
|
|
404
407
|
ALLOWED_FIRST_ABSOLUTE_SYMLINKS = {
|
|
405
408
|
"tmp": Path("/private/tmp"),
|
|
@@ -991,6 +994,64 @@ def _stream_contains_nonfinite(value: Any) -> bool:
|
|
|
991
994
|
return False
|
|
992
995
|
|
|
993
996
|
|
|
997
|
+
def _stream_is_post_result_task_cleanup(
|
|
998
|
+
terminal_payload: dict[str, Any], events: list[dict[str, Any]],
|
|
999
|
+
) -> bool:
|
|
1000
|
+
"""Recognize Claude Code's exact bounded background-task shutdown tail."""
|
|
1001
|
+
if len(events) != 3:
|
|
1002
|
+
return False
|
|
1003
|
+
session_id = terminal_payload.get("session_id")
|
|
1004
|
+
if not isinstance(session_id, str) or not session_id:
|
|
1005
|
+
return False
|
|
1006
|
+
changed, updated, notification = events
|
|
1007
|
+
if (
|
|
1008
|
+
set(changed) != {"type", "subtype", "session_id", "tasks", "uuid"}
|
|
1009
|
+
or changed.get("type") != "system"
|
|
1010
|
+
or changed.get("subtype") != "background_tasks_changed"
|
|
1011
|
+
or changed.get("session_id") != session_id
|
|
1012
|
+
or changed.get("tasks") != []
|
|
1013
|
+
or not isinstance(changed.get("uuid"), str)
|
|
1014
|
+
or not changed["uuid"]
|
|
1015
|
+
):
|
|
1016
|
+
return False
|
|
1017
|
+
patch = updated.get("patch")
|
|
1018
|
+
task_id = updated.get("task_id")
|
|
1019
|
+
if (
|
|
1020
|
+
set(updated) != {"type", "subtype", "session_id", "task_id", "patch", "uuid"}
|
|
1021
|
+
or updated.get("type") != "system"
|
|
1022
|
+
or updated.get("subtype") != "task_updated"
|
|
1023
|
+
or updated.get("session_id") != session_id
|
|
1024
|
+
or not isinstance(task_id, str)
|
|
1025
|
+
or not task_id
|
|
1026
|
+
or not isinstance(patch, dict)
|
|
1027
|
+
or set(patch) != {"end_time", "status"}
|
|
1028
|
+
or isinstance(patch.get("end_time"), bool)
|
|
1029
|
+
or not isinstance(patch.get("end_time"), int)
|
|
1030
|
+
or patch["end_time"] < 0
|
|
1031
|
+
or patch.get("status") != "killed"
|
|
1032
|
+
or not isinstance(updated.get("uuid"), str)
|
|
1033
|
+
or not updated["uuid"]
|
|
1034
|
+
):
|
|
1035
|
+
return False
|
|
1036
|
+
return (
|
|
1037
|
+
set(notification) == {
|
|
1038
|
+
"type", "subtype", "session_id", "task_id", "tool_use_id",
|
|
1039
|
+
"status", "summary", "output_file", "uuid",
|
|
1040
|
+
}
|
|
1041
|
+
and notification.get("type") == "system"
|
|
1042
|
+
and notification.get("subtype") == "task_notification"
|
|
1043
|
+
and notification.get("session_id") == session_id
|
|
1044
|
+
and notification.get("task_id") == task_id
|
|
1045
|
+
and isinstance(notification.get("tool_use_id"), str)
|
|
1046
|
+
and bool(notification["tool_use_id"])
|
|
1047
|
+
and notification.get("status") == "stopped"
|
|
1048
|
+
and isinstance(notification.get("summary"), str)
|
|
1049
|
+
and isinstance(notification.get("output_file"), str)
|
|
1050
|
+
and isinstance(notification.get("uuid"), str)
|
|
1051
|
+
and bool(notification["uuid"])
|
|
1052
|
+
)
|
|
1053
|
+
|
|
1054
|
+
|
|
994
1055
|
def parse_claude_stream_output(
|
|
995
1056
|
stdout: bytes | str,
|
|
996
1057
|
*,
|
|
@@ -1046,6 +1107,7 @@ def parse_claude_stream_output(
|
|
|
1046
1107
|
terminal_payload: dict[str, Any] | None = None
|
|
1047
1108
|
terminal_result_code: str | None = None
|
|
1048
1109
|
terminal_status: str | None = None
|
|
1110
|
+
post_result_events: list[dict[str, Any]] = []
|
|
1049
1111
|
for raw_line in physical_lines:
|
|
1050
1112
|
# CR in a CRLF record is a delimiter byte, not part of the JSON content.
|
|
1051
1113
|
line = raw_line[:-1] if raw_line.endswith(b"\r") else raw_line
|
|
@@ -1104,11 +1166,14 @@ def parse_claude_stream_output(
|
|
|
1104
1166
|
|
|
1105
1167
|
is_result = event.get("type") == "result"
|
|
1106
1168
|
if terminal_payload is not None:
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1169
|
+
if is_result:
|
|
1170
|
+
return _stream_result(
|
|
1171
|
+
"invalid_stream",
|
|
1172
|
+
result_code="invalid_stream",
|
|
1173
|
+
error_code="stream_duplicate_result",
|
|
1174
|
+
)
|
|
1175
|
+
post_result_events.append(event)
|
|
1176
|
+
continue
|
|
1112
1177
|
if not is_result:
|
|
1113
1178
|
continue
|
|
1114
1179
|
|
|
@@ -1144,6 +1209,12 @@ def parse_claude_stream_output(
|
|
|
1144
1209
|
result_code="missing_terminal",
|
|
1145
1210
|
error_code="stream_missing_terminal",
|
|
1146
1211
|
)
|
|
1212
|
+
if post_result_events and not _stream_is_post_result_task_cleanup(
|
|
1213
|
+
terminal_payload, post_result_events,
|
|
1214
|
+
):
|
|
1215
|
+
return _stream_result(
|
|
1216
|
+
"invalid_stream", result_code="invalid_stream", error_code="stream_post_result",
|
|
1217
|
+
)
|
|
1147
1218
|
return _stream_result(
|
|
1148
1219
|
terminal_status,
|
|
1149
1220
|
result_code=terminal_result_code,
|
|
@@ -1484,8 +1555,11 @@ def _measurement_parse_variant(
|
|
|
1484
1555
|
if any(item not in MEASUREMENT_DOCUMENTED_HOOK_EVENTS for item in required_event_classes):
|
|
1485
1556
|
raise SystemExit(f"{owner}.hook_events.required_event_classes contains unsupported hook event")
|
|
1486
1557
|
ordered_classes = tuple(dict.fromkeys(event for event, _command in registered_bindings))
|
|
1487
|
-
|
|
1488
|
-
|
|
1558
|
+
required_set = set(required_event_classes)
|
|
1559
|
+
if required_event_classes != tuple(event for event in ordered_classes if event in required_set):
|
|
1560
|
+
raise SystemExit(
|
|
1561
|
+
f"{owner}.hook_events.required_event_classes must be an ordered subset of registered hook events"
|
|
1562
|
+
)
|
|
1489
1563
|
if variant_name == "baseline" and (registered_bindings or required_event_classes):
|
|
1490
1564
|
raise SystemExit(f"{owner} baseline hook configuration must be empty")
|
|
1491
1565
|
|
|
@@ -1638,7 +1712,7 @@ def _measurement_validate_treatment_bindings(spec: MeasurementVariant) -> dict[s
|
|
|
1638
1712
|
raise SystemExit("measurement baseline and treatment settings differ outside registered hooks")
|
|
1639
1713
|
binding_set = set(spec.registered_bindings)
|
|
1640
1714
|
occurrences = {binding: 0 for binding in spec.registered_bindings}
|
|
1641
|
-
for event in spec.
|
|
1715
|
+
for event in dict.fromkeys(event for event, _command in spec.registered_bindings):
|
|
1642
1716
|
registrations = hooks.get(event)
|
|
1643
1717
|
if not isinstance(registrations, list) or not registrations:
|
|
1644
1718
|
raise SystemExit("measurement baseline and treatment settings differ outside registered hooks")
|
|
@@ -2632,7 +2706,12 @@ def collect_self_hosted_metrics(payload: Any) -> dict[str, Any] | None:
|
|
|
2632
2706
|
return None
|
|
2633
2707
|
|
|
2634
2708
|
|
|
2635
|
-
def _measurement_child_env(
|
|
2709
|
+
def _measurement_child_env(
|
|
2710
|
+
spec: MeasurementVariant,
|
|
2711
|
+
context: MeasurementRunContext | None = None,
|
|
2712
|
+
*,
|
|
2713
|
+
existing_login_home: Path | None = None,
|
|
2714
|
+
) -> dict[str, str]:
|
|
2636
2715
|
env: dict[str, str] = {}
|
|
2637
2716
|
for name in spec.environment_allow:
|
|
2638
2717
|
# Names were validated before this function; do not inspect values for
|
|
@@ -2644,14 +2723,15 @@ def _measurement_child_env(spec: MeasurementVariant, context: MeasurementRunCont
|
|
|
2644
2723
|
env["PATH"] = os.defpath
|
|
2645
2724
|
if context is not None:
|
|
2646
2725
|
env.update({
|
|
2647
|
-
"HOME": str(context.home),
|
|
2726
|
+
"HOME": str(existing_login_home or context.home),
|
|
2648
2727
|
"XDG_CONFIG_HOME": str(context.xdg_config),
|
|
2649
2728
|
"XDG_CACHE_HOME": str(context.xdg_cache),
|
|
2650
2729
|
"XDG_DATA_HOME": str(context.xdg_data),
|
|
2651
2730
|
"XDG_STATE_HOME": str(context.xdg_state),
|
|
2652
2731
|
"TMPDIR": str(context.tmp),
|
|
2653
|
-
"CLAUDE_CONFIG_DIR": str(context.session),
|
|
2654
2732
|
})
|
|
2733
|
+
if existing_login_home is None:
|
|
2734
|
+
env["CLAUDE_CONFIG_DIR"] = str(context.session)
|
|
2655
2735
|
return env
|
|
2656
2736
|
|
|
2657
2737
|
|
|
@@ -2697,7 +2777,7 @@ def validate_measurement_cli_capabilities(claude_bin: str, spec: MeasurementVari
|
|
|
2697
2777
|
[executable, "--help"],
|
|
2698
2778
|
cwd=cwd,
|
|
2699
2779
|
timeout_seconds=10,
|
|
2700
|
-
max_output_bytes=
|
|
2780
|
+
max_output_bytes=MEASUREMENT_CLI_PROBE_OUTPUT_MAX_BYTES,
|
|
2701
2781
|
env=env,
|
|
2702
2782
|
)
|
|
2703
2783
|
except (OSError, subprocess.TimeoutExpired, ValueError) as exc:
|
|
@@ -3061,6 +3141,7 @@ def run_task_checker_study(
|
|
|
3061
3141
|
workspace: Path,
|
|
3062
3142
|
*,
|
|
3063
3143
|
env: dict[str, str],
|
|
3144
|
+
interpreter_binding: Mapping[str, Any] | None = None,
|
|
3064
3145
|
) -> str:
|
|
3065
3146
|
"""Run the content-bound success checker outside the measured workspace.
|
|
3066
3147
|
|
|
@@ -3076,6 +3157,14 @@ def run_task_checker_study(
|
|
|
3076
3157
|
payload = task.success_checker_bytes
|
|
3077
3158
|
if not payload:
|
|
3078
3159
|
return "success_checker_infra_invalid"
|
|
3160
|
+
checker_executable = sys.executable
|
|
3161
|
+
if interpreter_binding is not None:
|
|
3162
|
+
try:
|
|
3163
|
+
checker_executable = _benchmark_study_v2_assert_python_binding(
|
|
3164
|
+
interpreter_binding, require_current=False,
|
|
3165
|
+
)
|
|
3166
|
+
except (OSError, TypeError, ValueError):
|
|
3167
|
+
return "success_checker_infra_invalid"
|
|
3079
3168
|
private_root: str | None = None
|
|
3080
3169
|
try:
|
|
3081
3170
|
private_root = tempfile.mkdtemp(prefix="contextguard-bench-checker-")
|
|
@@ -3091,7 +3180,7 @@ def run_task_checker_study(
|
|
|
3091
3180
|
# PYTHONHOME/PYTHONSTARTUP 또는 sys.path[0] 로 들어오면 심어둔 sitecustomize.py 가
|
|
3092
3181
|
# 판정기 안에서 실행되어 위협 모델이 무너진다. 환경도 물려받지 않고 최소로 만든다.
|
|
3093
3182
|
result = run_bounded_command(
|
|
3094
|
-
[
|
|
3183
|
+
[checker_executable, "-I", str(checker_path)],
|
|
3095
3184
|
cwd=workspace,
|
|
3096
3185
|
timeout_seconds=600,
|
|
3097
3186
|
max_output_bytes=SUCCESS_COMMAND_OUTPUT_MAX_BYTES,
|
|
@@ -3349,9 +3438,11 @@ def normalize_measurement_hook_events(raw: bytes) -> list[dict[str, Any]]:
|
|
|
3349
3438
|
def _measurement_resolve_terminal_status(
|
|
3350
3439
|
*, raw_byte_limit: bool, raw_line_limit: bool, raw_line_byte_limit: bool,
|
|
3351
3440
|
process_status: str, stream_status: str, hook_result: dict[str, Any],
|
|
3352
|
-
arm: str,
|
|
3441
|
+
arm: str, allowed_event_classes: tuple[str, ...],
|
|
3442
|
+
required_event_classes: tuple[str, ...],
|
|
3353
3443
|
) -> str:
|
|
3354
3444
|
completed_classes = {item["hook_event"] for item in hook_result["hooks"]}
|
|
3445
|
+
allowed_classes = set(allowed_event_classes)
|
|
3355
3446
|
required_classes = set(required_event_classes)
|
|
3356
3447
|
hook_process_failed = any(
|
|
3357
3448
|
item["hook_process_outcome"] != "success" or item["hook_process_exit_code"] not in (None, 0)
|
|
@@ -3381,11 +3472,13 @@ def _measurement_resolve_terminal_status(
|
|
|
3381
3472
|
):
|
|
3382
3473
|
if hook_result.get("classification") == status or status in hook_result.get("failure_flags", ()):
|
|
3383
3474
|
return status
|
|
3384
|
-
|
|
3475
|
+
hook_arms = {"treatment", "legacy_trim", "bash_reference_v1"}
|
|
3476
|
+
unmodified_arms = {"baseline", "host_unmodified"}
|
|
3477
|
+
if arm in hook_arms and completed_classes - allowed_classes:
|
|
3385
3478
|
return "unexpected_hook_event_class"
|
|
3386
|
-
if arm
|
|
3479
|
+
if arm in unmodified_arms and hook_result["observed"]:
|
|
3387
3480
|
return "baseline_hook_contamination"
|
|
3388
|
-
if arm
|
|
3481
|
+
if arm in hook_arms and required_classes - completed_classes:
|
|
3389
3482
|
return "missing_required_hook_event_class"
|
|
3390
3483
|
if hook_process_failed:
|
|
3391
3484
|
return "hook_process_failure"
|
|
@@ -3429,12 +3522,13 @@ def _measurement_receipt(
|
|
|
3429
3522
|
) -> dict[str, Any]:
|
|
3430
3523
|
raw_sha256 = hashlib.sha256(raw).hexdigest()
|
|
3431
3524
|
raw_lines = len(raw.splitlines())
|
|
3432
|
-
|
|
3525
|
+
allowed_classes = tuple(dict.fromkeys(event for event, _command in spec.pair_registered_bindings))
|
|
3526
|
+
required_classes = spec.required_event_classes
|
|
3433
3527
|
counts = collections.Counter(item["hook_event"] for item in hook_result["hooks"])
|
|
3434
3528
|
event_class_counts = [
|
|
3435
3529
|
{"hook_event": event, "count": counts[event]}
|
|
3436
3530
|
for event in MEASUREMENT_DOCUMENTED_HOOK_EVENTS
|
|
3437
|
-
if counts[event] or event in
|
|
3531
|
+
if counts[event] or event in allowed_classes
|
|
3438
3532
|
]
|
|
3439
3533
|
settings_relative = Path("session") / spec.settings_file.name
|
|
3440
3534
|
return {
|
|
@@ -3503,6 +3597,10 @@ def _run_measurement_fixture_locked(
|
|
|
3503
3597
|
locked_root_fd: int,
|
|
3504
3598
|
on_process_started: Callable[[], None] | None = None,
|
|
3505
3599
|
measurement_study: bool = False,
|
|
3600
|
+
workspace_overlay: Path | None = None,
|
|
3601
|
+
on_workspace_prepared: Callable[[Path], None] | None = None,
|
|
3602
|
+
checker_interpreter_binding: Mapping[str, Any] | None = None,
|
|
3603
|
+
existing_login_home: Path | None = None,
|
|
3506
3604
|
) -> RunResult:
|
|
3507
3605
|
spec = variant.measurement
|
|
3508
3606
|
assert spec is not None
|
|
@@ -3522,6 +3620,18 @@ def _run_measurement_fixture_locked(
|
|
|
3522
3620
|
"load_task_fixture_trees for every task declaring fixture_tree"
|
|
3523
3621
|
)
|
|
3524
3622
|
reset_task_fixture_tree(task.fixture_tree_entries or (), context.workspace)
|
|
3623
|
+
if workspace_overlay is not None:
|
|
3624
|
+
destination = context.workspace / BENCHMARK_STUDY_V2_OVERLAY_NAME
|
|
3625
|
+
if destination.exists() or destination.is_symlink():
|
|
3626
|
+
raise SystemExit("measurement candidate overlay destination already exists")
|
|
3627
|
+
shutil.copytree(
|
|
3628
|
+
workspace_overlay,
|
|
3629
|
+
destination,
|
|
3630
|
+
symlinks=True,
|
|
3631
|
+
copy_function=shutil.copy2,
|
|
3632
|
+
)
|
|
3633
|
+
if on_workspace_prepared is not None:
|
|
3634
|
+
on_workspace_prepared(context.workspace)
|
|
3525
3635
|
settings_snapshot = context.session / spec.settings_file.name
|
|
3526
3636
|
_measurement_write_exclusive(settings_snapshot, spec.settings_source_bytes)
|
|
3527
3637
|
try:
|
|
@@ -3535,7 +3645,9 @@ def _run_measurement_fixture_locked(
|
|
|
3535
3645
|
variant,
|
|
3536
3646
|
measurement_settings_file=settings_snapshot,
|
|
3537
3647
|
)
|
|
3538
|
-
env = _measurement_child_env(
|
|
3648
|
+
env = _measurement_child_env(
|
|
3649
|
+
spec, context, existing_login_home=existing_login_home,
|
|
3650
|
+
)
|
|
3539
3651
|
try:
|
|
3540
3652
|
try:
|
|
3541
3653
|
proc = run_bounded_command(
|
|
@@ -3582,6 +3694,9 @@ def _run_measurement_fixture_locked(
|
|
|
3582
3694
|
stream_status=parsed.status,
|
|
3583
3695
|
hook_result=hook_result,
|
|
3584
3696
|
arm=variant.name,
|
|
3697
|
+
allowed_event_classes=tuple(
|
|
3698
|
+
dict.fromkeys(event for event, _command in spec.pair_registered_bindings)
|
|
3699
|
+
),
|
|
3585
3700
|
required_event_classes=spec.required_event_classes,
|
|
3586
3701
|
)
|
|
3587
3702
|
|
|
@@ -3660,6 +3775,7 @@ def _run_measurement_fixture_locked(
|
|
|
3660
3775
|
)
|
|
3661
3776
|
checker_classification = run_task_checker_study(
|
|
3662
3777
|
task, context.workspace, env=env,
|
|
3778
|
+
interpreter_binding=checker_interpreter_binding,
|
|
3663
3779
|
)
|
|
3664
3780
|
else:
|
|
3665
3781
|
checker_classification = run_success_command_study(task, project_root, env=env)
|
|
@@ -7402,7 +7518,8 @@ def _verify_existing_measurement_run(
|
|
|
7402
7518
|
hooks = receipt.get("hooks")
|
|
7403
7519
|
if not isinstance(hooks, list) or summary.get("completed_lifecycle_count") != len(hooks):
|
|
7404
7520
|
raise ValueError("hook count")
|
|
7405
|
-
|
|
7521
|
+
allowed_classes = list(dict.fromkeys(event for event, _command in spec.pair_registered_bindings))
|
|
7522
|
+
required_classes = list(spec.required_event_classes)
|
|
7406
7523
|
if summary.get("required_event_classes") != required_classes:
|
|
7407
7524
|
raise ValueError("required hook classes")
|
|
7408
7525
|
for count_name in ("observed_lifecycle_count", "completed_lifecycle_count"):
|
|
@@ -7443,7 +7560,7 @@ def _verify_existing_measurement_run(
|
|
|
7443
7560
|
expected_counts = [
|
|
7444
7561
|
{"hook_event": event, "count": counts[event]}
|
|
7445
7562
|
for event in MEASUREMENT_DOCUMENTED_HOOK_EVENTS
|
|
7446
|
-
if counts[event] or event in
|
|
7563
|
+
if counts[event] or event in allowed_classes
|
|
7447
7564
|
]
|
|
7448
7565
|
if summary.get("event_class_counts") != expected_counts:
|
|
7449
7566
|
raise ValueError("event class counts")
|
|
@@ -7482,6 +7599,7 @@ def _verify_existing_measurement_run(
|
|
|
7482
7599
|
stream_status=reparsed_stream.status,
|
|
7483
7600
|
hook_result=reparsed_hooks,
|
|
7484
7601
|
arm=spec.identity.arm,
|
|
7602
|
+
allowed_event_classes=tuple(allowed_classes),
|
|
7485
7603
|
required_event_classes=spec.required_event_classes,
|
|
7486
7604
|
)
|
|
7487
7605
|
expected_receipt = _measurement_receipt(
|
|
@@ -8063,8 +8181,10 @@ def parse_measurement_terminal_usage(raw: bytes | str) -> dict[str, int]:
|
|
|
8063
8181
|
if parsed.payload is None or parsed.payload.get("type") != "result":
|
|
8064
8182
|
raise ValueError("measurement terminal usage requires one terminal result record")
|
|
8065
8183
|
usage = parsed.payload.get("usage")
|
|
8066
|
-
|
|
8067
|
-
|
|
8184
|
+
# Claude Code may add diagnostic telemetry beside the four accounting
|
|
8185
|
+
# buckets. Only the frozen required buckets participate in the estimator.
|
|
8186
|
+
if not isinstance(usage, dict) or not set(MEASUREMENT_STUDY_USAGE_KEYS).issubset(usage):
|
|
8187
|
+
raise ValueError("measurement terminal usage must contain all four required buckets")
|
|
8068
8188
|
result: dict[str, int] = {}
|
|
8069
8189
|
total = 0
|
|
8070
8190
|
for key in MEASUREMENT_STUDY_USAGE_KEYS:
|
|
@@ -8681,6 +8801,10 @@ def append_study_attempt_event(path: Path, event: Mapping[str, Any]) -> None:
|
|
|
8681
8801
|
fd = _open_regular_no_symlink(path, flags, 0o600)
|
|
8682
8802
|
os.fchmod(fd, 0o600)
|
|
8683
8803
|
_measurement_write_fd(fd, payload)
|
|
8804
|
+
# A provider launch may follow this reservation immediately. Persist the
|
|
8805
|
+
# directory entry as well as the file bytes so a power loss cannot make
|
|
8806
|
+
# an already-consumed identity appear unreserved after restart.
|
|
8807
|
+
os.fsync(parent_fd)
|
|
8684
8808
|
finally:
|
|
8685
8809
|
if fd >= 0:
|
|
8686
8810
|
os.close(fd)
|
|
@@ -8932,7 +9056,7 @@ def run_measurement_cli_probes(
|
|
|
8932
9056
|
[claude_bin, flag],
|
|
8933
9057
|
cwd=Path("<probe-root>/cwd"),
|
|
8934
9058
|
timeout_seconds=10.0,
|
|
8935
|
-
max_output_bytes=
|
|
9059
|
+
max_output_bytes=MEASUREMENT_CLI_PROBE_OUTPUT_MAX_BYTES,
|
|
8936
9060
|
env=dict(env),
|
|
8937
9061
|
)
|
|
8938
9062
|
for flag in ("--version", "--help")
|
|
@@ -8968,7 +9092,16 @@ def run_measurement_cli_probes(
|
|
|
8968
9092
|
root, paths = create_measurement_probe_layout()
|
|
8969
9093
|
try:
|
|
8970
9094
|
validate_measurement_probe_layout(root, paths)
|
|
8971
|
-
|
|
9095
|
+
runtime_directories = [str(executable_path.parent)]
|
|
9096
|
+
invoked = shutil.which(claude_bin)
|
|
9097
|
+
if invoked is not None:
|
|
9098
|
+
runtime_directories.append(str(Path(invoked).absolute().parent))
|
|
9099
|
+
for runtime_name in ("node", "python3"):
|
|
9100
|
+
runtime = shutil.which(runtime_name)
|
|
9101
|
+
if runtime is not None:
|
|
9102
|
+
runtime_directories.append(str(Path(runtime).absolute().parent))
|
|
9103
|
+
runtime_directories.extend(("/usr/bin", "/bin", "/usr/sbin", "/sbin"))
|
|
9104
|
+
path_value = os.pathsep.join(dict.fromkeys(runtime_directories))
|
|
8972
9105
|
env = {
|
|
8973
9106
|
"PATH": path_value,
|
|
8974
9107
|
"LANG": "C",
|
|
@@ -8988,7 +9121,7 @@ def run_measurement_cli_probes(
|
|
|
8988
9121
|
[executable, flag],
|
|
8989
9122
|
cwd=paths["cwd"],
|
|
8990
9123
|
timeout_seconds=10,
|
|
8991
|
-
max_output_bytes=
|
|
9124
|
+
max_output_bytes=MEASUREMENT_CLI_PROBE_OUTPUT_MAX_BYTES,
|
|
8992
9125
|
env=env,
|
|
8993
9126
|
))
|
|
8994
9127
|
version_raw = _study_validate_probe_output(results[0], kind="version")
|
|
@@ -9286,6 +9419,7 @@ def build_measurement_study_manifest(
|
|
|
9286
9419
|
"record_type": "result",
|
|
9287
9420
|
"object_path": "$.usage",
|
|
9288
9421
|
"keys": list(MEASUREMENT_STUDY_USAGE_KEYS),
|
|
9422
|
+
"additional_fields": "ignored_not_counted_v1",
|
|
9289
9423
|
"integer_grammar": "0|[1-9][0-9]*",
|
|
9290
9424
|
"maximum": MAX_USAGE_TOKEN_COUNT,
|
|
9291
9425
|
"formula": "P=input_tokens+cache_creation_input_tokens+cache_read_input_tokens+output_tokens",
|
|
@@ -10452,85 +10586,4064 @@ def run_measurement_study_action(
|
|
|
10452
10586
|
return 0
|
|
10453
10587
|
|
|
10454
10588
|
|
|
10455
|
-
|
|
10456
|
-
|
|
10457
|
-
|
|
10458
|
-
|
|
10459
|
-
|
|
10460
|
-
|
|
10461
|
-
|
|
10462
|
-
|
|
10463
|
-
|
|
10464
|
-
|
|
10465
|
-
|
|
10466
|
-
|
|
10467
|
-
|
|
10468
|
-
|
|
10469
|
-
|
|
10470
|
-
|
|
10471
|
-
|
|
10472
|
-
|
|
10473
|
-
|
|
10474
|
-
|
|
10475
|
-
|
|
10476
|
-
|
|
10477
|
-
|
|
10478
|
-
|
|
10479
|
-
|
|
10480
|
-
|
|
10481
|
-
|
|
10482
|
-
|
|
10483
|
-
|
|
10484
|
-
|
|
10485
|
-
|
|
10486
|
-
|
|
10487
|
-
|
|
10488
|
-
|
|
10489
|
-
|
|
10490
|
-
|
|
10491
|
-
|
|
10589
|
+
# V2 is a separately-versioned analytical surface. It intentionally does not
|
|
10590
|
+
# alter the frozen S001--S003 runner, manifest, or report contracts above.
|
|
10591
|
+
BENCHMARK_STUDY_V2_PLAN_SCHEMA_VERSION = "contextguard.bench.study-plan.v2"
|
|
10592
|
+
BENCHMARK_STUDY_V2_SCHEDULE_ALGORITHM = "splitmix64-blocked-three-arm-v1"
|
|
10593
|
+
BENCHMARK_STUDY_V2_ARMS = (
|
|
10594
|
+
"host_unmodified", "legacy_trim", "bash_reference_v1",
|
|
10595
|
+
)
|
|
10596
|
+
BENCHMARK_STUDY_V2_PRIMARY_CONTRAST = ("host_unmodified", "bash_reference_v1")
|
|
10597
|
+
BENCHMARK_STUDY_V2_DIAGNOSTIC_CONTRAST = ("legacy_trim", "bash_reference_v1")
|
|
10598
|
+
BENCHMARK_STUDY_V2_RETRY_POLICY = "retain_valid_unfavorable_attempts_v1"
|
|
10599
|
+
BENCHMARK_STUDY_V2_EVIDENCE_FORBIDDEN_KEYS = frozenset({
|
|
10600
|
+
"prompt", "output", "command", "command_hash", "command_sha256", "path",
|
|
10601
|
+
"project_id", "capabilities", "credential", "credentials", "token", "secret",
|
|
10602
|
+
})
|
|
10603
|
+
BENCHMARK_STUDY_V2_HANDLE_RE = re.compile(r"(?i)\bcgr1p(?:[_-]|\b)")
|
|
10604
|
+
BENCHMARK_STUDY_V2_REVISION_KEYS = frozenset({
|
|
10605
|
+
"backend_revision", "model_revision", "cli_version",
|
|
10606
|
+
})
|
|
10607
|
+
BENCHMARK_STUDY_V2_REVISION_RE = re.compile(
|
|
10608
|
+
r"[A-Za-z0-9][A-Za-z0-9._+:/@-]{0,127}"
|
|
10609
|
+
)
|
|
10610
|
+
BENCHMARK_STUDY_V2_SECRET_SHAPE_RE = re.compile(
|
|
10611
|
+
r"(?i)(?:"
|
|
10612
|
+
r"\bsk-[A-Za-z0-9_-]{16,}"
|
|
10613
|
+
r"|\b[rs]k_(?:live|test)_[A-Za-z0-9]{16,}"
|
|
10614
|
+
r"|\bgithub_pat_[A-Za-z0-9_]{16,}"
|
|
10615
|
+
r"|\bgh[pousr]_[A-Za-z0-9]{16,}"
|
|
10616
|
+
r"|\bnpm_[A-Za-z0-9]{16,}"
|
|
10617
|
+
r"|\bxox[baprs]-[A-Za-z0-9-]{16,}"
|
|
10618
|
+
r"|\bA[KS]IA[0-9A-Z]{16}\b"
|
|
10619
|
+
r"|\bAIza[0-9A-Za-z_-]{20,}"
|
|
10620
|
+
r"|\bya29\.[0-9A-Za-z_-]{16,}"
|
|
10621
|
+
r"|\bbearer\s+[0-9A-Za-z._~+/-]+=*"
|
|
10622
|
+
r"|-----BEGIN [A-Z ]*PRIVATE KEY-----"
|
|
10623
|
+
r")"
|
|
10624
|
+
)
|
|
10625
|
+
BENCHMARK_STUDY_V2_CHECKER_BINDING_DOMAIN = (
|
|
10626
|
+
"contextguard.bench.v2.checker-binding.v1"
|
|
10627
|
+
)
|
|
10628
|
+
BENCHMARK_STUDY_V2_CORPUS_TASK_ORDER_DOMAIN = (
|
|
10629
|
+
"contextguard.bench.v2.corpus-task-order.v1"
|
|
10630
|
+
)
|
|
10492
10631
|
|
|
10493
|
-
|
|
10494
|
-
|
|
10495
|
-
|
|
10496
|
-
|
|
10497
|
-
|
|
10632
|
+
|
|
10633
|
+
def _benchmark_study_v2_seed(seed: int | str) -> int:
|
|
10634
|
+
if isinstance(seed, str):
|
|
10635
|
+
if re.fullmatch(r"0x[0-9A-F]{16}", seed) is None:
|
|
10636
|
+
raise ValueError("v2 schedule seed must be frozen uppercase 64-bit hex")
|
|
10637
|
+
return int(seed, 16)
|
|
10638
|
+
if isinstance(seed, bool) or not isinstance(seed, int) or not 0 <= seed <= SPLITMIX64_MASK:
|
|
10639
|
+
raise ValueError("v2 schedule seed must be an unsigned 64-bit integer")
|
|
10640
|
+
return seed
|
|
10641
|
+
|
|
10642
|
+
|
|
10643
|
+
def _benchmark_study_v2_task_ids(task_ids: Sequence[str]) -> list[str]:
|
|
10644
|
+
normalized = list(task_ids)
|
|
10645
|
+
if len(normalized) != 12 or len(set(normalized)) != 12:
|
|
10646
|
+
raise ValueError("v2 study requires exactly 12 unique ordered task ids")
|
|
10647
|
+
if any(not isinstance(task_id, str) or not task_id for task_id in normalized):
|
|
10648
|
+
raise ValueError("v2 study task ids must be non-empty strings")
|
|
10649
|
+
return normalized
|
|
10650
|
+
|
|
10651
|
+
|
|
10652
|
+
def generate_benchmark_study_v2_schedule(
|
|
10653
|
+
task_ids: Sequence[str], *, repetitions: int, schedule_seed: int | str,
|
|
10654
|
+
) -> list[dict[str, Any]]:
|
|
10655
|
+
"""Create the pre-randomized 3-arm order for each task/repetition block."""
|
|
10656
|
+
tasks = _benchmark_study_v2_task_ids(task_ids)
|
|
10657
|
+
if repetitions != 3:
|
|
10658
|
+
raise ValueError("v2 study requires exactly three repetitions")
|
|
10659
|
+
state = _benchmark_study_v2_seed(schedule_seed)
|
|
10660
|
+
schedule: list[dict[str, Any]] = []
|
|
10661
|
+
for task_id in tasks:
|
|
10662
|
+
for repetition in range(repetitions):
|
|
10663
|
+
arm_order = list(BENCHMARK_STUDY_V2_ARMS)
|
|
10664
|
+
for index in range(len(arm_order) - 1, 0, -1):
|
|
10665
|
+
state, selected = splitmix64_bounded(state, index + 1)
|
|
10666
|
+
arm_order[index], arm_order[selected] = arm_order[selected], arm_order[index]
|
|
10667
|
+
schedule.append({
|
|
10668
|
+
"block_id": _study_domain_hash(
|
|
10669
|
+
"contextguard.bench.v2.block-id.v1", [task_id, repetition],
|
|
10670
|
+
),
|
|
10671
|
+
"task_id": task_id,
|
|
10672
|
+
"repetition": repetition,
|
|
10673
|
+
"arm_order": arm_order,
|
|
10674
|
+
})
|
|
10675
|
+
return schedule
|
|
10676
|
+
|
|
10677
|
+
|
|
10678
|
+
def generate_benchmark_study_v2_slots(
|
|
10679
|
+
task_ids: Sequence[str], schedule: Sequence[Mapping[str, Any]], *,
|
|
10680
|
+
candidate_hash: str, namespace: str,
|
|
10681
|
+
) -> list[dict[str, Any]]:
|
|
10682
|
+
"""Materialize immutable initial/retry identities without replacement."""
|
|
10683
|
+
tasks = _benchmark_study_v2_task_ids(task_ids)
|
|
10684
|
+
if SHA256_HEX_PATTERN.fullmatch(candidate_hash) is None:
|
|
10685
|
+
raise ValueError("v2 candidate hash is invalid")
|
|
10686
|
+
if not isinstance(namespace, str) or not MEASUREMENT_ID_NAMESPACE_RE.fullmatch(namespace):
|
|
10687
|
+
raise ValueError("v2 namespace is invalid")
|
|
10688
|
+
expected_blocks = [(task, repetition) for task in tasks for repetition in range(3)]
|
|
10689
|
+
if [(row.get("task_id"), row.get("repetition")) for row in schedule] != expected_blocks:
|
|
10690
|
+
raise ValueError("v2 schedule task/repetition order drift")
|
|
10691
|
+
initial: list[dict[str, Any]] = []
|
|
10692
|
+
retry: list[dict[str, Any]] = []
|
|
10693
|
+
seen: set[str] = set()
|
|
10694
|
+
for block in schedule:
|
|
10695
|
+
arm_order = block.get("arm_order")
|
|
10696
|
+
if not isinstance(arm_order, list) or set(arm_order) != set(BENCHMARK_STUDY_V2_ARMS) or len(arm_order) != 3:
|
|
10697
|
+
raise ValueError("v2 block arm order is invalid")
|
|
10698
|
+
for arm in arm_order:
|
|
10699
|
+
for attempt, state, destination in ((0, "planned", initial), (1, "conditional", retry)):
|
|
10700
|
+
run_id = MeasurementIdentity(
|
|
10701
|
+
candidate_hash=candidate_hash, repetition=int(block["repetition"]),
|
|
10702
|
+
arm=arm, attempt=attempt, namespace=namespace,
|
|
10703
|
+
).run_id(str(block["task_id"]))
|
|
10704
|
+
if run_id in seen:
|
|
10705
|
+
raise ValueError("v2 run identity collision")
|
|
10706
|
+
seen.add(run_id)
|
|
10707
|
+
destination.append({
|
|
10708
|
+
"block_id": block["block_id"], "task_id": block["task_id"],
|
|
10709
|
+
"repetition": block["repetition"], "arm": arm, "attempt": attempt,
|
|
10710
|
+
"run_id": run_id, "state": state,
|
|
10711
|
+
})
|
|
10712
|
+
slots = initial + retry
|
|
10713
|
+
validate_benchmark_study_v2_slots(slots, task_ids=tasks)
|
|
10714
|
+
return slots
|
|
10715
|
+
|
|
10716
|
+
|
|
10717
|
+
def validate_benchmark_study_v2_slots(
|
|
10718
|
+
slots: Sequence[Mapping[str, Any]], *, task_ids: Sequence[str],
|
|
10719
|
+
) -> None:
|
|
10720
|
+
tasks = _benchmark_study_v2_task_ids(task_ids)
|
|
10721
|
+
if len(slots) != 216:
|
|
10722
|
+
raise ValueError("v2 study requires exactly 216 immutable slots")
|
|
10723
|
+
expected = {
|
|
10724
|
+
(task, repetition, arm, attempt)
|
|
10725
|
+
for task in tasks for repetition in range(3)
|
|
10726
|
+
for arm in BENCHMARK_STUDY_V2_ARMS for attempt in (0, 1)
|
|
10727
|
+
}
|
|
10728
|
+
observed: set[tuple[str, int, str, int]] = set()
|
|
10729
|
+
run_ids: set[str] = set()
|
|
10730
|
+
required = {"block_id", "task_id", "repetition", "arm", "attempt", "run_id", "state"}
|
|
10731
|
+
for slot in slots:
|
|
10732
|
+
if set(slot) != required:
|
|
10733
|
+
raise ValueError("v2 slot schema mismatch")
|
|
10734
|
+
key = (slot["task_id"], slot["repetition"], slot["arm"], slot["attempt"])
|
|
10735
|
+
if key in observed or key not in expected or slot["run_id"] in run_ids:
|
|
10736
|
+
raise ValueError("v2 slot identity mismatch")
|
|
10737
|
+
if slot["state"] != ("planned" if slot["attempt"] == 0 else "conditional"):
|
|
10738
|
+
raise ValueError("v2 slot state mismatch")
|
|
10739
|
+
if not isinstance(slot["run_id"], str) or SHA256_HEX_PATTERN.fullmatch(slot["run_id"]) is None:
|
|
10740
|
+
raise ValueError("v2 slot run id is invalid")
|
|
10741
|
+
observed.add(key)
|
|
10742
|
+
run_ids.add(slot["run_id"])
|
|
10743
|
+
if observed != expected:
|
|
10744
|
+
raise ValueError("v2 slot coverage mismatch")
|
|
10745
|
+
|
|
10746
|
+
|
|
10747
|
+
def benchmark_study_v2_contrasts(_values: Mapping[str, Any] | None = None) -> dict[str, list[str]]:
|
|
10748
|
+
"""Expose the single product contrast separately from its diagnostic control."""
|
|
10749
|
+
return {
|
|
10750
|
+
"primary": list(BENCHMARK_STUDY_V2_PRIMARY_CONTRAST),
|
|
10751
|
+
"diagnostic": list(BENCHMARK_STUDY_V2_DIAGNOSTIC_CONTRAST),
|
|
10752
|
+
}
|
|
10753
|
+
|
|
10754
|
+
|
|
10755
|
+
def _benchmark_study_v2_cluster_interval(values_by_task: Sequence[Sequence[float]]) -> dict[str, Any]:
|
|
10756
|
+
task_count = len(values_by_task)
|
|
10757
|
+
if task_count < 2 or any(len(row) != 3 for row in values_by_task):
|
|
10758
|
+
raise ValueError("v2 task-cluster interval requires task x 3 values")
|
|
10759
|
+
task_means = [sum(float(value) for value in row) / 3.0 for row in values_by_task]
|
|
10760
|
+
state = MEASUREMENT_STUDY_INFERENCE_SEED
|
|
10761
|
+
estimates: list[float] = []
|
|
10762
|
+
for _ in range(MEASUREMENT_STUDY_BOOTSTRAP_RESAMPLES):
|
|
10763
|
+
total = 0.0
|
|
10764
|
+
for _ in range(task_count):
|
|
10765
|
+
state, index = splitmix64_bounded(state, task_count)
|
|
10766
|
+
total += task_means[index]
|
|
10767
|
+
estimates.append(total / task_count)
|
|
10768
|
+
return {
|
|
10769
|
+
"method": "task_cluster_bootstrap_v2",
|
|
10770
|
+
"point": sum(task_means) / task_count,
|
|
10771
|
+
"q025": float(type7_quantile(estimates, 0.025)),
|
|
10772
|
+
"q975": float(type7_quantile(estimates, 0.975)),
|
|
10773
|
+
"task_count": task_count,
|
|
10774
|
+
"resamples": MEASUREMENT_STUDY_BOOTSTRAP_RESAMPLES,
|
|
10775
|
+
}
|
|
10776
|
+
|
|
10777
|
+
|
|
10778
|
+
def infer_benchmark_study_v2_binary(
|
|
10779
|
+
rows: Sequence[Mapping[str, Any]], *, task_order: Sequence[str], ni_margin: float = 0.10,
|
|
10780
|
+
) -> dict[str, Any]:
|
|
10781
|
+
"""Exact task-cluster sign-permutation inference for the product contrast."""
|
|
10782
|
+
tasks = _benchmark_study_v2_task_ids(task_order)
|
|
10783
|
+
if not isinstance(ni_margin, (int, float)) or isinstance(ni_margin, bool) or not 0 <= ni_margin < 1:
|
|
10784
|
+
raise ValueError("v2 non-inferiority margin is invalid")
|
|
10785
|
+
units: dict[tuple[str, int, str], bool] = {}
|
|
10786
|
+
for row in rows:
|
|
10787
|
+
task_id, repetition, arm, success = row.get("task_id"), row.get("repetition"), row.get("arm"), row.get("success")
|
|
10788
|
+
if task_id not in tasks or repetition not in (0, 1, 2) or arm not in BENCHMARK_STUDY_V2_PRIMARY_CONTRAST or not isinstance(success, bool):
|
|
10789
|
+
raise ValueError("v2 binary outcome identity is invalid")
|
|
10790
|
+
key = (str(task_id), int(repetition), str(arm))
|
|
10791
|
+
if key in units:
|
|
10792
|
+
raise ValueError("duplicate v2 binary outcome")
|
|
10793
|
+
units[key] = success
|
|
10794
|
+
expected = {
|
|
10795
|
+
(task, repetition, arm) for task in tasks for repetition in range(3)
|
|
10796
|
+
for arm in BENCHMARK_STUDY_V2_PRIMARY_CONTRAST
|
|
10797
|
+
}
|
|
10798
|
+
if set(units) != expected:
|
|
10799
|
+
raise ValueError("v2 binary outcome coverage is incomplete")
|
|
10800
|
+
task_deltas = [
|
|
10801
|
+
sum(
|
|
10802
|
+
int(units[(task, repetition, "bash_reference_v1")])
|
|
10803
|
+
- int(units[(task, repetition, "host_unmodified")])
|
|
10804
|
+
for repetition in range(3)
|
|
10805
|
+
) / 3.0
|
|
10806
|
+
for task in tasks
|
|
10807
|
+
]
|
|
10808
|
+
point = sum(task_deltas) / len(tasks)
|
|
10809
|
+
all_success = all(units.values())
|
|
10810
|
+
# At the NI boundary, reference-minus-host plus the frozen margin has mean
|
|
10811
|
+
# zero. Sign-flip that centered task effect, never the nested run rows.
|
|
10812
|
+
centered_deltas = [value + float(ni_margin) for value in task_deltas]
|
|
10813
|
+
outcomes = []
|
|
10814
|
+
for mask in range(1 << len(tasks)):
|
|
10815
|
+
outcomes.append(sum(
|
|
10816
|
+
(-value if mask & (1 << index) else value)
|
|
10817
|
+
for index, value in enumerate(centered_deltas)
|
|
10818
|
+
) / len(tasks))
|
|
10819
|
+
observed_statistic = point + float(ni_margin)
|
|
10820
|
+
p_value = (
|
|
10821
|
+
sum(value >= observed_statistic for value in outcomes) + 1
|
|
10822
|
+
) / (len(outcomes) + 1)
|
|
10823
|
+
return {
|
|
10824
|
+
"method": "exact_task_cluster_sign_permutation_v1",
|
|
10825
|
+
"contrast": list(BENCHMARK_STUDY_V2_PRIMARY_CONTRAST),
|
|
10826
|
+
"task_ids_sha256": _study_domain_hash(
|
|
10827
|
+
"contextguard.bench.v2.task-order.v1", tasks,
|
|
10828
|
+
),
|
|
10829
|
+
"point": point,
|
|
10830
|
+
"ni_margin": float(ni_margin),
|
|
10831
|
+
"p_value": p_value,
|
|
10832
|
+
"task_count": len(tasks),
|
|
10833
|
+
"degenerate_all_success": all_success,
|
|
10834
|
+
"noninferiority_pass": bool(not all_success and point > -float(ni_margin) and p_value < 0.05),
|
|
10835
|
+
}
|
|
10836
|
+
|
|
10837
|
+
|
|
10838
|
+
def compute_benchmark_study_v2_effects(
|
|
10839
|
+
records: Sequence[Mapping[str, Any]], *, task_order: Sequence[str],
|
|
10840
|
+
) -> dict[str, Any]:
|
|
10841
|
+
"""Retain every valid terminal attempt and derive task-clustered effects."""
|
|
10842
|
+
tasks = _benchmark_study_v2_task_ids(task_order)
|
|
10843
|
+
grouped: dict[tuple[str, int, str], list[Mapping[str, Any]]] = collections.defaultdict(list)
|
|
10844
|
+
for record in records:
|
|
10845
|
+
task_id, repetition, arm = record.get("task_id"), record.get("repetition"), record.get("arm")
|
|
10846
|
+
if task_id not in tasks or repetition not in (0, 1, 2) or arm not in BENCHMARK_STUDY_V2_ARMS:
|
|
10847
|
+
raise ValueError("v2 effect record identity is invalid")
|
|
10848
|
+
grouped[(str(task_id), int(repetition), str(arm))].append(record)
|
|
10849
|
+
token_deltas: list[list[float]] = []
|
|
10850
|
+
diagnostic_token_deltas: list[list[float]] = []
|
|
10851
|
+
metric_deltas: dict[str, list[list[float]]] = {"correction": [], "retrieval": []}
|
|
10852
|
+
diagnostic_metric_deltas: dict[str, list[list[float]]] = {
|
|
10853
|
+
"correction": [], "retrieval": [],
|
|
10854
|
+
}
|
|
10855
|
+
metric_available = {"correction": True, "retrieval": True}
|
|
10856
|
+
retained_unfavorable = 0
|
|
10857
|
+
for task in tasks:
|
|
10858
|
+
per_task: list[float] = []
|
|
10859
|
+
diagnostic_per_task: list[float] = []
|
|
10860
|
+
per_task_metrics: dict[str, list[float]] = {"correction": [], "retrieval": []}
|
|
10861
|
+
diagnostic_per_task_metrics: dict[str, list[float]] = {
|
|
10862
|
+
"correction": [], "retrieval": [],
|
|
10863
|
+
}
|
|
10864
|
+
for repetition in range(3):
|
|
10865
|
+
costs: dict[str, float] = {}
|
|
10866
|
+
metrics: dict[str, dict[str, float]] = {"correction": {}, "retrieval": {}}
|
|
10867
|
+
for arm in BENCHMARK_STUDY_V2_ARMS:
|
|
10868
|
+
attempts = sorted(grouped.get((task, repetition, arm), ()), key=lambda row: int(row.get("attempt", -1)))
|
|
10869
|
+
if not attempts or [row.get("attempt") for row in attempts] not in ([0], [0, 1]):
|
|
10870
|
+
raise ValueError("v2 attempts are incomplete or replaced")
|
|
10871
|
+
values = []
|
|
10872
|
+
for row in attempts:
|
|
10873
|
+
token = row.get("tokens")
|
|
10874
|
+
if (
|
|
10875
|
+
isinstance(token, bool)
|
|
10876
|
+
or not isinstance(token, (int, float))
|
|
10877
|
+
or not math.isfinite(float(token))
|
|
10878
|
+
or token < 0
|
|
10879
|
+
):
|
|
10880
|
+
raise ValueError("v2 token value is invalid")
|
|
10881
|
+
if row.get("terminal_status") != "success" or row.get("success") is not True:
|
|
10882
|
+
retained_unfavorable += 1
|
|
10883
|
+
values.append(float(token))
|
|
10884
|
+
costs[arm] = sum(values)
|
|
10885
|
+
for metric in metrics:
|
|
10886
|
+
attempt_values = [row.get(metric) for row in attempts]
|
|
10887
|
+
if any(
|
|
10888
|
+
isinstance(value, bool)
|
|
10889
|
+
or not isinstance(value, (int, float))
|
|
10890
|
+
or not math.isfinite(float(value))
|
|
10891
|
+
or value < 0
|
|
10892
|
+
for value in attempt_values
|
|
10893
|
+
):
|
|
10894
|
+
metric_available[metric] = False
|
|
10895
|
+
continue
|
|
10896
|
+
metrics[metric][arm] = sum(
|
|
10897
|
+
float(value) for value in attempt_values
|
|
10898
|
+
)
|
|
10899
|
+
per_task.append(costs["host_unmodified"] - costs["bash_reference_v1"])
|
|
10900
|
+
diagnostic_per_task.append(
|
|
10901
|
+
costs["legacy_trim"] - costs["bash_reference_v1"]
|
|
10902
|
+
)
|
|
10903
|
+
for metric, values in metrics.items():
|
|
10904
|
+
if set(values) == set(BENCHMARK_STUDY_V2_ARMS):
|
|
10905
|
+
per_task_metrics[metric].append(
|
|
10906
|
+
values["host_unmodified"] - values["bash_reference_v1"]
|
|
10907
|
+
)
|
|
10908
|
+
diagnostic_per_task_metrics[metric].append(
|
|
10909
|
+
values["legacy_trim"] - values["bash_reference_v1"]
|
|
10910
|
+
)
|
|
10911
|
+
token_deltas.append(per_task)
|
|
10912
|
+
diagnostic_token_deltas.append(diagnostic_per_task)
|
|
10913
|
+
for metric in metric_deltas:
|
|
10914
|
+
if len(per_task_metrics[metric]) == 3:
|
|
10915
|
+
metric_deltas[metric].append(per_task_metrics[metric])
|
|
10916
|
+
diagnostic_metric_deltas[metric].append(
|
|
10917
|
+
diagnostic_per_task_metrics[metric]
|
|
10918
|
+
)
|
|
10919
|
+
else:
|
|
10920
|
+
metric_available[metric] = False
|
|
10921
|
+
metric_effects = {
|
|
10922
|
+
f"{metric}_effect": (
|
|
10923
|
+
_benchmark_study_v2_cluster_interval(metric_deltas[metric])
|
|
10924
|
+
if metric_available[metric] and len(metric_deltas[metric]) == len(tasks)
|
|
10925
|
+
else {"method": "unavailable", "point": None, "q025": None, "q975": None}
|
|
10926
|
+
)
|
|
10927
|
+
for metric in metric_deltas
|
|
10928
|
+
}
|
|
10929
|
+
diagnostic_metric_effects = {
|
|
10930
|
+
f"diagnostic_{metric}_effect": (
|
|
10931
|
+
_benchmark_study_v2_cluster_interval(diagnostic_metric_deltas[metric])
|
|
10932
|
+
if metric_available[metric]
|
|
10933
|
+
and len(diagnostic_metric_deltas[metric]) == len(tasks)
|
|
10934
|
+
else {"method": "unavailable", "point": None, "q025": None, "q975": None}
|
|
10935
|
+
)
|
|
10936
|
+
for metric in diagnostic_metric_deltas
|
|
10937
|
+
}
|
|
10938
|
+
return {
|
|
10939
|
+
"primary_contrast": list(BENCHMARK_STUDY_V2_PRIMARY_CONTRAST),
|
|
10940
|
+
"diagnostic_contrast": list(BENCHMARK_STUDY_V2_DIAGNOSTIC_CONTRAST),
|
|
10941
|
+
"task_ids_sha256": _study_domain_hash(
|
|
10942
|
+
"contextguard.bench.v2.task-order.v1", tasks,
|
|
10943
|
+
),
|
|
10944
|
+
"retained_unfavorable_runs": retained_unfavorable,
|
|
10945
|
+
"token_effect": _benchmark_study_v2_cluster_interval(token_deltas),
|
|
10946
|
+
"diagnostic_token_effect": _benchmark_study_v2_cluster_interval(
|
|
10947
|
+
diagnostic_token_deltas
|
|
10948
|
+
),
|
|
10949
|
+
"quality_gate": False,
|
|
10950
|
+
"failure_gate": False,
|
|
10951
|
+
"correction_gate": False,
|
|
10952
|
+
"retrieval_gate": False,
|
|
10953
|
+
"shifted_cost_gate": False,
|
|
10954
|
+
**metric_effects,
|
|
10955
|
+
**diagnostic_metric_effects,
|
|
10956
|
+
}
|
|
10957
|
+
|
|
10958
|
+
|
|
10959
|
+
def make_benchmark_study_v2_plan(
|
|
10960
|
+
*, schedule_seed: str, required_task_count: int, corpus_sha256: str = "0" * 64,
|
|
10961
|
+
checker_sha256: str = "0" * 64, task_ids_sha256: str = "0" * 64,
|
|
10962
|
+
ni_margin: float = 0.10,
|
|
10963
|
+
) -> dict[str, Any]:
|
|
10964
|
+
"""Build the immutable, a-priori v2 analysis plan for a frozen corpus."""
|
|
10965
|
+
plan = {
|
|
10966
|
+
"schema_version": BENCHMARK_STUDY_V2_PLAN_SCHEMA_VERSION,
|
|
10967
|
+
"arms": list(BENCHMARK_STUDY_V2_ARMS), "schedule_seed": schedule_seed,
|
|
10968
|
+
"repetitions": 3, "max_attempts_per_arm_unit": 2,
|
|
10969
|
+
"retry_policy": BENCHMARK_STUDY_V2_RETRY_POLICY,
|
|
10970
|
+
"corpus_sha256": corpus_sha256, "checker_sha256": checker_sha256,
|
|
10971
|
+
"task_ids_sha256": task_ids_sha256,
|
|
10972
|
+
"primary_contrast": list(BENCHMARK_STUDY_V2_PRIMARY_CONTRAST),
|
|
10973
|
+
"diagnostic_contrast": list(BENCHMARK_STUDY_V2_DIAGNOSTIC_CONTRAST),
|
|
10974
|
+
"noninferiority_margin": ni_margin,
|
|
10975
|
+
"power": {
|
|
10976
|
+
"claim_capable": False,
|
|
10977
|
+
"method": "not_estimated_without_independent_effect_model_v1",
|
|
10978
|
+
"reason": "fixed_12_task_corpus_is_descriptive_only",
|
|
10979
|
+
"required_task_count": required_task_count,
|
|
10980
|
+
},
|
|
10981
|
+
"exclusions": "none_after_schedule_except_prelaunch_refusal_v1",
|
|
10982
|
+
"missing_data": "incomplete_primary_pair_is_descriptive_only_v1",
|
|
10983
|
+
"contamination": "any_contamination_blocks_claim_v1",
|
|
10984
|
+
"stopping": "fixed_task_count_no_optional_stopping_v1",
|
|
10985
|
+
"model_cli_fields": ["model_revision", "backend_revision", "cli_version"],
|
|
10986
|
+
"gates": ["quality", "failure", "correction", "retrieval", "shifted_cost"],
|
|
10987
|
+
}
|
|
10988
|
+
validate_benchmark_study_v2_plan(plan)
|
|
10989
|
+
return plan
|
|
10990
|
+
|
|
10991
|
+
|
|
10992
|
+
def validate_benchmark_study_v2_plan(plan: Mapping[str, Any]) -> None:
|
|
10993
|
+
required = {
|
|
10994
|
+
"schema_version", "arms", "schedule_seed", "repetitions", "max_attempts_per_arm_unit", "retry_policy",
|
|
10995
|
+
"corpus_sha256", "checker_sha256", "task_ids_sha256", "primary_contrast", "diagnostic_contrast", "noninferiority_margin",
|
|
10996
|
+
"power", "exclusions", "missing_data", "contamination", "stopping", "model_cli_fields", "gates",
|
|
10997
|
+
}
|
|
10998
|
+
if set(plan) != required or plan.get("schema_version") != BENCHMARK_STUDY_V2_PLAN_SCHEMA_VERSION:
|
|
10999
|
+
raise ValueError("v2 study plan schema mismatch")
|
|
11000
|
+
_benchmark_study_v2_seed(plan["schedule_seed"])
|
|
11001
|
+
if plan["arms"] != list(BENCHMARK_STUDY_V2_ARMS) or plan["primary_contrast"] != list(BENCHMARK_STUDY_V2_PRIMARY_CONTRAST) or plan["diagnostic_contrast"] != list(BENCHMARK_STUDY_V2_DIAGNOSTIC_CONTRAST):
|
|
11002
|
+
raise ValueError("v2 study arms or contrasts drifted")
|
|
11003
|
+
if plan["repetitions"] != 3 or plan["max_attempts_per_arm_unit"] != 2 or plan["retry_policy"] != BENCHMARK_STUDY_V2_RETRY_POLICY:
|
|
11004
|
+
raise ValueError("v2 study retry contract drifted")
|
|
11005
|
+
if any(not isinstance(plan[key], str) or SHA256_HEX_PATTERN.fullmatch(plan[key]) is None for key in ("corpus_sha256", "checker_sha256", "task_ids_sha256")):
|
|
11006
|
+
raise ValueError("v2 corpus/checker/task-order binding is invalid")
|
|
11007
|
+
if not isinstance(plan["noninferiority_margin"], (int, float)) or isinstance(plan["noninferiority_margin"], bool) or not 0 <= plan["noninferiority_margin"] < 1:
|
|
11008
|
+
raise ValueError("v2 non-inferiority margin is invalid")
|
|
11009
|
+
power = plan["power"]
|
|
11010
|
+
if (
|
|
11011
|
+
not isinstance(power, Mapping)
|
|
11012
|
+
or set(power) != {"claim_capable", "method", "reason", "required_task_count"}
|
|
11013
|
+
or power.get("claim_capable") is not False
|
|
11014
|
+
or power.get("method") != "not_estimated_without_independent_effect_model_v1"
|
|
11015
|
+
or power.get("reason") != "fixed_12_task_corpus_is_descriptive_only"
|
|
11016
|
+
or power.get("required_task_count") != 12
|
|
11017
|
+
):
|
|
11018
|
+
raise ValueError("v2 descriptive sample-size contract is unavailable or invalid")
|
|
11019
|
+
if plan["model_cli_fields"] != ["model_revision", "backend_revision", "cli_version"] or plan["gates"] != ["quality", "failure", "correction", "retrieval", "shifted_cost"]:
|
|
11020
|
+
raise ValueError("v2 provenance or gate contract drifted")
|
|
11021
|
+
frozen_text = {
|
|
11022
|
+
"exclusions": "none_after_schedule_except_prelaunch_refusal_v1",
|
|
11023
|
+
"missing_data": "incomplete_primary_pair_is_descriptive_only_v1",
|
|
11024
|
+
"contamination": "any_contamination_blocks_claim_v1",
|
|
11025
|
+
"stopping": "fixed_task_count_no_optional_stopping_v1",
|
|
11026
|
+
}
|
|
11027
|
+
if any(plan[key] != expected for key, expected in frozen_text.items()):
|
|
11028
|
+
raise ValueError("v2 study plan operational rule drifted")
|
|
11029
|
+
|
|
11030
|
+
|
|
11031
|
+
def load_benchmark_study_v2_plan(path: Path) -> dict[str, Any]:
|
|
11032
|
+
"""Load only canonical JSON so a plan's signed-by-bytes form is stable."""
|
|
11033
|
+
raw = _read_bytes_no_follow(path, max_bytes=100_000)
|
|
11034
|
+
try:
|
|
11035
|
+
value = json.loads(raw.decode("utf-8"))
|
|
11036
|
+
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
11037
|
+
raise ValueError("v2 study plan is invalid JSON") from exc
|
|
11038
|
+
if not isinstance(value, dict) or raw != _study_canonical_json_bytes(value):
|
|
11039
|
+
raise ValueError("v2 study plan must be canonical JSON")
|
|
11040
|
+
validate_benchmark_study_v2_plan(value)
|
|
11041
|
+
return value
|
|
11042
|
+
|
|
11043
|
+
|
|
11044
|
+
def validate_benchmark_study_v2_bindings(
|
|
11045
|
+
plan: Mapping[str, Any], *, corpus_bytes: bytes,
|
|
11046
|
+
checker_binding: Mapping[str, Any],
|
|
11047
|
+
) -> None:
|
|
11048
|
+
"""Bind the raw corpus and domain-separated ordered checker inventory."""
|
|
11049
|
+
validate_benchmark_study_v2_plan(plan)
|
|
11050
|
+
if not isinstance(corpus_bytes, bytes):
|
|
11051
|
+
raise ValueError("v2 corpus/checker binding bytes are invalid")
|
|
11052
|
+
validate_benchmark_study_v2_checker_binding(checker_binding)
|
|
11053
|
+
try:
|
|
11054
|
+
task_ids = _benchmark_study_v2_task_ids_from_corpus(corpus_bytes)
|
|
11055
|
+
except ValueError as exc:
|
|
11056
|
+
raise ValueError("v2 corpus/checker/task-order binding drift") from exc
|
|
11057
|
+
if (
|
|
11058
|
+
_study_sha256_bytes(corpus_bytes) != plan["corpus_sha256"]
|
|
11059
|
+
or checker_binding["sha256"] != plan["checker_sha256"]
|
|
11060
|
+
or _benchmark_study_v2_task_ids_sha256(task_ids)
|
|
11061
|
+
!= plan["task_ids_sha256"]
|
|
11062
|
+
):
|
|
11063
|
+
raise ValueError("v2 corpus/checker/task-order binding drift")
|
|
11064
|
+
|
|
11065
|
+
|
|
11066
|
+
def validate_benchmark_study_v2_checker_binding(
|
|
11067
|
+
binding: Mapping[str, Any],
|
|
11068
|
+
) -> None:
|
|
11069
|
+
"""Validate the filename/size/content inventory before trusting its digest."""
|
|
11070
|
+
if not isinstance(binding, Mapping) or set(binding) != {
|
|
11071
|
+
"domain", "files", "sha256",
|
|
11072
|
+
}:
|
|
11073
|
+
raise ValueError("v2 checker binding schema is invalid")
|
|
11074
|
+
files = binding["files"]
|
|
11075
|
+
if (
|
|
11076
|
+
binding["domain"] != BENCHMARK_STUDY_V2_CHECKER_BINDING_DOMAIN
|
|
11077
|
+
or not isinstance(files, list)
|
|
11078
|
+
or len(files) != 12
|
|
11079
|
+
):
|
|
11080
|
+
raise ValueError("v2 checker binding inventory is invalid")
|
|
11081
|
+
filenames: list[str] = []
|
|
11082
|
+
for entry in files:
|
|
11083
|
+
if not isinstance(entry, Mapping) or set(entry) != {
|
|
11084
|
+
"filename", "size", "sha256",
|
|
11085
|
+
}:
|
|
11086
|
+
raise ValueError("v2 checker binding entry is invalid")
|
|
11087
|
+
filename, size, digest = (
|
|
11088
|
+
entry["filename"], entry["size"], entry["sha256"],
|
|
11089
|
+
)
|
|
11090
|
+
if (
|
|
11091
|
+
not isinstance(filename, str)
|
|
11092
|
+
or not filename.endswith(".py")
|
|
11093
|
+
or Path(filename).name != filename
|
|
11094
|
+
or isinstance(size, bool)
|
|
11095
|
+
or not isinstance(size, int)
|
|
11096
|
+
or not 0 <= size <= MAX_FIXTURE_FILE_BYTES
|
|
11097
|
+
or not isinstance(digest, str)
|
|
11098
|
+
or SHA256_HEX_PATTERN.fullmatch(digest) is None
|
|
11099
|
+
):
|
|
11100
|
+
raise ValueError("v2 checker binding entry is invalid")
|
|
11101
|
+
filenames.append(filename)
|
|
11102
|
+
if filenames != sorted(filenames) or len(set(filenames)) != len(filenames):
|
|
11103
|
+
raise ValueError("v2 checker binding order is invalid")
|
|
11104
|
+
expected = _study_domain_hash(
|
|
11105
|
+
BENCHMARK_STUDY_V2_CHECKER_BINDING_DOMAIN, files,
|
|
10498
11106
|
)
|
|
10499
|
-
if
|
|
10500
|
-
|
|
10501
|
-
|
|
10502
|
-
|
|
10503
|
-
|
|
11107
|
+
if binding["sha256"] != expected:
|
|
11108
|
+
raise ValueError("v2 checker binding digest is invalid")
|
|
11109
|
+
|
|
11110
|
+
|
|
11111
|
+
def validate_benchmark_study_v2_evidence_metadata(metadata: Mapping[str, Any]) -> None:
|
|
11112
|
+
"""Fail closed before potentially sensitive execution evidence reaches a report."""
|
|
11113
|
+
def visit(value: Any, key: str = "") -> None:
|
|
11114
|
+
key_lower = key.lower()
|
|
11115
|
+
# `tokens` is the aggregate study metric, not a credential-shaped token.
|
|
11116
|
+
# All other token-bearing field names remain forbidden.
|
|
11117
|
+
if key_lower == "tokens" and (
|
|
11118
|
+
isinstance(value, bool)
|
|
11119
|
+
or not isinstance(value, (int, float))
|
|
11120
|
+
or not math.isfinite(float(value))
|
|
11121
|
+
or value < 0
|
|
11122
|
+
):
|
|
11123
|
+
raise ValueError("unsafe evidence token metric is invalid")
|
|
11124
|
+
if (
|
|
11125
|
+
key_lower != "tokens"
|
|
11126
|
+
and any(forbidden in key_lower for forbidden in BENCHMARK_STUDY_V2_EVIDENCE_FORBIDDEN_KEYS)
|
|
11127
|
+
):
|
|
11128
|
+
raise ValueError("unsafe evidence field is forbidden")
|
|
11129
|
+
if isinstance(value, str):
|
|
11130
|
+
if (
|
|
11131
|
+
BENCHMARK_STUDY_V2_HANDLE_RE.search(value)
|
|
11132
|
+
or BENCHMARK_STUDY_V2_SECRET_SHAPE_RE.search(value)
|
|
11133
|
+
):
|
|
11134
|
+
raise ValueError("unsafe evidence secret-shaped value is forbidden")
|
|
11135
|
+
if (
|
|
11136
|
+
key_lower in BENCHMARK_STUDY_V2_REVISION_KEYS
|
|
11137
|
+
and BENCHMARK_STUDY_V2_REVISION_RE.fullmatch(value) is None
|
|
11138
|
+
):
|
|
11139
|
+
raise ValueError("unsafe evidence revision format is invalid")
|
|
11140
|
+
if isinstance(value, Mapping):
|
|
11141
|
+
for child_key, child in value.items():
|
|
11142
|
+
if not isinstance(child_key, str):
|
|
11143
|
+
raise ValueError("unsafe evidence key is invalid")
|
|
11144
|
+
visit(child, child_key)
|
|
11145
|
+
elif isinstance(value, (list, tuple)):
|
|
11146
|
+
for child in value:
|
|
11147
|
+
visit(child, key)
|
|
11148
|
+
visit(metadata)
|
|
11149
|
+
|
|
11150
|
+
|
|
11151
|
+
def redact_benchmark_study_v2_evidence(value: str) -> str:
|
|
11152
|
+
"""Safe display helper for planted/accidental cgr1p handles; reports still reject them."""
|
|
11153
|
+
return BENCHMARK_STUDY_V2_HANDLE_RE.sub("[REDACTED_HANDLE]", value)
|
|
11154
|
+
|
|
11155
|
+
|
|
11156
|
+
def evaluate_benchmark_study_v2_claim_readiness(
|
|
11157
|
+
*, plan: Mapping[str, Any], task_ids: Sequence[str], binary_inference: Mapping[str, Any],
|
|
11158
|
+
effects: Mapping[str, Any], provenance: Mapping[str, Any],
|
|
11159
|
+
binary_rows: Sequence[Mapping[str, Any]] | None = None,
|
|
11160
|
+
effect_records: Sequence[Mapping[str, Any]] | None = None,
|
|
11161
|
+
) -> dict[str, Any]:
|
|
11162
|
+
"""Return a fail-closed product-claim gate; diagnostic results are ignored."""
|
|
11163
|
+
validate_benchmark_study_v2_plan(plan)
|
|
11164
|
+
unique_tasks = list(dict.fromkeys(task_ids))
|
|
11165
|
+
task_order: list[str] | None = None
|
|
11166
|
+
try:
|
|
11167
|
+
task_order = _benchmark_study_v2_task_ids(task_ids)
|
|
11168
|
+
except (TypeError, ValueError):
|
|
11169
|
+
pass
|
|
11170
|
+
power_ready = bool(
|
|
11171
|
+
plan["power"].get("claim_capable") is True
|
|
11172
|
+
and len(unique_tasks) >= int(plan["power"]["required_task_count"])
|
|
11173
|
+
)
|
|
11174
|
+
provenance_safe = True
|
|
11175
|
+
try:
|
|
11176
|
+
validate_benchmark_study_v2_evidence_metadata(provenance)
|
|
11177
|
+
except (TypeError, ValueError):
|
|
11178
|
+
provenance_safe = False
|
|
11179
|
+
provider_ready = bool(
|
|
11180
|
+
provenance_safe
|
|
11181
|
+
and provenance.get("source") == "provider_export"
|
|
11182
|
+
and provenance.get("complete_provider_export") is True
|
|
11183
|
+
and isinstance(provenance.get("backend_revision"), str) and provenance["backend_revision"]
|
|
11184
|
+
and isinstance(provenance.get("model_revision"), str) and provenance["model_revision"]
|
|
11185
|
+
and isinstance(provenance.get("cli_version"), str) and provenance["cli_version"]
|
|
11186
|
+
)
|
|
11187
|
+
recomputed_inference: dict[str, Any] | None = None
|
|
11188
|
+
if task_order is not None and binary_rows is not None:
|
|
11189
|
+
try:
|
|
11190
|
+
recomputed_inference = infer_benchmark_study_v2_binary(
|
|
11191
|
+
binary_rows,
|
|
11192
|
+
task_order=task_order,
|
|
11193
|
+
ni_margin=float(plan["noninferiority_margin"]),
|
|
10504
11194
|
)
|
|
10505
|
-
|
|
10506
|
-
|
|
10507
|
-
|
|
10508
|
-
|
|
10509
|
-
|
|
10510
|
-
|
|
10511
|
-
|
|
10512
|
-
|
|
10513
|
-
("--report-json", args.report_json is not None),
|
|
10514
|
-
("--dashboard-md", args.dashboard_md is not None),
|
|
10515
|
-
("--csv", args.csv is not None),
|
|
10516
|
-
("--baseline-variant", args.baseline_variant != "baseline"),
|
|
10517
|
-
) if active
|
|
10518
|
-
]
|
|
10519
|
-
if conflicts:
|
|
10520
|
-
parser.error(f"measurement study mode conflicts with {', '.join(conflicts)}")
|
|
10521
|
-
return run_measurement_study_action(args)
|
|
10522
|
-
args.csv = args.csv or Path("bench/results.csv")
|
|
10523
|
-
validate_distinct_output_paths(args.csv, args.ledger_jsonl, args.report_json, args.dashboard_md)
|
|
11195
|
+
except (AttributeError, TypeError, ValueError):
|
|
11196
|
+
pass
|
|
11197
|
+
binary_ready = bool(
|
|
11198
|
+
recomputed_inference is not None
|
|
11199
|
+
and dict(binary_inference) == recomputed_inference
|
|
11200
|
+
and recomputed_inference["degenerate_all_success"] is False
|
|
11201
|
+
and recomputed_inference["noninferiority_pass"] is True
|
|
11202
|
+
)
|
|
10524
11203
|
|
|
10525
|
-
|
|
10526
|
-
|
|
10527
|
-
|
|
10528
|
-
|
|
10529
|
-
|
|
10530
|
-
|
|
10531
|
-
|
|
10532
|
-
|
|
10533
|
-
|
|
11204
|
+
recomputed_effects: dict[str, Any] | None = None
|
|
11205
|
+
if task_order is not None and effect_records is not None:
|
|
11206
|
+
try:
|
|
11207
|
+
recomputed_effects = compute_benchmark_study_v2_effects(
|
|
11208
|
+
effect_records, task_order=task_order,
|
|
11209
|
+
)
|
|
11210
|
+
except (AttributeError, TypeError, ValueError):
|
|
11211
|
+
pass
|
|
11212
|
+
bound_effect_fields = (
|
|
11213
|
+
"primary_contrast", "diagnostic_contrast", "task_ids_sha256",
|
|
11214
|
+
"retained_unfavorable_runs", "token_effect", "diagnostic_token_effect",
|
|
11215
|
+
"correction_effect", "diagnostic_correction_effect",
|
|
11216
|
+
"retrieval_effect", "diagnostic_retrieval_effect",
|
|
11217
|
+
)
|
|
11218
|
+
effects_bound = bool(
|
|
11219
|
+
recomputed_effects is not None
|
|
11220
|
+
and all(
|
|
11221
|
+
effects.get(field) == recomputed_effects.get(field)
|
|
11222
|
+
for field in bound_effect_fields
|
|
11223
|
+
)
|
|
11224
|
+
)
|
|
11225
|
+
|
|
11226
|
+
def interval_gate(field: str, *, strict: bool) -> bool:
|
|
11227
|
+
if recomputed_effects is None:
|
|
11228
|
+
return False
|
|
11229
|
+
interval = recomputed_effects.get(field)
|
|
11230
|
+
if not isinstance(interval, Mapping) or interval.get("method") != "task_cluster_bootstrap_v2":
|
|
11231
|
+
return False
|
|
11232
|
+
lower = interval.get("q025")
|
|
11233
|
+
if isinstance(lower, bool) or not isinstance(lower, (int, float)) or not math.isfinite(float(lower)):
|
|
11234
|
+
return False
|
|
11235
|
+
return bool(lower > 0 if strict else lower >= 0)
|
|
11236
|
+
|
|
11237
|
+
derived_gates = {
|
|
11238
|
+
"quality": binary_ready,
|
|
11239
|
+
"failure": binary_ready,
|
|
11240
|
+
"correction": interval_gate("correction_effect", strict=False),
|
|
11241
|
+
"retrieval": interval_gate("retrieval_effect", strict=False),
|
|
11242
|
+
"shifted_cost": interval_gate("token_effect", strict=True),
|
|
11243
|
+
}
|
|
11244
|
+
effect_ready = bool(effects_bound and all(derived_gates.values()))
|
|
11245
|
+
contamination_ready = provenance.get("contaminated") is False
|
|
11246
|
+
mixed_versions_ready = provenance.get("mixed_versions") is False
|
|
11247
|
+
missing_data_ready = provenance.get("missing_primary_data") is False
|
|
11248
|
+
unmet = []
|
|
11249
|
+
for name, value in (("power", power_ready), ("provider_provenance", provider_ready), ("binary_inference", binary_ready), ("effect_gates", effect_ready), ("contamination", contamination_ready), ("mixed_versions", mixed_versions_ready), ("missing_data", missing_data_ready)):
|
|
11250
|
+
if not value:
|
|
11251
|
+
unmet.append(name)
|
|
11252
|
+
return {
|
|
11253
|
+
"claim_ready": not unmet,
|
|
11254
|
+
"descriptive_only": bool(unmet),
|
|
11255
|
+
"unmet_gates": unmet,
|
|
11256
|
+
"primary_contrast": list(BENCHMARK_STUDY_V2_PRIMARY_CONTRAST),
|
|
11257
|
+
"diagnostic_contrast": list(BENCHMARK_STUDY_V2_DIAGNOSTIC_CONTRAST),
|
|
11258
|
+
"derived_gates": derived_gates,
|
|
11259
|
+
"backend_revision": provenance.get("backend_revision") if provider_ready else "unavailable",
|
|
11260
|
+
"model_revision": provenance.get("model_revision") if provider_ready else "unavailable",
|
|
11261
|
+
}
|
|
11262
|
+
|
|
11263
|
+
|
|
11264
|
+
BENCHMARK_STUDY_V2_EXEC_MANIFEST_SCHEMA_VERSION = "contextguard.bench.study-manifest.v6"
|
|
11265
|
+
BENCHMARK_STUDY_V2_ATTEMPT_SCHEMA_VERSION = "contextguard.bench.study-attempt.v4"
|
|
11266
|
+
BENCHMARK_STUDY_V2_BOUNDED_FAILURE_RESULT_CODES = frozenset({
|
|
11267
|
+
"error_max_turns",
|
|
11268
|
+
})
|
|
11269
|
+
BENCHMARK_STUDY_V2_BOUNDED_FAILURE_CHECKER_STATUS = (
|
|
11270
|
+
"not_run_provider_bounded_failure_v1"
|
|
11271
|
+
)
|
|
11272
|
+
BENCHMARK_STUDY_V2_REPORT_SCHEMA_VERSION = "contextguard.bench.study-report.v4"
|
|
11273
|
+
BENCHMARK_STUDY_V2_INVALID_DECISION_SCHEMA_VERSION = (
|
|
11274
|
+
"contextguard.bench.study-invalid-decision.v1"
|
|
11275
|
+
)
|
|
11276
|
+
BENCHMARK_STUDY_V2_CANDIDATE_SCHEMA_VERSION = "contextguard-npm-candidate-set/v1"
|
|
11277
|
+
BENCHMARK_STUDY_V2_CANDIDATE_NAMES = (
|
|
11278
|
+
"@ictechgy/context-guard-receipt", "@ictechgy/context-guard",
|
|
11279
|
+
)
|
|
11280
|
+
BENCHMARK_STUDY_V2_OVERLAY_NAME = "node_modules"
|
|
11281
|
+
BENCHMARK_STUDY_V2_REWRITE_COMMAND = (
|
|
11282
|
+
"./node_modules/.bin/context-guard-rewrite-bash"
|
|
11283
|
+
)
|
|
11284
|
+
BENCHMARK_STUDY_V2_CLI_BINDING_SCHEMA_VERSION = (
|
|
11285
|
+
"contextguard.bench.cli-binding.v2"
|
|
11286
|
+
)
|
|
11287
|
+
BENCHMARK_STUDY_V2_AUTH_CONTEXT_SCHEMA_VERSION = (
|
|
11288
|
+
"contextguard.bench.auth-context.v1"
|
|
11289
|
+
)
|
|
11290
|
+
BENCHMARK_STUDY_V2_CLI_CAPABILITIES = (
|
|
11291
|
+
"--settings", "--setting-sources", "--include-hook-events",
|
|
11292
|
+
"--no-session-persistence", "stream-json",
|
|
11293
|
+
)
|
|
11294
|
+
BENCHMARK_STUDY_V2_CLI_MAX_BYTES = 1_073_741_824
|
|
11295
|
+
BENCHMARK_STUDY_V2_CANARY_EVENT_SCHEMA_VERSION = (
|
|
11296
|
+
"contextguard.bench.canary-event.v2"
|
|
11297
|
+
)
|
|
11298
|
+
BENCHMARK_STUDY_V2_CANARY_EVIDENCE_SCHEMA_VERSION = (
|
|
11299
|
+
"contextguard.bench.canary-evidence.v3"
|
|
11300
|
+
)
|
|
11301
|
+
BENCHMARK_STUDY_V2_CANARY_ARMS = ("legacy_trim", "bash_reference_v1")
|
|
11302
|
+
BENCHMARK_STUDY_V2_CANARY_TASK_ID = "contextguard-v2-bash-canary"
|
|
11303
|
+
BENCHMARK_STUDY_V2_CANARY_MAX_BUDGET_USD = 0.75
|
|
11304
|
+
BENCHMARK_STUDY_V2_CANARY_MARKER = b"contextguard-v2-host-pretooluse-canary\n"
|
|
11305
|
+
BENCHMARK_STUDY_V2_CANARY_COMMAND = (
|
|
11306
|
+
"python3 -c 'from pathlib import Path;"
|
|
11307
|
+
"Path(\"contextguard-v2-canary.txt\").write_bytes("
|
|
11308
|
+
"b\"contextguard-v2-host-pretooluse-canary\\n\")'"
|
|
11309
|
+
)
|
|
11310
|
+
BENCHMARK_STUDY_V2_CANARY_PROMPT = (
|
|
11311
|
+
"Use the Bash tool exactly once to run this command, then reply done: "
|
|
11312
|
+
+ BENCHMARK_STUDY_V2_CANARY_COMMAND
|
|
11313
|
+
)
|
|
11314
|
+
BENCHMARK_STUDY_V2_CANARY_CHECKER = (
|
|
11315
|
+
b"from pathlib import Path\n"
|
|
11316
|
+
b"try:\n"
|
|
11317
|
+
b" value = Path('contextguard-v2-canary.txt').read_bytes()\n"
|
|
11318
|
+
b"except OSError:\n"
|
|
11319
|
+
b" raise SystemExit(1)\n"
|
|
11320
|
+
b"raise SystemExit(0 if value == b'contextguard-v2-host-pretooluse-canary\\n' else 1)\n"
|
|
11321
|
+
)
|
|
11322
|
+
|
|
11323
|
+
|
|
11324
|
+
def _benchmark_study_v2_canary_contract() -> dict[str, Any]:
|
|
11325
|
+
fixture = b"This workspace is used only for the discarded v2 Bash-hook canary.\n"
|
|
11326
|
+
return {
|
|
11327
|
+
"schema_version": "contextguard.bench.canary-contract.v2",
|
|
11328
|
+
"task_id": BENCHMARK_STUDY_V2_CANARY_TASK_ID,
|
|
11329
|
+
"prompt_sha256": _study_sha256_bytes(
|
|
11330
|
+
BENCHMARK_STUDY_V2_CANARY_PROMPT.encode("utf-8")
|
|
11331
|
+
),
|
|
11332
|
+
"fixture_sha256": _study_sha256_bytes(fixture),
|
|
11333
|
+
"checker_sha256": _study_sha256_bytes(BENCHMARK_STUDY_V2_CANARY_CHECKER),
|
|
11334
|
+
"marker_sha256": _study_sha256_bytes(BENCHMARK_STUDY_V2_CANARY_MARKER),
|
|
11335
|
+
"arms": list(BENCHMARK_STUDY_V2_CANARY_ARMS),
|
|
11336
|
+
"required_event_classes": ["PreToolUse"],
|
|
11337
|
+
"max_budget_usd": BENCHMARK_STUDY_V2_CANARY_MAX_BUDGET_USD,
|
|
11338
|
+
"provider_calls": 2,
|
|
11339
|
+
"discarded": True,
|
|
11340
|
+
"excluded_from_analysis": True,
|
|
11341
|
+
}
|
|
11342
|
+
|
|
11343
|
+
|
|
11344
|
+
def _benchmark_study_v2_canary_task() -> TaskFixture:
|
|
11345
|
+
fixture = b"This workspace is used only for the discarded v2 Bash-hook canary.\n"
|
|
11346
|
+
return TaskFixture(
|
|
11347
|
+
id=BENCHMARK_STUDY_V2_CANARY_TASK_ID,
|
|
11348
|
+
prompt=BENCHMARK_STUDY_V2_CANARY_PROMPT,
|
|
11349
|
+
model="sonnet", max_turns=2,
|
|
11350
|
+
max_budget_usd=BENCHMARK_STUDY_V2_CANARY_MAX_BUDGET_USD,
|
|
11351
|
+
allowed_tools=["Bash"],
|
|
11352
|
+
fixture_tree="inline-canary-fixture",
|
|
11353
|
+
success_checker="inline-canary-checker.py",
|
|
11354
|
+
fixture_tree_entries=(
|
|
11355
|
+
FixtureTreeEntry(path="CANARY.md", data=fixture, executable=False),
|
|
11356
|
+
),
|
|
11357
|
+
success_checker_bytes=BENCHMARK_STUDY_V2_CANARY_CHECKER,
|
|
11358
|
+
)
|
|
11359
|
+
|
|
11360
|
+
|
|
11361
|
+
def _benchmark_study_v2_output_root(path: Path) -> Path:
|
|
11362
|
+
if path.is_symlink():
|
|
11363
|
+
raise ValueError("v2 output root must not be a symlink")
|
|
11364
|
+
try:
|
|
11365
|
+
return path.resolve(strict=False)
|
|
11366
|
+
except OSError as exc:
|
|
11367
|
+
raise ValueError("v2 output root is unavailable") from exc
|
|
11368
|
+
|
|
11369
|
+
|
|
11370
|
+
@contextmanager
|
|
11371
|
+
def _benchmark_study_v2_action_lock(output_root: Path) -> Iterable[None]:
|
|
11372
|
+
"""Serialize every canary/run/resume/analyze mutation for one study root."""
|
|
11373
|
+
root = _benchmark_study_v2_output_root(output_root)
|
|
11374
|
+
try:
|
|
11375
|
+
root_fd = _ensure_directory_no_symlink(root, create=False)
|
|
11376
|
+
except (OSError, SystemExit, ValueError) as exc:
|
|
11377
|
+
raise ValueError(
|
|
11378
|
+
"v2 executable study requires a prepared output root"
|
|
11379
|
+
) from exc
|
|
11380
|
+
lock_fd = -1
|
|
11381
|
+
locked = False
|
|
11382
|
+
try:
|
|
11383
|
+
if fcntl is None:
|
|
11384
|
+
raise ValueError("v2 lifecycle locking is unavailable")
|
|
11385
|
+
flags = os.O_RDWR | os.O_CREAT
|
|
11386
|
+
if hasattr(os, "O_CLOEXEC"):
|
|
11387
|
+
flags |= os.O_CLOEXEC
|
|
11388
|
+
lock_fd = _open_regular_no_symlink(
|
|
11389
|
+
root / ".study-v2-action.lock", flags, 0o600,
|
|
11390
|
+
)
|
|
11391
|
+
os.fchmod(lock_fd, 0o600)
|
|
11392
|
+
os.fsync(root_fd)
|
|
11393
|
+
try:
|
|
11394
|
+
fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
11395
|
+
except (BlockingIOError, OSError) as exc:
|
|
11396
|
+
raise ValueError("another v2 lifecycle action is already active") from exc
|
|
11397
|
+
locked = True
|
|
11398
|
+
yield
|
|
11399
|
+
finally:
|
|
11400
|
+
if locked:
|
|
11401
|
+
fcntl.flock(lock_fd, fcntl.LOCK_UN)
|
|
11402
|
+
if lock_fd >= 0:
|
|
11403
|
+
os.close(lock_fd)
|
|
11404
|
+
os.close(root_fd)
|
|
11405
|
+
|
|
11406
|
+
|
|
11407
|
+
def _benchmark_study_v2_validate_cli_binding(binding: Mapping[str, Any]) -> None:
|
|
11408
|
+
required = {
|
|
11409
|
+
"schema_version", "executable", "executable_bytes",
|
|
11410
|
+
"executable_sha256", "bundle", "probe",
|
|
11411
|
+
}
|
|
11412
|
+
probe = binding.get("probe")
|
|
11413
|
+
bundle = binding.get("bundle")
|
|
11414
|
+
if (
|
|
11415
|
+
set(binding) != required
|
|
11416
|
+
or binding.get("schema_version")
|
|
11417
|
+
!= BENCHMARK_STUDY_V2_CLI_BINDING_SCHEMA_VERSION
|
|
11418
|
+
or not isinstance(binding.get("executable"), str)
|
|
11419
|
+
or not Path(str(binding.get("executable"))).is_absolute()
|
|
11420
|
+
or isinstance(binding.get("executable_bytes"), bool)
|
|
11421
|
+
or not isinstance(binding.get("executable_bytes"), int)
|
|
11422
|
+
or not 0 < int(binding["executable_bytes"]) <= BENCHMARK_STUDY_V2_CLI_MAX_BYTES
|
|
11423
|
+
or not isinstance(binding.get("executable_sha256"), str)
|
|
11424
|
+
or SHA256_HEX_PATTERN.fullmatch(str(binding["executable_sha256"])) is None
|
|
11425
|
+
or not isinstance(probe, Mapping)
|
|
11426
|
+
or probe.get("schema_version") != MEASUREMENT_CLI_PROBE_SCHEMA_VERSION
|
|
11427
|
+
or probe.get("executable") != binding.get("executable")
|
|
11428
|
+
or probe.get("capabilities") != sorted(BENCHMARK_STUDY_V2_CLI_CAPABILITIES)
|
|
11429
|
+
or "version" in probe
|
|
11430
|
+
or not isinstance(bundle, Mapping)
|
|
11431
|
+
or set(bundle) != {
|
|
11432
|
+
"scope", "root", "file_count", "total_bytes", "sha256",
|
|
11433
|
+
}
|
|
11434
|
+
or bundle.get("scope") != "single-native-executable-v1"
|
|
11435
|
+
or not isinstance(bundle.get("root"), str)
|
|
11436
|
+
or not Path(str(bundle["root"])).is_absolute()
|
|
11437
|
+
or isinstance(bundle.get("file_count"), bool)
|
|
11438
|
+
or not isinstance(bundle.get("file_count"), int)
|
|
11439
|
+
or not 0 < bundle["file_count"] <= 100_000
|
|
11440
|
+
or isinstance(bundle.get("total_bytes"), bool)
|
|
11441
|
+
or not isinstance(bundle.get("total_bytes"), int)
|
|
11442
|
+
or not 0 < bundle["total_bytes"] <= 2_147_483_648
|
|
11443
|
+
or not isinstance(bundle.get("sha256"), str)
|
|
11444
|
+
or SHA256_HEX_PATTERN.fullmatch(bundle["sha256"]) is None
|
|
11445
|
+
or bundle.get("root") != binding.get("executable")
|
|
11446
|
+
or bundle.get("file_count") != 1
|
|
11447
|
+
or bundle.get("total_bytes") != binding.get("executable_bytes")
|
|
11448
|
+
or bundle.get("sha256") != binding.get("executable_sha256")
|
|
11449
|
+
):
|
|
11450
|
+
raise ValueError("v2 CLI binding schema mismatch")
|
|
11451
|
+
|
|
11452
|
+
|
|
11453
|
+
def _benchmark_study_v2_cli_stat_guard(path: Path) -> dict[str, int | str]:
|
|
11454
|
+
fd = _open_regular_no_symlink(path)
|
|
11455
|
+
try:
|
|
11456
|
+
item = os.fstat(fd)
|
|
11457
|
+
if item.st_size <= 0 or item.st_size > BENCHMARK_STUDY_V2_CLI_MAX_BYTES:
|
|
11458
|
+
raise ValueError("v2 CLI executable size is unsupported")
|
|
11459
|
+
return {
|
|
11460
|
+
"executable": str(path.resolve(strict=True)),
|
|
11461
|
+
"device": int(item.st_dev), "inode": int(item.st_ino),
|
|
11462
|
+
"bytes": int(item.st_size),
|
|
11463
|
+
"mtime_ns": int(item.st_mtime_ns), "ctime_ns": int(item.st_ctime_ns),
|
|
11464
|
+
}
|
|
11465
|
+
finally:
|
|
11466
|
+
os.close(fd)
|
|
11467
|
+
|
|
11468
|
+
|
|
11469
|
+
def _benchmark_study_v2_cli_file_binding(
|
|
11470
|
+
path: Path,
|
|
11471
|
+
) -> tuple[dict[str, Any], dict[str, int | str]]:
|
|
11472
|
+
"""Hash a large native CLI through a bounded no-follow fd without buffering it."""
|
|
11473
|
+
fd = _open_regular_no_symlink(path)
|
|
11474
|
+
try:
|
|
11475
|
+
before = os.fstat(fd)
|
|
11476
|
+
if before.st_size <= 0 or before.st_size > BENCHMARK_STUDY_V2_CLI_MAX_BYTES:
|
|
11477
|
+
raise ValueError("v2 CLI executable size is unsupported")
|
|
11478
|
+
digest = hashlib.sha256()
|
|
11479
|
+
total = 0
|
|
11480
|
+
while True:
|
|
11481
|
+
chunk = os.read(fd, 1024 * 1024)
|
|
11482
|
+
if not chunk:
|
|
11483
|
+
break
|
|
11484
|
+
total += len(chunk)
|
|
11485
|
+
if total > BENCHMARK_STUDY_V2_CLI_MAX_BYTES:
|
|
11486
|
+
raise ValueError("v2 CLI executable size is unsupported")
|
|
11487
|
+
digest.update(chunk)
|
|
11488
|
+
after = os.fstat(fd)
|
|
11489
|
+
stat_fields = ("st_dev", "st_ino", "st_size", "st_mtime_ns", "st_ctime_ns")
|
|
11490
|
+
if total != before.st_size or any(
|
|
11491
|
+
getattr(before, field) != getattr(after, field) for field in stat_fields
|
|
11492
|
+
):
|
|
11493
|
+
raise ValueError("v2 CLI executable changed while hashing")
|
|
11494
|
+
executable = str(path.resolve(strict=True))
|
|
11495
|
+
return (
|
|
11496
|
+
{
|
|
11497
|
+
"executable": executable,
|
|
11498
|
+
"executable_bytes": total,
|
|
11499
|
+
"executable_sha256": digest.hexdigest(),
|
|
11500
|
+
},
|
|
11501
|
+
{
|
|
11502
|
+
"executable": executable,
|
|
11503
|
+
"device": int(after.st_dev), "inode": int(after.st_ino),
|
|
11504
|
+
"bytes": int(after.st_size),
|
|
11505
|
+
"mtime_ns": int(after.st_mtime_ns),
|
|
11506
|
+
"ctime_ns": int(after.st_ctime_ns),
|
|
11507
|
+
},
|
|
11508
|
+
)
|
|
11509
|
+
finally:
|
|
11510
|
+
os.close(fd)
|
|
11511
|
+
|
|
11512
|
+
|
|
11513
|
+
def _benchmark_study_v2_read_executable_shebang(path: Path) -> str | None:
|
|
11514
|
+
fd = _open_regular_no_symlink(path)
|
|
11515
|
+
try:
|
|
11516
|
+
raw = os.read(fd, 4096)
|
|
11517
|
+
finally:
|
|
11518
|
+
os.close(fd)
|
|
11519
|
+
first = raw.splitlines()[0] if raw else b""
|
|
11520
|
+
if not first.startswith(b"#!"):
|
|
11521
|
+
return None
|
|
11522
|
+
try:
|
|
11523
|
+
value = first[2:].decode("utf-8", "strict").strip()
|
|
11524
|
+
except UnicodeDecodeError:
|
|
11525
|
+
raise ValueError("v2 CLI shebang is not UTF-8") from None
|
|
11526
|
+
if not value or len(value) > 512:
|
|
11527
|
+
raise ValueError("v2 CLI shebang is invalid")
|
|
11528
|
+
return value
|
|
11529
|
+
|
|
11530
|
+
|
|
11531
|
+
def _benchmark_study_v2_cli_bundle_binding(
|
|
11532
|
+
executable: Path, shebang: str | None,
|
|
11533
|
+
) -> dict[str, Any]:
|
|
11534
|
+
if shebang is not None:
|
|
11535
|
+
raise ValueError("v2 executable study requires a native executable")
|
|
11536
|
+
file_binding, _guard = _benchmark_study_v2_cli_file_binding(executable)
|
|
11537
|
+
return {
|
|
11538
|
+
"scope": "single-native-executable-v1",
|
|
11539
|
+
"root": str(executable), "file_count": 1,
|
|
11540
|
+
"total_bytes": file_binding["executable_bytes"],
|
|
11541
|
+
"sha256": file_binding["executable_sha256"],
|
|
11542
|
+
}
|
|
11543
|
+
|
|
11544
|
+
|
|
11545
|
+
def _benchmark_study_v2_cli_bundle_stat_guard(
|
|
11546
|
+
bundle: Mapping[str, Any],
|
|
11547
|
+
) -> dict[str, Any]:
|
|
11548
|
+
"""Detect a native executable replacement between action hash and launch."""
|
|
11549
|
+
if bundle.get("scope") != "single-native-executable-v1":
|
|
11550
|
+
raise ValueError("v2 CLI bundle scope is invalid")
|
|
11551
|
+
item = _benchmark_study_v2_cli_stat_guard(Path(str(bundle["root"])))
|
|
11552
|
+
return {
|
|
11553
|
+
"entry_count": 1,
|
|
11554
|
+
"sha256": _study_domain_hash(
|
|
11555
|
+
"contextguard.bench.v2.cli-bundle-stat.v1", item,
|
|
11556
|
+
),
|
|
11557
|
+
}
|
|
11558
|
+
|
|
11559
|
+
|
|
11560
|
+
def _benchmark_study_v2_execution_environment(
|
|
11561
|
+
cli_binding: Mapping[str, Any],
|
|
11562
|
+
) -> dict[str, Any]:
|
|
11563
|
+
"""Freeze the PATH and interpreters used by the CLI and Python hook."""
|
|
11564
|
+
_benchmark_study_v2_validate_cli_binding(cli_binding)
|
|
11565
|
+
names = ["python3"]
|
|
11566
|
+
bindings: list[dict[str, Any]] = []
|
|
11567
|
+
lookup_directories: list[str] = []
|
|
11568
|
+
for name in dict.fromkeys(names):
|
|
11569
|
+
lookup = shutil.which(name)
|
|
11570
|
+
if lookup is None:
|
|
11571
|
+
raise ValueError(f"v2 required runtime unavailable: {name}")
|
|
11572
|
+
lookup_path = Path(lookup).absolute()
|
|
11573
|
+
resolved = lookup_path.resolve(strict=True)
|
|
11574
|
+
file_binding, _guard = _benchmark_study_v2_cli_file_binding(resolved)
|
|
11575
|
+
lookup_directories.append(str(lookup_path.parent))
|
|
11576
|
+
bindings.append({
|
|
11577
|
+
"name": name, "lookup_path": str(lookup_path),
|
|
11578
|
+
**file_binding,
|
|
11579
|
+
})
|
|
11580
|
+
path_value = os.pathsep.join(dict.fromkeys(
|
|
11581
|
+
lookup_directories + ["/usr/bin", "/bin", "/usr/sbin", "/sbin"]
|
|
11582
|
+
))
|
|
11583
|
+
return {
|
|
11584
|
+
"schema_version": "contextguard.bench.execution-environment.v3",
|
|
11585
|
+
"values": {
|
|
11586
|
+
"PATH": path_value,
|
|
11587
|
+
"LANG": "C",
|
|
11588
|
+
"LC_ALL": "C",
|
|
11589
|
+
"PYTHONDONTWRITEBYTECODE": "1",
|
|
11590
|
+
},
|
|
11591
|
+
"runtime_bindings": bindings,
|
|
11592
|
+
}
|
|
11593
|
+
|
|
11594
|
+
|
|
11595
|
+
def _benchmark_study_v2_validate_execution_environment(
|
|
11596
|
+
binding: Mapping[str, Any],
|
|
11597
|
+
) -> None:
|
|
11598
|
+
values = binding.get("values")
|
|
11599
|
+
runtimes = binding.get("runtime_bindings")
|
|
11600
|
+
if (
|
|
11601
|
+
set(binding) != {"schema_version", "values", "runtime_bindings"}
|
|
11602
|
+
or binding.get("schema_version")
|
|
11603
|
+
!= "contextguard.bench.execution-environment.v3"
|
|
11604
|
+
or not isinstance(values, Mapping)
|
|
11605
|
+
or set(values) != {"PATH", "LANG", "LC_ALL", "PYTHONDONTWRITEBYTECODE"}
|
|
11606
|
+
or values.get("LANG") != "C" or values.get("LC_ALL") != "C"
|
|
11607
|
+
or values.get("PYTHONDONTWRITEBYTECODE") != "1"
|
|
11608
|
+
or not isinstance(values.get("PATH"), str) or not values["PATH"]
|
|
11609
|
+
or any(
|
|
11610
|
+
not part or not Path(part).is_absolute()
|
|
11611
|
+
for part in str(values["PATH"]).split(os.pathsep)
|
|
11612
|
+
)
|
|
11613
|
+
or not isinstance(runtimes, list) or not runtimes
|
|
11614
|
+
):
|
|
11615
|
+
raise ValueError("v2 execution environment binding schema mismatch")
|
|
11616
|
+
seen: set[str] = set()
|
|
11617
|
+
for runtime in runtimes:
|
|
11618
|
+
if (
|
|
11619
|
+
not isinstance(runtime, Mapping)
|
|
11620
|
+
or set(runtime) != {
|
|
11621
|
+
"name", "lookup_path", "executable", "executable_bytes",
|
|
11622
|
+
"executable_sha256",
|
|
11623
|
+
}
|
|
11624
|
+
or not isinstance(runtime.get("name"), str)
|
|
11625
|
+
or not re.fullmatch(r"[A-Za-z0-9_.+-]+", str(runtime["name"]))
|
|
11626
|
+
or runtime["name"] in seen
|
|
11627
|
+
or not isinstance(runtime.get("lookup_path"), str)
|
|
11628
|
+
or not Path(str(runtime["lookup_path"])).is_absolute()
|
|
11629
|
+
or not isinstance(runtime.get("executable"), str)
|
|
11630
|
+
or not Path(str(runtime["executable"])).is_absolute()
|
|
11631
|
+
or isinstance(runtime.get("executable_bytes"), bool)
|
|
11632
|
+
or not isinstance(runtime.get("executable_bytes"), int)
|
|
11633
|
+
or not 0 < runtime["executable_bytes"] <= BENCHMARK_STUDY_V2_CLI_MAX_BYTES
|
|
11634
|
+
or not isinstance(runtime.get("executable_sha256"), str)
|
|
11635
|
+
or SHA256_HEX_PATTERN.fullmatch(runtime["executable_sha256"]) is None
|
|
11636
|
+
):
|
|
11637
|
+
raise ValueError("v2 runtime interpreter binding schema mismatch")
|
|
11638
|
+
seen.add(str(runtime["name"]))
|
|
11639
|
+
|
|
11640
|
+
|
|
11641
|
+
def _benchmark_study_v2_assert_execution_environment(
|
|
11642
|
+
binding: Mapping[str, Any],
|
|
11643
|
+
) -> dict[str, dict[str, int | str]]:
|
|
11644
|
+
_benchmark_study_v2_validate_execution_environment(binding)
|
|
11645
|
+
path_value = str(binding["values"]["PATH"])
|
|
11646
|
+
guards: dict[str, dict[str, int | str]] = {}
|
|
11647
|
+
for runtime in binding["runtime_bindings"]:
|
|
11648
|
+
found = shutil.which(str(runtime["name"]), path=path_value)
|
|
11649
|
+
if found is None or str(Path(found).absolute()) != runtime["lookup_path"]:
|
|
11650
|
+
raise ValueError("v2 runtime interpreter lookup drift")
|
|
11651
|
+
current, _guard = _benchmark_study_v2_cli_file_binding(
|
|
11652
|
+
Path(str(runtime["executable"]))
|
|
11653
|
+
)
|
|
11654
|
+
expected = {
|
|
11655
|
+
key: runtime[key]
|
|
11656
|
+
for key in ("executable", "executable_bytes", "executable_sha256")
|
|
11657
|
+
}
|
|
11658
|
+
if current != expected or Path(found).resolve(strict=True) != Path(
|
|
11659
|
+
str(runtime["executable"])
|
|
11660
|
+
):
|
|
11661
|
+
raise ValueError("v2 runtime interpreter binding drift")
|
|
11662
|
+
guards[str(runtime["name"])] = _benchmark_study_v2_cli_stat_guard(
|
|
11663
|
+
Path(str(runtime["executable"]))
|
|
11664
|
+
)
|
|
11665
|
+
return guards
|
|
11666
|
+
|
|
11667
|
+
|
|
11668
|
+
def _benchmark_study_v2_assert_runtime_stat_guards(
|
|
11669
|
+
binding: Mapping[str, Any],
|
|
11670
|
+
guards: Mapping[str, Mapping[str, int | str]],
|
|
11671
|
+
) -> None:
|
|
11672
|
+
_benchmark_study_v2_validate_execution_environment(binding)
|
|
11673
|
+
runtimes = binding["runtime_bindings"]
|
|
11674
|
+
if set(guards) != {str(runtime["name"]) for runtime in runtimes}:
|
|
11675
|
+
raise ValueError("v2 runtime interpreter guard mismatch")
|
|
11676
|
+
for runtime in runtimes:
|
|
11677
|
+
name = str(runtime["name"])
|
|
11678
|
+
guard = guards[name]
|
|
11679
|
+
found = shutil.which(
|
|
11680
|
+
name, path=str(binding["values"]["PATH"]),
|
|
11681
|
+
)
|
|
11682
|
+
if (
|
|
11683
|
+
found is None
|
|
11684
|
+
or str(Path(found).absolute()) != runtime["lookup_path"]
|
|
11685
|
+
or Path(found).resolve(strict=True) != Path(str(runtime["executable"]))
|
|
11686
|
+
or guard.get("executable") != runtime["executable"]
|
|
11687
|
+
or guard.get("bytes") != runtime["executable_bytes"]
|
|
11688
|
+
or _benchmark_study_v2_cli_stat_guard(
|
|
11689
|
+
Path(str(runtime["executable"]))
|
|
11690
|
+
) != dict(guard)
|
|
11691
|
+
):
|
|
11692
|
+
raise ValueError("v2 runtime interpreter changed before launch")
|
|
11693
|
+
|
|
11694
|
+
|
|
11695
|
+
def _benchmark_study_v2_auth_home_binding(home: Path) -> tuple[Path, dict[str, Any]]:
|
|
11696
|
+
if "CLAUDE_CONFIG_DIR" in os.environ:
|
|
11697
|
+
raise ValueError("v2 existing-login mode requires CLAUDE_CONFIG_DIR to be unset")
|
|
11698
|
+
if not home.is_absolute() or "\0" in str(home):
|
|
11699
|
+
raise ValueError("v2 existing-login HOME must be an absolute path")
|
|
11700
|
+
resolved = home.resolve(strict=True)
|
|
11701
|
+
directory_fd = _ensure_directory_no_symlink(resolved, create=False)
|
|
11702
|
+
try:
|
|
11703
|
+
item = os.fstat(directory_fd)
|
|
11704
|
+
finally:
|
|
11705
|
+
os.close(directory_fd)
|
|
11706
|
+
mode = stat.S_IMODE(item.st_mode)
|
|
11707
|
+
if (
|
|
11708
|
+
not stat.S_ISDIR(item.st_mode)
|
|
11709
|
+
or item.st_uid != os.geteuid()
|
|
11710
|
+
or mode & 0o022
|
|
11711
|
+
):
|
|
11712
|
+
raise ValueError("v2 existing-login HOME is not a private owned directory")
|
|
11713
|
+
return resolved, {
|
|
11714
|
+
"device": item.st_dev,
|
|
11715
|
+
"inode": item.st_ino,
|
|
11716
|
+
"uid": item.st_uid,
|
|
11717
|
+
"mode": mode,
|
|
11718
|
+
}
|
|
11719
|
+
|
|
11720
|
+
|
|
11721
|
+
def _benchmark_study_v2_validate_auth_context(binding: Mapping[str, Any]) -> None:
|
|
11722
|
+
if (
|
|
11723
|
+
set(binding) != {
|
|
11724
|
+
"schema_version", "mode", "home_path_sha256", "home_stat",
|
|
11725
|
+
"identity_sha256", "auth_method", "api_provider",
|
|
11726
|
+
"credential_environment",
|
|
11727
|
+
}
|
|
11728
|
+
or binding.get("schema_version")
|
|
11729
|
+
!= BENCHMARK_STUDY_V2_AUTH_CONTEXT_SCHEMA_VERSION
|
|
11730
|
+
or binding.get("mode") != "existing_cli_login_v1"
|
|
11731
|
+
or not isinstance(binding.get("home_path_sha256"), str)
|
|
11732
|
+
or SHA256_HEX_PATTERN.fullmatch(str(binding["home_path_sha256"])) is None
|
|
11733
|
+
or not isinstance(binding.get("identity_sha256"), str)
|
|
11734
|
+
or SHA256_HEX_PATTERN.fullmatch(str(binding["identity_sha256"])) is None
|
|
11735
|
+
or binding.get("auth_method") != "claude.ai"
|
|
11736
|
+
or binding.get("api_provider") != "firstParty"
|
|
11737
|
+
or binding.get("credential_environment") != "forbidden"
|
|
11738
|
+
):
|
|
11739
|
+
raise ValueError("v2 auth context binding schema mismatch")
|
|
11740
|
+
home_stat = binding.get("home_stat")
|
|
11741
|
+
if (
|
|
11742
|
+
not isinstance(home_stat, Mapping)
|
|
11743
|
+
or set(home_stat) != {"device", "inode", "uid", "mode"}
|
|
11744
|
+
or any(isinstance(value, bool) or not isinstance(value, int) or value < 0
|
|
11745
|
+
for value in home_stat.values())
|
|
11746
|
+
or int(home_stat["uid"]) != os.geteuid()
|
|
11747
|
+
or int(home_stat["mode"]) & 0o022
|
|
11748
|
+
):
|
|
11749
|
+
raise ValueError("v2 auth HOME binding schema mismatch")
|
|
11750
|
+
|
|
11751
|
+
|
|
11752
|
+
def _benchmark_study_v2_auth_context(
|
|
11753
|
+
claude_bin: str,
|
|
11754
|
+
execution_environment: Mapping[str, Any],
|
|
11755
|
+
auth_home: Path,
|
|
11756
|
+
) -> tuple[Path, dict[str, Any]]:
|
|
11757
|
+
_benchmark_study_v2_validate_execution_environment(execution_environment)
|
|
11758
|
+
resolved_home, home_stat = _benchmark_study_v2_auth_home_binding(auth_home)
|
|
11759
|
+
with tempfile.TemporaryDirectory(prefix="contextguard-v2-auth-probe-") as temporary:
|
|
11760
|
+
root = Path(temporary)
|
|
11761
|
+
os.chmod(root, 0o700)
|
|
11762
|
+
paths = {
|
|
11763
|
+
name: root / name
|
|
11764
|
+
for name in (
|
|
11765
|
+
"cwd", "xdg-config", "xdg-cache", "xdg-data", "xdg-state", "tmp",
|
|
11766
|
+
)
|
|
11767
|
+
}
|
|
11768
|
+
for path in paths.values():
|
|
11769
|
+
path.mkdir(mode=0o700)
|
|
11770
|
+
env = dict(execution_environment["values"])
|
|
11771
|
+
env.update({
|
|
11772
|
+
"HOME": str(resolved_home),
|
|
11773
|
+
"XDG_CONFIG_HOME": str(paths["xdg-config"]),
|
|
11774
|
+
"XDG_CACHE_HOME": str(paths["xdg-cache"]),
|
|
11775
|
+
"XDG_DATA_HOME": str(paths["xdg-data"]),
|
|
11776
|
+
"XDG_STATE_HOME": str(paths["xdg-state"]),
|
|
11777
|
+
"TMPDIR": str(paths["tmp"]),
|
|
11778
|
+
"NO_COLOR": "1",
|
|
11779
|
+
})
|
|
11780
|
+
result = run_bounded_command(
|
|
11781
|
+
[executable_argv0(claude_bin), "auth", "status", "--json"],
|
|
11782
|
+
cwd=paths["cwd"], timeout_seconds=10,
|
|
11783
|
+
max_output_bytes=MEASUREMENT_CLI_PROBE_OUTPUT_MAX_BYTES, env=env,
|
|
11784
|
+
)
|
|
11785
|
+
if (
|
|
11786
|
+
result.returncode != 0 or result.timed_out or result.output_truncated
|
|
11787
|
+
or result.launch_error or result.stderr_bytes
|
|
11788
|
+
or not 0 < len(result.stdout_bytes) <= 4096
|
|
11789
|
+
):
|
|
11790
|
+
raise ValueError("v2 existing Claude login is unavailable")
|
|
11791
|
+
try:
|
|
11792
|
+
status_payload = json.loads(result.stdout_bytes.decode("utf-8", "strict"))
|
|
11793
|
+
except (UnicodeDecodeError, json.JSONDecodeError):
|
|
11794
|
+
raise ValueError("v2 existing Claude login status is invalid") from None
|
|
11795
|
+
identity_keys = {
|
|
11796
|
+
"loggedIn", "authMethod", "apiProvider", "email", "orgId",
|
|
11797
|
+
"orgName", "subscriptionType",
|
|
11798
|
+
}
|
|
11799
|
+
if (
|
|
11800
|
+
not isinstance(status_payload, Mapping)
|
|
11801
|
+
or set(status_payload) != identity_keys
|
|
11802
|
+
or status_payload.get("loggedIn") is not True
|
|
11803
|
+
or status_payload.get("authMethod") != "claude.ai"
|
|
11804
|
+
or status_payload.get("apiProvider") != "firstParty"
|
|
11805
|
+
or any(
|
|
11806
|
+
not isinstance(status_payload.get(key), str)
|
|
11807
|
+
or len(str(status_payload[key]).encode("utf-8")) > 1024
|
|
11808
|
+
or "\0" in str(status_payload[key])
|
|
11809
|
+
for key in ("email", "orgId", "orgName", "subscriptionType")
|
|
11810
|
+
)
|
|
11811
|
+
):
|
|
11812
|
+
raise ValueError("v2 existing Claude login status is unsupported")
|
|
11813
|
+
private_identity = {
|
|
11814
|
+
key: status_payload[key]
|
|
11815
|
+
for key in (
|
|
11816
|
+
"authMethod", "apiProvider", "email", "orgId", "orgName",
|
|
11817
|
+
"subscriptionType",
|
|
11818
|
+
)
|
|
11819
|
+
}
|
|
11820
|
+
binding = {
|
|
11821
|
+
"schema_version": BENCHMARK_STUDY_V2_AUTH_CONTEXT_SCHEMA_VERSION,
|
|
11822
|
+
"mode": "existing_cli_login_v1",
|
|
11823
|
+
"home_path_sha256": _study_domain_hash(
|
|
11824
|
+
"contextguard.bench.v2.auth-home-path.v1", str(resolved_home),
|
|
11825
|
+
),
|
|
11826
|
+
"home_stat": home_stat,
|
|
11827
|
+
"identity_sha256": _study_domain_hash(
|
|
11828
|
+
"contextguard.bench.v2.auth-identity.v1", private_identity,
|
|
11829
|
+
),
|
|
11830
|
+
"auth_method": "claude.ai",
|
|
11831
|
+
"api_provider": "firstParty",
|
|
11832
|
+
"credential_environment": "forbidden",
|
|
11833
|
+
}
|
|
11834
|
+
_benchmark_study_v2_validate_auth_context(binding)
|
|
11835
|
+
return resolved_home, binding
|
|
11836
|
+
|
|
11837
|
+
|
|
11838
|
+
def _benchmark_study_v2_assert_auth_context(
|
|
11839
|
+
claude_bin: str,
|
|
11840
|
+
execution_environment: Mapping[str, Any],
|
|
11841
|
+
expected: Mapping[str, Any],
|
|
11842
|
+
auth_home: Path,
|
|
11843
|
+
) -> Path:
|
|
11844
|
+
_benchmark_study_v2_validate_auth_context(expected)
|
|
11845
|
+
try:
|
|
11846
|
+
resolved_home, actual = _benchmark_study_v2_auth_context(
|
|
11847
|
+
claude_bin, execution_environment, auth_home,
|
|
11848
|
+
)
|
|
11849
|
+
except (OSError, SystemExit, TypeError, ValueError) as exc:
|
|
11850
|
+
raise ValueError("v2 existing Claude login binding drift") from exc
|
|
11851
|
+
if actual != dict(expected):
|
|
11852
|
+
raise ValueError("v2 existing Claude login binding drift")
|
|
11853
|
+
return resolved_home
|
|
11854
|
+
|
|
11855
|
+
|
|
11856
|
+
def _benchmark_study_v2_cli_binding(claude_bin: str) -> dict[str, Any]:
|
|
11857
|
+
"""Bind the exact executable bytes and isolated version/help capability probes."""
|
|
11858
|
+
executable = Path(executable_argv0(claude_bin))
|
|
11859
|
+
file_binding, stat_guard = _benchmark_study_v2_cli_file_binding(executable)
|
|
11860
|
+
shebang = _benchmark_study_v2_read_executable_shebang(executable)
|
|
11861
|
+
if shebang is not None:
|
|
11862
|
+
raise ValueError(
|
|
11863
|
+
"v2 executable study requires a native executable; script launcher "
|
|
11864
|
+
"dependency closure cannot be proven"
|
|
11865
|
+
)
|
|
11866
|
+
bundle = _benchmark_study_v2_cli_bundle_binding(executable, shebang)
|
|
11867
|
+
probe = run_measurement_cli_probes(
|
|
11868
|
+
str(executable), BENCHMARK_STUDY_V2_CLI_CAPABILITIES,
|
|
11869
|
+
)
|
|
11870
|
+
# Bind exact version bytes by hash/length without persisting arbitrary CLI
|
|
11871
|
+
# display text that could contain a secret-shaped value.
|
|
11872
|
+
probe.pop("version", None)
|
|
11873
|
+
if _benchmark_study_v2_cli_stat_guard(executable) != stat_guard:
|
|
11874
|
+
raise ValueError("v2 CLI executable changed during capability probes")
|
|
11875
|
+
binding = {
|
|
11876
|
+
"schema_version": BENCHMARK_STUDY_V2_CLI_BINDING_SCHEMA_VERSION,
|
|
11877
|
+
**file_binding,
|
|
11878
|
+
"bundle": bundle,
|
|
11879
|
+
"probe": probe,
|
|
11880
|
+
}
|
|
11881
|
+
_benchmark_study_v2_validate_cli_binding(binding)
|
|
11882
|
+
return binding
|
|
11883
|
+
|
|
11884
|
+
|
|
11885
|
+
def _benchmark_study_v2_assert_cli_binding(
|
|
11886
|
+
claude_bin: str, expected: Mapping[str, Any],
|
|
11887
|
+
) -> dict[str, Any]:
|
|
11888
|
+
"""Refuse a changed or incompatible CLI before reserving any identity."""
|
|
11889
|
+
_benchmark_study_v2_validate_cli_binding(expected)
|
|
11890
|
+
try:
|
|
11891
|
+
actual = _benchmark_study_v2_cli_binding(claude_bin)
|
|
11892
|
+
except (OSError, SystemExit, TypeError, ValueError) as exc:
|
|
11893
|
+
raise ValueError("v2 CLI binding drift") from exc
|
|
11894
|
+
if actual != dict(expected):
|
|
11895
|
+
raise ValueError("v2 CLI binding drift")
|
|
11896
|
+
guard: dict[str, Any] = _benchmark_study_v2_cli_stat_guard(
|
|
11897
|
+
Path(str(actual["executable"]))
|
|
11898
|
+
)
|
|
11899
|
+
guard["bundle_stat_guard"] = _benchmark_study_v2_cli_bundle_stat_guard(
|
|
11900
|
+
actual["bundle"]
|
|
11901
|
+
)
|
|
11902
|
+
return guard
|
|
11903
|
+
|
|
11904
|
+
|
|
11905
|
+
def _benchmark_study_v2_assert_cli_executable_bytes(
|
|
11906
|
+
expected: Mapping[str, Any], stat_guard: Mapping[str, Any],
|
|
11907
|
+
) -> None:
|
|
11908
|
+
"""Repeat a cheap post-hash file-identity check before provider reservation."""
|
|
11909
|
+
_benchmark_study_v2_validate_cli_binding(expected)
|
|
11910
|
+
if (
|
|
11911
|
+
stat_guard.get("executable") != expected["executable"]
|
|
11912
|
+
or stat_guard.get("bytes") != expected["executable_bytes"]
|
|
11913
|
+
or _benchmark_study_v2_cli_stat_guard(
|
|
11914
|
+
Path(str(expected["executable"]))
|
|
11915
|
+
) != {
|
|
11916
|
+
key: stat_guard[key]
|
|
11917
|
+
for key in ("executable", "device", "inode", "bytes", "mtime_ns", "ctime_ns")
|
|
11918
|
+
}
|
|
11919
|
+
or stat_guard.get("bundle_stat_guard")
|
|
11920
|
+
!= _benchmark_study_v2_cli_bundle_stat_guard(expected["bundle"])
|
|
11921
|
+
):
|
|
11922
|
+
raise ValueError("v2 CLI executable bytes drift")
|
|
11923
|
+
|
|
11924
|
+
|
|
11925
|
+
def _benchmark_study_v2_python_binding() -> dict[str, Any]:
|
|
11926
|
+
invoked_python = Path(sys.executable).absolute()
|
|
11927
|
+
resolved_python = invoked_python.resolve(strict=True)
|
|
11928
|
+
python_file, _guard = _benchmark_study_v2_cli_file_binding(resolved_python)
|
|
11929
|
+
return {
|
|
11930
|
+
"invoked_path": str(invoked_python),
|
|
11931
|
+
**python_file,
|
|
11932
|
+
"implementation": sys.implementation.name,
|
|
11933
|
+
"cache_tag": sys.implementation.cache_tag,
|
|
11934
|
+
"version_info": [
|
|
11935
|
+
sys.version_info.major, sys.version_info.minor,
|
|
11936
|
+
sys.version_info.micro, sys.version_info.releaselevel,
|
|
11937
|
+
sys.version_info.serial,
|
|
11938
|
+
],
|
|
11939
|
+
"version_sha256": _study_sha256_bytes(sys.version.encode("utf-8")),
|
|
11940
|
+
"flags_sha256": _study_domain_hash(
|
|
11941
|
+
"contextguard.bench.v2.python-flags.v1", list(sys.flags),
|
|
11942
|
+
),
|
|
11943
|
+
"prefix": sys.prefix,
|
|
11944
|
+
"base_prefix": sys.base_prefix,
|
|
11945
|
+
}
|
|
11946
|
+
|
|
11947
|
+
|
|
11948
|
+
def _benchmark_study_v2_runner_binding() -> dict[str, Any]:
|
|
11949
|
+
path = Path(__file__).resolve(strict=True)
|
|
11950
|
+
raw = _read_bytes_no_follow(path, max_bytes=4_000_000)
|
|
11951
|
+
return {
|
|
11952
|
+
"path": str(path), "bytes": len(raw),
|
|
11953
|
+
"sha256": _study_sha256_bytes(raw),
|
|
11954
|
+
"python": _benchmark_study_v2_python_binding(),
|
|
11955
|
+
}
|
|
11956
|
+
|
|
11957
|
+
|
|
11958
|
+
def _benchmark_study_v2_assert_python_binding(
|
|
11959
|
+
binding: Mapping[str, Any], *, require_current: bool,
|
|
11960
|
+
) -> str:
|
|
11961
|
+
required = {
|
|
11962
|
+
"invoked_path", "executable", "executable_bytes", "executable_sha256",
|
|
11963
|
+
"implementation", "cache_tag", "version_info", "version_sha256",
|
|
11964
|
+
"flags_sha256", "prefix", "base_prefix",
|
|
11965
|
+
}
|
|
11966
|
+
if (
|
|
11967
|
+
set(binding) != required
|
|
11968
|
+
or not isinstance(binding.get("invoked_path"), str)
|
|
11969
|
+
or not Path(str(binding["invoked_path"])).is_absolute()
|
|
11970
|
+
or not isinstance(binding.get("executable"), str)
|
|
11971
|
+
or not Path(str(binding["executable"])).is_absolute()
|
|
11972
|
+
or isinstance(binding.get("executable_bytes"), bool)
|
|
11973
|
+
or not isinstance(binding.get("executable_bytes"), int)
|
|
11974
|
+
or not 0 < int(binding["executable_bytes"]) <= BENCHMARK_STUDY_V2_CLI_MAX_BYTES
|
|
11975
|
+
or not isinstance(binding.get("executable_sha256"), str)
|
|
11976
|
+
or SHA256_HEX_PATTERN.fullmatch(str(binding["executable_sha256"])) is None
|
|
11977
|
+
or not isinstance(binding.get("implementation"), str)
|
|
11978
|
+
or not binding["implementation"]
|
|
11979
|
+
or not isinstance(binding.get("cache_tag"), (str, type(None)))
|
|
11980
|
+
or not isinstance(binding.get("version_info"), list)
|
|
11981
|
+
or len(binding["version_info"]) != 5
|
|
11982
|
+
or any(
|
|
11983
|
+
isinstance(value, bool) or not isinstance(value, (int, str))
|
|
11984
|
+
for value in binding["version_info"]
|
|
11985
|
+
)
|
|
11986
|
+
or not isinstance(binding.get("version_sha256"), str)
|
|
11987
|
+
or SHA256_HEX_PATTERN.fullmatch(str(binding["version_sha256"])) is None
|
|
11988
|
+
or not isinstance(binding.get("flags_sha256"), str)
|
|
11989
|
+
or SHA256_HEX_PATTERN.fullmatch(str(binding["flags_sha256"])) is None
|
|
11990
|
+
or not isinstance(binding.get("prefix"), str)
|
|
11991
|
+
or not isinstance(binding.get("base_prefix"), str)
|
|
11992
|
+
):
|
|
11993
|
+
raise ValueError("v2 runner Python binding schema mismatch")
|
|
11994
|
+
executable = Path(str(binding["executable"]))
|
|
11995
|
+
current_file, _guard = _benchmark_study_v2_cli_file_binding(executable)
|
|
11996
|
+
expected_file = {
|
|
11997
|
+
key: binding[key]
|
|
11998
|
+
for key in ("executable", "executable_bytes", "executable_sha256")
|
|
11999
|
+
}
|
|
12000
|
+
if current_file != expected_file:
|
|
12001
|
+
raise ValueError("v2 runner Python binding drift")
|
|
12002
|
+
if require_current:
|
|
12003
|
+
invoked = Path(sys.executable).absolute()
|
|
12004
|
+
try:
|
|
12005
|
+
resolved = invoked.resolve(strict=True)
|
|
12006
|
+
except OSError as exc:
|
|
12007
|
+
raise ValueError("v2 runner Python binding drift") from exc
|
|
12008
|
+
if (
|
|
12009
|
+
str(invoked) != binding["invoked_path"]
|
|
12010
|
+
or str(resolved) != binding["executable"]
|
|
12011
|
+
or sys.implementation.name != binding["implementation"]
|
|
12012
|
+
or sys.implementation.cache_tag != binding["cache_tag"]
|
|
12013
|
+
or [
|
|
12014
|
+
sys.version_info.major, sys.version_info.minor,
|
|
12015
|
+
sys.version_info.micro, sys.version_info.releaselevel,
|
|
12016
|
+
sys.version_info.serial,
|
|
12017
|
+
] != binding["version_info"]
|
|
12018
|
+
or _study_sha256_bytes(sys.version.encode("utf-8"))
|
|
12019
|
+
!= binding["version_sha256"]
|
|
12020
|
+
or _study_domain_hash(
|
|
12021
|
+
"contextguard.bench.v2.python-flags.v1", list(sys.flags),
|
|
12022
|
+
) != binding["flags_sha256"]
|
|
12023
|
+
or sys.prefix != binding["prefix"]
|
|
12024
|
+
or sys.base_prefix != binding["base_prefix"]
|
|
12025
|
+
):
|
|
12026
|
+
raise ValueError("v2 runner Python binding drift")
|
|
12027
|
+
return str(executable)
|
|
12028
|
+
|
|
12029
|
+
|
|
12030
|
+
def _benchmark_study_v2_read_canonical(path: Path, *, owner: str, maximum: int) -> tuple[dict[str, Any], bytes]:
|
|
12031
|
+
raw = _read_bytes_no_follow(path, max_bytes=maximum)
|
|
12032
|
+
try:
|
|
12033
|
+
value = json.loads(
|
|
12034
|
+
raw.decode("utf-8"), object_pairs_hook=_measurement_object_no_duplicates,
|
|
12035
|
+
parse_constant=_stream_reject_nonfinite,
|
|
12036
|
+
)
|
|
12037
|
+
except (UnicodeDecodeError, json.JSONDecodeError, ValueError) as exc:
|
|
12038
|
+
raise ValueError(f"{owner} is invalid JSON") from exc
|
|
12039
|
+
if not isinstance(value, dict) or raw != _study_canonical_json_bytes(value):
|
|
12040
|
+
raise ValueError(f"{owner} must be exact canonical JSON bytes")
|
|
12041
|
+
return value, raw
|
|
12042
|
+
|
|
12043
|
+
|
|
12044
|
+
def verify_benchmark_study_v2_candidate(
|
|
12045
|
+
manifest_path: Path, *, checksum_path: Path | None = None,
|
|
12046
|
+
expected_manifest_sha256: str | None = None,
|
|
12047
|
+
expected_commit_sha: str | None = None,
|
|
12048
|
+
) -> dict[str, Any]:
|
|
12049
|
+
"""Verify the build-once candidate as inert bytes; never import its code."""
|
|
12050
|
+
manifest, manifest_raw = _benchmark_study_v2_read_canonical(
|
|
12051
|
+
manifest_path, owner="v2 candidate manifest", maximum=1_000_000,
|
|
12052
|
+
)
|
|
12053
|
+
required = {
|
|
12054
|
+
"build_policy", "commit_sha", "exact_dependency", "packages",
|
|
12055
|
+
"policy_sha256", "receipt_package_files_sha256", "protocol",
|
|
12056
|
+
"repository", "schema_version", "tool_versions",
|
|
12057
|
+
}
|
|
12058
|
+
if set(manifest) != required or manifest.get("schema_version") != BENCHMARK_STUDY_V2_CANDIDATE_SCHEMA_VERSION:
|
|
12059
|
+
raise ValueError("v2 candidate manifest schema mismatch")
|
|
12060
|
+
if manifest.get("build_policy") != {
|
|
12061
|
+
"ignore_scripts": True, "lockfiles": [], "network": "offline",
|
|
12062
|
+
"package_build_count": 1,
|
|
12063
|
+
}:
|
|
12064
|
+
raise ValueError("v2 candidate build policy mismatch")
|
|
12065
|
+
if manifest.get("protocol") != {
|
|
12066
|
+
"maximum": 1, "minimum": 1, "name": "bash_reference_v1",
|
|
12067
|
+
}:
|
|
12068
|
+
raise ValueError("v2 candidate protocol mismatch")
|
|
12069
|
+
if (
|
|
12070
|
+
manifest.get("repository") != "ictechgy/context-guard"
|
|
12071
|
+
or not isinstance(manifest.get("commit_sha"), str)
|
|
12072
|
+
or re.fullmatch(r"[0-9a-f]{40}", manifest["commit_sha"]) is None
|
|
12073
|
+
or not isinstance(manifest.get("tool_versions"), dict)
|
|
12074
|
+
or not manifest["tool_versions"]
|
|
12075
|
+
or any(
|
|
12076
|
+
not isinstance(key, str) or not key or len(key) > 64
|
|
12077
|
+
or not isinstance(value, str) or not value or len(value) > 128
|
|
12078
|
+
for key, value in manifest["tool_versions"].items()
|
|
12079
|
+
)
|
|
12080
|
+
or any(
|
|
12081
|
+
not isinstance(manifest.get(key), str)
|
|
12082
|
+
or SHA256_HEX_PATTERN.fullmatch(manifest[key]) is None
|
|
12083
|
+
for key in ("policy_sha256", "receipt_package_files_sha256")
|
|
12084
|
+
)
|
|
12085
|
+
):
|
|
12086
|
+
raise ValueError("v2 candidate build provenance is invalid")
|
|
12087
|
+
manifest_sha256 = _study_sha256_bytes(manifest_raw)
|
|
12088
|
+
if expected_manifest_sha256 is not None and manifest_sha256 != expected_manifest_sha256:
|
|
12089
|
+
raise ValueError("v2 candidate canonical manifest hash drift")
|
|
12090
|
+
if expected_commit_sha is not None and (
|
|
12091
|
+
re.fullmatch(r"[0-9a-f]{40}", expected_commit_sha) is None
|
|
12092
|
+
or manifest["commit_sha"] != expected_commit_sha
|
|
12093
|
+
):
|
|
12094
|
+
raise ValueError(
|
|
12095
|
+
"v2 candidate commit does not match approved source revision"
|
|
12096
|
+
)
|
|
12097
|
+
packages = manifest.get("packages")
|
|
12098
|
+
if not isinstance(packages, list) or len(packages) != 2:
|
|
12099
|
+
raise ValueError("v2 candidate must bind exactly two tarballs")
|
|
12100
|
+
records: list[dict[str, Any]] = []
|
|
12101
|
+
checksum_rows: list[str] = []
|
|
12102
|
+
for index, package in enumerate(packages):
|
|
12103
|
+
if not isinstance(package, dict) or set(package) != {
|
|
12104
|
+
"filename", "integrity", "name", "sha256", "size_bytes", "version",
|
|
12105
|
+
}:
|
|
12106
|
+
raise ValueError("v2 candidate tarball record schema mismatch")
|
|
12107
|
+
filename = package.get("filename")
|
|
12108
|
+
digest = package.get("sha256")
|
|
12109
|
+
size = package.get("size_bytes")
|
|
12110
|
+
if (
|
|
12111
|
+
package.get("name") != BENCHMARK_STUDY_V2_CANDIDATE_NAMES[index]
|
|
12112
|
+
or not isinstance(filename, str) or Path(filename).name != filename
|
|
12113
|
+
or not isinstance(digest, str) or SHA256_HEX_PATTERN.fullmatch(digest) is None
|
|
12114
|
+
or isinstance(size, bool) or not isinstance(size, int) or size <= 0
|
|
12115
|
+
or not isinstance(package.get("version"), str) or not package["version"]
|
|
12116
|
+
):
|
|
12117
|
+
raise ValueError("v2 candidate tarball identity is invalid")
|
|
12118
|
+
tarball_path = manifest_path.parent / filename
|
|
12119
|
+
raw = _read_bytes_no_follow(tarball_path, max_bytes=100_000_000)
|
|
12120
|
+
sri = "sha512-" + base64.b64encode(hashlib.sha512(raw).digest()).decode("ascii")
|
|
12121
|
+
if len(raw) != size or _study_sha256_bytes(raw) != digest or package.get("integrity") != sri:
|
|
12122
|
+
raise ValueError("v2 candidate tarball size, SHA-256, or SRI mismatch")
|
|
12123
|
+
checksum_rows.append(f"{digest} {filename}\n")
|
|
12124
|
+
records.append({**package, "path": str(tarball_path.resolve())})
|
|
12125
|
+
if len({record["filename"] for record in records}) != 2 or manifest.get("exact_dependency") != {
|
|
12126
|
+
"name": BENCHMARK_STUDY_V2_CANDIDATE_NAMES[0],
|
|
12127
|
+
"version": records[0]["version"],
|
|
12128
|
+
}:
|
|
12129
|
+
raise ValueError("v2 candidate exact dependency binding mismatch")
|
|
12130
|
+
checksum = checksum_path or manifest_path.with_name("candidate-sha256sums.txt")
|
|
12131
|
+
checksum_raw = _read_bytes_no_follow(checksum, max_bytes=10_000)
|
|
12132
|
+
expected_checksum = "".join(checksum_rows).encode("ascii")
|
|
12133
|
+
if checksum_raw != expected_checksum:
|
|
12134
|
+
raise ValueError("v2 candidate checksum document mismatch")
|
|
12135
|
+
return {
|
|
12136
|
+
"manifest_path": str(manifest_path.resolve()),
|
|
12137
|
+
"manifest_sha256": manifest_sha256,
|
|
12138
|
+
"manifest_bytes": len(manifest_raw),
|
|
12139
|
+
"checksum_path": str(checksum.resolve()),
|
|
12140
|
+
"checksum_sha256": _study_sha256_bytes(checksum_raw),
|
|
12141
|
+
"commit_sha": manifest["commit_sha"],
|
|
12142
|
+
"packages": records,
|
|
12143
|
+
}
|
|
12144
|
+
|
|
12145
|
+
|
|
12146
|
+
def _benchmark_study_v2_verify_retained_ref(
|
|
12147
|
+
retained_ref: str, expected_commit_sha: str,
|
|
12148
|
+
) -> dict[str, str]:
|
|
12149
|
+
if (
|
|
12150
|
+
not isinstance(retained_ref, str)
|
|
12151
|
+
or len(retained_ref) > 200
|
|
12152
|
+
or re.fullmatch(
|
|
12153
|
+
r"refs/heads/candidate/[A-Za-z0-9][A-Za-z0-9._/-]*",
|
|
12154
|
+
retained_ref,
|
|
12155
|
+
) is None
|
|
12156
|
+
or ".." in retained_ref
|
|
12157
|
+
or "//" in retained_ref
|
|
12158
|
+
or retained_ref.endswith(("/", ".lock"))
|
|
12159
|
+
or re.fullmatch(r"[0-9a-f]{40}", expected_commit_sha) is None
|
|
12160
|
+
):
|
|
12161
|
+
raise ValueError("v2 retained ref binding is invalid")
|
|
12162
|
+
git = Path("/usr/bin/git")
|
|
12163
|
+
if not git.is_file() or not os.access(git, os.X_OK):
|
|
12164
|
+
raise ValueError("v2 retained ref verifier is unavailable")
|
|
12165
|
+
with tempfile.TemporaryDirectory(prefix="contextguard-v2-git-home-") as temp:
|
|
12166
|
+
os.chmod(temp, 0o700)
|
|
12167
|
+
result = run_bounded_command(
|
|
12168
|
+
[
|
|
12169
|
+
str(git), "-c", "credential.helper=", "-c",
|
|
12170
|
+
"core.askPass=", "ls-remote", "--exit-code", "--refs",
|
|
12171
|
+
"https://github.com/ictechgy/context-guard.git", retained_ref,
|
|
12172
|
+
],
|
|
12173
|
+
cwd=Path(temp), timeout_seconds=30, max_output_bytes=10_000,
|
|
12174
|
+
env={
|
|
12175
|
+
"GIT_ASKPASS": "/usr/bin/false",
|
|
12176
|
+
"GIT_CONFIG_GLOBAL": "/dev/null",
|
|
12177
|
+
"GIT_CONFIG_NOSYSTEM": "1",
|
|
12178
|
+
"GIT_TERMINAL_PROMPT": "0",
|
|
12179
|
+
"HOME": temp,
|
|
12180
|
+
"LC_ALL": "C",
|
|
12181
|
+
"PATH": "/usr/bin:/bin",
|
|
12182
|
+
"SSH_ASKPASS": "/usr/bin/false",
|
|
12183
|
+
},
|
|
12184
|
+
)
|
|
12185
|
+
expected_output = f"{expected_commit_sha}\t{retained_ref}\n"
|
|
12186
|
+
if (
|
|
12187
|
+
result.returncode != 0
|
|
12188
|
+
or result.timed_out
|
|
12189
|
+
or result.output_truncated
|
|
12190
|
+
or result.stderr
|
|
12191
|
+
or result.stdout != expected_output
|
|
12192
|
+
):
|
|
12193
|
+
raise ValueError("v2 retained ref does not resolve to approved source")
|
|
12194
|
+
return {
|
|
12195
|
+
"commit_sha": expected_commit_sha,
|
|
12196
|
+
"ref": retained_ref,
|
|
12197
|
+
"repository": "ictechgy/context-guard",
|
|
12198
|
+
"verification": "git-ls-remote-v1",
|
|
12199
|
+
}
|
|
12200
|
+
|
|
12201
|
+
|
|
12202
|
+
def _benchmark_study_v2_tarball_inventory(path: Path) -> dict[str, Any]:
|
|
12203
|
+
"""Inventory regular npm package members without extracting candidate code."""
|
|
12204
|
+
files: list[dict[str, Any]] = []
|
|
12205
|
+
seen: set[str] = set()
|
|
12206
|
+
total_bytes = 0
|
|
12207
|
+
try:
|
|
12208
|
+
with tarfile.open(path, mode="r:gz") as archive:
|
|
12209
|
+
for index, member in enumerate(archive):
|
|
12210
|
+
if index >= 10_000:
|
|
12211
|
+
raise ValueError("v2 candidate tarball has too many members")
|
|
12212
|
+
raw_parts = member.name.split("/")
|
|
12213
|
+
if (
|
|
12214
|
+
not raw_parts
|
|
12215
|
+
or raw_parts[0] != "package"
|
|
12216
|
+
or any(part in {"", ".", ".."} for part in raw_parts)
|
|
12217
|
+
or any(
|
|
12218
|
+
ord(character) < 0x20 or ord(character) == 0x7F
|
|
12219
|
+
for character in member.name
|
|
12220
|
+
)
|
|
12221
|
+
):
|
|
12222
|
+
raise ValueError("v2 candidate tarball member path is invalid")
|
|
12223
|
+
if len(raw_parts) == 1:
|
|
12224
|
+
if not member.isdir():
|
|
12225
|
+
raise ValueError("v2 candidate tarball package root is invalid")
|
|
12226
|
+
continue
|
|
12227
|
+
relative = PurePosixPath(*raw_parts[1:]).as_posix()
|
|
12228
|
+
if relative in seen:
|
|
12229
|
+
raise ValueError("v2 candidate tarball has duplicate members")
|
|
12230
|
+
if member.isdir():
|
|
12231
|
+
continue
|
|
12232
|
+
if not member.isreg() or member.size < 0 or member.size > 100_000_000:
|
|
12233
|
+
raise ValueError("v2 candidate tarball member type is unsupported")
|
|
12234
|
+
stream = archive.extractfile(member)
|
|
12235
|
+
if stream is None:
|
|
12236
|
+
raise ValueError("v2 candidate tarball member is unreadable")
|
|
12237
|
+
raw = stream.read(member.size + 1)
|
|
12238
|
+
if len(raw) != member.size:
|
|
12239
|
+
raise ValueError("v2 candidate tarball member size drift")
|
|
12240
|
+
total_bytes += len(raw)
|
|
12241
|
+
if total_bytes > 100_000_000:
|
|
12242
|
+
raise ValueError("v2 candidate tarball expanded size exceeds limit")
|
|
12243
|
+
seen.add(relative)
|
|
12244
|
+
files.append({
|
|
12245
|
+
"path": relative,
|
|
12246
|
+
"bytes": len(raw),
|
|
12247
|
+
"sha256": _study_sha256_bytes(raw),
|
|
12248
|
+
"executable": bool(member.mode & 0o111),
|
|
12249
|
+
"kind": "file",
|
|
12250
|
+
"target": None,
|
|
12251
|
+
})
|
|
12252
|
+
except (OSError, tarfile.TarError) as exc:
|
|
12253
|
+
raise ValueError("v2 candidate tarball is not a readable npm archive") from exc
|
|
12254
|
+
files.sort(key=lambda item: item["path"])
|
|
12255
|
+
if not files:
|
|
12256
|
+
raise ValueError("v2 candidate tarball package is empty")
|
|
12257
|
+
return {
|
|
12258
|
+
"files": files,
|
|
12259
|
+
"file_count": len(files),
|
|
12260
|
+
"sha256": _study_domain_hash("contextguard.bench.v2.inventory.v1", files),
|
|
12261
|
+
}
|
|
12262
|
+
|
|
12263
|
+
|
|
12264
|
+
def _benchmark_study_v2_verify_installed_packages(
|
|
12265
|
+
overlay_root: Path, candidate: Mapping[str, Any],
|
|
12266
|
+
) -> dict[str, Any]:
|
|
12267
|
+
"""Bind the executed package bytes to the two already-verified tarballs."""
|
|
12268
|
+
bindings: list[dict[str, Any]] = []
|
|
12269
|
+
installed_documents: dict[str, dict[str, Any]] = {}
|
|
12270
|
+
allowed_overlay_paths: set[str] = set()
|
|
12271
|
+
for record in candidate["packages"]:
|
|
12272
|
+
name = record["name"]
|
|
12273
|
+
package_root = overlay_root.joinpath(*str(name).split("/"))
|
|
12274
|
+
expected = _benchmark_study_v2_tarball_inventory(Path(record["path"]))
|
|
12275
|
+
actual = _benchmark_study_v2_inventory(package_root)
|
|
12276
|
+
if actual != expected:
|
|
12277
|
+
raise ValueError("v2 installed package bytes differ from candidate tarball")
|
|
12278
|
+
package_raw = _read_bytes_no_follow(
|
|
12279
|
+
package_root / "package.json", max_bytes=128 * 1024,
|
|
12280
|
+
)
|
|
12281
|
+
try:
|
|
12282
|
+
document = json.loads(
|
|
12283
|
+
package_raw.decode("utf-8"),
|
|
12284
|
+
object_pairs_hook=_measurement_object_no_duplicates,
|
|
12285
|
+
)
|
|
12286
|
+
except (UnicodeDecodeError, json.JSONDecodeError, ValueError) as exc:
|
|
12287
|
+
raise ValueError("v2 installed package metadata is invalid") from exc
|
|
12288
|
+
if (
|
|
12289
|
+
not isinstance(document, dict)
|
|
12290
|
+
or document.get("name") != name
|
|
12291
|
+
or document.get("version") != record["version"]
|
|
12292
|
+
):
|
|
12293
|
+
raise ValueError("v2 installed package identity differs from candidate")
|
|
12294
|
+
installed_documents[str(name)] = document
|
|
12295
|
+
allowed_overlay_paths.update(
|
|
12296
|
+
f"{name}/{item['path']}" for item in expected["files"]
|
|
12297
|
+
)
|
|
12298
|
+
bindings.append({
|
|
12299
|
+
"name": name,
|
|
12300
|
+
"version": record["version"],
|
|
12301
|
+
"inventory_sha256": actual["sha256"],
|
|
12302
|
+
})
|
|
12303
|
+
root_document = installed_documents[BENCHMARK_STUDY_V2_CANDIDATE_NAMES[1]]
|
|
12304
|
+
receipt_record = candidate["packages"][0]
|
|
12305
|
+
if root_document.get("dependencies") != {
|
|
12306
|
+
BENCHMARK_STUDY_V2_CANDIDATE_NAMES[0]: receipt_record["version"],
|
|
12307
|
+
}:
|
|
12308
|
+
raise ValueError("v2 installed root exact dependency binding mismatch")
|
|
12309
|
+
required_bins = ("context-guard", "context-guard-rewrite-bash")
|
|
12310
|
+
root_bin_map = root_document.get("bin")
|
|
12311
|
+
if not isinstance(root_bin_map, dict) or any(
|
|
12312
|
+
not isinstance(root_bin_map.get(name), str) for name in required_bins
|
|
12313
|
+
):
|
|
12314
|
+
raise ValueError("v2 installed root package lacks required public bins")
|
|
12315
|
+
bin_bindings: list[dict[str, str]] = []
|
|
12316
|
+
seen_bin_names: set[str] = set()
|
|
12317
|
+
for package_name in sorted(installed_documents):
|
|
12318
|
+
bin_map = installed_documents[package_name].get("bin", {})
|
|
12319
|
+
if not isinstance(bin_map, dict):
|
|
12320
|
+
raise ValueError("v2 installed package bin metadata is invalid")
|
|
12321
|
+
package_root = overlay_root.joinpath(*package_name.split("/"))
|
|
12322
|
+
for bin_name, relative_target in sorted(bin_map.items()):
|
|
12323
|
+
if (
|
|
12324
|
+
not isinstance(bin_name, str)
|
|
12325
|
+
or not bin_name
|
|
12326
|
+
or "/" in bin_name
|
|
12327
|
+
or bin_name in {".", ".."}
|
|
12328
|
+
or bin_name in seen_bin_names
|
|
12329
|
+
or not isinstance(relative_target, str)
|
|
12330
|
+
):
|
|
12331
|
+
raise ValueError("v2 installed package bin metadata is invalid")
|
|
12332
|
+
target_parts = relative_target.split("/")
|
|
12333
|
+
target_path = PurePosixPath(relative_target)
|
|
12334
|
+
if (
|
|
12335
|
+
target_path.is_absolute()
|
|
12336
|
+
or not target_parts
|
|
12337
|
+
or any(part in {"", ".", ".."} for part in target_parts)
|
|
12338
|
+
):
|
|
12339
|
+
raise ValueError("v2 installed package bin target is unsafe")
|
|
12340
|
+
package_relative_target = (
|
|
12341
|
+
PurePosixPath(package_name) / target_path
|
|
12342
|
+
).as_posix()
|
|
12343
|
+
if package_relative_target not in allowed_overlay_paths:
|
|
12344
|
+
raise ValueError("v2 installed package bin target is not in its tarball")
|
|
12345
|
+
link_relative = f".bin/{bin_name}"
|
|
12346
|
+
link = overlay_root / ".bin" / bin_name
|
|
12347
|
+
expected_raw_target = (
|
|
12348
|
+
PurePosixPath("..") / package_name / target_path
|
|
12349
|
+
).as_posix()
|
|
12350
|
+
try:
|
|
12351
|
+
raw_target = os.readlink(link)
|
|
12352
|
+
resolved = link.resolve(strict=True)
|
|
12353
|
+
expected_target = (package_root / Path(*target_parts)).resolve(
|
|
12354
|
+
strict=True
|
|
12355
|
+
)
|
|
12356
|
+
except OSError as exc:
|
|
12357
|
+
raise ValueError("v2 installed public bin link is unavailable") from exc
|
|
12358
|
+
if raw_target != expected_raw_target or resolved != expected_target:
|
|
12359
|
+
raise ValueError(
|
|
12360
|
+
"v2 installed public bin link differs from package metadata"
|
|
12361
|
+
)
|
|
12362
|
+
seen_bin_names.add(bin_name)
|
|
12363
|
+
allowed_overlay_paths.add(link_relative)
|
|
12364
|
+
bin_bindings.append({
|
|
12365
|
+
"name": bin_name,
|
|
12366
|
+
"package": package_name,
|
|
12367
|
+
"target": expected_raw_target,
|
|
12368
|
+
})
|
|
12369
|
+
actual_overlay = _benchmark_study_v2_inventory(overlay_root)
|
|
12370
|
+
actual_overlay_paths = {item["path"] for item in actual_overlay["files"]}
|
|
12371
|
+
if actual_overlay_paths - allowed_overlay_paths:
|
|
12372
|
+
raise ValueError("v2 candidate install contains an unverified overlay path")
|
|
12373
|
+
if actual_overlay_paths != allowed_overlay_paths:
|
|
12374
|
+
raise ValueError("v2 candidate install is missing a verified overlay path")
|
|
12375
|
+
return {
|
|
12376
|
+
"packages": bindings,
|
|
12377
|
+
"bins": bin_bindings,
|
|
12378
|
+
"sha256": _study_domain_hash(
|
|
12379
|
+
"contextguard.bench.v2.installed-packages.v2",
|
|
12380
|
+
{"packages": bindings, "bins": bin_bindings},
|
|
12381
|
+
),
|
|
12382
|
+
}
|
|
12383
|
+
|
|
12384
|
+
|
|
12385
|
+
def _benchmark_study_v2_inventory(root: Path, *, reject_symlinks: bool = False) -> dict[str, Any]:
|
|
12386
|
+
if root.is_symlink() or not root.is_dir():
|
|
12387
|
+
raise ValueError("v2 inventory root must be a real directory")
|
|
12388
|
+
files: list[dict[str, Any]] = []
|
|
12389
|
+
for path in sorted(root.rglob("*"), key=lambda item: item.relative_to(root).as_posix()):
|
|
12390
|
+
rel = path.relative_to(root).as_posix()
|
|
12391
|
+
mode = os.lstat(path).st_mode
|
|
12392
|
+
if stat.S_ISLNK(mode):
|
|
12393
|
+
if reject_symlinks:
|
|
12394
|
+
raise ValueError("v2 physical overlay must not contain symlinks")
|
|
12395
|
+
raw_target = os.readlink(path)
|
|
12396
|
+
if Path(raw_target).is_absolute():
|
|
12397
|
+
raise ValueError("v2 candidate overlay symlink must be relative")
|
|
12398
|
+
target = path.resolve(strict=True)
|
|
12399
|
+
try:
|
|
12400
|
+
target.relative_to(root.resolve(strict=True))
|
|
12401
|
+
except ValueError:
|
|
12402
|
+
raise ValueError("v2 candidate overlay symlink escapes the install root") from None
|
|
12403
|
+
if not target.is_file():
|
|
12404
|
+
raise ValueError("v2 candidate overlay symlink must resolve to an internal file")
|
|
12405
|
+
raw = _read_bytes_no_follow(target, max_bytes=100_000_000)
|
|
12406
|
+
executable = bool(target.stat().st_mode & 0o111)
|
|
12407
|
+
kind = "symlink"
|
|
12408
|
+
elif stat.S_ISREG(mode):
|
|
12409
|
+
raw = _read_bytes_no_follow(path, max_bytes=100_000_000)
|
|
12410
|
+
executable = bool(mode & 0o111)
|
|
12411
|
+
raw_target = None
|
|
12412
|
+
kind = "file"
|
|
12413
|
+
elif stat.S_ISDIR(mode):
|
|
12414
|
+
continue
|
|
12415
|
+
else:
|
|
12416
|
+
raise ValueError("v2 inventory contains an unsupported filesystem entry")
|
|
12417
|
+
files.append({
|
|
12418
|
+
"path": rel, "bytes": len(raw), "sha256": _study_sha256_bytes(raw),
|
|
12419
|
+
"executable": executable, "kind": kind, "target": raw_target,
|
|
12420
|
+
})
|
|
12421
|
+
if not files:
|
|
12422
|
+
raise ValueError("v2 candidate overlay is empty")
|
|
12423
|
+
return {
|
|
12424
|
+
"files": files,
|
|
12425
|
+
"file_count": len(files),
|
|
12426
|
+
"sha256": _study_domain_hash("contextguard.bench.v2.inventory.v1", files),
|
|
12427
|
+
}
|
|
12428
|
+
|
|
12429
|
+
|
|
12430
|
+
def _benchmark_study_v2_verify_physical_copy(
|
|
12431
|
+
source: Path, destination: Path, *, expected: Mapping[str, Any] | None = None,
|
|
12432
|
+
) -> dict[str, Any]:
|
|
12433
|
+
source_inventory = _benchmark_study_v2_inventory(source)
|
|
12434
|
+
if expected is not None and source_inventory != dict(expected):
|
|
12435
|
+
raise ValueError("v2 staged candidate changed before the attempt copy")
|
|
12436
|
+
destination_inventory = _benchmark_study_v2_inventory(destination)
|
|
12437
|
+
if destination_inventory != source_inventory:
|
|
12438
|
+
raise ValueError("v2 candidate overlay copy differs from installed candidate")
|
|
12439
|
+
for item in source_inventory["files"]:
|
|
12440
|
+
if item["kind"] != "file":
|
|
12441
|
+
continue
|
|
12442
|
+
source_path = source / item["path"]
|
|
12443
|
+
destination_path = destination / item["path"]
|
|
12444
|
+
if source_path.stat().st_dev == destination_path.stat().st_dev and source_path.stat().st_ino == destination_path.stat().st_ino:
|
|
12445
|
+
raise ValueError("v2 candidate overlay contains a hardlink to the staged install")
|
|
12446
|
+
return destination_inventory
|
|
12447
|
+
|
|
12448
|
+
|
|
12449
|
+
def _benchmark_study_v2_expected_pre_workspace(
|
|
12450
|
+
task: TaskFixture, *, install_root: Path,
|
|
12451
|
+
expected_overlay_inventory: Mapping[str, Any],
|
|
12452
|
+
) -> dict[str, Any]:
|
|
12453
|
+
"""Re-materialize the bound cold workspace without invoking candidate code."""
|
|
12454
|
+
with tempfile.TemporaryDirectory(prefix="contextguard-v2-pre-workspace-") as temp:
|
|
12455
|
+
workspace = Path(temp).resolve()
|
|
12456
|
+
os.chmod(workspace, 0o700)
|
|
12457
|
+
reset_task_fixture_tree(task.fixture_tree_entries or (), workspace)
|
|
12458
|
+
overlay = workspace / BENCHMARK_STUDY_V2_OVERLAY_NAME
|
|
12459
|
+
shutil.copytree(
|
|
12460
|
+
install_root, overlay, symlinks=True, copy_function=shutil.copy2,
|
|
12461
|
+
)
|
|
12462
|
+
_benchmark_study_v2_verify_physical_copy(
|
|
12463
|
+
install_root, overlay, expected=expected_overlay_inventory,
|
|
12464
|
+
)
|
|
12465
|
+
return _benchmark_study_v2_inventory(workspace)
|
|
12466
|
+
|
|
12467
|
+
|
|
12468
|
+
def _benchmark_study_v2_settings() -> dict[str, dict[str, Any]]:
|
|
12469
|
+
common: dict[str, Any] = {
|
|
12470
|
+
"model": "sonnet",
|
|
12471
|
+
"permissions": {"allow": ["Bash", "Edit", "Glob", "Grep", "Read", "Write"]},
|
|
12472
|
+
}
|
|
12473
|
+
result = {"host_unmodified": json.loads(json.dumps(common))}
|
|
12474
|
+
for arm, suffix in (("legacy_trim", ""), ("bash_reference_v1", " --bash-reference-v1")):
|
|
12475
|
+
settings = json.loads(json.dumps(common))
|
|
12476
|
+
settings["hooks"] = {
|
|
12477
|
+
"PreToolUse": [{
|
|
12478
|
+
"matcher": "Bash",
|
|
12479
|
+
"hooks": [{
|
|
12480
|
+
"type": "command",
|
|
12481
|
+
"command": BENCHMARK_STUDY_V2_REWRITE_COMMAND + suffix,
|
|
12482
|
+
}],
|
|
12483
|
+
}],
|
|
12484
|
+
}
|
|
12485
|
+
result[arm] = settings
|
|
12486
|
+
return result
|
|
12487
|
+
|
|
12488
|
+
|
|
12489
|
+
def _benchmark_study_v2_write_settings(output_root: Path) -> dict[str, Any]:
|
|
12490
|
+
settings_root = output_root / "inputs" / "settings-v2"
|
|
12491
|
+
settings_root.mkdir(mode=0o700, parents=True)
|
|
12492
|
+
bindings: dict[str, Any] = {}
|
|
12493
|
+
for arm, value in _benchmark_study_v2_settings().items():
|
|
12494
|
+
path = settings_root / f"{arm}.settings.json"
|
|
12495
|
+
raw = _study_canonical_json_bytes(value)
|
|
12496
|
+
_measurement_write_exclusive(path, raw)
|
|
12497
|
+
bindings[arm] = {
|
|
12498
|
+
"path": str(path.resolve()), "sha256": _study_sha256_bytes(raw),
|
|
12499
|
+
"bytes": len(raw),
|
|
12500
|
+
"bash_hook": arm != "host_unmodified",
|
|
12501
|
+
"bash_reference_v1": arm == "bash_reference_v1",
|
|
12502
|
+
}
|
|
12503
|
+
return bindings
|
|
12504
|
+
|
|
12505
|
+
|
|
12506
|
+
def _benchmark_study_v2_install_candidate(
|
|
12507
|
+
*, npm_bin: str, candidate: Mapping[str, Any], output_root: Path,
|
|
12508
|
+
) -> tuple[Path, dict[str, Any]]:
|
|
12509
|
+
install_root = output_root / "candidate-install"
|
|
12510
|
+
install_root.mkdir(mode=0o700)
|
|
12511
|
+
isolated = output_root / "npm-isolation"
|
|
12512
|
+
isolated.mkdir(mode=0o700)
|
|
12513
|
+
for name in ("home", "cache", "tmp"):
|
|
12514
|
+
(isolated / name).mkdir(mode=0o700)
|
|
12515
|
+
root_package = next(
|
|
12516
|
+
record for record in candidate["packages"]
|
|
12517
|
+
if record["name"] == "@ictechgy/context-guard"
|
|
12518
|
+
)
|
|
12519
|
+
receipt_package = next(
|
|
12520
|
+
record for record in candidate["packages"]
|
|
12521
|
+
if record["name"] == "@ictechgy/context-guard-receipt"
|
|
12522
|
+
)
|
|
12523
|
+
npm_executable = executable_argv0(npm_bin)
|
|
12524
|
+
runtime_directories = [str(Path(npm_executable).parent)]
|
|
12525
|
+
node_executable = shutil.which("node")
|
|
12526
|
+
if node_executable is not None:
|
|
12527
|
+
runtime_directories.append(str(Path(node_executable).resolve().parent))
|
|
12528
|
+
runtime_directories.extend(os.defpath.split(os.pathsep))
|
|
12529
|
+
runtime_path = os.pathsep.join(dict.fromkeys(runtime_directories))
|
|
12530
|
+
argv = [
|
|
12531
|
+
npm_executable, "install", "--offline", "--ignore-scripts",
|
|
12532
|
+
"--no-audit", "--fund=false", "--package-lock=false",
|
|
12533
|
+
"--prefix", str(install_root.resolve()),
|
|
12534
|
+
str(root_package["path"]), str(receipt_package["path"]),
|
|
12535
|
+
]
|
|
12536
|
+
env = {
|
|
12537
|
+
"PATH": runtime_path, "HOME": str((isolated / "home").resolve()),
|
|
12538
|
+
"TMPDIR": str((isolated / "tmp").resolve()),
|
|
12539
|
+
"NPM_CONFIG_CACHE": str((isolated / "cache").resolve()),
|
|
12540
|
+
"NPM_CONFIG_OFFLINE": "true", "NPM_CONFIG_IGNORE_SCRIPTS": "true",
|
|
12541
|
+
"NPM_CONFIG_AUDIT": "false", "NPM_CONFIG_FUND": "false",
|
|
12542
|
+
"NPM_CONFIG_PACKAGE_LOCK": "false",
|
|
12543
|
+
"NPM_CONFIG_UPDATE_NOTIFIER": "false",
|
|
12544
|
+
"NPM_CONFIG_REGISTRY": "https://registry.invalid/",
|
|
12545
|
+
}
|
|
12546
|
+
result = run_bounded_command(
|
|
12547
|
+
argv, cwd=output_root, timeout_seconds=180,
|
|
12548
|
+
max_output_bytes=MEASUREMENT_CLI_PROBE_OUTPUT_MAX_BYTES, env=env,
|
|
12549
|
+
)
|
|
12550
|
+
if result.returncode != 0 or result.timed_out or result.output_truncated:
|
|
12551
|
+
raise ValueError("v2 candidate offline npm install failed")
|
|
12552
|
+
overlay_root = install_root / "node_modules"
|
|
12553
|
+
hidden_lock = overlay_root / ".package-lock.json"
|
|
12554
|
+
hidden_lock_removed = False
|
|
12555
|
+
try:
|
|
12556
|
+
hidden_lock_mode = os.lstat(hidden_lock).st_mode
|
|
12557
|
+
except FileNotFoundError:
|
|
12558
|
+
pass
|
|
12559
|
+
else:
|
|
12560
|
+
if not stat.S_ISREG(hidden_lock_mode):
|
|
12561
|
+
raise ValueError("v2 npm hidden lockfile has an unsupported type")
|
|
12562
|
+
hidden_lock.unlink()
|
|
12563
|
+
hidden_lock_removed = True
|
|
12564
|
+
inventory = _benchmark_study_v2_inventory(overlay_root)
|
|
12565
|
+
installed_packages = _benchmark_study_v2_verify_installed_packages(
|
|
12566
|
+
overlay_root, candidate,
|
|
12567
|
+
)
|
|
12568
|
+
receipt = {
|
|
12569
|
+
"schema_version": "contextguard.bench.candidate-install.v2",
|
|
12570
|
+
"install_count": 1,
|
|
12571
|
+
"network": "offline",
|
|
12572
|
+
"ignore_scripts": True,
|
|
12573
|
+
"no_audit": True,
|
|
12574
|
+
"fund": False,
|
|
12575
|
+
"hidden_lockfile_removed": hidden_lock_removed,
|
|
12576
|
+
"candidate_manifest_sha256": candidate["manifest_sha256"],
|
|
12577
|
+
"argv_policy": [
|
|
12578
|
+
"install", "--offline", "--ignore-scripts", "--no-audit",
|
|
12579
|
+
"--fund=false", "--package-lock=false",
|
|
12580
|
+
],
|
|
12581
|
+
"inventory": inventory,
|
|
12582
|
+
"installed_packages": installed_packages,
|
|
12583
|
+
}
|
|
12584
|
+
_study_write_private(output_root / "candidate-install-receipt.json", receipt)
|
|
12585
|
+
return overlay_root, receipt
|
|
12586
|
+
|
|
12587
|
+
|
|
12588
|
+
def prepare_benchmark_study_v2_executable(
|
|
12589
|
+
*, output_root: Path, plan_path: Path, tasks_path: Path, checkers_dir: Path,
|
|
12590
|
+
candidate_manifest_path: Path, candidate_checksum_path: Path | None,
|
|
12591
|
+
expected_candidate_hash: str, npm_bin: str, claude_bin: str,
|
|
12592
|
+
auth_home: Path, approved_source_commit: str,
|
|
12593
|
+
retained_ref: str | None, offline_rehearsal: bool,
|
|
12594
|
+
) -> dict[str, Any]:
|
|
12595
|
+
output_root = _benchmark_study_v2_output_root(output_root)
|
|
12596
|
+
if output_root.exists() and (
|
|
12597
|
+
output_root.is_symlink() or not output_root.is_dir() or any(output_root.iterdir())
|
|
12598
|
+
):
|
|
12599
|
+
raise ValueError("v2 prepare output root must be new or empty")
|
|
12600
|
+
# Candidate validation is deliberately first and performs no candidate import.
|
|
12601
|
+
candidate = verify_benchmark_study_v2_candidate(
|
|
12602
|
+
candidate_manifest_path, checksum_path=candidate_checksum_path,
|
|
12603
|
+
expected_manifest_sha256=expected_candidate_hash,
|
|
12604
|
+
expected_commit_sha=approved_source_commit,
|
|
12605
|
+
)
|
|
12606
|
+
if type(offline_rehearsal) is not bool:
|
|
12607
|
+
raise TypeError("v2 offline rehearsal flag must be boolean")
|
|
12608
|
+
if offline_rehearsal:
|
|
12609
|
+
if retained_ref is not None:
|
|
12610
|
+
raise ValueError("v2 offline rehearsal cannot bind a retained ref")
|
|
12611
|
+
source_ref_binding: dict[str, Any] = {
|
|
12612
|
+
"commit_sha": approved_source_commit,
|
|
12613
|
+
"ref": None,
|
|
12614
|
+
"repository": "ictechgy/context-guard",
|
|
12615
|
+
"verification": "offline-rehearsal-unverified-v1",
|
|
12616
|
+
}
|
|
12617
|
+
else:
|
|
12618
|
+
if retained_ref is None:
|
|
12619
|
+
raise ValueError("v2 live prepare requires a retained ref")
|
|
12620
|
+
source_ref_binding = _benchmark_study_v2_verify_retained_ref(
|
|
12621
|
+
retained_ref, approved_source_commit,
|
|
12622
|
+
)
|
|
12623
|
+
plan = load_benchmark_study_v2_plan(plan_path)
|
|
12624
|
+
corpus_bytes = _read_bytes_no_follow(tasks_path, max_bytes=MAX_FIXTURE_FILE_BYTES)
|
|
12625
|
+
checker_binding = benchmark_study_v2_checker_binding(checkers_dir)
|
|
12626
|
+
validate_benchmark_study_v2_bindings(
|
|
12627
|
+
plan, corpus_bytes=corpus_bytes, checker_binding=checker_binding,
|
|
12628
|
+
)
|
|
12629
|
+
tasks = parse_tasks(tasks_path)
|
|
12630
|
+
load_task_fixture_trees(tasks, task_file_dir=tasks_path.parent)
|
|
12631
|
+
task_definitions = [
|
|
12632
|
+
_study_task_manifest(task, tasks_path.parent) for task in tasks
|
|
12633
|
+
]
|
|
12634
|
+
task_ids = _benchmark_study_v2_task_ids_from_corpus(corpus_bytes)
|
|
12635
|
+
if [task.id for task in tasks] != task_ids:
|
|
12636
|
+
raise ValueError("v2 parsed task order differs from the bound corpus")
|
|
12637
|
+
cli_binding = _benchmark_study_v2_cli_binding(claude_bin)
|
|
12638
|
+
execution_environment = _benchmark_study_v2_execution_environment(cli_binding)
|
|
12639
|
+
_benchmark_study_v2_assert_execution_environment(execution_environment)
|
|
12640
|
+
_resolved_auth_home, auth_context = _benchmark_study_v2_auth_context(
|
|
12641
|
+
claude_bin, execution_environment, auth_home,
|
|
12642
|
+
)
|
|
12643
|
+
runner_binding = _benchmark_study_v2_runner_binding()
|
|
12644
|
+
schedule = generate_benchmark_study_v2_schedule(
|
|
12645
|
+
task_ids, repetitions=3, schedule_seed=plan["schedule_seed"],
|
|
12646
|
+
)
|
|
12647
|
+
slots = generate_benchmark_study_v2_slots(
|
|
12648
|
+
task_ids, schedule, candidate_hash=candidate["manifest_sha256"],
|
|
12649
|
+
namespace=BENCHMARK_STUDY_V2_NAMESPACE,
|
|
12650
|
+
)
|
|
12651
|
+
output_root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
12652
|
+
os.chmod(output_root, 0o700)
|
|
12653
|
+
settings = _benchmark_study_v2_write_settings(output_root)
|
|
12654
|
+
install_root, install_receipt = _benchmark_study_v2_install_candidate(
|
|
12655
|
+
npm_bin=npm_bin, candidate=candidate, output_root=output_root,
|
|
12656
|
+
)
|
|
12657
|
+
manifest = {
|
|
12658
|
+
"schema_version": BENCHMARK_STUDY_V2_EXEC_MANIFEST_SCHEMA_VERSION,
|
|
12659
|
+
"plan": plan,
|
|
12660
|
+
"plan_sha256": _study_sha256_bytes(_study_canonical_json_bytes(plan)),
|
|
12661
|
+
"inputs": {
|
|
12662
|
+
"plan_path": str(plan_path.resolve()),
|
|
12663
|
+
"tasks_path": str(tasks_path.resolve()),
|
|
12664
|
+
"tasks_sha256": _study_sha256_bytes(corpus_bytes),
|
|
12665
|
+
"task_definitions": task_definitions,
|
|
12666
|
+
"checkers_dir": str(checkers_dir.resolve()),
|
|
12667
|
+
"checker_binding": checker_binding,
|
|
12668
|
+
"cli_binding": cli_binding,
|
|
12669
|
+
"execution_environment": execution_environment,
|
|
12670
|
+
"auth_context": auth_context,
|
|
12671
|
+
"runner_binding": runner_binding,
|
|
12672
|
+
"canary_contract": _benchmark_study_v2_canary_contract(),
|
|
12673
|
+
"approved_source_commit": approved_source_commit,
|
|
12674
|
+
"source_ref_binding": source_ref_binding,
|
|
12675
|
+
"candidate": candidate,
|
|
12676
|
+
"candidate_install_root": str(install_root.resolve()),
|
|
12677
|
+
"candidate_install_receipt_sha256": _study_sha256_bytes(
|
|
12678
|
+
_study_canonical_json_bytes(install_receipt)
|
|
12679
|
+
),
|
|
12680
|
+
"candidate_overlay_inventory": install_receipt["inventory"],
|
|
12681
|
+
"settings": settings,
|
|
12682
|
+
"namespace": BENCHMARK_STUDY_V2_NAMESPACE,
|
|
12683
|
+
"task_ids": task_ids,
|
|
12684
|
+
"task_ids_sha256": _benchmark_study_v2_task_ids_sha256(task_ids),
|
|
12685
|
+
},
|
|
12686
|
+
"schedule": schedule,
|
|
12687
|
+
"slots": slots,
|
|
12688
|
+
"execution": {
|
|
12689
|
+
"identities": 216, "initial_calls": 108,
|
|
12690
|
+
"retry": "exactly_after_valid_initial_failure_v1",
|
|
12691
|
+
"resume": "never_replay_launched_identity_v1",
|
|
12692
|
+
"attempt_schema_version": BENCHMARK_STUDY_V2_ATTEMPT_SCHEMA_VERSION,
|
|
12693
|
+
"invalid_decision_schema_version": (
|
|
12694
|
+
BENCHMARK_STUDY_V2_INVALID_DECISION_SCHEMA_VERSION
|
|
12695
|
+
),
|
|
12696
|
+
"candidate_imported": False, "candidate_install_count": 1,
|
|
12697
|
+
"overlay_copy": "physical_copy_no_hardlinks_v1",
|
|
12698
|
+
},
|
|
12699
|
+
}
|
|
12700
|
+
_study_write_private(output_root / "study-manifest.json", manifest)
|
|
12701
|
+
return manifest
|
|
12702
|
+
|
|
12703
|
+
|
|
12704
|
+
def load_benchmark_study_v2_executable_manifest(output_root: Path, *, revalidate_external: bool) -> tuple[dict[str, Any], str]:
|
|
12705
|
+
output_root = _benchmark_study_v2_output_root(output_root)
|
|
12706
|
+
manifest_path = output_root / "study-manifest.json"
|
|
12707
|
+
if not manifest_path.exists():
|
|
12708
|
+
raise ValueError("v2 executable study requires a prepared output root")
|
|
12709
|
+
manifest, raw = _benchmark_study_v2_read_canonical(
|
|
12710
|
+
manifest_path, owner="v2 executable manifest", maximum=2_000_000,
|
|
12711
|
+
)
|
|
12712
|
+
if set(manifest) != {"schema_version", "plan", "plan_sha256", "inputs", "schedule", "slots", "execution"} or manifest.get("schema_version") != BENCHMARK_STUDY_V2_EXEC_MANIFEST_SCHEMA_VERSION:
|
|
12713
|
+
raise ValueError("v2 executable manifest schema mismatch")
|
|
12714
|
+
validate_benchmark_study_v2_plan(manifest["plan"])
|
|
12715
|
+
if manifest["plan_sha256"] != _study_sha256_bytes(_study_canonical_json_bytes(manifest["plan"])):
|
|
12716
|
+
raise ValueError("v2 executable plan binding mismatch")
|
|
12717
|
+
inputs = manifest["inputs"]
|
|
12718
|
+
required_inputs = {
|
|
12719
|
+
"plan_path", "tasks_path", "tasks_sha256", "task_definitions",
|
|
12720
|
+
"checkers_dir", "checker_binding", "cli_binding", "execution_environment",
|
|
12721
|
+
"auth_context", "runner_binding", "canary_contract", "candidate",
|
|
12722
|
+
"approved_source_commit",
|
|
12723
|
+
"source_ref_binding",
|
|
12724
|
+
"candidate_install_root", "candidate_install_receipt_sha256",
|
|
12725
|
+
"candidate_overlay_inventory", "settings", "namespace", "task_ids",
|
|
12726
|
+
"task_ids_sha256",
|
|
12727
|
+
}
|
|
12728
|
+
if set(inputs) != required_inputs:
|
|
12729
|
+
raise ValueError("v2 executable input schema mismatch")
|
|
12730
|
+
_benchmark_study_v2_validate_cli_binding(inputs["cli_binding"])
|
|
12731
|
+
_benchmark_study_v2_validate_execution_environment(
|
|
12732
|
+
inputs["execution_environment"]
|
|
12733
|
+
)
|
|
12734
|
+
_benchmark_study_v2_validate_auth_context(inputs["auth_context"])
|
|
12735
|
+
if inputs["runner_binding"] != _benchmark_study_v2_runner_binding():
|
|
12736
|
+
raise ValueError("v2 benchmark runner binding drift")
|
|
12737
|
+
if inputs["canary_contract"] != _benchmark_study_v2_canary_contract():
|
|
12738
|
+
raise ValueError("v2 canary contract binding mismatch")
|
|
12739
|
+
if (
|
|
12740
|
+
not isinstance(inputs["approved_source_commit"], str)
|
|
12741
|
+
or re.fullmatch(r"[0-9a-f]{40}", inputs["approved_source_commit"]) is None
|
|
12742
|
+
or inputs["candidate"].get("commit_sha")
|
|
12743
|
+
!= inputs["approved_source_commit"]
|
|
12744
|
+
):
|
|
12745
|
+
raise ValueError("v2 approved source revision binding mismatch")
|
|
12746
|
+
source_ref_binding = inputs["source_ref_binding"]
|
|
12747
|
+
if (
|
|
12748
|
+
not isinstance(source_ref_binding, Mapping)
|
|
12749
|
+
or set(source_ref_binding) != {
|
|
12750
|
+
"commit_sha", "ref", "repository", "verification",
|
|
12751
|
+
}
|
|
12752
|
+
or source_ref_binding.get("commit_sha")
|
|
12753
|
+
!= inputs["approved_source_commit"]
|
|
12754
|
+
or source_ref_binding.get("repository") != "ictechgy/context-guard"
|
|
12755
|
+
or source_ref_binding.get("verification") not in {
|
|
12756
|
+
"git-ls-remote-v1", "offline-rehearsal-unverified-v1",
|
|
12757
|
+
}
|
|
12758
|
+
or (
|
|
12759
|
+
source_ref_binding.get("verification") == "git-ls-remote-v1"
|
|
12760
|
+
and (
|
|
12761
|
+
not isinstance(source_ref_binding.get("ref"), str)
|
|
12762
|
+
or re.fullmatch(
|
|
12763
|
+
r"refs/heads/candidate/[A-Za-z0-9][A-Za-z0-9._/-]*",
|
|
12764
|
+
source_ref_binding["ref"],
|
|
12765
|
+
) is None
|
|
12766
|
+
)
|
|
12767
|
+
)
|
|
12768
|
+
or (
|
|
12769
|
+
source_ref_binding.get("verification")
|
|
12770
|
+
== "offline-rehearsal-unverified-v1"
|
|
12771
|
+
and source_ref_binding.get("ref") is not None
|
|
12772
|
+
)
|
|
12773
|
+
):
|
|
12774
|
+
raise ValueError("v2 retained source ref binding mismatch")
|
|
12775
|
+
if (
|
|
12776
|
+
not isinstance(inputs["task_definitions"], list)
|
|
12777
|
+
or len(inputs["task_definitions"]) != 12
|
|
12778
|
+
):
|
|
12779
|
+
raise ValueError("v2 executable task definition binding mismatch")
|
|
12780
|
+
task_ids = _benchmark_study_v2_task_ids(inputs["task_ids"])
|
|
12781
|
+
task_definition_ids = [
|
|
12782
|
+
item.get("id") if isinstance(item, Mapping) else None
|
|
12783
|
+
for item in inputs["task_definitions"]
|
|
12784
|
+
]
|
|
12785
|
+
if (
|
|
12786
|
+
inputs.get("namespace") != BENCHMARK_STUDY_V2_NAMESPACE
|
|
12787
|
+
or inputs.get("tasks_sha256") != manifest["plan"]["corpus_sha256"]
|
|
12788
|
+
or not isinstance(inputs.get("checker_binding"), Mapping)
|
|
12789
|
+
or inputs["checker_binding"].get("sha256")
|
|
12790
|
+
!= manifest["plan"]["checker_sha256"]
|
|
12791
|
+
or inputs.get("task_ids_sha256") != manifest["plan"]["task_ids_sha256"]
|
|
12792
|
+
or inputs.get("task_ids_sha256")
|
|
12793
|
+
!= _benchmark_study_v2_task_ids_sha256(task_ids)
|
|
12794
|
+
or task_definition_ids != task_ids
|
|
12795
|
+
):
|
|
12796
|
+
raise ValueError("v2 executable task order or namespace binding mismatch")
|
|
12797
|
+
if manifest["execution"] != {
|
|
12798
|
+
"identities": 216, "initial_calls": 108,
|
|
12799
|
+
"retry": "exactly_after_valid_initial_failure_v1",
|
|
12800
|
+
"resume": "never_replay_launched_identity_v1",
|
|
12801
|
+
"attempt_schema_version": BENCHMARK_STUDY_V2_ATTEMPT_SCHEMA_VERSION,
|
|
12802
|
+
"invalid_decision_schema_version": (
|
|
12803
|
+
BENCHMARK_STUDY_V2_INVALID_DECISION_SCHEMA_VERSION
|
|
12804
|
+
),
|
|
12805
|
+
"candidate_imported": False, "candidate_install_count": 1,
|
|
12806
|
+
"overlay_copy": "physical_copy_no_hardlinks_v1",
|
|
12807
|
+
}:
|
|
12808
|
+
raise ValueError("v2 executable lifecycle contract drift")
|
|
12809
|
+
settings = inputs.get("settings")
|
|
12810
|
+
expected_settings = _benchmark_study_v2_settings()
|
|
12811
|
+
if not isinstance(settings, Mapping) or set(settings) != set(BENCHMARK_STUDY_V2_ARMS):
|
|
12812
|
+
raise ValueError("v2 settings arm binding mismatch")
|
|
12813
|
+
for arm in BENCHMARK_STUDY_V2_ARMS:
|
|
12814
|
+
binding = settings[arm]
|
|
12815
|
+
expected_path = output_root / "inputs" / "settings-v2" / f"{arm}.settings.json"
|
|
12816
|
+
if (
|
|
12817
|
+
not isinstance(binding, Mapping)
|
|
12818
|
+
or set(binding) != {
|
|
12819
|
+
"path", "sha256", "bytes", "bash_hook", "bash_reference_v1",
|
|
12820
|
+
}
|
|
12821
|
+
or binding.get("path") != str(expected_path.resolve())
|
|
12822
|
+
or binding.get("bash_hook") is not (arm != "host_unmodified")
|
|
12823
|
+
or binding.get("bash_reference_v1") is not (arm == "bash_reference_v1")
|
|
12824
|
+
):
|
|
12825
|
+
raise ValueError(f"v2 {arm} settings binding mismatch")
|
|
12826
|
+
raw_settings = _read_bytes_no_follow(expected_path, max_bytes=100_000)
|
|
12827
|
+
expected_raw = _study_canonical_json_bytes(expected_settings[arm])
|
|
12828
|
+
if (
|
|
12829
|
+
raw_settings != expected_raw
|
|
12830
|
+
or binding.get("sha256") != _study_sha256_bytes(expected_raw)
|
|
12831
|
+
or binding.get("bytes") != len(expected_raw)
|
|
12832
|
+
):
|
|
12833
|
+
raise ValueError(f"v2 {arm} settings policy drift")
|
|
12834
|
+
install_receipt, install_receipt_raw = _benchmark_study_v2_read_canonical(
|
|
12835
|
+
output_root / "candidate-install-receipt.json",
|
|
12836
|
+
owner="v2 candidate install receipt", maximum=2_000_000,
|
|
12837
|
+
)
|
|
12838
|
+
if (
|
|
12839
|
+
set(install_receipt) != {
|
|
12840
|
+
"schema_version", "install_count", "network", "ignore_scripts",
|
|
12841
|
+
"no_audit", "fund", "hidden_lockfile_removed",
|
|
12842
|
+
"candidate_manifest_sha256", "argv_policy", "inventory",
|
|
12843
|
+
"installed_packages",
|
|
12844
|
+
}
|
|
12845
|
+
or install_receipt.get("schema_version")
|
|
12846
|
+
!= "contextguard.bench.candidate-install.v2"
|
|
12847
|
+
or install_receipt.get("install_count") != 1
|
|
12848
|
+
or install_receipt.get("network") != "offline"
|
|
12849
|
+
or install_receipt.get("ignore_scripts") is not True
|
|
12850
|
+
or install_receipt.get("no_audit") is not True
|
|
12851
|
+
or install_receipt.get("fund") is not False
|
|
12852
|
+
or not isinstance(install_receipt.get("hidden_lockfile_removed"), bool)
|
|
12853
|
+
or install_receipt.get("argv_policy") != [
|
|
12854
|
+
"install", "--offline", "--ignore-scripts", "--no-audit",
|
|
12855
|
+
"--fund=false", "--package-lock=false",
|
|
12856
|
+
]
|
|
12857
|
+
or _study_sha256_bytes(install_receipt_raw)
|
|
12858
|
+
!= inputs.get("candidate_install_receipt_sha256")
|
|
12859
|
+
or install_receipt.get("candidate_manifest_sha256")
|
|
12860
|
+
!= inputs.get("candidate", {}).get("manifest_sha256")
|
|
12861
|
+
or install_receipt.get("inventory")
|
|
12862
|
+
!= inputs.get("candidate_overlay_inventory")
|
|
12863
|
+
):
|
|
12864
|
+
raise ValueError("v2 candidate install receipt binding mismatch")
|
|
12865
|
+
expected_schedule = generate_benchmark_study_v2_schedule(
|
|
12866
|
+
task_ids, repetitions=3, schedule_seed=manifest["plan"]["schedule_seed"],
|
|
12867
|
+
)
|
|
12868
|
+
expected_slots = generate_benchmark_study_v2_slots(
|
|
12869
|
+
task_ids, expected_schedule,
|
|
12870
|
+
candidate_hash=inputs["candidate"]["manifest_sha256"],
|
|
12871
|
+
namespace=inputs["namespace"],
|
|
12872
|
+
)
|
|
12873
|
+
if manifest["schedule"] != expected_schedule or manifest["slots"] != expected_slots:
|
|
12874
|
+
raise ValueError("v2 executable schedule or identity drift")
|
|
12875
|
+
if revalidate_external:
|
|
12876
|
+
_benchmark_study_v2_assert_execution_environment(
|
|
12877
|
+
inputs["execution_environment"]
|
|
12878
|
+
)
|
|
12879
|
+
external_plan = load_benchmark_study_v2_plan(Path(inputs["plan_path"]))
|
|
12880
|
+
if external_plan != manifest["plan"]:
|
|
12881
|
+
raise ValueError("v2 external study plan binding drift")
|
|
12882
|
+
corpus = _read_bytes_no_follow(Path(inputs["tasks_path"]), max_bytes=MAX_FIXTURE_FILE_BYTES)
|
|
12883
|
+
if _study_sha256_bytes(corpus) != inputs["tasks_sha256"]:
|
|
12884
|
+
raise ValueError("v2 task corpus binding drift")
|
|
12885
|
+
checker = benchmark_study_v2_checker_binding(Path(inputs["checkers_dir"]))
|
|
12886
|
+
if checker != inputs["checker_binding"]:
|
|
12887
|
+
raise ValueError("v2 checker inventory binding drift")
|
|
12888
|
+
validate_benchmark_study_v2_bindings(
|
|
12889
|
+
manifest["plan"], corpus_bytes=corpus, checker_binding=checker,
|
|
12890
|
+
)
|
|
12891
|
+
tasks = parse_tasks(Path(inputs["tasks_path"]))
|
|
12892
|
+
load_task_fixture_trees(
|
|
12893
|
+
tasks, task_file_dir=Path(inputs["tasks_path"]).parent,
|
|
12894
|
+
)
|
|
12895
|
+
task_definitions = [
|
|
12896
|
+
_study_task_manifest(task, Path(inputs["tasks_path"]).parent)
|
|
12897
|
+
for task in tasks
|
|
12898
|
+
]
|
|
12899
|
+
if task_definitions != inputs["task_definitions"]:
|
|
12900
|
+
raise ValueError("v2 task fixture or checker binding drift")
|
|
12901
|
+
if [task.id for task in tasks] != task_ids:
|
|
12902
|
+
raise ValueError("v2 external task order binding drift")
|
|
12903
|
+
candidate = verify_benchmark_study_v2_candidate(
|
|
12904
|
+
Path(inputs["candidate"]["manifest_path"]),
|
|
12905
|
+
checksum_path=Path(inputs["candidate"]["checksum_path"]),
|
|
12906
|
+
expected_manifest_sha256=inputs["candidate"]["manifest_sha256"],
|
|
12907
|
+
expected_commit_sha=inputs["approved_source_commit"],
|
|
12908
|
+
)
|
|
12909
|
+
if candidate != inputs["candidate"]:
|
|
12910
|
+
raise ValueError("v2 candidate binding drift")
|
|
12911
|
+
if _benchmark_study_v2_verify_installed_packages(
|
|
12912
|
+
Path(inputs["candidate_install_root"]), candidate,
|
|
12913
|
+
) != install_receipt["installed_packages"]:
|
|
12914
|
+
raise ValueError("v2 installed candidate package binding drift")
|
|
12915
|
+
if _benchmark_study_v2_inventory(Path(inputs["candidate_install_root"])) != inputs["candidate_overlay_inventory"]:
|
|
12916
|
+
raise ValueError("v2 installed candidate inventory drift")
|
|
12917
|
+
for arm, binding in inputs["settings"].items():
|
|
12918
|
+
raw_settings = _read_bytes_no_follow(Path(binding["path"]), max_bytes=100_000)
|
|
12919
|
+
if _study_sha256_bytes(raw_settings) != binding["sha256"] or len(raw_settings) != binding["bytes"]:
|
|
12920
|
+
raise ValueError(f"v2 {arm} settings drift")
|
|
12921
|
+
return manifest, _study_sha256_bytes(raw)
|
|
12922
|
+
|
|
12923
|
+
|
|
12924
|
+
def _benchmark_study_v2_variants(manifest: Mapping[str, Any], output_root: Path) -> dict[str, Variant]:
|
|
12925
|
+
output_root = _benchmark_study_v2_output_root(output_root)
|
|
12926
|
+
result: dict[str, Variant] = {}
|
|
12927
|
+
artifact_root = output_root / "artifacts"
|
|
12928
|
+
for arm in BENCHMARK_STUDY_V2_ARMS:
|
|
12929
|
+
binding = manifest["inputs"]["settings"][arm]
|
|
12930
|
+
settings_path = Path(binding["path"])
|
|
12931
|
+
settings_raw = _read_bytes_no_follow(settings_path, max_bytes=100_000)
|
|
12932
|
+
settings_payload = json.loads(settings_raw)
|
|
12933
|
+
command = BENCHMARK_STUDY_V2_REWRITE_COMMAND + (
|
|
12934
|
+
" --bash-reference-v1" if arm == "bash_reference_v1" else ""
|
|
12935
|
+
)
|
|
12936
|
+
registered = () if arm == "host_unmodified" else (("PreToolUse", command),)
|
|
12937
|
+
identity = MeasurementIdentity(
|
|
12938
|
+
candidate_hash=manifest["inputs"]["candidate"]["manifest_sha256"],
|
|
12939
|
+
repetition=0, arm=arm, attempt=0,
|
|
12940
|
+
namespace=manifest["inputs"]["namespace"],
|
|
12941
|
+
)
|
|
12942
|
+
result[arm] = Variant(
|
|
12943
|
+
name=arm,
|
|
12944
|
+
measurement=MeasurementVariant(
|
|
12945
|
+
settings_file=settings_path,
|
|
12946
|
+
setting_sources=("project",), environment_allow=(),
|
|
12947
|
+
environment_overrides=tuple(
|
|
12948
|
+
(name, str(manifest["inputs"]["execution_environment"]["values"][name]))
|
|
12949
|
+
for name in sorted(
|
|
12950
|
+
manifest["inputs"]["execution_environment"]["values"]
|
|
12951
|
+
)
|
|
12952
|
+
),
|
|
12953
|
+
workspace_mode="isolated",
|
|
12954
|
+
session_mode="isolated", session_persistence="disabled",
|
|
12955
|
+
hook_events_enabled=True, registered_bindings=registered,
|
|
12956
|
+
required_event_classes=(), pair_registered_bindings=registered,
|
|
12957
|
+
cli_capabilities=(
|
|
12958
|
+
"--settings", "--setting-sources", "--include-hook-events",
|
|
12959
|
+
"--no-session-persistence", "stream-json",
|
|
12960
|
+
),
|
|
12961
|
+
identity=identity, artifact_root=artifact_root,
|
|
12962
|
+
settings_payload=settings_payload, settings_source_bytes=settings_raw,
|
|
12963
|
+
),
|
|
12964
|
+
)
|
|
12965
|
+
return result
|
|
12966
|
+
|
|
12967
|
+
|
|
12968
|
+
def _benchmark_study_v2_canary_variants(
|
|
12969
|
+
manifest: Mapping[str, Any], output_root: Path,
|
|
12970
|
+
) -> dict[str, Variant]:
|
|
12971
|
+
analytic = _benchmark_study_v2_variants(manifest, output_root)
|
|
12972
|
+
result: dict[str, Variant] = {}
|
|
12973
|
+
for arm in BENCHMARK_STUDY_V2_CANARY_ARMS:
|
|
12974
|
+
base = analytic[arm].measurement
|
|
12975
|
+
assert base is not None
|
|
12976
|
+
identity = MeasurementIdentity(
|
|
12977
|
+
candidate_hash=manifest["inputs"]["candidate"]["manifest_sha256"],
|
|
12978
|
+
repetition=0, arm=arm, attempt=0,
|
|
12979
|
+
namespace=f"{manifest['inputs']['namespace']}.canary",
|
|
12980
|
+
)
|
|
12981
|
+
result[arm] = Variant(
|
|
12982
|
+
name=arm,
|
|
12983
|
+
measurement=replace(
|
|
12984
|
+
base,
|
|
12985
|
+
required_event_classes=("PreToolUse",),
|
|
12986
|
+
identity=identity,
|
|
12987
|
+
artifact_root=output_root / "canary-artifacts",
|
|
12988
|
+
),
|
|
12989
|
+
)
|
|
12990
|
+
return result
|
|
12991
|
+
|
|
12992
|
+
|
|
12993
|
+
def _benchmark_study_v2_canary_base_event(
|
|
12994
|
+
*, arm: str, run_id: str, manifest_sha256: str, state: str,
|
|
12995
|
+
**extra: Any,
|
|
12996
|
+
) -> dict[str, Any]:
|
|
12997
|
+
event = {
|
|
12998
|
+
"schema_version": BENCHMARK_STUDY_V2_CANARY_EVENT_SCHEMA_VERSION,
|
|
12999
|
+
"manifest_sha256": manifest_sha256,
|
|
13000
|
+
"arm": arm, "run_id": run_id, "state": state,
|
|
13001
|
+
}
|
|
13002
|
+
event.update(extra)
|
|
13003
|
+
return event
|
|
13004
|
+
|
|
13005
|
+
|
|
13006
|
+
def _benchmark_study_v2_read_canary_events(
|
|
13007
|
+
path: Path, *, manifest_sha256: str,
|
|
13008
|
+
variants: Mapping[str, Variant],
|
|
13009
|
+
) -> list[dict[str, Any]]:
|
|
13010
|
+
if not path.exists():
|
|
13011
|
+
return []
|
|
13012
|
+
raw = _measurement_read_private_file(path, maximum=200_000)
|
|
13013
|
+
expected_run_ids = {}
|
|
13014
|
+
for arm, variant in variants.items():
|
|
13015
|
+
spec = variant.measurement
|
|
13016
|
+
assert spec is not None
|
|
13017
|
+
expected_run_ids[arm] = spec.identity.run_id(BENCHMARK_STUDY_V2_CANARY_TASK_ID)
|
|
13018
|
+
base_keys = {"schema_version", "manifest_sha256", "arm", "run_id", "state"}
|
|
13019
|
+
workspace_keys = base_keys | {
|
|
13020
|
+
"pre_workspace_inventory_sha256", "pre_overlay_inventory_sha256",
|
|
13021
|
+
}
|
|
13022
|
+
terminal_keys = base_keys | {
|
|
13023
|
+
"passed", "measurement_terminal_status", "checker_status",
|
|
13024
|
+
"receipt_sha256", "pre_workspace_inventory_sha256",
|
|
13025
|
+
"post_workspace_inventory_sha256", "pre_overlay_inventory_sha256",
|
|
13026
|
+
"post_overlay_inventory_sha256", "pretooluse_event_count",
|
|
13027
|
+
}
|
|
13028
|
+
states: dict[str, list[str]] = collections.defaultdict(list)
|
|
13029
|
+
rows: list[dict[str, Any]] = []
|
|
13030
|
+
for line in raw.splitlines():
|
|
13031
|
+
try:
|
|
13032
|
+
row = json.loads(
|
|
13033
|
+
line.decode("utf-8"),
|
|
13034
|
+
object_pairs_hook=_measurement_object_no_duplicates,
|
|
13035
|
+
parse_constant=_stream_reject_nonfinite,
|
|
13036
|
+
)
|
|
13037
|
+
except (UnicodeDecodeError, json.JSONDecodeError, ValueError) as exc:
|
|
13038
|
+
raise ValueError("v2 canary event ledger is invalid JSONL") from exc
|
|
13039
|
+
if not isinstance(row, dict) or line + b"\n" != _study_canonical_json_bytes(row):
|
|
13040
|
+
raise ValueError("v2 canary event ledger must be canonical JSONL")
|
|
13041
|
+
arm = row.get("arm")
|
|
13042
|
+
state = row.get("state")
|
|
13043
|
+
if (
|
|
13044
|
+
row.get("schema_version") != BENCHMARK_STUDY_V2_CANARY_EVENT_SCHEMA_VERSION
|
|
13045
|
+
or row.get("manifest_sha256") != manifest_sha256
|
|
13046
|
+
or arm not in BENCHMARK_STUDY_V2_CANARY_ARMS
|
|
13047
|
+
or row.get("run_id") != expected_run_ids.get(str(arm))
|
|
13048
|
+
or state not in {"launch_reserved", "workspace_prepared", "launched", "terminal"}
|
|
13049
|
+
):
|
|
13050
|
+
raise ValueError("v2 canary event binding mismatch")
|
|
13051
|
+
expected_keys = (
|
|
13052
|
+
workspace_keys if state == "workspace_prepared"
|
|
13053
|
+
else terminal_keys if state == "terminal" else base_keys
|
|
13054
|
+
)
|
|
13055
|
+
if set(row) != expected_keys:
|
|
13056
|
+
raise ValueError("v2 canary event schema mismatch")
|
|
13057
|
+
previous = states[str(arm)]
|
|
13058
|
+
expected_previous = {
|
|
13059
|
+
"launch_reserved": [],
|
|
13060
|
+
"workspace_prepared": ["launch_reserved"],
|
|
13061
|
+
"launched": ["launch_reserved", "workspace_prepared"],
|
|
13062
|
+
"terminal": ["launch_reserved", "workspace_prepared", "launched"],
|
|
13063
|
+
}[str(state)]
|
|
13064
|
+
if previous != expected_previous:
|
|
13065
|
+
raise ValueError("v2 canary event state transition is invalid")
|
|
13066
|
+
if state in {"workspace_prepared", "terminal"}:
|
|
13067
|
+
hash_fields = (
|
|
13068
|
+
("pre_workspace_inventory_sha256", "pre_overlay_inventory_sha256")
|
|
13069
|
+
if state == "workspace_prepared" else (
|
|
13070
|
+
"receipt_sha256", "pre_workspace_inventory_sha256",
|
|
13071
|
+
"post_workspace_inventory_sha256", "pre_overlay_inventory_sha256",
|
|
13072
|
+
"post_overlay_inventory_sha256",
|
|
13073
|
+
)
|
|
13074
|
+
)
|
|
13075
|
+
if any(
|
|
13076
|
+
not isinstance(row.get(field), str)
|
|
13077
|
+
or SHA256_HEX_PATTERN.fullmatch(str(row[field])) is None
|
|
13078
|
+
for field in hash_fields
|
|
13079
|
+
):
|
|
13080
|
+
raise ValueError("v2 canary evidence hash is invalid")
|
|
13081
|
+
if state == "terminal" and (
|
|
13082
|
+
not isinstance(row.get("passed"), bool)
|
|
13083
|
+
or row.get("measurement_terminal_status") not in {
|
|
13084
|
+
"success", "raw_byte_limit", "raw_line_limit", "raw_line_byte_limit",
|
|
13085
|
+
"process_timeout", "process_launch_error", "process_error",
|
|
13086
|
+
"terminal_error", "missing_terminal", "invalid_stream",
|
|
13087
|
+
"hook_payload_limit", "hook_lifecycle_limit", "invalid_hook_lifecycle",
|
|
13088
|
+
"unexpected_hook_event_class", "missing_required_hook_event_class",
|
|
13089
|
+
"hook_process_failure",
|
|
13090
|
+
}
|
|
13091
|
+
or row.get("checker_status") not in {
|
|
13092
|
+
"task_success", "valid_task_failure_v1", "success_checker_infra_invalid",
|
|
13093
|
+
"not_run",
|
|
13094
|
+
}
|
|
13095
|
+
or isinstance(row.get("pretooluse_event_count"), bool)
|
|
13096
|
+
or not isinstance(row.get("pretooluse_event_count"), int)
|
|
13097
|
+
or row["pretooluse_event_count"] < 0
|
|
13098
|
+
):
|
|
13099
|
+
raise ValueError("v2 canary terminal evidence is invalid")
|
|
13100
|
+
previous.append(str(state))
|
|
13101
|
+
rows.append(row)
|
|
13102
|
+
return rows
|
|
13103
|
+
|
|
13104
|
+
|
|
13105
|
+
def _benchmark_study_v2_derive_canary_terminal(
|
|
13106
|
+
*, manifest: Mapping[str, Any], manifest_sha256: str, output_root: Path,
|
|
13107
|
+
arm: str, variant: Variant, task: TaskFixture,
|
|
13108
|
+
workspace_event: Mapping[str, Any],
|
|
13109
|
+
) -> dict[str, Any]:
|
|
13110
|
+
spec = variant.measurement
|
|
13111
|
+
assert spec is not None
|
|
13112
|
+
run_id = spec.identity.run_id(task.id)
|
|
13113
|
+
receipt = _verify_existing_measurement_run(
|
|
13114
|
+
spec, task.id, run_id, require_index=True,
|
|
13115
|
+
)
|
|
13116
|
+
context = _measurement_existing_context(spec, run_id)
|
|
13117
|
+
receipt_raw = _measurement_read_private_file(context.receipt_path)
|
|
13118
|
+
expected_overlay = manifest["inputs"]["candidate_overlay_inventory"]
|
|
13119
|
+
expected_pre = _benchmark_study_v2_expected_pre_workspace(
|
|
13120
|
+
task,
|
|
13121
|
+
install_root=Path(manifest["inputs"]["candidate_install_root"]),
|
|
13122
|
+
expected_overlay_inventory=expected_overlay,
|
|
13123
|
+
)
|
|
13124
|
+
post_workspace = _benchmark_study_v2_inventory(context.workspace)
|
|
13125
|
+
post_overlay = _benchmark_study_v2_inventory(
|
|
13126
|
+
context.workspace / BENCHMARK_STUDY_V2_OVERLAY_NAME,
|
|
13127
|
+
)
|
|
13128
|
+
checker = (
|
|
13129
|
+
run_task_checker_study(
|
|
13130
|
+
task, context.workspace, env={},
|
|
13131
|
+
interpreter_binding=manifest["inputs"]["runner_binding"]["python"],
|
|
13132
|
+
)
|
|
13133
|
+
if receipt["terminal_status"] == "success" else "not_run"
|
|
13134
|
+
)
|
|
13135
|
+
summary = receipt["hook_summary"]
|
|
13136
|
+
counts = {
|
|
13137
|
+
row["hook_event"]: row["count"]
|
|
13138
|
+
for row in summary["event_class_counts"]
|
|
13139
|
+
}
|
|
13140
|
+
pretooluse_count = int(counts.get("PreToolUse", 0))
|
|
13141
|
+
pre_bound = bool(
|
|
13142
|
+
workspace_event.get("pre_workspace_inventory_sha256") == expected_pre["sha256"]
|
|
13143
|
+
and workspace_event.get("pre_overlay_inventory_sha256") == expected_overlay["sha256"]
|
|
13144
|
+
)
|
|
13145
|
+
hooks_valid = bool(
|
|
13146
|
+
summary["required_event_classes"] == ["PreToolUse"]
|
|
13147
|
+
and pretooluse_count >= 1
|
|
13148
|
+
and set(counts) == {"PreToolUse"}
|
|
13149
|
+
and all(
|
|
13150
|
+
hook["hook_event"] == "PreToolUse"
|
|
13151
|
+
and hook["hook_process_outcome"] == "success"
|
|
13152
|
+
and hook["hook_process_exit_code"] in (None, 0)
|
|
13153
|
+
for hook in receipt["hooks"]
|
|
13154
|
+
)
|
|
13155
|
+
)
|
|
13156
|
+
passed = bool(
|
|
13157
|
+
receipt["terminal_status"] == "success"
|
|
13158
|
+
and checker == "task_success" and hooks_valid and pre_bound
|
|
13159
|
+
and post_overlay == expected_overlay
|
|
13160
|
+
)
|
|
13161
|
+
return _benchmark_study_v2_canary_base_event(
|
|
13162
|
+
arm=arm, run_id=run_id, manifest_sha256=manifest_sha256,
|
|
13163
|
+
state="terminal", passed=passed,
|
|
13164
|
+
measurement_terminal_status=receipt["terminal_status"],
|
|
13165
|
+
checker_status=checker,
|
|
13166
|
+
receipt_sha256=_study_sha256_bytes(receipt_raw),
|
|
13167
|
+
pre_workspace_inventory_sha256=str(
|
|
13168
|
+
workspace_event["pre_workspace_inventory_sha256"]
|
|
13169
|
+
),
|
|
13170
|
+
post_workspace_inventory_sha256=post_workspace["sha256"],
|
|
13171
|
+
pre_overlay_inventory_sha256=str(
|
|
13172
|
+
workspace_event["pre_overlay_inventory_sha256"]
|
|
13173
|
+
),
|
|
13174
|
+
post_overlay_inventory_sha256=post_overlay["sha256"],
|
|
13175
|
+
pretooluse_event_count=pretooluse_count,
|
|
13176
|
+
)
|
|
13177
|
+
|
|
13178
|
+
|
|
13179
|
+
def _benchmark_study_v2_expected_canary_evidence(
|
|
13180
|
+
*, manifest: Mapping[str, Any], manifest_sha256: str, output_root: Path,
|
|
13181
|
+
rows: Sequence[Mapping[str, Any]], variants: Mapping[str, Variant],
|
|
13182
|
+
) -> dict[str, Any]:
|
|
13183
|
+
task = _benchmark_study_v2_canary_task()
|
|
13184
|
+
by_arm: dict[str, list[Mapping[str, Any]]] = {
|
|
13185
|
+
arm: [row for row in rows if row["arm"] == arm]
|
|
13186
|
+
for arm in BENCHMARK_STUDY_V2_CANARY_ARMS
|
|
13187
|
+
}
|
|
13188
|
+
records: list[dict[str, Any]] = []
|
|
13189
|
+
for arm in BENCHMARK_STUDY_V2_CANARY_ARMS:
|
|
13190
|
+
arm_rows = by_arm[arm]
|
|
13191
|
+
if [row["state"] for row in arm_rows] != [
|
|
13192
|
+
"launch_reserved", "workspace_prepared", "launched", "terminal",
|
|
13193
|
+
]:
|
|
13194
|
+
raise ValueError("v2 canary evidence is incomplete")
|
|
13195
|
+
recomputed = _benchmark_study_v2_derive_canary_terminal(
|
|
13196
|
+
manifest=manifest, manifest_sha256=manifest_sha256,
|
|
13197
|
+
output_root=output_root, arm=arm, variant=variants[arm], task=task,
|
|
13198
|
+
workspace_event=arm_rows[1],
|
|
13199
|
+
)
|
|
13200
|
+
if recomputed != dict(arm_rows[-1]) or recomputed["passed"] is not True:
|
|
13201
|
+
raise ValueError("v2 canary terminal evidence did not pass")
|
|
13202
|
+
spec = variants[arm].measurement
|
|
13203
|
+
assert spec is not None
|
|
13204
|
+
records.append({
|
|
13205
|
+
"arm": arm, "run_id": recomputed["run_id"], "passed": True,
|
|
13206
|
+
"settings_sha256": manifest["inputs"]["settings"][arm]["sha256"],
|
|
13207
|
+
"settings_binding_set_sha256": _measurement_binding_set_sha256(
|
|
13208
|
+
spec.registered_bindings
|
|
13209
|
+
),
|
|
13210
|
+
"required_event_classes": ["PreToolUse"],
|
|
13211
|
+
"pretooluse_event_count": recomputed["pretooluse_event_count"],
|
|
13212
|
+
"receipt_sha256": recomputed["receipt_sha256"],
|
|
13213
|
+
"pre_overlay_inventory_sha256": recomputed[
|
|
13214
|
+
"pre_overlay_inventory_sha256"
|
|
13215
|
+
],
|
|
13216
|
+
"post_overlay_inventory_sha256": recomputed[
|
|
13217
|
+
"post_overlay_inventory_sha256"
|
|
13218
|
+
],
|
|
13219
|
+
})
|
|
13220
|
+
return {
|
|
13221
|
+
"schema_version": BENCHMARK_STUDY_V2_CANARY_EVIDENCE_SCHEMA_VERSION,
|
|
13222
|
+
"manifest_sha256": manifest_sha256,
|
|
13223
|
+
"canary_contract_sha256": _study_domain_hash(
|
|
13224
|
+
"contextguard.bench.v2.canary-contract.v1",
|
|
13225
|
+
manifest["inputs"]["canary_contract"],
|
|
13226
|
+
),
|
|
13227
|
+
"cli_binding_sha256": _study_domain_hash(
|
|
13228
|
+
"contextguard.bench.v2.cli-binding.v1",
|
|
13229
|
+
manifest["inputs"]["cli_binding"],
|
|
13230
|
+
),
|
|
13231
|
+
"auth_context_sha256": _study_domain_hash(
|
|
13232
|
+
"contextguard.bench.v2.auth-context.v1",
|
|
13233
|
+
manifest["inputs"]["auth_context"],
|
|
13234
|
+
),
|
|
13235
|
+
"candidate_manifest_sha256": manifest["inputs"]["candidate"][
|
|
13236
|
+
"manifest_sha256"
|
|
13237
|
+
],
|
|
13238
|
+
"candidate_overlay_sha256": manifest["inputs"][
|
|
13239
|
+
"candidate_overlay_inventory"
|
|
13240
|
+
]["sha256"],
|
|
13241
|
+
"discarded": True, "excluded_from_analysis": True,
|
|
13242
|
+
"provider_calls": 2, "arms": records,
|
|
13243
|
+
}
|
|
13244
|
+
|
|
13245
|
+
|
|
13246
|
+
def _benchmark_study_v2_verify_canary_evidence(
|
|
13247
|
+
*, manifest: Mapping[str, Any], manifest_sha256: str, output_root: Path,
|
|
13248
|
+
) -> tuple[dict[str, Any], str]:
|
|
13249
|
+
variants = _benchmark_study_v2_canary_variants(manifest, output_root)
|
|
13250
|
+
rows = _benchmark_study_v2_read_canary_events(
|
|
13251
|
+
output_root / "canary-events.jsonl",
|
|
13252
|
+
manifest_sha256=manifest_sha256, variants=variants,
|
|
13253
|
+
)
|
|
13254
|
+
expected = _benchmark_study_v2_expected_canary_evidence(
|
|
13255
|
+
manifest=manifest, manifest_sha256=manifest_sha256,
|
|
13256
|
+
output_root=output_root, rows=rows, variants=variants,
|
|
13257
|
+
)
|
|
13258
|
+
observed, raw = _benchmark_study_v2_read_canonical(
|
|
13259
|
+
output_root / "canary-evidence.json",
|
|
13260
|
+
owner="v2 canary evidence", maximum=200_000,
|
|
13261
|
+
)
|
|
13262
|
+
if observed != expected:
|
|
13263
|
+
raise ValueError("v2 canary evidence binding mismatch")
|
|
13264
|
+
return observed, _study_sha256_bytes(raw)
|
|
13265
|
+
|
|
13266
|
+
|
|
13267
|
+
def _benchmark_study_v2_run_canary_arm(
|
|
13268
|
+
*, manifest: Mapping[str, Any], manifest_sha256: str, output_root: Path,
|
|
13269
|
+
arm: str, variant: Variant, task: TaskFixture, claude_bin: str,
|
|
13270
|
+
cli_stat_guard: Mapping[str, Any],
|
|
13271
|
+
runtime_stat_guards: Mapping[str, Mapping[str, int | str]],
|
|
13272
|
+
auth_home: Path,
|
|
13273
|
+
) -> dict[str, Any]:
|
|
13274
|
+
spec = variant.measurement
|
|
13275
|
+
assert spec is not None
|
|
13276
|
+
run_id = spec.identity.run_id(task.id)
|
|
13277
|
+
ledger_path = output_root / "canary-events.jsonl"
|
|
13278
|
+
_benchmark_study_v2_assert_cli_executable_bytes(
|
|
13279
|
+
manifest["inputs"]["cli_binding"], cli_stat_guard,
|
|
13280
|
+
)
|
|
13281
|
+
_benchmark_study_v2_assert_runtime_stat_guards(
|
|
13282
|
+
manifest["inputs"]["execution_environment"], runtime_stat_guards,
|
|
13283
|
+
)
|
|
13284
|
+
resolved_auth_home = _benchmark_study_v2_assert_auth_context(
|
|
13285
|
+
claude_bin, manifest["inputs"]["execution_environment"],
|
|
13286
|
+
manifest["inputs"]["auth_context"], auth_home,
|
|
13287
|
+
)
|
|
13288
|
+
append_study_attempt_event(
|
|
13289
|
+
ledger_path,
|
|
13290
|
+
_benchmark_study_v2_canary_base_event(
|
|
13291
|
+
arm=arm, run_id=run_id, manifest_sha256=manifest_sha256,
|
|
13292
|
+
state="launch_reserved",
|
|
13293
|
+
),
|
|
13294
|
+
)
|
|
13295
|
+
pre: dict[str, Any] = {}
|
|
13296
|
+
|
|
13297
|
+
def prepared(workspace: Path) -> None:
|
|
13298
|
+
overlay = workspace / BENCHMARK_STUDY_V2_OVERLAY_NAME
|
|
13299
|
+
overlay_inventory = _benchmark_study_v2_verify_physical_copy(
|
|
13300
|
+
Path(manifest["inputs"]["candidate_install_root"]), overlay,
|
|
13301
|
+
expected=manifest["inputs"]["candidate_overlay_inventory"],
|
|
13302
|
+
)
|
|
13303
|
+
workspace_inventory = _benchmark_study_v2_inventory(workspace)
|
|
13304
|
+
pre.update({"overlay": overlay_inventory, "workspace": workspace_inventory})
|
|
13305
|
+
append_study_attempt_event(
|
|
13306
|
+
ledger_path,
|
|
13307
|
+
_benchmark_study_v2_canary_base_event(
|
|
13308
|
+
arm=arm, run_id=run_id, manifest_sha256=manifest_sha256,
|
|
13309
|
+
state="workspace_prepared",
|
|
13310
|
+
pre_workspace_inventory_sha256=workspace_inventory["sha256"],
|
|
13311
|
+
pre_overlay_inventory_sha256=overlay_inventory["sha256"],
|
|
13312
|
+
),
|
|
13313
|
+
)
|
|
13314
|
+
|
|
13315
|
+
def launched() -> None:
|
|
13316
|
+
append_study_attempt_event(
|
|
13317
|
+
ledger_path,
|
|
13318
|
+
_benchmark_study_v2_canary_base_event(
|
|
13319
|
+
arm=arm, run_id=run_id, manifest_sha256=manifest_sha256,
|
|
13320
|
+
state="launched",
|
|
13321
|
+
),
|
|
13322
|
+
)
|
|
13323
|
+
|
|
13324
|
+
root_fd = _ensure_directory_no_symlink(spec.artifact_root, create=True)
|
|
13325
|
+
try:
|
|
13326
|
+
os.fchmod(root_fd, 0o700)
|
|
13327
|
+
if fcntl is not None:
|
|
13328
|
+
fcntl.flock(root_fd, fcntl.LOCK_EX)
|
|
13329
|
+
_run_measurement_fixture_locked(
|
|
13330
|
+
task, variant, claude_bin, Path(manifest_sha256),
|
|
13331
|
+
locked_root_fd=root_fd, on_process_started=launched,
|
|
13332
|
+
measurement_study=True,
|
|
13333
|
+
workspace_overlay=Path(manifest["inputs"]["candidate_install_root"]),
|
|
13334
|
+
on_workspace_prepared=prepared,
|
|
13335
|
+
checker_interpreter_binding=manifest["inputs"]["runner_binding"][
|
|
13336
|
+
"python"
|
|
13337
|
+
],
|
|
13338
|
+
existing_login_home=resolved_auth_home,
|
|
13339
|
+
)
|
|
13340
|
+
finally:
|
|
13341
|
+
os.close(root_fd)
|
|
13342
|
+
if set(pre) != {"overlay", "workspace"}:
|
|
13343
|
+
raise ValueError("v2 canary workspace was not prepared")
|
|
13344
|
+
workspace_event = _benchmark_study_v2_canary_base_event(
|
|
13345
|
+
arm=arm, run_id=run_id, manifest_sha256=manifest_sha256,
|
|
13346
|
+
state="workspace_prepared",
|
|
13347
|
+
pre_workspace_inventory_sha256=pre["workspace"]["sha256"],
|
|
13348
|
+
pre_overlay_inventory_sha256=pre["overlay"]["sha256"],
|
|
13349
|
+
)
|
|
13350
|
+
terminal = _benchmark_study_v2_derive_canary_terminal(
|
|
13351
|
+
manifest=manifest, manifest_sha256=manifest_sha256,
|
|
13352
|
+
output_root=output_root, arm=arm, variant=variant, task=task,
|
|
13353
|
+
workspace_event=workspace_event,
|
|
13354
|
+
)
|
|
13355
|
+
append_study_attempt_event(ledger_path, terminal)
|
|
13356
|
+
if terminal["passed"] is not True:
|
|
13357
|
+
raise ValueError(f"v2 {arm} host PreToolUse canary failed")
|
|
13358
|
+
return terminal
|
|
13359
|
+
|
|
13360
|
+
|
|
13361
|
+
def execute_benchmark_study_v2_canary(
|
|
13362
|
+
*, output_root: Path, claude_bin: str, auth_home: Path,
|
|
13363
|
+
) -> dict[str, Any]:
|
|
13364
|
+
with _benchmark_study_v2_action_lock(output_root):
|
|
13365
|
+
return _execute_benchmark_study_v2_canary_unlocked(
|
|
13366
|
+
output_root=output_root, claude_bin=claude_bin,
|
|
13367
|
+
auth_home=auth_home,
|
|
13368
|
+
)
|
|
13369
|
+
|
|
13370
|
+
|
|
13371
|
+
def _execute_benchmark_study_v2_canary_unlocked(
|
|
13372
|
+
*, output_root: Path, claude_bin: str, auth_home: Path,
|
|
13373
|
+
) -> dict[str, Any]:
|
|
13374
|
+
output_root = _benchmark_study_v2_output_root(output_root)
|
|
13375
|
+
manifest, manifest_sha256 = load_benchmark_study_v2_executable_manifest(
|
|
13376
|
+
output_root, revalidate_external=True,
|
|
13377
|
+
)
|
|
13378
|
+
attempts_path = output_root / "attempts.jsonl"
|
|
13379
|
+
if attempts_path.exists() and attempts_path.stat().st_size:
|
|
13380
|
+
raise ValueError("v2 canary must finish before analytic attempts")
|
|
13381
|
+
cli_stat_guard = _benchmark_study_v2_assert_cli_binding(
|
|
13382
|
+
claude_bin, manifest["inputs"]["cli_binding"],
|
|
13383
|
+
)
|
|
13384
|
+
runtime_stat_guards = _benchmark_study_v2_assert_execution_environment(
|
|
13385
|
+
manifest["inputs"]["execution_environment"]
|
|
13386
|
+
)
|
|
13387
|
+
bound_claude_bin = str(cli_stat_guard["executable"])
|
|
13388
|
+
resolved_auth_home = _benchmark_study_v2_assert_auth_context(
|
|
13389
|
+
bound_claude_bin, manifest["inputs"]["execution_environment"],
|
|
13390
|
+
manifest["inputs"]["auth_context"], auth_home,
|
|
13391
|
+
)
|
|
13392
|
+
variants = _benchmark_study_v2_canary_variants(manifest, output_root)
|
|
13393
|
+
task = _benchmark_study_v2_canary_task()
|
|
13394
|
+
ledger_path = output_root / "canary-events.jsonl"
|
|
13395
|
+
launched_now = 0
|
|
13396
|
+
for arm in BENCHMARK_STUDY_V2_CANARY_ARMS:
|
|
13397
|
+
rows = _benchmark_study_v2_read_canary_events(
|
|
13398
|
+
ledger_path, manifest_sha256=manifest_sha256, variants=variants,
|
|
13399
|
+
)
|
|
13400
|
+
arm_rows = [row for row in rows if row["arm"] == arm]
|
|
13401
|
+
if arm_rows and arm_rows[-1]["state"] == "terminal":
|
|
13402
|
+
if arm_rows[-1]["passed"] is not True:
|
|
13403
|
+
raise ValueError(f"v2 {arm} canary terminal did not pass")
|
|
13404
|
+
continue
|
|
13405
|
+
if arm_rows:
|
|
13406
|
+
if arm_rows[-1]["state"] != "launched":
|
|
13407
|
+
raise ValueError("v2 canary reserved identity cannot be replayed")
|
|
13408
|
+
try:
|
|
13409
|
+
recovered = _benchmark_study_v2_derive_canary_terminal(
|
|
13410
|
+
manifest=manifest, manifest_sha256=manifest_sha256,
|
|
13411
|
+
output_root=output_root, arm=arm, variant=variants[arm],
|
|
13412
|
+
task=task, workspace_event=arm_rows[1],
|
|
13413
|
+
)
|
|
13414
|
+
except (OSError, SystemExit, TypeError, ValueError) as exc:
|
|
13415
|
+
raise ValueError("v2 launched canary cannot be safely recovered") from exc
|
|
13416
|
+
append_study_attempt_event(ledger_path, recovered)
|
|
13417
|
+
if recovered["passed"] is not True:
|
|
13418
|
+
raise ValueError(f"v2 {arm} recovered canary did not pass")
|
|
13419
|
+
continue
|
|
13420
|
+
_benchmark_study_v2_run_canary_arm(
|
|
13421
|
+
manifest=manifest, manifest_sha256=manifest_sha256,
|
|
13422
|
+
output_root=output_root, arm=arm, variant=variants[arm], task=task,
|
|
13423
|
+
claude_bin=bound_claude_bin, cli_stat_guard=cli_stat_guard,
|
|
13424
|
+
runtime_stat_guards=runtime_stat_guards,
|
|
13425
|
+
auth_home=resolved_auth_home,
|
|
13426
|
+
)
|
|
13427
|
+
launched_now += 1
|
|
13428
|
+
rows = _benchmark_study_v2_read_canary_events(
|
|
13429
|
+
ledger_path, manifest_sha256=manifest_sha256, variants=variants,
|
|
13430
|
+
)
|
|
13431
|
+
expected = _benchmark_study_v2_expected_canary_evidence(
|
|
13432
|
+
manifest=manifest, manifest_sha256=manifest_sha256,
|
|
13433
|
+
output_root=output_root, rows=rows, variants=variants,
|
|
13434
|
+
)
|
|
13435
|
+
evidence_path = output_root / "canary-evidence.json"
|
|
13436
|
+
if evidence_path.exists():
|
|
13437
|
+
observed, _raw = _benchmark_study_v2_read_canonical(
|
|
13438
|
+
evidence_path, owner="v2 canary evidence", maximum=200_000,
|
|
13439
|
+
)
|
|
13440
|
+
if observed != expected:
|
|
13441
|
+
raise ValueError("v2 canary evidence binding mismatch")
|
|
13442
|
+
else:
|
|
13443
|
+
_study_write_private(evidence_path, expected)
|
|
13444
|
+
_observed, evidence_sha256 = _benchmark_study_v2_verify_canary_evidence(
|
|
13445
|
+
manifest=manifest, manifest_sha256=manifest_sha256,
|
|
13446
|
+
output_root=output_root,
|
|
13447
|
+
)
|
|
13448
|
+
return {
|
|
13449
|
+
"provider_process_calls": launched_now,
|
|
13450
|
+
"discarded_provider_calls": 2,
|
|
13451
|
+
"canary_evidence_sha256": evidence_sha256,
|
|
13452
|
+
}
|
|
13453
|
+
|
|
13454
|
+
|
|
13455
|
+
def _benchmark_study_v2_bounded_failure_usage(
|
|
13456
|
+
*, receipt: Mapping[str, Any], raw: bytes, arm: str,
|
|
13457
|
+
allowed_event_classes: Sequence[str],
|
|
13458
|
+
required_event_classes: Sequence[str],
|
|
13459
|
+
) -> dict[str, int] | None:
|
|
13460
|
+
"""Return usage only for an exact policy-bounded provider task failure."""
|
|
13461
|
+
if (
|
|
13462
|
+
receipt.get("process_status") != "exited_nonzero"
|
|
13463
|
+
or receipt.get("terminal_status") != "process_error"
|
|
13464
|
+
):
|
|
13465
|
+
return None
|
|
13466
|
+
parsed = parse_claude_stream_output(
|
|
13467
|
+
raw, max_line_bytes=MEASUREMENT_RAW_MAX_LINE_BYTES,
|
|
13468
|
+
)
|
|
13469
|
+
if (
|
|
13470
|
+
parsed.status != "terminal_error"
|
|
13471
|
+
or parsed.result_code not in BENCHMARK_STUDY_V2_BOUNDED_FAILURE_RESULT_CODES
|
|
13472
|
+
):
|
|
13473
|
+
return None
|
|
13474
|
+
hooks = _parse_measurement_hook_events(raw)
|
|
13475
|
+
completed_classes = {item["hook_event"] for item in hooks["hooks"]}
|
|
13476
|
+
allowed_classes = set(allowed_event_classes)
|
|
13477
|
+
required_classes = set(required_event_classes)
|
|
13478
|
+
hook_arms = {"legacy_trim", "bash_reference_v1"}
|
|
13479
|
+
if (
|
|
13480
|
+
hooks["classification"] is not None
|
|
13481
|
+
or hooks["failure_flags"]
|
|
13482
|
+
or (arm in hook_arms and completed_classes - allowed_classes)
|
|
13483
|
+
or (arm == "host_unmodified" and hooks["observed"])
|
|
13484
|
+
or required_classes - completed_classes
|
|
13485
|
+
or any(
|
|
13486
|
+
item["hook_process_outcome"] != "success"
|
|
13487
|
+
or item["hook_process_exit_code"] not in (None, 0)
|
|
13488
|
+
for item in hooks["hooks"]
|
|
13489
|
+
)
|
|
13490
|
+
):
|
|
13491
|
+
return None
|
|
13492
|
+
try:
|
|
13493
|
+
return parse_measurement_terminal_usage(raw)
|
|
13494
|
+
except ValueError:
|
|
13495
|
+
return None
|
|
13496
|
+
|
|
13497
|
+
|
|
13498
|
+
def _benchmark_study_v2_revalidate_terminal_evidence(
|
|
13499
|
+
*, manifest: Mapping[str, Any], output_root: Path,
|
|
13500
|
+
rows: Sequence[Mapping[str, Any]], tasks_by_id: Mapping[str, TaskFixture],
|
|
13501
|
+
variants: Mapping[str, Variant],
|
|
13502
|
+
) -> None:
|
|
13503
|
+
slots = {slot["run_id"]: slot for slot in manifest["slots"]}
|
|
13504
|
+
install_root = Path(manifest["inputs"]["candidate_install_root"])
|
|
13505
|
+
expected_overlay = manifest["inputs"]["candidate_overlay_inventory"]
|
|
13506
|
+
expected_pre_by_task: dict[str, dict[str, Any]] = {}
|
|
13507
|
+
for row in rows:
|
|
13508
|
+
if row["state"] != "terminal" or row["terminal_status"] == "recovered_process_status_unknown":
|
|
13509
|
+
continue
|
|
13510
|
+
slot = slots[row["run_id"]]
|
|
13511
|
+
task_id = str(slot["task_id"])
|
|
13512
|
+
if task_id not in expected_pre_by_task:
|
|
13513
|
+
expected_pre_by_task[task_id] = _benchmark_study_v2_expected_pre_workspace(
|
|
13514
|
+
tasks_by_id[task_id], install_root=install_root,
|
|
13515
|
+
expected_overlay_inventory=expected_overlay,
|
|
13516
|
+
)
|
|
13517
|
+
if (
|
|
13518
|
+
row["pre_workspace_inventory_sha256"]
|
|
13519
|
+
!= expected_pre_by_task[task_id]["sha256"]
|
|
13520
|
+
):
|
|
13521
|
+
raise ValueError("v2 pre-launch workspace inventory drift")
|
|
13522
|
+
study_variant = _study_variant_for_slot(variants[slot["arm"]], slot)
|
|
13523
|
+
spec = study_variant.measurement
|
|
13524
|
+
assert spec is not None
|
|
13525
|
+
receipt = _verify_existing_measurement_run(
|
|
13526
|
+
spec, task_id, str(slot["run_id"]), require_index=True,
|
|
13527
|
+
)
|
|
13528
|
+
context = _measurement_existing_context(spec, str(slot["run_id"]))
|
|
13529
|
+
receipt_raw = _measurement_read_private_file(context.receipt_path)
|
|
13530
|
+
raw = _measurement_read_private_raw(context.raw_path)
|
|
13531
|
+
if _study_sha256_bytes(receipt_raw) != row["receipt_sha256"]:
|
|
13532
|
+
raise ValueError("v2 attempt receipt hash differs from immutable receipt")
|
|
13533
|
+
if receipt["terminal_status"] != row["provider_terminal_status"]:
|
|
13534
|
+
raise ValueError("v2 attempt provider terminal differs from immutable receipt")
|
|
13535
|
+
bounded_failure_usage = _benchmark_study_v2_bounded_failure_usage(
|
|
13536
|
+
receipt=receipt, raw=raw, arm=str(slot["arm"]),
|
|
13537
|
+
allowed_event_classes=tuple(
|
|
13538
|
+
dict.fromkeys(
|
|
13539
|
+
event for event, _command in spec.pair_registered_bindings
|
|
13540
|
+
)
|
|
13541
|
+
),
|
|
13542
|
+
required_event_classes=spec.required_event_classes,
|
|
13543
|
+
)
|
|
13544
|
+
if receipt["terminal_status"] == "success":
|
|
13545
|
+
usage = parse_measurement_terminal_usage(raw)
|
|
13546
|
+
expected_buckets = {
|
|
13547
|
+
key: usage[key] for key in MEASUREMENT_STUDY_USAGE_KEYS
|
|
13548
|
+
}
|
|
13549
|
+
checker = run_task_checker_study(
|
|
13550
|
+
tasks_by_id[task_id], context.workspace, env={},
|
|
13551
|
+
interpreter_binding=manifest["inputs"]["runner_binding"][
|
|
13552
|
+
"python"
|
|
13553
|
+
],
|
|
13554
|
+
)
|
|
13555
|
+
elif bounded_failure_usage is not None:
|
|
13556
|
+
expected_buckets = {
|
|
13557
|
+
key: bounded_failure_usage[key]
|
|
13558
|
+
for key in MEASUREMENT_STUDY_USAGE_KEYS
|
|
13559
|
+
}
|
|
13560
|
+
checker = BENCHMARK_STUDY_V2_BOUNDED_FAILURE_CHECKER_STATUS
|
|
13561
|
+
else:
|
|
13562
|
+
expected_buckets = {key: 0 for key in MEASUREMENT_STUDY_USAGE_KEYS}
|
|
13563
|
+
checker = "not_run"
|
|
13564
|
+
if row["token_buckets"] != expected_buckets or row["primary_tokens"] != sum(expected_buckets.values()):
|
|
13565
|
+
raise ValueError("v2 attempt tokens differ from provider terminal usage")
|
|
13566
|
+
if row["checker_status"] != checker:
|
|
13567
|
+
raise ValueError("v2 attempt checker status differs from the bound checker")
|
|
13568
|
+
derived = (
|
|
13569
|
+
"success" if receipt["terminal_status"] == "success" and checker == "task_success"
|
|
13570
|
+
else "valid_task_failure_v1"
|
|
13571
|
+
if receipt["terminal_status"] == "success" and checker == "valid_task_failure_v1"
|
|
13572
|
+
else "valid_task_failure_v1"
|
|
13573
|
+
if bounded_failure_usage is not None
|
|
13574
|
+
else "study_infra_invalid"
|
|
13575
|
+
)
|
|
13576
|
+
if (
|
|
13577
|
+
row["post_overlay_inventory_sha256"]
|
|
13578
|
+
!= row["pre_overlay_inventory_sha256"]
|
|
13579
|
+
):
|
|
13580
|
+
derived = "study_infra_invalid"
|
|
13581
|
+
if row["terminal_status"] != derived or row["success"] is not (derived == "success"):
|
|
13582
|
+
raise ValueError("v2 attempt outcome was not derived from provider plus checker")
|
|
13583
|
+
workspace_inventory = _benchmark_study_v2_inventory(context.workspace)
|
|
13584
|
+
overlay_inventory = _benchmark_study_v2_inventory(
|
|
13585
|
+
context.workspace / BENCHMARK_STUDY_V2_OVERLAY_NAME,
|
|
13586
|
+
)
|
|
13587
|
+
overlay_drifted = (
|
|
13588
|
+
overlay_inventory != manifest["inputs"]["candidate_overlay_inventory"]
|
|
13589
|
+
)
|
|
13590
|
+
if (
|
|
13591
|
+
workspace_inventory["sha256"] != row["post_workspace_inventory_sha256"]
|
|
13592
|
+
or overlay_inventory["sha256"] != row["post_overlay_inventory_sha256"]
|
|
13593
|
+
or (
|
|
13594
|
+
overlay_drifted
|
|
13595
|
+
and row["terminal_status"] != "study_infra_invalid"
|
|
13596
|
+
)
|
|
13597
|
+
):
|
|
13598
|
+
raise ValueError("v2 terminal workspace or overlay inventory drift")
|
|
13599
|
+
|
|
13600
|
+
|
|
13601
|
+
def _benchmark_study_v2_read_attempts(path: Path, *, manifest: Mapping[str, Any], manifest_sha256: str) -> list[dict[str, Any]]:
|
|
13602
|
+
if not path.exists():
|
|
13603
|
+
return []
|
|
13604
|
+
raw = _measurement_read_private_file(path, maximum=4_000_000)
|
|
13605
|
+
slots = {slot["run_id"]: slot for slot in manifest["slots"]}
|
|
13606
|
+
rows: list[dict[str, Any]] = []
|
|
13607
|
+
states: dict[str, list[str]] = collections.defaultdict(list)
|
|
13608
|
+
base_keys = {
|
|
13609
|
+
"schema_version", "manifest_sha256", "run_id", "task_id",
|
|
13610
|
+
"repetition", "arm", "attempt", "state",
|
|
13611
|
+
}
|
|
13612
|
+
terminal_keys = base_keys | {
|
|
13613
|
+
"terminal_status", "provider_terminal_status", "checker_status",
|
|
13614
|
+
"success", "token_buckets", "primary_tokens", "correction",
|
|
13615
|
+
"retrieval", "shifted_cost", "pre_workspace_inventory_sha256",
|
|
13616
|
+
"post_workspace_inventory_sha256", "pre_overlay_inventory_sha256",
|
|
13617
|
+
"post_overlay_inventory_sha256", "receipt_sha256",
|
|
13618
|
+
}
|
|
13619
|
+
blocked_keys = base_keys | {"reason"}
|
|
13620
|
+
for line in raw.splitlines():
|
|
13621
|
+
try:
|
|
13622
|
+
row = json.loads(line.decode("utf-8"), object_pairs_hook=_measurement_object_no_duplicates)
|
|
13623
|
+
except (UnicodeDecodeError, json.JSONDecodeError, ValueError) as exc:
|
|
13624
|
+
raise ValueError("v2 attempt index is invalid JSONL") from exc
|
|
13625
|
+
if not isinstance(row, dict) or line + b"\n" != _study_canonical_json_bytes(row):
|
|
13626
|
+
raise ValueError("v2 attempt index must be canonical JSONL")
|
|
13627
|
+
slot = slots.get(row.get("run_id"))
|
|
13628
|
+
if row.get("schema_version") != BENCHMARK_STUDY_V2_ATTEMPT_SCHEMA_VERSION or row.get("manifest_sha256") != manifest_sha256 or slot is None:
|
|
13629
|
+
raise ValueError("v2 attempt index binding mismatch")
|
|
13630
|
+
if any(row.get(key) != slot.get(key) for key in ("task_id", "repetition", "arm", "attempt")):
|
|
13631
|
+
raise ValueError("v2 attempt identity mismatch")
|
|
13632
|
+
state = row.get("state")
|
|
13633
|
+
if state not in {
|
|
13634
|
+
"launch_reserved", "launched", "terminal", "not_needed",
|
|
13635
|
+
"blocked_study_invalid",
|
|
13636
|
+
}:
|
|
13637
|
+
raise ValueError("v2 attempt state is invalid")
|
|
13638
|
+
expected_keys = (
|
|
13639
|
+
terminal_keys if state == "terminal"
|
|
13640
|
+
else blocked_keys if state == "blocked_study_invalid"
|
|
13641
|
+
else base_keys
|
|
13642
|
+
)
|
|
13643
|
+
if set(row) != expected_keys:
|
|
13644
|
+
raise ValueError("v2 attempt state has an inexact key schema")
|
|
13645
|
+
previous = states[row["run_id"]]
|
|
13646
|
+
if (
|
|
13647
|
+
(state == "launch_reserved" and previous)
|
|
13648
|
+
or (state == "launched" and previous != ["launch_reserved"])
|
|
13649
|
+
or (state == "terminal" and previous not in (["launch_reserved"], ["launch_reserved", "launched"]))
|
|
13650
|
+
or (state in {"not_needed", "blocked_study_invalid"} and previous)
|
|
13651
|
+
):
|
|
13652
|
+
raise ValueError("v2 attempt state transition is invalid")
|
|
13653
|
+
if state in {"not_needed", "blocked_study_invalid"} and row["attempt"] != 1:
|
|
13654
|
+
raise ValueError("v2 non-launched final state is only valid for a retry")
|
|
13655
|
+
if state == "blocked_study_invalid" and row["reason"] != "initial_study_invalid":
|
|
13656
|
+
raise ValueError("v2 blocked retry reason is invalid")
|
|
13657
|
+
previous.append(state)
|
|
13658
|
+
if state == "terminal":
|
|
13659
|
+
status = row["terminal_status"]
|
|
13660
|
+
if status not in {
|
|
13661
|
+
"success", "valid_task_failure_v1", "study_infra_invalid",
|
|
13662
|
+
} or not isinstance(row["success"], bool) or row["success"] is not (status == "success"):
|
|
13663
|
+
raise ValueError("v2 attempt success/classification binding is invalid")
|
|
13664
|
+
buckets = row["token_buckets"]
|
|
13665
|
+
if not isinstance(buckets, dict) or set(buckets) != set(MEASUREMENT_STUDY_USAGE_KEYS):
|
|
13666
|
+
raise ValueError("v2 attempt token bucket schema is invalid")
|
|
13667
|
+
total = 0
|
|
13668
|
+
for key in MEASUREMENT_STUDY_USAGE_KEYS:
|
|
13669
|
+
value = buckets[key]
|
|
13670
|
+
if isinstance(value, bool) or not isinstance(value, int) or not 0 <= value <= MAX_USAGE_TOKEN_COUNT:
|
|
13671
|
+
raise ValueError("v2 attempt token bucket is invalid")
|
|
13672
|
+
total += value
|
|
13673
|
+
if total > MAX_USAGE_TOKEN_COUNT or row["primary_tokens"] != total:
|
|
13674
|
+
raise ValueError("v2 attempt primary token sum is invalid")
|
|
13675
|
+
if any(row[field] is not None for field in ("correction", "retrieval", "shifted_cost")):
|
|
13676
|
+
raise ValueError("v2 absent observer must remain null")
|
|
13677
|
+
hash_fields = (
|
|
13678
|
+
"pre_workspace_inventory_sha256", "post_workspace_inventory_sha256",
|
|
13679
|
+
"pre_overlay_inventory_sha256", "post_overlay_inventory_sha256",
|
|
13680
|
+
"receipt_sha256",
|
|
13681
|
+
)
|
|
13682
|
+
if any(
|
|
13683
|
+
not isinstance(row[field], str)
|
|
13684
|
+
or SHA256_HEX_PATTERN.fullmatch(row[field]) is None
|
|
13685
|
+
for field in hash_fields
|
|
13686
|
+
):
|
|
13687
|
+
raise ValueError("v2 terminal evidence hash is invalid")
|
|
13688
|
+
expected_overlay_sha256 = manifest["inputs"][
|
|
13689
|
+
"candidate_overlay_inventory"
|
|
13690
|
+
]["sha256"]
|
|
13691
|
+
if (
|
|
13692
|
+
row["pre_overlay_inventory_sha256"] != expected_overlay_sha256
|
|
13693
|
+
or (
|
|
13694
|
+
row["post_overlay_inventory_sha256"]
|
|
13695
|
+
!= row["pre_overlay_inventory_sha256"]
|
|
13696
|
+
and status != "study_infra_invalid"
|
|
13697
|
+
)
|
|
13698
|
+
):
|
|
13699
|
+
raise ValueError("v2 terminal overlay binding is invalid")
|
|
13700
|
+
if status == "success" and not (
|
|
13701
|
+
row["provider_terminal_status"] == "success"
|
|
13702
|
+
and row["checker_status"] == "task_success"
|
|
13703
|
+
):
|
|
13704
|
+
raise ValueError("v2 successful outcome lacks provider/checker evidence")
|
|
13705
|
+
if status == "valid_task_failure_v1" and not (
|
|
13706
|
+
(
|
|
13707
|
+
row["provider_terminal_status"] == "success"
|
|
13708
|
+
and row["checker_status"] == "valid_task_failure_v1"
|
|
13709
|
+
)
|
|
13710
|
+
or (
|
|
13711
|
+
row["provider_terminal_status"] == "process_error"
|
|
13712
|
+
and row["checker_status"]
|
|
13713
|
+
== BENCHMARK_STUDY_V2_BOUNDED_FAILURE_CHECKER_STATUS
|
|
13714
|
+
)
|
|
13715
|
+
):
|
|
13716
|
+
raise ValueError("v2 valid failure lacks provider/checker evidence")
|
|
13717
|
+
rows.append(row)
|
|
13718
|
+
terminal_by_unit = {
|
|
13719
|
+
(row["task_id"], row["repetition"], row["arm"]): row
|
|
13720
|
+
for row in rows
|
|
13721
|
+
if row["attempt"] == 0 and row["state"] == "terminal"
|
|
13722
|
+
}
|
|
13723
|
+
for row in rows:
|
|
13724
|
+
if row["attempt"] != 1:
|
|
13725
|
+
continue
|
|
13726
|
+
initial = terminal_by_unit.get((row["task_id"], row["repetition"], row["arm"]))
|
|
13727
|
+
if row["state"] == "not_needed" and (
|
|
13728
|
+
initial is None or initial["terminal_status"] != "success"
|
|
13729
|
+
):
|
|
13730
|
+
raise ValueError("v2 not-needed retry lacks a successful initial")
|
|
13731
|
+
if row["state"] == "blocked_study_invalid" and (
|
|
13732
|
+
initial is None
|
|
13733
|
+
or initial["terminal_status"] != "study_infra_invalid"
|
|
13734
|
+
):
|
|
13735
|
+
raise ValueError("v2 blocked retry lacks an invalid initial")
|
|
13736
|
+
if row["state"] in {"launch_reserved", "launched", "terminal"} and (
|
|
13737
|
+
initial is None or initial["terminal_status"] != "valid_task_failure_v1"
|
|
13738
|
+
):
|
|
13739
|
+
raise ValueError("v2 launched retry lacks a valid failed initial")
|
|
13740
|
+
return rows
|
|
13741
|
+
|
|
13742
|
+
|
|
13743
|
+
def _benchmark_study_v2_event(slot: Mapping[str, Any], manifest_sha256: str, state: str, **extra: Any) -> dict[str, Any]:
|
|
13744
|
+
event = {
|
|
13745
|
+
"schema_version": BENCHMARK_STUDY_V2_ATTEMPT_SCHEMA_VERSION,
|
|
13746
|
+
"manifest_sha256": manifest_sha256, "run_id": slot["run_id"],
|
|
13747
|
+
"task_id": slot["task_id"], "repetition": slot["repetition"],
|
|
13748
|
+
"arm": slot["arm"], "attempt": slot["attempt"], "state": state,
|
|
13749
|
+
}
|
|
13750
|
+
event.update(extra)
|
|
13751
|
+
return event
|
|
13752
|
+
|
|
13753
|
+
|
|
13754
|
+
def _benchmark_study_v2_run_slot(
|
|
13755
|
+
*, slot: Mapping[str, Any], task: TaskFixture, variant: Variant,
|
|
13756
|
+
claude_bin: str, attempts_path: Path, manifest_sha256: str,
|
|
13757
|
+
install_root: Path, expected_overlay_inventory: Mapping[str, Any],
|
|
13758
|
+
cli_binding: Mapping[str, Any],
|
|
13759
|
+
cli_stat_guard: Mapping[str, Any],
|
|
13760
|
+
execution_environment: Mapping[str, Any],
|
|
13761
|
+
runtime_stat_guards: Mapping[str, Mapping[str, int | str]],
|
|
13762
|
+
checker_interpreter_binding: Mapping[str, Any],
|
|
13763
|
+
auth_context: Mapping[str, Any],
|
|
13764
|
+
auth_home: Path,
|
|
13765
|
+
) -> str:
|
|
13766
|
+
study_variant = _study_variant_for_slot(variant, slot)
|
|
13767
|
+
spec = study_variant.measurement
|
|
13768
|
+
assert spec is not None
|
|
13769
|
+
pre: dict[str, Any] = {}
|
|
13770
|
+
|
|
13771
|
+
def prepared(workspace: Path) -> None:
|
|
13772
|
+
overlay = workspace / BENCHMARK_STUDY_V2_OVERLAY_NAME
|
|
13773
|
+
pre["overlay"] = _benchmark_study_v2_verify_physical_copy(
|
|
13774
|
+
install_root, overlay, expected=expected_overlay_inventory,
|
|
13775
|
+
)
|
|
13776
|
+
pre["workspace"] = _benchmark_study_v2_inventory(workspace)
|
|
13777
|
+
|
|
13778
|
+
def launched() -> None:
|
|
13779
|
+
append_study_attempt_event(
|
|
13780
|
+
attempts_path,
|
|
13781
|
+
_benchmark_study_v2_event(slot, manifest_sha256, "launched"),
|
|
13782
|
+
)
|
|
13783
|
+
|
|
13784
|
+
# Durable reservation precedes Popen. Resume treats even a reservation-only
|
|
13785
|
+
# identity as consumed/unknown, so a provider that started while launched
|
|
13786
|
+
# accounting failed can never be invoked twice.
|
|
13787
|
+
_benchmark_study_v2_assert_cli_executable_bytes(cli_binding, cli_stat_guard)
|
|
13788
|
+
_benchmark_study_v2_assert_runtime_stat_guards(
|
|
13789
|
+
execution_environment, runtime_stat_guards,
|
|
13790
|
+
)
|
|
13791
|
+
resolved_auth_home = _benchmark_study_v2_assert_auth_context(
|
|
13792
|
+
claude_bin, execution_environment, auth_context, auth_home,
|
|
13793
|
+
)
|
|
13794
|
+
append_study_attempt_event(
|
|
13795
|
+
attempts_path,
|
|
13796
|
+
_benchmark_study_v2_event(slot, manifest_sha256, "launch_reserved"),
|
|
13797
|
+
)
|
|
13798
|
+
root_fd = _ensure_directory_no_symlink(spec.artifact_root, create=True)
|
|
13799
|
+
try:
|
|
13800
|
+
os.fchmod(root_fd, 0o700)
|
|
13801
|
+
if fcntl is not None:
|
|
13802
|
+
fcntl.flock(root_fd, fcntl.LOCK_EX)
|
|
13803
|
+
result = _run_measurement_fixture_locked(
|
|
13804
|
+
task, study_variant, claude_bin, Path(manifest_sha256),
|
|
13805
|
+
locked_root_fd=root_fd, on_process_started=launched,
|
|
13806
|
+
measurement_study=True, workspace_overlay=install_root,
|
|
13807
|
+
on_workspace_prepared=prepared,
|
|
13808
|
+
checker_interpreter_binding=checker_interpreter_binding,
|
|
13809
|
+
existing_login_home=resolved_auth_home,
|
|
13810
|
+
)
|
|
13811
|
+
finally:
|
|
13812
|
+
os.close(root_fd)
|
|
13813
|
+
context = _measurement_existing_context(spec, str(slot["run_id"]))
|
|
13814
|
+
post_workspace = _benchmark_study_v2_inventory(context.workspace)
|
|
13815
|
+
post_overlay = _benchmark_study_v2_inventory(
|
|
13816
|
+
context.workspace / BENCHMARK_STUDY_V2_OVERLAY_NAME,
|
|
13817
|
+
)
|
|
13818
|
+
if _benchmark_study_v2_inventory(install_root) != dict(expected_overlay_inventory):
|
|
13819
|
+
raise ValueError("v2 staged candidate changed during an attempt")
|
|
13820
|
+
receipt_raw = _measurement_read_private_file(context.receipt_path)
|
|
13821
|
+
receipt = _measurement_parse_canonical_json_bytes(receipt_raw, owner="v2 measurement receipt")
|
|
13822
|
+
provider_terminal = str(receipt["terminal_status"])
|
|
13823
|
+
raw = _measurement_read_private_raw(context.raw_path)
|
|
13824
|
+
bounded_failure_usage = _benchmark_study_v2_bounded_failure_usage(
|
|
13825
|
+
receipt=receipt, raw=raw, arm=str(slot["arm"]),
|
|
13826
|
+
allowed_event_classes=tuple(
|
|
13827
|
+
dict.fromkeys(
|
|
13828
|
+
event for event, _command in spec.pair_registered_bindings
|
|
13829
|
+
)
|
|
13830
|
+
),
|
|
13831
|
+
required_event_classes=spec.required_event_classes,
|
|
13832
|
+
)
|
|
13833
|
+
checker_status = (
|
|
13834
|
+
result.notes if provider_terminal == "success"
|
|
13835
|
+
else BENCHMARK_STUDY_V2_BOUNDED_FAILURE_CHECKER_STATUS
|
|
13836
|
+
if bounded_failure_usage is not None
|
|
13837
|
+
else "not_run"
|
|
13838
|
+
)
|
|
13839
|
+
if provider_terminal == "success" and checker_status == "task_success":
|
|
13840
|
+
classification = "success"
|
|
13841
|
+
elif provider_terminal == "success" and checker_status == "valid_task_failure_v1":
|
|
13842
|
+
classification = "valid_task_failure_v1"
|
|
13843
|
+
elif bounded_failure_usage is not None:
|
|
13844
|
+
classification = "valid_task_failure_v1"
|
|
13845
|
+
else:
|
|
13846
|
+
classification = "study_infra_invalid"
|
|
13847
|
+
if post_overlay != pre.get("overlay"):
|
|
13848
|
+
classification = "study_infra_invalid"
|
|
13849
|
+
token_buckets = (
|
|
13850
|
+
{
|
|
13851
|
+
key: bounded_failure_usage[key]
|
|
13852
|
+
for key in MEASUREMENT_STUDY_USAGE_KEYS
|
|
13853
|
+
}
|
|
13854
|
+
if bounded_failure_usage is not None
|
|
13855
|
+
else {
|
|
13856
|
+
"input_tokens": result.tokens["input_tokens"],
|
|
13857
|
+
"cache_creation_input_tokens": result.tokens["cache_creation"],
|
|
13858
|
+
"cache_read_input_tokens": result.tokens["cache_read"],
|
|
13859
|
+
"output_tokens": result.tokens["output_tokens"],
|
|
13860
|
+
}
|
|
13861
|
+
)
|
|
13862
|
+
append_study_attempt_event(
|
|
13863
|
+
attempts_path,
|
|
13864
|
+
_benchmark_study_v2_event(
|
|
13865
|
+
slot, manifest_sha256, "terminal",
|
|
13866
|
+
terminal_status=classification,
|
|
13867
|
+
provider_terminal_status=provider_terminal,
|
|
13868
|
+
checker_status=checker_status,
|
|
13869
|
+
success=classification == "success",
|
|
13870
|
+
token_buckets=token_buckets,
|
|
13871
|
+
primary_tokens=sum(token_buckets.values()),
|
|
13872
|
+
correction=None, retrieval=None, shifted_cost=None,
|
|
13873
|
+
pre_workspace_inventory_sha256=pre["workspace"]["sha256"],
|
|
13874
|
+
post_workspace_inventory_sha256=post_workspace["sha256"],
|
|
13875
|
+
pre_overlay_inventory_sha256=pre["overlay"]["sha256"],
|
|
13876
|
+
post_overlay_inventory_sha256=post_overlay["sha256"],
|
|
13877
|
+
receipt_sha256=_study_sha256_bytes(receipt_raw),
|
|
13878
|
+
),
|
|
13879
|
+
)
|
|
13880
|
+
return classification
|
|
13881
|
+
|
|
13882
|
+
|
|
13883
|
+
def execute_benchmark_study_v2(
|
|
13884
|
+
*, output_root: Path, claude_bin: str, resume: bool, auth_home: Path,
|
|
13885
|
+
) -> dict[str, int]:
|
|
13886
|
+
with _benchmark_study_v2_action_lock(output_root):
|
|
13887
|
+
return _execute_benchmark_study_v2_unlocked(
|
|
13888
|
+
output_root=output_root, claude_bin=claude_bin, resume=resume,
|
|
13889
|
+
auth_home=auth_home,
|
|
13890
|
+
)
|
|
13891
|
+
|
|
13892
|
+
|
|
13893
|
+
def _execute_benchmark_study_v2_unlocked(
|
|
13894
|
+
*, output_root: Path, claude_bin: str, resume: bool, auth_home: Path,
|
|
13895
|
+
) -> dict[str, int]:
|
|
13896
|
+
output_root = _benchmark_study_v2_output_root(output_root)
|
|
13897
|
+
# Revalidate every inert candidate byte and installed overlay before a provider launch.
|
|
13898
|
+
manifest, manifest_sha256 = load_benchmark_study_v2_executable_manifest(
|
|
13899
|
+
output_root, revalidate_external=True,
|
|
13900
|
+
)
|
|
13901
|
+
cli_stat_guard = _benchmark_study_v2_assert_cli_binding(
|
|
13902
|
+
claude_bin, manifest["inputs"]["cli_binding"],
|
|
13903
|
+
)
|
|
13904
|
+
runtime_stat_guards = _benchmark_study_v2_assert_execution_environment(
|
|
13905
|
+
manifest["inputs"]["execution_environment"]
|
|
13906
|
+
)
|
|
13907
|
+
bound_claude_bin = str(cli_stat_guard["executable"])
|
|
13908
|
+
resolved_auth_home = _benchmark_study_v2_assert_auth_context(
|
|
13909
|
+
bound_claude_bin, manifest["inputs"]["execution_environment"],
|
|
13910
|
+
manifest["inputs"]["auth_context"], auth_home,
|
|
13911
|
+
)
|
|
13912
|
+
attempts_path = output_root / "attempts.jsonl"
|
|
13913
|
+
if not resume and attempts_path.exists() and attempts_path.stat().st_size:
|
|
13914
|
+
raise ValueError("v2 run requires an absent or empty attempt index")
|
|
13915
|
+
if resume and not attempts_path.exists():
|
|
13916
|
+
raise ValueError("v2 resume requires an existing attempt index")
|
|
13917
|
+
_canary_evidence, _canary_evidence_sha256 = (
|
|
13918
|
+
_benchmark_study_v2_verify_canary_evidence(
|
|
13919
|
+
manifest=manifest, manifest_sha256=manifest_sha256,
|
|
13920
|
+
output_root=output_root,
|
|
13921
|
+
)
|
|
13922
|
+
)
|
|
13923
|
+
tasks = parse_tasks(Path(manifest["inputs"]["tasks_path"]))
|
|
13924
|
+
load_task_fixture_trees(tasks, task_file_dir=Path(manifest["inputs"]["tasks_path"]).parent)
|
|
13925
|
+
tasks_by_id = {task.id: task for task in tasks}
|
|
13926
|
+
variants = _benchmark_study_v2_variants(manifest, output_root)
|
|
13927
|
+
rows = _benchmark_study_v2_read_attempts(
|
|
13928
|
+
attempts_path, manifest=manifest, manifest_sha256=manifest_sha256,
|
|
13929
|
+
)
|
|
13930
|
+
_benchmark_study_v2_revalidate_terminal_evidence(
|
|
13931
|
+
manifest=manifest, output_root=output_root, rows=rows,
|
|
13932
|
+
tasks_by_id=tasks_by_id, variants=variants,
|
|
13933
|
+
)
|
|
13934
|
+
launched = {
|
|
13935
|
+
row["run_id"] for row in rows
|
|
13936
|
+
if row["state"] in {"launch_reserved", "launched", "terminal"}
|
|
13937
|
+
}
|
|
13938
|
+
accounted = {row["run_id"] for row in rows}
|
|
13939
|
+
terminal = {row["run_id"]: row for row in rows if row["state"] == "terminal"}
|
|
13940
|
+
ambiguous_run_ids = sorted(launched - set(terminal))
|
|
13941
|
+
if ambiguous_run_ids:
|
|
13942
|
+
raise ValueError(
|
|
13943
|
+
"v2 ambiguous provider process state permanently blocks this study root"
|
|
13944
|
+
)
|
|
13945
|
+
retry_by_unit = {
|
|
13946
|
+
(slot["task_id"], slot["repetition"], slot["arm"]): slot
|
|
13947
|
+
for slot in manifest["slots"] if slot["attempt"] == 1
|
|
13948
|
+
}
|
|
13949
|
+
install_root = Path(manifest["inputs"]["candidate_install_root"])
|
|
13950
|
+
calls_before = len(launched)
|
|
13951
|
+
for initial in (slot for slot in manifest["slots"] if slot["attempt"] == 0):
|
|
13952
|
+
if initial["run_id"] not in launched:
|
|
13953
|
+
_benchmark_study_v2_run_slot(
|
|
13954
|
+
slot=initial, task=tasks_by_id[initial["task_id"]],
|
|
13955
|
+
variant=variants[initial["arm"]], claude_bin=bound_claude_bin,
|
|
13956
|
+
attempts_path=attempts_path, manifest_sha256=manifest_sha256,
|
|
13957
|
+
install_root=install_root,
|
|
13958
|
+
expected_overlay_inventory=manifest["inputs"]["candidate_overlay_inventory"],
|
|
13959
|
+
cli_binding=manifest["inputs"]["cli_binding"],
|
|
13960
|
+
cli_stat_guard=cli_stat_guard,
|
|
13961
|
+
execution_environment=manifest["inputs"]["execution_environment"],
|
|
13962
|
+
runtime_stat_guards=runtime_stat_guards,
|
|
13963
|
+
checker_interpreter_binding=manifest["inputs"]["runner_binding"][
|
|
13964
|
+
"python"
|
|
13965
|
+
],
|
|
13966
|
+
auth_context=manifest["inputs"]["auth_context"],
|
|
13967
|
+
auth_home=resolved_auth_home,
|
|
13968
|
+
)
|
|
13969
|
+
launched.add(initial["run_id"])
|
|
13970
|
+
terminal = {
|
|
13971
|
+
row["run_id"]: row for row in _benchmark_study_v2_read_attempts(
|
|
13972
|
+
attempts_path, manifest=manifest, manifest_sha256=manifest_sha256,
|
|
13973
|
+
) if row["state"] == "terminal"
|
|
13974
|
+
}
|
|
13975
|
+
initial_row = terminal.get(initial["run_id"])
|
|
13976
|
+
if initial_row is None:
|
|
13977
|
+
continue
|
|
13978
|
+
retry = retry_by_unit[(initial["task_id"], initial["repetition"], initial["arm"])]
|
|
13979
|
+
if initial_row.get("terminal_status") != "valid_task_failure_v1":
|
|
13980
|
+
state = (
|
|
13981
|
+
"not_needed"
|
|
13982
|
+
if initial_row.get("terminal_status") == "success"
|
|
13983
|
+
else "blocked_study_invalid"
|
|
13984
|
+
)
|
|
13985
|
+
if retry["run_id"] not in accounted:
|
|
13986
|
+
extra = {"reason": "initial_study_invalid"} if state == "blocked_study_invalid" else {}
|
|
13987
|
+
append_study_attempt_event(
|
|
13988
|
+
attempts_path,
|
|
13989
|
+
_benchmark_study_v2_event(
|
|
13990
|
+
retry, manifest_sha256, state, **extra,
|
|
13991
|
+
),
|
|
13992
|
+
)
|
|
13993
|
+
accounted.add(retry["run_id"])
|
|
13994
|
+
if state == "blocked_study_invalid":
|
|
13995
|
+
raise ValueError(
|
|
13996
|
+
"v2 terminal infrastructure-invalid evidence permanently "
|
|
13997
|
+
"blocks later provider launches"
|
|
13998
|
+
)
|
|
13999
|
+
continue
|
|
14000
|
+
if retry["run_id"] in accounted:
|
|
14001
|
+
continue
|
|
14002
|
+
# A failed retry is retained, but it never stops later scheduled blocks.
|
|
14003
|
+
_benchmark_study_v2_run_slot(
|
|
14004
|
+
slot=retry, task=tasks_by_id[retry["task_id"]],
|
|
14005
|
+
variant=variants[retry["arm"]], claude_bin=bound_claude_bin,
|
|
14006
|
+
attempts_path=attempts_path, manifest_sha256=manifest_sha256,
|
|
14007
|
+
install_root=install_root,
|
|
14008
|
+
expected_overlay_inventory=manifest["inputs"]["candidate_overlay_inventory"],
|
|
14009
|
+
cli_binding=manifest["inputs"]["cli_binding"],
|
|
14010
|
+
cli_stat_guard=cli_stat_guard,
|
|
14011
|
+
execution_environment=manifest["inputs"]["execution_environment"],
|
|
14012
|
+
runtime_stat_guards=runtime_stat_guards,
|
|
14013
|
+
checker_interpreter_binding=manifest["inputs"]["runner_binding"][
|
|
14014
|
+
"python"
|
|
14015
|
+
],
|
|
14016
|
+
auth_context=manifest["inputs"]["auth_context"],
|
|
14017
|
+
auth_home=resolved_auth_home,
|
|
14018
|
+
)
|
|
14019
|
+
launched.add(retry["run_id"])
|
|
14020
|
+
accounted.add(retry["run_id"])
|
|
14021
|
+
terminal = {
|
|
14022
|
+
row["run_id"]: row for row in _benchmark_study_v2_read_attempts(
|
|
14023
|
+
attempts_path, manifest=manifest, manifest_sha256=manifest_sha256,
|
|
14024
|
+
) if row["state"] == "terminal"
|
|
14025
|
+
}
|
|
14026
|
+
final_rows = _benchmark_study_v2_read_attempts(
|
|
14027
|
+
attempts_path, manifest=manifest, manifest_sha256=manifest_sha256,
|
|
14028
|
+
)
|
|
14029
|
+
final_states = {row["run_id"]: row["state"] for row in final_rows}
|
|
14030
|
+
return {
|
|
14031
|
+
"provider_process_calls": len(launched) - calls_before,
|
|
14032
|
+
"launched_identities": len(launched),
|
|
14033
|
+
"accounted_identities": len(final_states),
|
|
14034
|
+
}
|
|
14035
|
+
|
|
14036
|
+
|
|
14037
|
+
def analyze_benchmark_study_v2_executable(
|
|
14038
|
+
*, output_root: Path, claude_bin: str,
|
|
14039
|
+
) -> dict[str, Any]:
|
|
14040
|
+
with _benchmark_study_v2_action_lock(output_root):
|
|
14041
|
+
return _analyze_benchmark_study_v2_executable_unlocked(
|
|
14042
|
+
output_root=output_root, claude_bin=claude_bin,
|
|
14043
|
+
)
|
|
14044
|
+
|
|
14045
|
+
|
|
14046
|
+
def _benchmark_study_v2_persist_analysis(
|
|
14047
|
+
*, output_root: Path, report: Mapping[str, Any],
|
|
14048
|
+
) -> str:
|
|
14049
|
+
report_name = (
|
|
14050
|
+
"study-invalid-decision.json"
|
|
14051
|
+
if report.get("schema_version")
|
|
14052
|
+
== BENCHMARK_STUDY_V2_INVALID_DECISION_SCHEMA_VERSION
|
|
14053
|
+
else "study-report.json"
|
|
14054
|
+
)
|
|
14055
|
+
path = output_root / report_name
|
|
14056
|
+
expected = _study_canonical_json_bytes(report)
|
|
14057
|
+
if path.exists():
|
|
14058
|
+
observed = _measurement_read_private_file(path, maximum=4_000_000)
|
|
14059
|
+
if observed != expected:
|
|
14060
|
+
raise ValueError("v2 persisted analysis binding mismatch")
|
|
14061
|
+
else:
|
|
14062
|
+
_measurement_write_exclusive(path, expected)
|
|
14063
|
+
return report_name
|
|
14064
|
+
|
|
14065
|
+
|
|
14066
|
+
def _benchmark_study_v2_persist_invalid_after_refusal(
|
|
14067
|
+
*, output_root: Path, claude_bin: str,
|
|
14068
|
+
) -> None:
|
|
14069
|
+
try:
|
|
14070
|
+
report = analyze_benchmark_study_v2_executable(
|
|
14071
|
+
output_root=output_root, claude_bin=claude_bin,
|
|
14072
|
+
)
|
|
14073
|
+
if (
|
|
14074
|
+
report.get("schema_version")
|
|
14075
|
+
== BENCHMARK_STUDY_V2_INVALID_DECISION_SCHEMA_VERSION
|
|
14076
|
+
):
|
|
14077
|
+
_benchmark_study_v2_persist_analysis(
|
|
14078
|
+
output_root=output_root, report=report,
|
|
14079
|
+
)
|
|
14080
|
+
except (OSError, SystemExit, TypeError, ValueError):
|
|
14081
|
+
# The original refusal remains authoritative when damaged or incomplete
|
|
14082
|
+
# evidence cannot safely support a canonical P1-X decision.
|
|
14083
|
+
print(
|
|
14084
|
+
"v2 canonical P1-X unavailable after refusal",
|
|
14085
|
+
file=sys.stderr,
|
|
14086
|
+
)
|
|
14087
|
+
return
|
|
14088
|
+
|
|
14089
|
+
|
|
14090
|
+
def _benchmark_study_v2_ledger_binding(
|
|
14091
|
+
path: Path, *, maximum: int,
|
|
14092
|
+
) -> dict[str, Any]:
|
|
14093
|
+
if not path.exists():
|
|
14094
|
+
return {"bytes": 0, "record_count": 0, "sha256": None}
|
|
14095
|
+
raw = _measurement_read_private_file(path, maximum=maximum)
|
|
14096
|
+
return {
|
|
14097
|
+
"bytes": len(raw), "record_count": len(raw.splitlines()),
|
|
14098
|
+
"sha256": _study_sha256_bytes(raw),
|
|
14099
|
+
}
|
|
14100
|
+
|
|
14101
|
+
|
|
14102
|
+
def _benchmark_study_v2_invalid_canary_decision(
|
|
14103
|
+
*, output_root: Path, manifest: Mapping[str, Any],
|
|
14104
|
+
manifest_sha256: str, rows: Sequence[Mapping[str, Any]],
|
|
14105
|
+
) -> dict[str, Any] | None:
|
|
14106
|
+
final_by_arm = {str(row["arm"]): row for row in rows}
|
|
14107
|
+
ambiguous = [
|
|
14108
|
+
final_by_arm[arm] for arm in BENCHMARK_STUDY_V2_CANARY_ARMS
|
|
14109
|
+
if arm in final_by_arm and final_by_arm[arm]["state"] != "terminal"
|
|
14110
|
+
]
|
|
14111
|
+
failed = [
|
|
14112
|
+
final_by_arm[arm] for arm in BENCHMARK_STUDY_V2_CANARY_ARMS
|
|
14113
|
+
if arm in final_by_arm
|
|
14114
|
+
and final_by_arm[arm]["state"] == "terminal"
|
|
14115
|
+
and final_by_arm[arm]["passed"] is not True
|
|
14116
|
+
]
|
|
14117
|
+
if not ambiguous and not failed:
|
|
14118
|
+
return None
|
|
14119
|
+
return {
|
|
14120
|
+
"schema_version": BENCHMARK_STUDY_V2_INVALID_DECISION_SCHEMA_VERSION,
|
|
14121
|
+
"study_version": "v2", "decision": "P1-X",
|
|
14122
|
+
"stop_reason": (
|
|
14123
|
+
"ambiguous_canary_process_state" if ambiguous
|
|
14124
|
+
else "failed_canary_terminal_evidence"
|
|
14125
|
+
),
|
|
14126
|
+
"manifest_sha256": manifest_sha256,
|
|
14127
|
+
"attempt_schema_version": BENCHMARK_STUDY_V2_ATTEMPT_SCHEMA_VERSION,
|
|
14128
|
+
"consumed_identity_count": len(final_by_arm),
|
|
14129
|
+
"accounted_identity_count": sum(
|
|
14130
|
+
row["state"] == "terminal" for row in final_by_arm.values()
|
|
14131
|
+
),
|
|
14132
|
+
"ambiguous_identities": [{
|
|
14133
|
+
"arm": row["arm"], "run_id": row["run_id"],
|
|
14134
|
+
"state": row["state"],
|
|
14135
|
+
} for row in ambiguous],
|
|
14136
|
+
"failed_canary_identities": [{
|
|
14137
|
+
"arm": row["arm"], "run_id": row["run_id"],
|
|
14138
|
+
"state": row["state"],
|
|
14139
|
+
} for row in failed],
|
|
14140
|
+
"ledgers": {
|
|
14141
|
+
"attempts": _benchmark_study_v2_ledger_binding(
|
|
14142
|
+
output_root / "attempts.jsonl", maximum=4_000_000,
|
|
14143
|
+
),
|
|
14144
|
+
"canary_events": _benchmark_study_v2_ledger_binding(
|
|
14145
|
+
output_root / "canary-events.jsonl", maximum=200_000,
|
|
14146
|
+
),
|
|
14147
|
+
},
|
|
14148
|
+
"canary_evidence_sha256": None,
|
|
14149
|
+
"descriptive_only": True, "claim_allowed": False, "claim": None,
|
|
14150
|
+
}
|
|
14151
|
+
|
|
14152
|
+
|
|
14153
|
+
def _benchmark_study_v2_invalid_analytic_decision(
|
|
14154
|
+
*, output_root: Path, manifest: Mapping[str, Any],
|
|
14155
|
+
manifest_sha256: str, rows: Sequence[Mapping[str, Any]],
|
|
14156
|
+
canary_evidence_sha256: str,
|
|
14157
|
+
) -> dict[str, Any] | None:
|
|
14158
|
+
final_by_run = {str(row["run_id"]): row for row in rows}
|
|
14159
|
+
ambiguous = [
|
|
14160
|
+
final_by_run[str(slot["run_id"])]
|
|
14161
|
+
for slot in manifest["slots"]
|
|
14162
|
+
if str(slot["run_id"]) in final_by_run
|
|
14163
|
+
and final_by_run[str(slot["run_id"])]["state"]
|
|
14164
|
+
in {"launch_reserved", "launched"}
|
|
14165
|
+
]
|
|
14166
|
+
failed = [
|
|
14167
|
+
row for row in final_by_run.values()
|
|
14168
|
+
if row["state"] == "terminal"
|
|
14169
|
+
and row["terminal_status"] == "study_infra_invalid"
|
|
14170
|
+
]
|
|
14171
|
+
if not ambiguous and not failed:
|
|
14172
|
+
return None
|
|
14173
|
+
attempts_path = output_root / "attempts.jsonl"
|
|
14174
|
+
return {
|
|
14175
|
+
"schema_version": BENCHMARK_STUDY_V2_INVALID_DECISION_SCHEMA_VERSION,
|
|
14176
|
+
"study_version": "v2", "decision": "P1-X",
|
|
14177
|
+
"stop_reason": (
|
|
14178
|
+
"ambiguous_analytic_process_state" if ambiguous
|
|
14179
|
+
else "terminal_analytic_infrastructure_invalid"
|
|
14180
|
+
),
|
|
14181
|
+
"manifest_sha256": manifest_sha256,
|
|
14182
|
+
"attempt_schema_version": BENCHMARK_STUDY_V2_ATTEMPT_SCHEMA_VERSION,
|
|
14183
|
+
"consumed_identity_count": sum(
|
|
14184
|
+
row["state"] in {"launch_reserved", "launched", "terminal"}
|
|
14185
|
+
for row in final_by_run.values()
|
|
14186
|
+
),
|
|
14187
|
+
"accounted_identity_count": len(final_by_run),
|
|
14188
|
+
"ambiguous_identities": [{
|
|
14189
|
+
"arm": row["arm"], "attempt": row["attempt"],
|
|
14190
|
+
"repetition": row["repetition"], "run_id": row["run_id"],
|
|
14191
|
+
"state": row["state"], "task_id": row["task_id"],
|
|
14192
|
+
} for row in ambiguous],
|
|
14193
|
+
"failed_analytic_identities": [{
|
|
14194
|
+
"arm": row["arm"], "attempt": row["attempt"],
|
|
14195
|
+
"repetition": row["repetition"], "run_id": row["run_id"],
|
|
14196
|
+
"state": row["state"], "task_id": row["task_id"],
|
|
14197
|
+
"terminal_status": row["terminal_status"],
|
|
14198
|
+
} for row in failed],
|
|
14199
|
+
"ledgers": {
|
|
14200
|
+
"attempts": _benchmark_study_v2_ledger_binding(
|
|
14201
|
+
attempts_path, maximum=4_000_000,
|
|
14202
|
+
),
|
|
14203
|
+
"canary_events": _benchmark_study_v2_ledger_binding(
|
|
14204
|
+
output_root / "canary-events.jsonl", maximum=200_000,
|
|
14205
|
+
),
|
|
14206
|
+
},
|
|
14207
|
+
"canary_evidence_sha256": canary_evidence_sha256,
|
|
14208
|
+
"descriptive_only": True, "claim_allowed": False, "claim": None,
|
|
14209
|
+
}
|
|
14210
|
+
|
|
14211
|
+
|
|
14212
|
+
def _analyze_benchmark_study_v2_executable_unlocked(
|
|
14213
|
+
*, output_root: Path, claude_bin: str,
|
|
14214
|
+
) -> dict[str, Any]:
|
|
14215
|
+
output_root = _benchmark_study_v2_output_root(output_root)
|
|
14216
|
+
manifest, manifest_sha256 = load_benchmark_study_v2_executable_manifest(
|
|
14217
|
+
output_root, revalidate_external=True,
|
|
14218
|
+
)
|
|
14219
|
+
_benchmark_study_v2_assert_cli_binding(
|
|
14220
|
+
claude_bin, manifest["inputs"]["cli_binding"],
|
|
14221
|
+
)
|
|
14222
|
+
canary_variants = _benchmark_study_v2_canary_variants(manifest, output_root)
|
|
14223
|
+
canary_rows = _benchmark_study_v2_read_canary_events(
|
|
14224
|
+
output_root / "canary-events.jsonl",
|
|
14225
|
+
manifest_sha256=manifest_sha256, variants=canary_variants,
|
|
14226
|
+
)
|
|
14227
|
+
invalid_canary_decision = _benchmark_study_v2_invalid_canary_decision(
|
|
14228
|
+
output_root=output_root, manifest=manifest,
|
|
14229
|
+
manifest_sha256=manifest_sha256, rows=canary_rows,
|
|
14230
|
+
)
|
|
14231
|
+
if invalid_canary_decision is not None:
|
|
14232
|
+
return invalid_canary_decision
|
|
14233
|
+
_canary_evidence, canary_evidence_sha256 = (
|
|
14234
|
+
_benchmark_study_v2_verify_canary_evidence(
|
|
14235
|
+
manifest=manifest, manifest_sha256=manifest_sha256,
|
|
14236
|
+
output_root=output_root,
|
|
14237
|
+
)
|
|
14238
|
+
)
|
|
14239
|
+
rows = _benchmark_study_v2_read_attempts(
|
|
14240
|
+
output_root / "attempts.jsonl", manifest=manifest,
|
|
14241
|
+
manifest_sha256=manifest_sha256,
|
|
14242
|
+
)
|
|
14243
|
+
tasks = parse_tasks(Path(manifest["inputs"]["tasks_path"]))
|
|
14244
|
+
load_task_fixture_trees(
|
|
14245
|
+
tasks, task_file_dir=Path(manifest["inputs"]["tasks_path"]).parent,
|
|
14246
|
+
)
|
|
14247
|
+
_benchmark_study_v2_revalidate_terminal_evidence(
|
|
14248
|
+
manifest=manifest, output_root=output_root, rows=rows,
|
|
14249
|
+
tasks_by_id={task.id: task for task in tasks},
|
|
14250
|
+
variants=_benchmark_study_v2_variants(manifest, output_root),
|
|
14251
|
+
)
|
|
14252
|
+
invalid_decision = _benchmark_study_v2_invalid_analytic_decision(
|
|
14253
|
+
output_root=output_root, manifest=manifest,
|
|
14254
|
+
manifest_sha256=manifest_sha256, rows=rows,
|
|
14255
|
+
canary_evidence_sha256=canary_evidence_sha256,
|
|
14256
|
+
)
|
|
14257
|
+
if invalid_decision is not None:
|
|
14258
|
+
return invalid_decision
|
|
14259
|
+
terminal_rows = [row for row in rows if row["state"] == "terminal"]
|
|
14260
|
+
final_states = {row["run_id"]: row["state"] for row in rows}
|
|
14261
|
+
if len(final_states) != 216 or any(
|
|
14262
|
+
state not in {"terminal", "not_needed", "blocked_study_invalid"}
|
|
14263
|
+
for state in final_states.values()
|
|
14264
|
+
):
|
|
14265
|
+
raise ValueError("v2 analysis requires final accounting for all 216 identities")
|
|
14266
|
+
identity_state_counts = dict(sorted(collections.Counter(final_states.values()).items()))
|
|
14267
|
+
if any(
|
|
14268
|
+
row["terminal_status"] not in {"success", "valid_task_failure_v1"}
|
|
14269
|
+
for row in terminal_rows
|
|
14270
|
+
):
|
|
14271
|
+
raise ValueError("v2 analysis refuses infrastructure-invalid or recovered attempts")
|
|
14272
|
+
initial = [row for row in terminal_rows if row["attempt"] == 0]
|
|
14273
|
+
if len(initial) != 108:
|
|
14274
|
+
raise ValueError("v2 analysis requires all 108 initial provider calls")
|
|
14275
|
+
by_unit = {(row["task_id"], row["repetition"], row["arm"]): row for row in initial}
|
|
14276
|
+
expected_retries = {
|
|
14277
|
+
key for key, row in by_unit.items()
|
|
14278
|
+
if row["terminal_status"] == "valid_task_failure_v1"
|
|
14279
|
+
}
|
|
14280
|
+
retry_rows = [row for row in terminal_rows if row["attempt"] == 1]
|
|
14281
|
+
if {(row["task_id"], row["repetition"], row["arm"]) for row in retry_rows} != expected_retries:
|
|
14282
|
+
raise ValueError("v2 analysis retry coverage is incomplete or replaced")
|
|
14283
|
+
effect_rows = [{
|
|
14284
|
+
"task_id": row["task_id"], "repetition": row["repetition"],
|
|
14285
|
+
"arm": row["arm"], "attempt": row["attempt"],
|
|
14286
|
+
"terminal_status": row["terminal_status"], "success": row["success"],
|
|
14287
|
+
"tokens": row["primary_tokens"], "correction": row["correction"],
|
|
14288
|
+
"retrieval": row["retrieval"],
|
|
14289
|
+
} for row in terminal_rows]
|
|
14290
|
+
terminal_by_unit = dict(by_unit)
|
|
14291
|
+
for row in retry_rows:
|
|
14292
|
+
terminal_by_unit[(row["task_id"], row["repetition"], row["arm"])] = row
|
|
14293
|
+
binary_rows = [
|
|
14294
|
+
{"task_id": row["task_id"], "repetition": row["repetition"],
|
|
14295
|
+
"arm": row["arm"], "success": row["success"]}
|
|
14296
|
+
for row in terminal_by_unit.values()
|
|
14297
|
+
if row["arm"] in BENCHMARK_STUDY_V2_PRIMARY_CONTRAST
|
|
14298
|
+
]
|
|
14299
|
+
inference = infer_benchmark_study_v2_binary(
|
|
14300
|
+
binary_rows, task_order=manifest["inputs"]["task_ids"],
|
|
14301
|
+
ni_margin=float(manifest["plan"]["noninferiority_margin"]),
|
|
14302
|
+
)
|
|
14303
|
+
effects = compute_benchmark_study_v2_effects(
|
|
14304
|
+
effect_rows, task_order=manifest["inputs"]["task_ids"],
|
|
14305
|
+
)
|
|
14306
|
+
unavailable = {"available": False, "value": None, "reason": "observer_absent"}
|
|
14307
|
+
return {
|
|
14308
|
+
"schema_version": BENCHMARK_STUDY_V2_REPORT_SCHEMA_VERSION,
|
|
14309
|
+
"study_version": "v2", "decision": "P1-F",
|
|
14310
|
+
"manifest_sha256": manifest_sha256,
|
|
14311
|
+
"record_count": len(terminal_rows), "initial_provider_calls": 108,
|
|
14312
|
+
"retry_provider_calls": len(retry_rows),
|
|
14313
|
+
"discarded_canary_provider_calls": 2,
|
|
14314
|
+
"identity_state_counts": identity_state_counts,
|
|
14315
|
+
"binary_inference": inference, "effects": effects,
|
|
14316
|
+
"observers": {
|
|
14317
|
+
"correction": dict(unavailable), "retrieval": dict(unavailable),
|
|
14318
|
+
"shifted_cost": dict(unavailable),
|
|
14319
|
+
},
|
|
14320
|
+
"descriptive_only": True, "claim_allowed": False, "claim": None,
|
|
14321
|
+
"claim_readiness": {
|
|
14322
|
+
"claim_ready": False, "descriptive_only": True,
|
|
14323
|
+
"claim_allowed": False, "unmet_gates": ["power"],
|
|
14324
|
+
},
|
|
14325
|
+
"provenance": {
|
|
14326
|
+
"source": "direct_cli_plus_bound_checker",
|
|
14327
|
+
"success_source": "provider_terminal_usage_and_bound_checker",
|
|
14328
|
+
"provider_success_boolean_trusted": False,
|
|
14329
|
+
"cli_binding_sha256": _study_domain_hash(
|
|
14330
|
+
"contextguard.bench.v2.cli-binding.v1",
|
|
14331
|
+
manifest["inputs"]["cli_binding"],
|
|
14332
|
+
),
|
|
14333
|
+
"auth_context_sha256": _study_domain_hash(
|
|
14334
|
+
"contextguard.bench.v2.auth-context.v1",
|
|
14335
|
+
manifest["inputs"]["auth_context"],
|
|
14336
|
+
),
|
|
14337
|
+
"cli_version_stdout_sha256": manifest["inputs"]["cli_binding"][
|
|
14338
|
+
"probe"
|
|
14339
|
+
]["version_stdout_sha256"],
|
|
14340
|
+
"backend_revision": "unavailable",
|
|
14341
|
+
"model_revision": "unavailable",
|
|
14342
|
+
"canary_evidence_sha256": canary_evidence_sha256,
|
|
14343
|
+
"canary_discarded_from_analysis": True,
|
|
14344
|
+
},
|
|
14345
|
+
}
|
|
14346
|
+
|
|
14347
|
+
|
|
14348
|
+
BENCHMARK_STUDY_V2_NAMESPACE = "contextguard.bench.v2"
|
|
14349
|
+
|
|
14350
|
+
|
|
14351
|
+
def _benchmark_study_v2_task_ids_from_corpus(corpus_bytes: bytes) -> list[str]:
|
|
14352
|
+
try:
|
|
14353
|
+
corpus = json.loads(corpus_bytes.decode("utf-8"))
|
|
14354
|
+
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
14355
|
+
raise ValueError("v2 task corpus is invalid JSON") from exc
|
|
14356
|
+
if not isinstance(corpus, list):
|
|
14357
|
+
raise ValueError("v2 task corpus is not a list")
|
|
14358
|
+
return _benchmark_study_v2_task_ids([
|
|
14359
|
+
row.get("id") if isinstance(row, Mapping) else None for row in corpus
|
|
14360
|
+
])
|
|
14361
|
+
|
|
14362
|
+
|
|
14363
|
+
def _benchmark_study_v2_task_ids_sha256(task_ids: Sequence[str]) -> str:
|
|
14364
|
+
return _study_domain_hash(
|
|
14365
|
+
BENCHMARK_STUDY_V2_CORPUS_TASK_ORDER_DOMAIN,
|
|
14366
|
+
_benchmark_study_v2_task_ids(task_ids),
|
|
14367
|
+
)
|
|
14368
|
+
|
|
14369
|
+
|
|
14370
|
+
def benchmark_study_v2_checker_binding(checkers_dir: Path) -> dict[str, Any]:
|
|
14371
|
+
"""Hash an ordered relative filename/size/content-digest checker inventory."""
|
|
14372
|
+
directory_fd = _ensure_directory_no_symlink(checkers_dir, create=False)
|
|
14373
|
+
try:
|
|
14374
|
+
names = sorted(
|
|
14375
|
+
entry.name for entry in checkers_dir.iterdir()
|
|
14376
|
+
if entry.name.endswith(".py") and entry.is_file() and not entry.is_symlink()
|
|
14377
|
+
)
|
|
14378
|
+
finally:
|
|
14379
|
+
os.close(directory_fd)
|
|
14380
|
+
if len(names) != 12:
|
|
14381
|
+
raise ValueError("v2 checker directory must contain exactly 12 regular Python files")
|
|
14382
|
+
files = []
|
|
14383
|
+
for name in names:
|
|
14384
|
+
raw = _read_bytes_no_follow(
|
|
14385
|
+
checkers_dir / name, max_bytes=MAX_FIXTURE_FILE_BYTES,
|
|
14386
|
+
)
|
|
14387
|
+
files.append({
|
|
14388
|
+
"filename": name,
|
|
14389
|
+
"size": len(raw),
|
|
14390
|
+
"sha256": _study_sha256_bytes(raw),
|
|
14391
|
+
})
|
|
14392
|
+
binding = {
|
|
14393
|
+
"domain": BENCHMARK_STUDY_V2_CHECKER_BINDING_DOMAIN,
|
|
14394
|
+
"files": files,
|
|
14395
|
+
"sha256": _study_domain_hash(
|
|
14396
|
+
BENCHMARK_STUDY_V2_CHECKER_BINDING_DOMAIN, files,
|
|
14397
|
+
),
|
|
14398
|
+
}
|
|
14399
|
+
validate_benchmark_study_v2_checker_binding(binding)
|
|
14400
|
+
return binding
|
|
14401
|
+
|
|
14402
|
+
|
|
14403
|
+
def main(argv: Sequence[str] | None = None) -> int:
|
|
14404
|
+
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
|
14405
|
+
parser.add_argument("--tasks", default=None, type=Path, help="task fixture JSON")
|
|
14406
|
+
parser.add_argument("--variants", default=None, type=Path, help="variant fixture JSON")
|
|
14407
|
+
parser.add_argument("--csv", default=None, type=Path,
|
|
14408
|
+
help="results CSV path (header is added on first write)")
|
|
14409
|
+
parser.add_argument("--task-id", default=None, help="run only the named task id")
|
|
14410
|
+
parser.add_argument("--variant", default=None, help="run only the named variant")
|
|
14411
|
+
parser.add_argument("--claude-bin", default=os.environ.get("CLAUDE_BIN", "claude"),
|
|
14412
|
+
help="claude CLI executable (default: $CLAUDE_BIN or 'claude')")
|
|
14413
|
+
parser.add_argument("--project-root", default=Path("."), type=Path,
|
|
14414
|
+
help="working directory used for success_command (default: cwd)")
|
|
14415
|
+
parser.add_argument("--dry-run", action="store_true",
|
|
14416
|
+
help="print the claude command without invoking it")
|
|
14417
|
+
parser.add_argument("--resume", action="store_true",
|
|
14418
|
+
help="skip (task_id, variant) rows already present in --csv")
|
|
14419
|
+
parser.add_argument("--ledger-jsonl", default=None, type=Path,
|
|
14420
|
+
help="optional JSONL ledger path for cost-shift accounting per run")
|
|
14421
|
+
parser.add_argument("--report-json", default=None, type=Path,
|
|
14422
|
+
help="optional A/B summary report JSON path generated from --csv after real runs")
|
|
14423
|
+
parser.add_argument("--dashboard-md", default=None, type=Path,
|
|
14424
|
+
help="optional Markdown dashboard path generated from the benchmark report")
|
|
14425
|
+
parser.add_argument("--evidence-jsonl", default=None, type=Path,
|
|
14426
|
+
help="optional validated run-evidence JSONL replay input; skips provider invocation")
|
|
14427
|
+
parser.add_argument("--baseline-variant", default="baseline",
|
|
14428
|
+
help="variant name used as the report baseline (default: baseline)")
|
|
14429
|
+
parser.add_argument("--measurement-study-plan", default=None, type=Path,
|
|
14430
|
+
help="exact S002 measurement study plan JSON")
|
|
14431
|
+
parser.add_argument(
|
|
14432
|
+
"--measurement-study-action",
|
|
14433
|
+
default=None,
|
|
14434
|
+
choices=("prepare", "run", "resume", "analyze"),
|
|
14435
|
+
help="S002 measurement study action",
|
|
14436
|
+
)
|
|
14437
|
+
parser.add_argument("--measurement-study-output-root", default=None, type=Path,
|
|
14438
|
+
help="private S002 measurement study artifact directory")
|
|
14439
|
+
parser.add_argument(
|
|
14440
|
+
"--study-v2-action", default=None,
|
|
14441
|
+
choices=("prepare", "canary", "run", "resume", "analyze"),
|
|
14442
|
+
help="prepare, canary, run, resume, or analyze the executable additive v2 study",
|
|
14443
|
+
)
|
|
14444
|
+
parser.add_argument("--study-v2-plan", default=None, type=Path,
|
|
14445
|
+
help="canonical v2 study plan used only by --study-v2-action prepare")
|
|
14446
|
+
parser.add_argument("--study-v2-tasks", default=None, type=Path,
|
|
14447
|
+
help="frozen v2 task corpus used only by --study-v2-action prepare")
|
|
14448
|
+
parser.add_argument("--study-v2-checkers-dir", default=None, type=Path,
|
|
14449
|
+
help="frozen v2 checker directory used only by --study-v2-action prepare")
|
|
14450
|
+
parser.add_argument("--study-v2-candidate-hash", default=None,
|
|
14451
|
+
help="exact candidate SHA-256 used only by --study-v2-action prepare")
|
|
14452
|
+
parser.add_argument("--study-v2-source-commit", default=None,
|
|
14453
|
+
help="approved 40-hex source commit used only by --study-v2-action prepare")
|
|
14454
|
+
parser.add_argument("--study-v2-retained-ref", default=None,
|
|
14455
|
+
help="retained candidate ref resolved during live v2 prepare")
|
|
14456
|
+
parser.add_argument(
|
|
14457
|
+
"--study-v2-offline-rehearsal", action="store_true",
|
|
14458
|
+
help="mark provider-free fake-host rehearsal; never use for live prepare",
|
|
14459
|
+
)
|
|
14460
|
+
parser.add_argument("--study-v2-output-root", default=None, type=Path,
|
|
14461
|
+
help="private executable v2 lifecycle directory")
|
|
14462
|
+
parser.add_argument("--study-v2-candidate-manifest", default=None, type=Path,
|
|
14463
|
+
help="canonical build-once npm candidate manifest used by v2 prepare")
|
|
14464
|
+
parser.add_argument("--study-v2-candidate-checksums", default=None, type=Path,
|
|
14465
|
+
help="exact candidate checksum document (default: manifest sibling)")
|
|
14466
|
+
parser.add_argument("--study-v2-npm-bin", default="npm",
|
|
14467
|
+
help="npm executable used once for the offline candidate install")
|
|
14468
|
+
parser.add_argument(
|
|
14469
|
+
"--study-v2-use-existing-login", action="store_true",
|
|
14470
|
+
help=(
|
|
14471
|
+
"allow executable v2 provider actions to reuse the exact CLI's "
|
|
14472
|
+
"existing first-party login without importing credential environment variables"
|
|
14473
|
+
),
|
|
14474
|
+
)
|
|
14475
|
+
args = parser.parse_args(argv)
|
|
14476
|
+
|
|
14477
|
+
require_no_follow_file_ops_supported()
|
|
14478
|
+
v2_values = (
|
|
14479
|
+
args.study_v2_action, args.study_v2_plan, args.study_v2_tasks,
|
|
14480
|
+
args.study_v2_checkers_dir, args.study_v2_candidate_hash,
|
|
14481
|
+
args.study_v2_source_commit, args.study_v2_retained_ref,
|
|
14482
|
+
args.study_v2_output_root, args.study_v2_candidate_manifest,
|
|
14483
|
+
args.study_v2_candidate_checksums,
|
|
14484
|
+
)
|
|
14485
|
+
if args.study_v2_use_existing_login and args.study_v2_action is None:
|
|
14486
|
+
parser.error("--study-v2-use-existing-login requires --study-v2-action")
|
|
14487
|
+
if args.study_v2_offline_rehearsal and args.study_v2_action is None:
|
|
14488
|
+
parser.error("--study-v2-offline-rehearsal requires --study-v2-action prepare")
|
|
14489
|
+
if any(value is not None for value in v2_values):
|
|
14490
|
+
if args.study_v2_action is None:
|
|
14491
|
+
parser.error("--study-v2-action is required when any --study-v2-* option is used")
|
|
14492
|
+
conflicts = [
|
|
14493
|
+
name for name, active in (
|
|
14494
|
+
("--tasks", args.tasks is not None), ("--variants", args.variants is not None),
|
|
14495
|
+
("--csv", args.csv is not None), ("--task-id", args.task_id is not None),
|
|
14496
|
+
("--variant", args.variant is not None), ("--dry-run", args.dry_run),
|
|
14497
|
+
("--resume", args.resume), ("--ledger-jsonl", args.ledger_jsonl is not None),
|
|
14498
|
+
("--report-json", args.report_json is not None),
|
|
14499
|
+
("--dashboard-md", args.dashboard_md is not None),
|
|
14500
|
+
("--evidence-jsonl", args.evidence_jsonl is not None),
|
|
14501
|
+
("--measurement-study-plan", args.measurement_study_plan is not None),
|
|
14502
|
+
("--measurement-study-action", args.measurement_study_action is not None),
|
|
14503
|
+
("--measurement-study-output-root", args.measurement_study_output_root is not None),
|
|
14504
|
+
) if active
|
|
14505
|
+
]
|
|
14506
|
+
if conflicts:
|
|
14507
|
+
parser.error(f"v2 executable mode conflicts with {', '.join(conflicts)}")
|
|
14508
|
+
if args.study_v2_output_root is None:
|
|
14509
|
+
parser.error("v2 executable actions require --study-v2-output-root")
|
|
14510
|
+
if (
|
|
14511
|
+
args.study_v2_action != "analyze"
|
|
14512
|
+
and not args.study_v2_use_existing_login
|
|
14513
|
+
):
|
|
14514
|
+
parser.error(
|
|
14515
|
+
"v2 prepare/canary/run/resume requires "
|
|
14516
|
+
"--study-v2-use-existing-login"
|
|
14517
|
+
)
|
|
14518
|
+
auth_home = Path(os.environ.get("HOME", ""))
|
|
14519
|
+
if args.study_v2_action == "prepare":
|
|
14520
|
+
required = (
|
|
14521
|
+
args.study_v2_plan, args.study_v2_tasks, args.study_v2_checkers_dir,
|
|
14522
|
+
args.study_v2_candidate_hash, args.study_v2_source_commit,
|
|
14523
|
+
args.study_v2_candidate_manifest,
|
|
14524
|
+
)
|
|
14525
|
+
forbidden = ()
|
|
14526
|
+
if args.study_v2_offline_rehearsal:
|
|
14527
|
+
if args.study_v2_retained_ref is not None:
|
|
14528
|
+
parser.error(
|
|
14529
|
+
"v2 offline rehearsal conflicts with --study-v2-retained-ref"
|
|
14530
|
+
)
|
|
14531
|
+
elif args.study_v2_retained_ref is None:
|
|
14532
|
+
parser.error("v2 live prepare requires --study-v2-retained-ref")
|
|
14533
|
+
else:
|
|
14534
|
+
required = (args.study_v2_output_root,)
|
|
14535
|
+
forbidden = (
|
|
14536
|
+
args.study_v2_plan, args.study_v2_tasks, args.study_v2_checkers_dir,
|
|
14537
|
+
args.study_v2_candidate_hash, args.study_v2_source_commit,
|
|
14538
|
+
args.study_v2_retained_ref, args.study_v2_candidate_manifest,
|
|
14539
|
+
args.study_v2_candidate_checksums,
|
|
14540
|
+
)
|
|
14541
|
+
if args.study_v2_offline_rehearsal:
|
|
14542
|
+
parser.error("--study-v2-offline-rehearsal is prepare-only")
|
|
14543
|
+
if not all(value is not None for value in required) or any(value is not None for value in forbidden):
|
|
14544
|
+
parser.error("v2 executable action has incomplete or conflicting arguments")
|
|
14545
|
+
try:
|
|
14546
|
+
if args.study_v2_action == "prepare":
|
|
14547
|
+
prepare_benchmark_study_v2_executable(
|
|
14548
|
+
output_root=args.study_v2_output_root,
|
|
14549
|
+
plan_path=args.study_v2_plan, tasks_path=args.study_v2_tasks,
|
|
14550
|
+
checkers_dir=args.study_v2_checkers_dir,
|
|
14551
|
+
candidate_manifest_path=args.study_v2_candidate_manifest,
|
|
14552
|
+
candidate_checksum_path=args.study_v2_candidate_checksums,
|
|
14553
|
+
expected_candidate_hash=args.study_v2_candidate_hash,
|
|
14554
|
+
npm_bin=args.study_v2_npm_bin,
|
|
14555
|
+
claude_bin=args.claude_bin,
|
|
14556
|
+
auth_home=auth_home,
|
|
14557
|
+
approved_source_commit=args.study_v2_source_commit,
|
|
14558
|
+
retained_ref=args.study_v2_retained_ref,
|
|
14559
|
+
offline_rehearsal=args.study_v2_offline_rehearsal,
|
|
14560
|
+
)
|
|
14561
|
+
print(f"prepared executable v2 study: {args.study_v2_output_root}")
|
|
14562
|
+
elif args.study_v2_action == "canary":
|
|
14563
|
+
summary = execute_benchmark_study_v2_canary(
|
|
14564
|
+
output_root=args.study_v2_output_root,
|
|
14565
|
+
claude_bin=args.claude_bin,
|
|
14566
|
+
auth_home=auth_home,
|
|
14567
|
+
)
|
|
14568
|
+
print(json.dumps(summary, sort_keys=True, separators=(",", ":")))
|
|
14569
|
+
elif args.study_v2_action in {"run", "resume"}:
|
|
14570
|
+
summary = execute_benchmark_study_v2(
|
|
14571
|
+
output_root=args.study_v2_output_root,
|
|
14572
|
+
claude_bin=args.claude_bin,
|
|
14573
|
+
resume=args.study_v2_action == "resume",
|
|
14574
|
+
auth_home=auth_home,
|
|
14575
|
+
)
|
|
14576
|
+
print(json.dumps(summary, sort_keys=True, separators=(",", ":")))
|
|
14577
|
+
else:
|
|
14578
|
+
report = analyze_benchmark_study_v2_executable(
|
|
14579
|
+
output_root=args.study_v2_output_root,
|
|
14580
|
+
claude_bin=args.claude_bin,
|
|
14581
|
+
)
|
|
14582
|
+
report_name = _benchmark_study_v2_persist_analysis(
|
|
14583
|
+
output_root=args.study_v2_output_root, report=report,
|
|
14584
|
+
)
|
|
14585
|
+
print(
|
|
14586
|
+
"analyzed executable v2 study: "
|
|
14587
|
+
f"{args.study_v2_output_root / report_name}"
|
|
14588
|
+
)
|
|
14589
|
+
if report_name == "study-invalid-decision.json":
|
|
14590
|
+
return 3
|
|
14591
|
+
return 0
|
|
14592
|
+
except (OSError, SystemExit, TypeError, ValueError) as exc:
|
|
14593
|
+
if args.study_v2_action in {"canary", "run", "resume"}:
|
|
14594
|
+
_benchmark_study_v2_persist_invalid_after_refusal(
|
|
14595
|
+
output_root=args.study_v2_output_root,
|
|
14596
|
+
claude_bin=args.claude_bin,
|
|
14597
|
+
)
|
|
14598
|
+
print(f"v2 executable study refused: {exc}", file=sys.stderr)
|
|
14599
|
+
return 2
|
|
14600
|
+
if args.tasks is None or args.variants is None:
|
|
14601
|
+
parser.error("--tasks and --variants are required outside study modes")
|
|
14602
|
+
study_values = (
|
|
14603
|
+
args.measurement_study_plan,
|
|
14604
|
+
args.measurement_study_action,
|
|
14605
|
+
args.measurement_study_output_root,
|
|
14606
|
+
)
|
|
14607
|
+
if any(value is not None for value in study_values):
|
|
14608
|
+
if not all(value is not None for value in study_values):
|
|
14609
|
+
parser.error(
|
|
14610
|
+
"--measurement-study-plan, --measurement-study-action, and "
|
|
14611
|
+
"--measurement-study-output-root are all-or-none"
|
|
14612
|
+
)
|
|
14613
|
+
conflicts = [
|
|
14614
|
+
name for name, active in (
|
|
14615
|
+
("--task-id", args.task_id is not None),
|
|
14616
|
+
("--variant", args.variant is not None),
|
|
14617
|
+
("--resume", args.resume),
|
|
14618
|
+
("--evidence-jsonl", args.evidence_jsonl is not None),
|
|
14619
|
+
("--dry-run", args.dry_run),
|
|
14620
|
+
("--ledger-jsonl", args.ledger_jsonl is not None),
|
|
14621
|
+
("--report-json", args.report_json is not None),
|
|
14622
|
+
("--dashboard-md", args.dashboard_md is not None),
|
|
14623
|
+
("--csv", args.csv is not None),
|
|
14624
|
+
("--baseline-variant", args.baseline_variant != "baseline"),
|
|
14625
|
+
) if active
|
|
14626
|
+
]
|
|
14627
|
+
if conflicts:
|
|
14628
|
+
parser.error(f"measurement study mode conflicts with {', '.join(conflicts)}")
|
|
14629
|
+
return run_measurement_study_action(args)
|
|
14630
|
+
args.csv = args.csv or Path("bench/results.csv")
|
|
14631
|
+
validate_distinct_output_paths(args.csv, args.ledger_jsonl, args.report_json, args.dashboard_md)
|
|
14632
|
+
|
|
14633
|
+
variants = parse_variants(args.variants)
|
|
14634
|
+
tasks = parse_tasks(args.tasks, variants=variants)
|
|
14635
|
+
targets = filter_targets(tasks, variants, args.task_id, args.variant)
|
|
14636
|
+
if not targets:
|
|
14637
|
+
if args.dry_run and (not tasks or not variants):
|
|
14638
|
+
print("completed 0 run(s) (dry-run; no CSV writes)")
|
|
14639
|
+
return 0
|
|
14640
|
+
print("no (task, variant) targets matched the filters", file=sys.stderr)
|
|
14641
|
+
return 1
|
|
14642
|
+
target_task_ids = {task.id for task, _variant in targets}
|
|
14643
|
+
load_task_fixture_trees(
|
|
14644
|
+
[task for task in tasks if task.id in target_task_ids],
|
|
14645
|
+
task_file_dir=args.tasks.parent,
|
|
14646
|
+
)
|
|
10534
14647
|
preflight_measurement_targets(
|
|
10535
14648
|
targets,
|
|
10536
14649
|
claude_bin=args.claude_bin,
|