@ictechgy/context-guard 0.4.16 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -54,6 +54,7 @@ dry-run 모드는 실제 호출은 하지 않고 어떤 명령이 실행될지
54
54
  from __future__ import annotations
55
55
 
56
56
  import argparse
57
+ import base64
57
58
  import collections
58
59
  from contextlib import contextmanager, nullcontext
59
60
  import csv
@@ -71,13 +72,14 @@ import stat
71
72
  import struct
72
73
  import subprocess
73
74
  import sys
75
+ import tarfile
74
76
  import tempfile
75
77
  import time
76
78
  import unicodedata
77
79
  from collections.abc import Iterable
78
80
  from dataclasses import dataclass, field, replace
79
81
  from fractions import Fraction
80
- from pathlib import Path
82
+ from pathlib import Path, PurePosixPath
81
83
  from typing import Any, Callable, Mapping, Sequence
82
84
 
83
85
  try:
@@ -400,6 +402,7 @@ MEASUREMENT_DOCUMENTED_HOOK_EVENTS = (
400
402
  )
401
403
  SUCCESS_COMMAND_OUTPUT_MAX_BYTES = 64_000
402
404
  VERSION_OUTPUT_MAX_BYTES = 16_000
405
+ MEASUREMENT_CLI_PROBE_OUTPUT_MAX_BYTES = 65_536
403
406
  PROCESS_TERMINATE_GRACE_SECONDS = 2.0
404
407
  ALLOWED_FIRST_ABSOLUTE_SYMLINKS = {
405
408
  "tmp": Path("/private/tmp"),
@@ -991,6 +994,64 @@ def _stream_contains_nonfinite(value: Any) -> bool:
991
994
  return False
992
995
 
993
996
 
997
+ def _stream_is_post_result_task_cleanup(
998
+ terminal_payload: dict[str, Any], events: list[dict[str, Any]],
999
+ ) -> bool:
1000
+ """Recognize Claude Code's exact bounded background-task shutdown tail."""
1001
+ if len(events) != 3:
1002
+ return False
1003
+ session_id = terminal_payload.get("session_id")
1004
+ if not isinstance(session_id, str) or not session_id:
1005
+ return False
1006
+ changed, updated, notification = events
1007
+ if (
1008
+ set(changed) != {"type", "subtype", "session_id", "tasks", "uuid"}
1009
+ or changed.get("type") != "system"
1010
+ or changed.get("subtype") != "background_tasks_changed"
1011
+ or changed.get("session_id") != session_id
1012
+ or changed.get("tasks") != []
1013
+ or not isinstance(changed.get("uuid"), str)
1014
+ or not changed["uuid"]
1015
+ ):
1016
+ return False
1017
+ patch = updated.get("patch")
1018
+ task_id = updated.get("task_id")
1019
+ if (
1020
+ set(updated) != {"type", "subtype", "session_id", "task_id", "patch", "uuid"}
1021
+ or updated.get("type") != "system"
1022
+ or updated.get("subtype") != "task_updated"
1023
+ or updated.get("session_id") != session_id
1024
+ or not isinstance(task_id, str)
1025
+ or not task_id
1026
+ or not isinstance(patch, dict)
1027
+ or set(patch) != {"end_time", "status"}
1028
+ or isinstance(patch.get("end_time"), bool)
1029
+ or not isinstance(patch.get("end_time"), int)
1030
+ or patch["end_time"] < 0
1031
+ or patch.get("status") != "killed"
1032
+ or not isinstance(updated.get("uuid"), str)
1033
+ or not updated["uuid"]
1034
+ ):
1035
+ return False
1036
+ return (
1037
+ set(notification) == {
1038
+ "type", "subtype", "session_id", "task_id", "tool_use_id",
1039
+ "status", "summary", "output_file", "uuid",
1040
+ }
1041
+ and notification.get("type") == "system"
1042
+ and notification.get("subtype") == "task_notification"
1043
+ and notification.get("session_id") == session_id
1044
+ and notification.get("task_id") == task_id
1045
+ and isinstance(notification.get("tool_use_id"), str)
1046
+ and bool(notification["tool_use_id"])
1047
+ and notification.get("status") == "stopped"
1048
+ and isinstance(notification.get("summary"), str)
1049
+ and isinstance(notification.get("output_file"), str)
1050
+ and isinstance(notification.get("uuid"), str)
1051
+ and bool(notification["uuid"])
1052
+ )
1053
+
1054
+
994
1055
  def parse_claude_stream_output(
995
1056
  stdout: bytes | str,
996
1057
  *,
@@ -1046,6 +1107,7 @@ def parse_claude_stream_output(
1046
1107
  terminal_payload: dict[str, Any] | None = None
1047
1108
  terminal_result_code: str | None = None
1048
1109
  terminal_status: str | None = None
1110
+ post_result_events: list[dict[str, Any]] = []
1049
1111
  for raw_line in physical_lines:
1050
1112
  # CR in a CRLF record is a delimiter byte, not part of the JSON content.
1051
1113
  line = raw_line[:-1] if raw_line.endswith(b"\r") else raw_line
@@ -1104,11 +1166,14 @@ def parse_claude_stream_output(
1104
1166
 
1105
1167
  is_result = event.get("type") == "result"
1106
1168
  if terminal_payload is not None:
1107
- return _stream_result(
1108
- "invalid_stream",
1109
- result_code="invalid_stream",
1110
- error_code="stream_duplicate_result" if is_result else "stream_post_result",
1111
- )
1169
+ if is_result:
1170
+ return _stream_result(
1171
+ "invalid_stream",
1172
+ result_code="invalid_stream",
1173
+ error_code="stream_duplicate_result",
1174
+ )
1175
+ post_result_events.append(event)
1176
+ continue
1112
1177
  if not is_result:
1113
1178
  continue
1114
1179
 
@@ -1144,6 +1209,12 @@ def parse_claude_stream_output(
1144
1209
  result_code="missing_terminal",
1145
1210
  error_code="stream_missing_terminal",
1146
1211
  )
1212
+ if post_result_events and not _stream_is_post_result_task_cleanup(
1213
+ terminal_payload, post_result_events,
1214
+ ):
1215
+ return _stream_result(
1216
+ "invalid_stream", result_code="invalid_stream", error_code="stream_post_result",
1217
+ )
1147
1218
  return _stream_result(
1148
1219
  terminal_status,
1149
1220
  result_code=terminal_result_code,
@@ -1484,8 +1555,11 @@ def _measurement_parse_variant(
1484
1555
  if any(item not in MEASUREMENT_DOCUMENTED_HOOK_EVENTS for item in required_event_classes):
1485
1556
  raise SystemExit(f"{owner}.hook_events.required_event_classes contains unsupported hook event")
1486
1557
  ordered_classes = tuple(dict.fromkeys(event for event, _command in registered_bindings))
1487
- if required_event_classes != ordered_classes:
1488
- raise SystemExit(f"{owner}.hook_events.required_event_classes must match binding order")
1558
+ required_set = set(required_event_classes)
1559
+ if required_event_classes != tuple(event for event in ordered_classes if event in required_set):
1560
+ raise SystemExit(
1561
+ f"{owner}.hook_events.required_event_classes must be an ordered subset of registered hook events"
1562
+ )
1489
1563
  if variant_name == "baseline" and (registered_bindings or required_event_classes):
1490
1564
  raise SystemExit(f"{owner} baseline hook configuration must be empty")
1491
1565
 
@@ -1638,7 +1712,7 @@ def _measurement_validate_treatment_bindings(spec: MeasurementVariant) -> dict[s
1638
1712
  raise SystemExit("measurement baseline and treatment settings differ outside registered hooks")
1639
1713
  binding_set = set(spec.registered_bindings)
1640
1714
  occurrences = {binding: 0 for binding in spec.registered_bindings}
1641
- for event in spec.required_event_classes:
1715
+ for event in dict.fromkeys(event for event, _command in spec.registered_bindings):
1642
1716
  registrations = hooks.get(event)
1643
1717
  if not isinstance(registrations, list) or not registrations:
1644
1718
  raise SystemExit("measurement baseline and treatment settings differ outside registered hooks")
@@ -2632,7 +2706,12 @@ def collect_self_hosted_metrics(payload: Any) -> dict[str, Any] | None:
2632
2706
  return None
2633
2707
 
2634
2708
 
2635
- def _measurement_child_env(spec: MeasurementVariant, context: MeasurementRunContext | None = None) -> dict[str, str]:
2709
+ def _measurement_child_env(
2710
+ spec: MeasurementVariant,
2711
+ context: MeasurementRunContext | None = None,
2712
+ *,
2713
+ existing_login_home: Path | None = None,
2714
+ ) -> dict[str, str]:
2636
2715
  env: dict[str, str] = {}
2637
2716
  for name in spec.environment_allow:
2638
2717
  # Names were validated before this function; do not inspect values for
@@ -2644,14 +2723,15 @@ def _measurement_child_env(spec: MeasurementVariant, context: MeasurementRunCont
2644
2723
  env["PATH"] = os.defpath
2645
2724
  if context is not None:
2646
2725
  env.update({
2647
- "HOME": str(context.home),
2726
+ "HOME": str(existing_login_home or context.home),
2648
2727
  "XDG_CONFIG_HOME": str(context.xdg_config),
2649
2728
  "XDG_CACHE_HOME": str(context.xdg_cache),
2650
2729
  "XDG_DATA_HOME": str(context.xdg_data),
2651
2730
  "XDG_STATE_HOME": str(context.xdg_state),
2652
2731
  "TMPDIR": str(context.tmp),
2653
- "CLAUDE_CONFIG_DIR": str(context.session),
2654
2732
  })
2733
+ if existing_login_home is None:
2734
+ env["CLAUDE_CONFIG_DIR"] = str(context.session)
2655
2735
  return env
2656
2736
 
2657
2737
 
@@ -2697,7 +2777,7 @@ def validate_measurement_cli_capabilities(claude_bin: str, spec: MeasurementVari
2697
2777
  [executable, "--help"],
2698
2778
  cwd=cwd,
2699
2779
  timeout_seconds=10,
2700
- max_output_bytes=VERSION_OUTPUT_MAX_BYTES,
2780
+ max_output_bytes=MEASUREMENT_CLI_PROBE_OUTPUT_MAX_BYTES,
2701
2781
  env=env,
2702
2782
  )
2703
2783
  except (OSError, subprocess.TimeoutExpired, ValueError) as exc:
@@ -3061,6 +3141,7 @@ def run_task_checker_study(
3061
3141
  workspace: Path,
3062
3142
  *,
3063
3143
  env: dict[str, str],
3144
+ interpreter_binding: Mapping[str, Any] | None = None,
3064
3145
  ) -> str:
3065
3146
  """Run the content-bound success checker outside the measured workspace.
3066
3147
 
@@ -3076,6 +3157,14 @@ def run_task_checker_study(
3076
3157
  payload = task.success_checker_bytes
3077
3158
  if not payload:
3078
3159
  return "success_checker_infra_invalid"
3160
+ checker_executable = sys.executable
3161
+ if interpreter_binding is not None:
3162
+ try:
3163
+ checker_executable = _benchmark_study_v2_assert_python_binding(
3164
+ interpreter_binding, require_current=False,
3165
+ )
3166
+ except (OSError, TypeError, ValueError):
3167
+ return "success_checker_infra_invalid"
3079
3168
  private_root: str | None = None
3080
3169
  try:
3081
3170
  private_root = tempfile.mkdtemp(prefix="contextguard-bench-checker-")
@@ -3091,7 +3180,7 @@ def run_task_checker_study(
3091
3180
  # PYTHONHOME/PYTHONSTARTUP 또는 sys.path[0] 로 들어오면 심어둔 sitecustomize.py 가
3092
3181
  # 판정기 안에서 실행되어 위협 모델이 무너진다. 환경도 물려받지 않고 최소로 만든다.
3093
3182
  result = run_bounded_command(
3094
- [sys.executable, "-I", str(checker_path)],
3183
+ [checker_executable, "-I", str(checker_path)],
3095
3184
  cwd=workspace,
3096
3185
  timeout_seconds=600,
3097
3186
  max_output_bytes=SUCCESS_COMMAND_OUTPUT_MAX_BYTES,
@@ -3349,9 +3438,11 @@ def normalize_measurement_hook_events(raw: bytes) -> list[dict[str, Any]]:
3349
3438
  def _measurement_resolve_terminal_status(
3350
3439
  *, raw_byte_limit: bool, raw_line_limit: bool, raw_line_byte_limit: bool,
3351
3440
  process_status: str, stream_status: str, hook_result: dict[str, Any],
3352
- arm: str, required_event_classes: tuple[str, ...],
3441
+ arm: str, allowed_event_classes: tuple[str, ...],
3442
+ required_event_classes: tuple[str, ...],
3353
3443
  ) -> str:
3354
3444
  completed_classes = {item["hook_event"] for item in hook_result["hooks"]}
3445
+ allowed_classes = set(allowed_event_classes)
3355
3446
  required_classes = set(required_event_classes)
3356
3447
  hook_process_failed = any(
3357
3448
  item["hook_process_outcome"] != "success" or item["hook_process_exit_code"] not in (None, 0)
@@ -3381,11 +3472,13 @@ def _measurement_resolve_terminal_status(
3381
3472
  ):
3382
3473
  if hook_result.get("classification") == status or status in hook_result.get("failure_flags", ()):
3383
3474
  return status
3384
- if arm == "treatment" and completed_classes - required_classes:
3475
+ hook_arms = {"treatment", "legacy_trim", "bash_reference_v1"}
3476
+ unmodified_arms = {"baseline", "host_unmodified"}
3477
+ if arm in hook_arms and completed_classes - allowed_classes:
3385
3478
  return "unexpected_hook_event_class"
3386
- if arm == "baseline" and hook_result["observed"]:
3479
+ if arm in unmodified_arms and hook_result["observed"]:
3387
3480
  return "baseline_hook_contamination"
3388
- if arm == "treatment" and required_classes - completed_classes:
3481
+ if arm in hook_arms and required_classes - completed_classes:
3389
3482
  return "missing_required_hook_event_class"
3390
3483
  if hook_process_failed:
3391
3484
  return "hook_process_failure"
@@ -3429,12 +3522,13 @@ def _measurement_receipt(
3429
3522
  ) -> dict[str, Any]:
3430
3523
  raw_sha256 = hashlib.sha256(raw).hexdigest()
3431
3524
  raw_lines = len(raw.splitlines())
3432
- required_classes = tuple(dict.fromkeys(event for event, _command in spec.pair_registered_bindings))
3525
+ allowed_classes = tuple(dict.fromkeys(event for event, _command in spec.pair_registered_bindings))
3526
+ required_classes = spec.required_event_classes
3433
3527
  counts = collections.Counter(item["hook_event"] for item in hook_result["hooks"])
3434
3528
  event_class_counts = [
3435
3529
  {"hook_event": event, "count": counts[event]}
3436
3530
  for event in MEASUREMENT_DOCUMENTED_HOOK_EVENTS
3437
- if counts[event] or event in required_classes
3531
+ if counts[event] or event in allowed_classes
3438
3532
  ]
3439
3533
  settings_relative = Path("session") / spec.settings_file.name
3440
3534
  return {
@@ -3503,6 +3597,10 @@ def _run_measurement_fixture_locked(
3503
3597
  locked_root_fd: int,
3504
3598
  on_process_started: Callable[[], None] | None = None,
3505
3599
  measurement_study: bool = False,
3600
+ workspace_overlay: Path | None = None,
3601
+ on_workspace_prepared: Callable[[Path], None] | None = None,
3602
+ checker_interpreter_binding: Mapping[str, Any] | None = None,
3603
+ existing_login_home: Path | None = None,
3506
3604
  ) -> RunResult:
3507
3605
  spec = variant.measurement
3508
3606
  assert spec is not None
@@ -3522,6 +3620,18 @@ def _run_measurement_fixture_locked(
3522
3620
  "load_task_fixture_trees for every task declaring fixture_tree"
3523
3621
  )
3524
3622
  reset_task_fixture_tree(task.fixture_tree_entries or (), context.workspace)
3623
+ if workspace_overlay is not None:
3624
+ destination = context.workspace / BENCHMARK_STUDY_V2_OVERLAY_NAME
3625
+ if destination.exists() or destination.is_symlink():
3626
+ raise SystemExit("measurement candidate overlay destination already exists")
3627
+ shutil.copytree(
3628
+ workspace_overlay,
3629
+ destination,
3630
+ symlinks=True,
3631
+ copy_function=shutil.copy2,
3632
+ )
3633
+ if on_workspace_prepared is not None:
3634
+ on_workspace_prepared(context.workspace)
3525
3635
  settings_snapshot = context.session / spec.settings_file.name
3526
3636
  _measurement_write_exclusive(settings_snapshot, spec.settings_source_bytes)
3527
3637
  try:
@@ -3535,7 +3645,9 @@ def _run_measurement_fixture_locked(
3535
3645
  variant,
3536
3646
  measurement_settings_file=settings_snapshot,
3537
3647
  )
3538
- env = _measurement_child_env(spec, context)
3648
+ env = _measurement_child_env(
3649
+ spec, context, existing_login_home=existing_login_home,
3650
+ )
3539
3651
  try:
3540
3652
  try:
3541
3653
  proc = run_bounded_command(
@@ -3582,6 +3694,9 @@ def _run_measurement_fixture_locked(
3582
3694
  stream_status=parsed.status,
3583
3695
  hook_result=hook_result,
3584
3696
  arm=variant.name,
3697
+ allowed_event_classes=tuple(
3698
+ dict.fromkeys(event for event, _command in spec.pair_registered_bindings)
3699
+ ),
3585
3700
  required_event_classes=spec.required_event_classes,
3586
3701
  )
3587
3702
 
@@ -3660,6 +3775,7 @@ def _run_measurement_fixture_locked(
3660
3775
  )
3661
3776
  checker_classification = run_task_checker_study(
3662
3777
  task, context.workspace, env=env,
3778
+ interpreter_binding=checker_interpreter_binding,
3663
3779
  )
3664
3780
  else:
3665
3781
  checker_classification = run_success_command_study(task, project_root, env=env)
@@ -7402,7 +7518,8 @@ def _verify_existing_measurement_run(
7402
7518
  hooks = receipt.get("hooks")
7403
7519
  if not isinstance(hooks, list) or summary.get("completed_lifecycle_count") != len(hooks):
7404
7520
  raise ValueError("hook count")
7405
- required_classes = list(dict.fromkeys(event for event, _command in spec.pair_registered_bindings))
7521
+ allowed_classes = list(dict.fromkeys(event for event, _command in spec.pair_registered_bindings))
7522
+ required_classes = list(spec.required_event_classes)
7406
7523
  if summary.get("required_event_classes") != required_classes:
7407
7524
  raise ValueError("required hook classes")
7408
7525
  for count_name in ("observed_lifecycle_count", "completed_lifecycle_count"):
@@ -7443,7 +7560,7 @@ def _verify_existing_measurement_run(
7443
7560
  expected_counts = [
7444
7561
  {"hook_event": event, "count": counts[event]}
7445
7562
  for event in MEASUREMENT_DOCUMENTED_HOOK_EVENTS
7446
- if counts[event] or event in required_classes
7563
+ if counts[event] or event in allowed_classes
7447
7564
  ]
7448
7565
  if summary.get("event_class_counts") != expected_counts:
7449
7566
  raise ValueError("event class counts")
@@ -7482,6 +7599,7 @@ def _verify_existing_measurement_run(
7482
7599
  stream_status=reparsed_stream.status,
7483
7600
  hook_result=reparsed_hooks,
7484
7601
  arm=spec.identity.arm,
7602
+ allowed_event_classes=tuple(allowed_classes),
7485
7603
  required_event_classes=spec.required_event_classes,
7486
7604
  )
7487
7605
  expected_receipt = _measurement_receipt(
@@ -8063,8 +8181,10 @@ def parse_measurement_terminal_usage(raw: bytes | str) -> dict[str, int]:
8063
8181
  if parsed.payload is None or parsed.payload.get("type") != "result":
8064
8182
  raise ValueError("measurement terminal usage requires one terminal result record")
8065
8183
  usage = parsed.payload.get("usage")
8066
- if not isinstance(usage, dict) or set(usage) != set(MEASUREMENT_STUDY_USAGE_KEYS):
8067
- raise ValueError("measurement terminal usage must contain the exact four buckets")
8184
+ # Claude Code may add diagnostic telemetry beside the four accounting
8185
+ # buckets. Only the frozen required buckets participate in the estimator.
8186
+ if not isinstance(usage, dict) or not set(MEASUREMENT_STUDY_USAGE_KEYS).issubset(usage):
8187
+ raise ValueError("measurement terminal usage must contain all four required buckets")
8068
8188
  result: dict[str, int] = {}
8069
8189
  total = 0
8070
8190
  for key in MEASUREMENT_STUDY_USAGE_KEYS:
@@ -8681,6 +8801,10 @@ def append_study_attempt_event(path: Path, event: Mapping[str, Any]) -> None:
8681
8801
  fd = _open_regular_no_symlink(path, flags, 0o600)
8682
8802
  os.fchmod(fd, 0o600)
8683
8803
  _measurement_write_fd(fd, payload)
8804
+ # A provider launch may follow this reservation immediately. Persist the
8805
+ # directory entry as well as the file bytes so a power loss cannot make
8806
+ # an already-consumed identity appear unreserved after restart.
8807
+ os.fsync(parent_fd)
8684
8808
  finally:
8685
8809
  if fd >= 0:
8686
8810
  os.close(fd)
@@ -8932,7 +9056,7 @@ def run_measurement_cli_probes(
8932
9056
  [claude_bin, flag],
8933
9057
  cwd=Path("<probe-root>/cwd"),
8934
9058
  timeout_seconds=10.0,
8935
- max_output_bytes=65_536,
9059
+ max_output_bytes=MEASUREMENT_CLI_PROBE_OUTPUT_MAX_BYTES,
8936
9060
  env=dict(env),
8937
9061
  )
8938
9062
  for flag in ("--version", "--help")
@@ -8968,7 +9092,16 @@ def run_measurement_cli_probes(
8968
9092
  root, paths = create_measurement_probe_layout()
8969
9093
  try:
8970
9094
  validate_measurement_probe_layout(root, paths)
8971
- path_value = f"{executable_path.parent}:/usr/bin:/bin:/usr/sbin:/sbin"
9095
+ runtime_directories = [str(executable_path.parent)]
9096
+ invoked = shutil.which(claude_bin)
9097
+ if invoked is not None:
9098
+ runtime_directories.append(str(Path(invoked).absolute().parent))
9099
+ for runtime_name in ("node", "python3"):
9100
+ runtime = shutil.which(runtime_name)
9101
+ if runtime is not None:
9102
+ runtime_directories.append(str(Path(runtime).absolute().parent))
9103
+ runtime_directories.extend(("/usr/bin", "/bin", "/usr/sbin", "/sbin"))
9104
+ path_value = os.pathsep.join(dict.fromkeys(runtime_directories))
8972
9105
  env = {
8973
9106
  "PATH": path_value,
8974
9107
  "LANG": "C",
@@ -8988,7 +9121,7 @@ def run_measurement_cli_probes(
8988
9121
  [executable, flag],
8989
9122
  cwd=paths["cwd"],
8990
9123
  timeout_seconds=10,
8991
- max_output_bytes=65_536,
9124
+ max_output_bytes=MEASUREMENT_CLI_PROBE_OUTPUT_MAX_BYTES,
8992
9125
  env=env,
8993
9126
  ))
8994
9127
  version_raw = _study_validate_probe_output(results[0], kind="version")
@@ -9286,6 +9419,7 @@ def build_measurement_study_manifest(
9286
9419
  "record_type": "result",
9287
9420
  "object_path": "$.usage",
9288
9421
  "keys": list(MEASUREMENT_STUDY_USAGE_KEYS),
9422
+ "additional_fields": "ignored_not_counted_v1",
9289
9423
  "integer_grammar": "0|[1-9][0-9]*",
9290
9424
  "maximum": MAX_USAGE_TOKEN_COUNT,
9291
9425
  "formula": "P=input_tokens+cache_creation_input_tokens+cache_read_input_tokens+output_tokens",
@@ -10452,85 +10586,4064 @@ def run_measurement_study_action(
10452
10586
  return 0
10453
10587
 
10454
10588
 
10455
- def main(argv: Sequence[str] | None = None) -> int:
10456
- parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
10457
- parser.add_argument("--tasks", required=True, type=Path, help="task fixture JSON")
10458
- parser.add_argument("--variants", required=True, type=Path, help="variant fixture JSON")
10459
- parser.add_argument("--csv", default=None, type=Path,
10460
- help="results CSV path (header is added on first write)")
10461
- parser.add_argument("--task-id", default=None, help="run only the named task id")
10462
- parser.add_argument("--variant", default=None, help="run only the named variant")
10463
- parser.add_argument("--claude-bin", default=os.environ.get("CLAUDE_BIN", "claude"),
10464
- help="claude CLI executable (default: $CLAUDE_BIN or 'claude')")
10465
- parser.add_argument("--project-root", default=Path("."), type=Path,
10466
- help="working directory used for success_command (default: cwd)")
10467
- parser.add_argument("--dry-run", action="store_true",
10468
- help="print the claude command without invoking it")
10469
- parser.add_argument("--resume", action="store_true",
10470
- help="skip (task_id, variant) rows already present in --csv")
10471
- parser.add_argument("--ledger-jsonl", default=None, type=Path,
10472
- help="optional JSONL ledger path for cost-shift accounting per run")
10473
- parser.add_argument("--report-json", default=None, type=Path,
10474
- help="optional A/B summary report JSON path generated from --csv after real runs")
10475
- parser.add_argument("--dashboard-md", default=None, type=Path,
10476
- help="optional Markdown dashboard path generated from the benchmark report")
10477
- parser.add_argument("--evidence-jsonl", default=None, type=Path,
10478
- help="optional validated run-evidence JSONL replay input; skips provider invocation")
10479
- parser.add_argument("--baseline-variant", default="baseline",
10480
- help="variant name used as the report baseline (default: baseline)")
10481
- parser.add_argument("--measurement-study-plan", default=None, type=Path,
10482
- help="exact S002 measurement study plan JSON")
10483
- parser.add_argument(
10484
- "--measurement-study-action",
10485
- default=None,
10486
- choices=("prepare", "run", "resume", "analyze"),
10487
- help="S002 measurement study action",
10488
- )
10489
- parser.add_argument("--measurement-study-output-root", default=None, type=Path,
10490
- help="private S002 measurement study artifact directory")
10491
- args = parser.parse_args(argv)
10589
+ # V2 is a separately-versioned analytical surface. It intentionally does not
10590
+ # alter the frozen S001--S003 runner, manifest, or report contracts above.
10591
+ BENCHMARK_STUDY_V2_PLAN_SCHEMA_VERSION = "contextguard.bench.study-plan.v2"
10592
+ BENCHMARK_STUDY_V2_SCHEDULE_ALGORITHM = "splitmix64-blocked-three-arm-v1"
10593
+ BENCHMARK_STUDY_V2_ARMS = (
10594
+ "host_unmodified", "legacy_trim", "bash_reference_v1",
10595
+ )
10596
+ BENCHMARK_STUDY_V2_PRIMARY_CONTRAST = ("host_unmodified", "bash_reference_v1")
10597
+ BENCHMARK_STUDY_V2_DIAGNOSTIC_CONTRAST = ("legacy_trim", "bash_reference_v1")
10598
+ BENCHMARK_STUDY_V2_RETRY_POLICY = "retain_valid_unfavorable_attempts_v1"
10599
+ BENCHMARK_STUDY_V2_EVIDENCE_FORBIDDEN_KEYS = frozenset({
10600
+ "prompt", "output", "command", "command_hash", "command_sha256", "path",
10601
+ "project_id", "capabilities", "credential", "credentials", "token", "secret",
10602
+ })
10603
+ BENCHMARK_STUDY_V2_HANDLE_RE = re.compile(r"(?i)\bcgr1p(?:[_-]|\b)")
10604
+ BENCHMARK_STUDY_V2_REVISION_KEYS = frozenset({
10605
+ "backend_revision", "model_revision", "cli_version",
10606
+ })
10607
+ BENCHMARK_STUDY_V2_REVISION_RE = re.compile(
10608
+ r"[A-Za-z0-9][A-Za-z0-9._+:/@-]{0,127}"
10609
+ )
10610
+ BENCHMARK_STUDY_V2_SECRET_SHAPE_RE = re.compile(
10611
+ r"(?i)(?:"
10612
+ r"\bsk-[A-Za-z0-9_-]{16,}"
10613
+ r"|\b[rs]k_(?:live|test)_[A-Za-z0-9]{16,}"
10614
+ r"|\bgithub_pat_[A-Za-z0-9_]{16,}"
10615
+ r"|\bgh[pousr]_[A-Za-z0-9]{16,}"
10616
+ r"|\bnpm_[A-Za-z0-9]{16,}"
10617
+ r"|\bxox[baprs]-[A-Za-z0-9-]{16,}"
10618
+ r"|\bA[KS]IA[0-9A-Z]{16}\b"
10619
+ r"|\bAIza[0-9A-Za-z_-]{20,}"
10620
+ r"|\bya29\.[0-9A-Za-z_-]{16,}"
10621
+ r"|\bbearer\s+[0-9A-Za-z._~+/-]+=*"
10622
+ r"|-----BEGIN [A-Z ]*PRIVATE KEY-----"
10623
+ r")"
10624
+ )
10625
+ BENCHMARK_STUDY_V2_CHECKER_BINDING_DOMAIN = (
10626
+ "contextguard.bench.v2.checker-binding.v1"
10627
+ )
10628
+ BENCHMARK_STUDY_V2_CORPUS_TASK_ORDER_DOMAIN = (
10629
+ "contextguard.bench.v2.corpus-task-order.v1"
10630
+ )
10492
10631
 
10493
- require_no_follow_file_ops_supported()
10494
- study_values = (
10495
- args.measurement_study_plan,
10496
- args.measurement_study_action,
10497
- args.measurement_study_output_root,
10632
+
10633
+ def _benchmark_study_v2_seed(seed: int | str) -> int:
10634
+ if isinstance(seed, str):
10635
+ if re.fullmatch(r"0x[0-9A-F]{16}", seed) is None:
10636
+ raise ValueError("v2 schedule seed must be frozen uppercase 64-bit hex")
10637
+ return int(seed, 16)
10638
+ if isinstance(seed, bool) or not isinstance(seed, int) or not 0 <= seed <= SPLITMIX64_MASK:
10639
+ raise ValueError("v2 schedule seed must be an unsigned 64-bit integer")
10640
+ return seed
10641
+
10642
+
10643
+ def _benchmark_study_v2_task_ids(task_ids: Sequence[str]) -> list[str]:
10644
+ normalized = list(task_ids)
10645
+ if len(normalized) != 12 or len(set(normalized)) != 12:
10646
+ raise ValueError("v2 study requires exactly 12 unique ordered task ids")
10647
+ if any(not isinstance(task_id, str) or not task_id for task_id in normalized):
10648
+ raise ValueError("v2 study task ids must be non-empty strings")
10649
+ return normalized
10650
+
10651
+
10652
+ def generate_benchmark_study_v2_schedule(
10653
+ task_ids: Sequence[str], *, repetitions: int, schedule_seed: int | str,
10654
+ ) -> list[dict[str, Any]]:
10655
+ """Create the pre-randomized 3-arm order for each task/repetition block."""
10656
+ tasks = _benchmark_study_v2_task_ids(task_ids)
10657
+ if repetitions != 3:
10658
+ raise ValueError("v2 study requires exactly three repetitions")
10659
+ state = _benchmark_study_v2_seed(schedule_seed)
10660
+ schedule: list[dict[str, Any]] = []
10661
+ for task_id in tasks:
10662
+ for repetition in range(repetitions):
10663
+ arm_order = list(BENCHMARK_STUDY_V2_ARMS)
10664
+ for index in range(len(arm_order) - 1, 0, -1):
10665
+ state, selected = splitmix64_bounded(state, index + 1)
10666
+ arm_order[index], arm_order[selected] = arm_order[selected], arm_order[index]
10667
+ schedule.append({
10668
+ "block_id": _study_domain_hash(
10669
+ "contextguard.bench.v2.block-id.v1", [task_id, repetition],
10670
+ ),
10671
+ "task_id": task_id,
10672
+ "repetition": repetition,
10673
+ "arm_order": arm_order,
10674
+ })
10675
+ return schedule
10676
+
10677
+
10678
+ def generate_benchmark_study_v2_slots(
10679
+ task_ids: Sequence[str], schedule: Sequence[Mapping[str, Any]], *,
10680
+ candidate_hash: str, namespace: str,
10681
+ ) -> list[dict[str, Any]]:
10682
+ """Materialize immutable initial/retry identities without replacement."""
10683
+ tasks = _benchmark_study_v2_task_ids(task_ids)
10684
+ if SHA256_HEX_PATTERN.fullmatch(candidate_hash) is None:
10685
+ raise ValueError("v2 candidate hash is invalid")
10686
+ if not isinstance(namespace, str) or not MEASUREMENT_ID_NAMESPACE_RE.fullmatch(namespace):
10687
+ raise ValueError("v2 namespace is invalid")
10688
+ expected_blocks = [(task, repetition) for task in tasks for repetition in range(3)]
10689
+ if [(row.get("task_id"), row.get("repetition")) for row in schedule] != expected_blocks:
10690
+ raise ValueError("v2 schedule task/repetition order drift")
10691
+ initial: list[dict[str, Any]] = []
10692
+ retry: list[dict[str, Any]] = []
10693
+ seen: set[str] = set()
10694
+ for block in schedule:
10695
+ arm_order = block.get("arm_order")
10696
+ if not isinstance(arm_order, list) or set(arm_order) != set(BENCHMARK_STUDY_V2_ARMS) or len(arm_order) != 3:
10697
+ raise ValueError("v2 block arm order is invalid")
10698
+ for arm in arm_order:
10699
+ for attempt, state, destination in ((0, "planned", initial), (1, "conditional", retry)):
10700
+ run_id = MeasurementIdentity(
10701
+ candidate_hash=candidate_hash, repetition=int(block["repetition"]),
10702
+ arm=arm, attempt=attempt, namespace=namespace,
10703
+ ).run_id(str(block["task_id"]))
10704
+ if run_id in seen:
10705
+ raise ValueError("v2 run identity collision")
10706
+ seen.add(run_id)
10707
+ destination.append({
10708
+ "block_id": block["block_id"], "task_id": block["task_id"],
10709
+ "repetition": block["repetition"], "arm": arm, "attempt": attempt,
10710
+ "run_id": run_id, "state": state,
10711
+ })
10712
+ slots = initial + retry
10713
+ validate_benchmark_study_v2_slots(slots, task_ids=tasks)
10714
+ return slots
10715
+
10716
+
10717
+ def validate_benchmark_study_v2_slots(
10718
+ slots: Sequence[Mapping[str, Any]], *, task_ids: Sequence[str],
10719
+ ) -> None:
10720
+ tasks = _benchmark_study_v2_task_ids(task_ids)
10721
+ if len(slots) != 216:
10722
+ raise ValueError("v2 study requires exactly 216 immutable slots")
10723
+ expected = {
10724
+ (task, repetition, arm, attempt)
10725
+ for task in tasks for repetition in range(3)
10726
+ for arm in BENCHMARK_STUDY_V2_ARMS for attempt in (0, 1)
10727
+ }
10728
+ observed: set[tuple[str, int, str, int]] = set()
10729
+ run_ids: set[str] = set()
10730
+ required = {"block_id", "task_id", "repetition", "arm", "attempt", "run_id", "state"}
10731
+ for slot in slots:
10732
+ if set(slot) != required:
10733
+ raise ValueError("v2 slot schema mismatch")
10734
+ key = (slot["task_id"], slot["repetition"], slot["arm"], slot["attempt"])
10735
+ if key in observed or key not in expected or slot["run_id"] in run_ids:
10736
+ raise ValueError("v2 slot identity mismatch")
10737
+ if slot["state"] != ("planned" if slot["attempt"] == 0 else "conditional"):
10738
+ raise ValueError("v2 slot state mismatch")
10739
+ if not isinstance(slot["run_id"], str) or SHA256_HEX_PATTERN.fullmatch(slot["run_id"]) is None:
10740
+ raise ValueError("v2 slot run id is invalid")
10741
+ observed.add(key)
10742
+ run_ids.add(slot["run_id"])
10743
+ if observed != expected:
10744
+ raise ValueError("v2 slot coverage mismatch")
10745
+
10746
+
10747
+ def benchmark_study_v2_contrasts(_values: Mapping[str, Any] | None = None) -> dict[str, list[str]]:
10748
+ """Expose the single product contrast separately from its diagnostic control."""
10749
+ return {
10750
+ "primary": list(BENCHMARK_STUDY_V2_PRIMARY_CONTRAST),
10751
+ "diagnostic": list(BENCHMARK_STUDY_V2_DIAGNOSTIC_CONTRAST),
10752
+ }
10753
+
10754
+
10755
+ def _benchmark_study_v2_cluster_interval(values_by_task: Sequence[Sequence[float]]) -> dict[str, Any]:
10756
+ task_count = len(values_by_task)
10757
+ if task_count < 2 or any(len(row) != 3 for row in values_by_task):
10758
+ raise ValueError("v2 task-cluster interval requires task x 3 values")
10759
+ task_means = [sum(float(value) for value in row) / 3.0 for row in values_by_task]
10760
+ state = MEASUREMENT_STUDY_INFERENCE_SEED
10761
+ estimates: list[float] = []
10762
+ for _ in range(MEASUREMENT_STUDY_BOOTSTRAP_RESAMPLES):
10763
+ total = 0.0
10764
+ for _ in range(task_count):
10765
+ state, index = splitmix64_bounded(state, task_count)
10766
+ total += task_means[index]
10767
+ estimates.append(total / task_count)
10768
+ return {
10769
+ "method": "task_cluster_bootstrap_v2",
10770
+ "point": sum(task_means) / task_count,
10771
+ "q025": float(type7_quantile(estimates, 0.025)),
10772
+ "q975": float(type7_quantile(estimates, 0.975)),
10773
+ "task_count": task_count,
10774
+ "resamples": MEASUREMENT_STUDY_BOOTSTRAP_RESAMPLES,
10775
+ }
10776
+
10777
+
10778
+ def infer_benchmark_study_v2_binary(
10779
+ rows: Sequence[Mapping[str, Any]], *, task_order: Sequence[str], ni_margin: float = 0.10,
10780
+ ) -> dict[str, Any]:
10781
+ """Exact task-cluster sign-permutation inference for the product contrast."""
10782
+ tasks = _benchmark_study_v2_task_ids(task_order)
10783
+ if not isinstance(ni_margin, (int, float)) or isinstance(ni_margin, bool) or not 0 <= ni_margin < 1:
10784
+ raise ValueError("v2 non-inferiority margin is invalid")
10785
+ units: dict[tuple[str, int, str], bool] = {}
10786
+ for row in rows:
10787
+ task_id, repetition, arm, success = row.get("task_id"), row.get("repetition"), row.get("arm"), row.get("success")
10788
+ if task_id not in tasks or repetition not in (0, 1, 2) or arm not in BENCHMARK_STUDY_V2_PRIMARY_CONTRAST or not isinstance(success, bool):
10789
+ raise ValueError("v2 binary outcome identity is invalid")
10790
+ key = (str(task_id), int(repetition), str(arm))
10791
+ if key in units:
10792
+ raise ValueError("duplicate v2 binary outcome")
10793
+ units[key] = success
10794
+ expected = {
10795
+ (task, repetition, arm) for task in tasks for repetition in range(3)
10796
+ for arm in BENCHMARK_STUDY_V2_PRIMARY_CONTRAST
10797
+ }
10798
+ if set(units) != expected:
10799
+ raise ValueError("v2 binary outcome coverage is incomplete")
10800
+ task_deltas = [
10801
+ sum(
10802
+ int(units[(task, repetition, "bash_reference_v1")])
10803
+ - int(units[(task, repetition, "host_unmodified")])
10804
+ for repetition in range(3)
10805
+ ) / 3.0
10806
+ for task in tasks
10807
+ ]
10808
+ point = sum(task_deltas) / len(tasks)
10809
+ all_success = all(units.values())
10810
+ # At the NI boundary, reference-minus-host plus the frozen margin has mean
10811
+ # zero. Sign-flip that centered task effect, never the nested run rows.
10812
+ centered_deltas = [value + float(ni_margin) for value in task_deltas]
10813
+ outcomes = []
10814
+ for mask in range(1 << len(tasks)):
10815
+ outcomes.append(sum(
10816
+ (-value if mask & (1 << index) else value)
10817
+ for index, value in enumerate(centered_deltas)
10818
+ ) / len(tasks))
10819
+ observed_statistic = point + float(ni_margin)
10820
+ p_value = (
10821
+ sum(value >= observed_statistic for value in outcomes) + 1
10822
+ ) / (len(outcomes) + 1)
10823
+ return {
10824
+ "method": "exact_task_cluster_sign_permutation_v1",
10825
+ "contrast": list(BENCHMARK_STUDY_V2_PRIMARY_CONTRAST),
10826
+ "task_ids_sha256": _study_domain_hash(
10827
+ "contextguard.bench.v2.task-order.v1", tasks,
10828
+ ),
10829
+ "point": point,
10830
+ "ni_margin": float(ni_margin),
10831
+ "p_value": p_value,
10832
+ "task_count": len(tasks),
10833
+ "degenerate_all_success": all_success,
10834
+ "noninferiority_pass": bool(not all_success and point > -float(ni_margin) and p_value < 0.05),
10835
+ }
10836
+
10837
+
10838
+ def compute_benchmark_study_v2_effects(
10839
+ records: Sequence[Mapping[str, Any]], *, task_order: Sequence[str],
10840
+ ) -> dict[str, Any]:
10841
+ """Retain every valid terminal attempt and derive task-clustered effects."""
10842
+ tasks = _benchmark_study_v2_task_ids(task_order)
10843
+ grouped: dict[tuple[str, int, str], list[Mapping[str, Any]]] = collections.defaultdict(list)
10844
+ for record in records:
10845
+ task_id, repetition, arm = record.get("task_id"), record.get("repetition"), record.get("arm")
10846
+ if task_id not in tasks or repetition not in (0, 1, 2) or arm not in BENCHMARK_STUDY_V2_ARMS:
10847
+ raise ValueError("v2 effect record identity is invalid")
10848
+ grouped[(str(task_id), int(repetition), str(arm))].append(record)
10849
+ token_deltas: list[list[float]] = []
10850
+ diagnostic_token_deltas: list[list[float]] = []
10851
+ metric_deltas: dict[str, list[list[float]]] = {"correction": [], "retrieval": []}
10852
+ diagnostic_metric_deltas: dict[str, list[list[float]]] = {
10853
+ "correction": [], "retrieval": [],
10854
+ }
10855
+ metric_available = {"correction": True, "retrieval": True}
10856
+ retained_unfavorable = 0
10857
+ for task in tasks:
10858
+ per_task: list[float] = []
10859
+ diagnostic_per_task: list[float] = []
10860
+ per_task_metrics: dict[str, list[float]] = {"correction": [], "retrieval": []}
10861
+ diagnostic_per_task_metrics: dict[str, list[float]] = {
10862
+ "correction": [], "retrieval": [],
10863
+ }
10864
+ for repetition in range(3):
10865
+ costs: dict[str, float] = {}
10866
+ metrics: dict[str, dict[str, float]] = {"correction": {}, "retrieval": {}}
10867
+ for arm in BENCHMARK_STUDY_V2_ARMS:
10868
+ attempts = sorted(grouped.get((task, repetition, arm), ()), key=lambda row: int(row.get("attempt", -1)))
10869
+ if not attempts or [row.get("attempt") for row in attempts] not in ([0], [0, 1]):
10870
+ raise ValueError("v2 attempts are incomplete or replaced")
10871
+ values = []
10872
+ for row in attempts:
10873
+ token = row.get("tokens")
10874
+ if (
10875
+ isinstance(token, bool)
10876
+ or not isinstance(token, (int, float))
10877
+ or not math.isfinite(float(token))
10878
+ or token < 0
10879
+ ):
10880
+ raise ValueError("v2 token value is invalid")
10881
+ if row.get("terminal_status") != "success" or row.get("success") is not True:
10882
+ retained_unfavorable += 1
10883
+ values.append(float(token))
10884
+ costs[arm] = sum(values)
10885
+ for metric in metrics:
10886
+ attempt_values = [row.get(metric) for row in attempts]
10887
+ if any(
10888
+ isinstance(value, bool)
10889
+ or not isinstance(value, (int, float))
10890
+ or not math.isfinite(float(value))
10891
+ or value < 0
10892
+ for value in attempt_values
10893
+ ):
10894
+ metric_available[metric] = False
10895
+ continue
10896
+ metrics[metric][arm] = sum(
10897
+ float(value) for value in attempt_values
10898
+ )
10899
+ per_task.append(costs["host_unmodified"] - costs["bash_reference_v1"])
10900
+ diagnostic_per_task.append(
10901
+ costs["legacy_trim"] - costs["bash_reference_v1"]
10902
+ )
10903
+ for metric, values in metrics.items():
10904
+ if set(values) == set(BENCHMARK_STUDY_V2_ARMS):
10905
+ per_task_metrics[metric].append(
10906
+ values["host_unmodified"] - values["bash_reference_v1"]
10907
+ )
10908
+ diagnostic_per_task_metrics[metric].append(
10909
+ values["legacy_trim"] - values["bash_reference_v1"]
10910
+ )
10911
+ token_deltas.append(per_task)
10912
+ diagnostic_token_deltas.append(diagnostic_per_task)
10913
+ for metric in metric_deltas:
10914
+ if len(per_task_metrics[metric]) == 3:
10915
+ metric_deltas[metric].append(per_task_metrics[metric])
10916
+ diagnostic_metric_deltas[metric].append(
10917
+ diagnostic_per_task_metrics[metric]
10918
+ )
10919
+ else:
10920
+ metric_available[metric] = False
10921
+ metric_effects = {
10922
+ f"{metric}_effect": (
10923
+ _benchmark_study_v2_cluster_interval(metric_deltas[metric])
10924
+ if metric_available[metric] and len(metric_deltas[metric]) == len(tasks)
10925
+ else {"method": "unavailable", "point": None, "q025": None, "q975": None}
10926
+ )
10927
+ for metric in metric_deltas
10928
+ }
10929
+ diagnostic_metric_effects = {
10930
+ f"diagnostic_{metric}_effect": (
10931
+ _benchmark_study_v2_cluster_interval(diagnostic_metric_deltas[metric])
10932
+ if metric_available[metric]
10933
+ and len(diagnostic_metric_deltas[metric]) == len(tasks)
10934
+ else {"method": "unavailable", "point": None, "q025": None, "q975": None}
10935
+ )
10936
+ for metric in diagnostic_metric_deltas
10937
+ }
10938
+ return {
10939
+ "primary_contrast": list(BENCHMARK_STUDY_V2_PRIMARY_CONTRAST),
10940
+ "diagnostic_contrast": list(BENCHMARK_STUDY_V2_DIAGNOSTIC_CONTRAST),
10941
+ "task_ids_sha256": _study_domain_hash(
10942
+ "contextguard.bench.v2.task-order.v1", tasks,
10943
+ ),
10944
+ "retained_unfavorable_runs": retained_unfavorable,
10945
+ "token_effect": _benchmark_study_v2_cluster_interval(token_deltas),
10946
+ "diagnostic_token_effect": _benchmark_study_v2_cluster_interval(
10947
+ diagnostic_token_deltas
10948
+ ),
10949
+ "quality_gate": False,
10950
+ "failure_gate": False,
10951
+ "correction_gate": False,
10952
+ "retrieval_gate": False,
10953
+ "shifted_cost_gate": False,
10954
+ **metric_effects,
10955
+ **diagnostic_metric_effects,
10956
+ }
10957
+
10958
+
10959
+ def make_benchmark_study_v2_plan(
10960
+ *, schedule_seed: str, required_task_count: int, corpus_sha256: str = "0" * 64,
10961
+ checker_sha256: str = "0" * 64, task_ids_sha256: str = "0" * 64,
10962
+ ni_margin: float = 0.10,
10963
+ ) -> dict[str, Any]:
10964
+ """Build the immutable, a-priori v2 analysis plan for a frozen corpus."""
10965
+ plan = {
10966
+ "schema_version": BENCHMARK_STUDY_V2_PLAN_SCHEMA_VERSION,
10967
+ "arms": list(BENCHMARK_STUDY_V2_ARMS), "schedule_seed": schedule_seed,
10968
+ "repetitions": 3, "max_attempts_per_arm_unit": 2,
10969
+ "retry_policy": BENCHMARK_STUDY_V2_RETRY_POLICY,
10970
+ "corpus_sha256": corpus_sha256, "checker_sha256": checker_sha256,
10971
+ "task_ids_sha256": task_ids_sha256,
10972
+ "primary_contrast": list(BENCHMARK_STUDY_V2_PRIMARY_CONTRAST),
10973
+ "diagnostic_contrast": list(BENCHMARK_STUDY_V2_DIAGNOSTIC_CONTRAST),
10974
+ "noninferiority_margin": ni_margin,
10975
+ "power": {
10976
+ "claim_capable": False,
10977
+ "method": "not_estimated_without_independent_effect_model_v1",
10978
+ "reason": "fixed_12_task_corpus_is_descriptive_only",
10979
+ "required_task_count": required_task_count,
10980
+ },
10981
+ "exclusions": "none_after_schedule_except_prelaunch_refusal_v1",
10982
+ "missing_data": "incomplete_primary_pair_is_descriptive_only_v1",
10983
+ "contamination": "any_contamination_blocks_claim_v1",
10984
+ "stopping": "fixed_task_count_no_optional_stopping_v1",
10985
+ "model_cli_fields": ["model_revision", "backend_revision", "cli_version"],
10986
+ "gates": ["quality", "failure", "correction", "retrieval", "shifted_cost"],
10987
+ }
10988
+ validate_benchmark_study_v2_plan(plan)
10989
+ return plan
10990
+
10991
+
10992
+ def validate_benchmark_study_v2_plan(plan: Mapping[str, Any]) -> None:
10993
+ required = {
10994
+ "schema_version", "arms", "schedule_seed", "repetitions", "max_attempts_per_arm_unit", "retry_policy",
10995
+ "corpus_sha256", "checker_sha256", "task_ids_sha256", "primary_contrast", "diagnostic_contrast", "noninferiority_margin",
10996
+ "power", "exclusions", "missing_data", "contamination", "stopping", "model_cli_fields", "gates",
10997
+ }
10998
+ if set(plan) != required or plan.get("schema_version") != BENCHMARK_STUDY_V2_PLAN_SCHEMA_VERSION:
10999
+ raise ValueError("v2 study plan schema mismatch")
11000
+ _benchmark_study_v2_seed(plan["schedule_seed"])
11001
+ if plan["arms"] != list(BENCHMARK_STUDY_V2_ARMS) or plan["primary_contrast"] != list(BENCHMARK_STUDY_V2_PRIMARY_CONTRAST) or plan["diagnostic_contrast"] != list(BENCHMARK_STUDY_V2_DIAGNOSTIC_CONTRAST):
11002
+ raise ValueError("v2 study arms or contrasts drifted")
11003
+ if plan["repetitions"] != 3 or plan["max_attempts_per_arm_unit"] != 2 or plan["retry_policy"] != BENCHMARK_STUDY_V2_RETRY_POLICY:
11004
+ raise ValueError("v2 study retry contract drifted")
11005
+ if any(not isinstance(plan[key], str) or SHA256_HEX_PATTERN.fullmatch(plan[key]) is None for key in ("corpus_sha256", "checker_sha256", "task_ids_sha256")):
11006
+ raise ValueError("v2 corpus/checker/task-order binding is invalid")
11007
+ if not isinstance(plan["noninferiority_margin"], (int, float)) or isinstance(plan["noninferiority_margin"], bool) or not 0 <= plan["noninferiority_margin"] < 1:
11008
+ raise ValueError("v2 non-inferiority margin is invalid")
11009
+ power = plan["power"]
11010
+ if (
11011
+ not isinstance(power, Mapping)
11012
+ or set(power) != {"claim_capable", "method", "reason", "required_task_count"}
11013
+ or power.get("claim_capable") is not False
11014
+ or power.get("method") != "not_estimated_without_independent_effect_model_v1"
11015
+ or power.get("reason") != "fixed_12_task_corpus_is_descriptive_only"
11016
+ or power.get("required_task_count") != 12
11017
+ ):
11018
+ raise ValueError("v2 descriptive sample-size contract is unavailable or invalid")
11019
+ if plan["model_cli_fields"] != ["model_revision", "backend_revision", "cli_version"] or plan["gates"] != ["quality", "failure", "correction", "retrieval", "shifted_cost"]:
11020
+ raise ValueError("v2 provenance or gate contract drifted")
11021
+ frozen_text = {
11022
+ "exclusions": "none_after_schedule_except_prelaunch_refusal_v1",
11023
+ "missing_data": "incomplete_primary_pair_is_descriptive_only_v1",
11024
+ "contamination": "any_contamination_blocks_claim_v1",
11025
+ "stopping": "fixed_task_count_no_optional_stopping_v1",
11026
+ }
11027
+ if any(plan[key] != expected for key, expected in frozen_text.items()):
11028
+ raise ValueError("v2 study plan operational rule drifted")
11029
+
11030
+
11031
+ def load_benchmark_study_v2_plan(path: Path) -> dict[str, Any]:
11032
+ """Load only canonical JSON so a plan's signed-by-bytes form is stable."""
11033
+ raw = _read_bytes_no_follow(path, max_bytes=100_000)
11034
+ try:
11035
+ value = json.loads(raw.decode("utf-8"))
11036
+ except (UnicodeDecodeError, json.JSONDecodeError) as exc:
11037
+ raise ValueError("v2 study plan is invalid JSON") from exc
11038
+ if not isinstance(value, dict) or raw != _study_canonical_json_bytes(value):
11039
+ raise ValueError("v2 study plan must be canonical JSON")
11040
+ validate_benchmark_study_v2_plan(value)
11041
+ return value
11042
+
11043
+
11044
+ def validate_benchmark_study_v2_bindings(
11045
+ plan: Mapping[str, Any], *, corpus_bytes: bytes,
11046
+ checker_binding: Mapping[str, Any],
11047
+ ) -> None:
11048
+ """Bind the raw corpus and domain-separated ordered checker inventory."""
11049
+ validate_benchmark_study_v2_plan(plan)
11050
+ if not isinstance(corpus_bytes, bytes):
11051
+ raise ValueError("v2 corpus/checker binding bytes are invalid")
11052
+ validate_benchmark_study_v2_checker_binding(checker_binding)
11053
+ try:
11054
+ task_ids = _benchmark_study_v2_task_ids_from_corpus(corpus_bytes)
11055
+ except ValueError as exc:
11056
+ raise ValueError("v2 corpus/checker/task-order binding drift") from exc
11057
+ if (
11058
+ _study_sha256_bytes(corpus_bytes) != plan["corpus_sha256"]
11059
+ or checker_binding["sha256"] != plan["checker_sha256"]
11060
+ or _benchmark_study_v2_task_ids_sha256(task_ids)
11061
+ != plan["task_ids_sha256"]
11062
+ ):
11063
+ raise ValueError("v2 corpus/checker/task-order binding drift")
11064
+
11065
+
11066
+ def validate_benchmark_study_v2_checker_binding(
11067
+ binding: Mapping[str, Any],
11068
+ ) -> None:
11069
+ """Validate the filename/size/content inventory before trusting its digest."""
11070
+ if not isinstance(binding, Mapping) or set(binding) != {
11071
+ "domain", "files", "sha256",
11072
+ }:
11073
+ raise ValueError("v2 checker binding schema is invalid")
11074
+ files = binding["files"]
11075
+ if (
11076
+ binding["domain"] != BENCHMARK_STUDY_V2_CHECKER_BINDING_DOMAIN
11077
+ or not isinstance(files, list)
11078
+ or len(files) != 12
11079
+ ):
11080
+ raise ValueError("v2 checker binding inventory is invalid")
11081
+ filenames: list[str] = []
11082
+ for entry in files:
11083
+ if not isinstance(entry, Mapping) or set(entry) != {
11084
+ "filename", "size", "sha256",
11085
+ }:
11086
+ raise ValueError("v2 checker binding entry is invalid")
11087
+ filename, size, digest = (
11088
+ entry["filename"], entry["size"], entry["sha256"],
11089
+ )
11090
+ if (
11091
+ not isinstance(filename, str)
11092
+ or not filename.endswith(".py")
11093
+ or Path(filename).name != filename
11094
+ or isinstance(size, bool)
11095
+ or not isinstance(size, int)
11096
+ or not 0 <= size <= MAX_FIXTURE_FILE_BYTES
11097
+ or not isinstance(digest, str)
11098
+ or SHA256_HEX_PATTERN.fullmatch(digest) is None
11099
+ ):
11100
+ raise ValueError("v2 checker binding entry is invalid")
11101
+ filenames.append(filename)
11102
+ if filenames != sorted(filenames) or len(set(filenames)) != len(filenames):
11103
+ raise ValueError("v2 checker binding order is invalid")
11104
+ expected = _study_domain_hash(
11105
+ BENCHMARK_STUDY_V2_CHECKER_BINDING_DOMAIN, files,
10498
11106
  )
10499
- if any(value is not None for value in study_values):
10500
- if not all(value is not None for value in study_values):
10501
- parser.error(
10502
- "--measurement-study-plan, --measurement-study-action, and "
10503
- "--measurement-study-output-root are all-or-none"
11107
+ if binding["sha256"] != expected:
11108
+ raise ValueError("v2 checker binding digest is invalid")
11109
+
11110
+
11111
+ def validate_benchmark_study_v2_evidence_metadata(metadata: Mapping[str, Any]) -> None:
11112
+ """Fail closed before potentially sensitive execution evidence reaches a report."""
11113
+ def visit(value: Any, key: str = "") -> None:
11114
+ key_lower = key.lower()
11115
+ # `tokens` is the aggregate study metric, not a credential-shaped token.
11116
+ # All other token-bearing field names remain forbidden.
11117
+ if key_lower == "tokens" and (
11118
+ isinstance(value, bool)
11119
+ or not isinstance(value, (int, float))
11120
+ or not math.isfinite(float(value))
11121
+ or value < 0
11122
+ ):
11123
+ raise ValueError("unsafe evidence token metric is invalid")
11124
+ if (
11125
+ key_lower != "tokens"
11126
+ and any(forbidden in key_lower for forbidden in BENCHMARK_STUDY_V2_EVIDENCE_FORBIDDEN_KEYS)
11127
+ ):
11128
+ raise ValueError("unsafe evidence field is forbidden")
11129
+ if isinstance(value, str):
11130
+ if (
11131
+ BENCHMARK_STUDY_V2_HANDLE_RE.search(value)
11132
+ or BENCHMARK_STUDY_V2_SECRET_SHAPE_RE.search(value)
11133
+ ):
11134
+ raise ValueError("unsafe evidence secret-shaped value is forbidden")
11135
+ if (
11136
+ key_lower in BENCHMARK_STUDY_V2_REVISION_KEYS
11137
+ and BENCHMARK_STUDY_V2_REVISION_RE.fullmatch(value) is None
11138
+ ):
11139
+ raise ValueError("unsafe evidence revision format is invalid")
11140
+ if isinstance(value, Mapping):
11141
+ for child_key, child in value.items():
11142
+ if not isinstance(child_key, str):
11143
+ raise ValueError("unsafe evidence key is invalid")
11144
+ visit(child, child_key)
11145
+ elif isinstance(value, (list, tuple)):
11146
+ for child in value:
11147
+ visit(child, key)
11148
+ visit(metadata)
11149
+
11150
+
11151
+ def redact_benchmark_study_v2_evidence(value: str) -> str:
11152
+ """Safe display helper for planted/accidental cgr1p handles; reports still reject them."""
11153
+ return BENCHMARK_STUDY_V2_HANDLE_RE.sub("[REDACTED_HANDLE]", value)
11154
+
11155
+
11156
+ def evaluate_benchmark_study_v2_claim_readiness(
11157
+ *, plan: Mapping[str, Any], task_ids: Sequence[str], binary_inference: Mapping[str, Any],
11158
+ effects: Mapping[str, Any], provenance: Mapping[str, Any],
11159
+ binary_rows: Sequence[Mapping[str, Any]] | None = None,
11160
+ effect_records: Sequence[Mapping[str, Any]] | None = None,
11161
+ ) -> dict[str, Any]:
11162
+ """Return a fail-closed product-claim gate; diagnostic results are ignored."""
11163
+ validate_benchmark_study_v2_plan(plan)
11164
+ unique_tasks = list(dict.fromkeys(task_ids))
11165
+ task_order: list[str] | None = None
11166
+ try:
11167
+ task_order = _benchmark_study_v2_task_ids(task_ids)
11168
+ except (TypeError, ValueError):
11169
+ pass
11170
+ power_ready = bool(
11171
+ plan["power"].get("claim_capable") is True
11172
+ and len(unique_tasks) >= int(plan["power"]["required_task_count"])
11173
+ )
11174
+ provenance_safe = True
11175
+ try:
11176
+ validate_benchmark_study_v2_evidence_metadata(provenance)
11177
+ except (TypeError, ValueError):
11178
+ provenance_safe = False
11179
+ provider_ready = bool(
11180
+ provenance_safe
11181
+ and provenance.get("source") == "provider_export"
11182
+ and provenance.get("complete_provider_export") is True
11183
+ and isinstance(provenance.get("backend_revision"), str) and provenance["backend_revision"]
11184
+ and isinstance(provenance.get("model_revision"), str) and provenance["model_revision"]
11185
+ and isinstance(provenance.get("cli_version"), str) and provenance["cli_version"]
11186
+ )
11187
+ recomputed_inference: dict[str, Any] | None = None
11188
+ if task_order is not None and binary_rows is not None:
11189
+ try:
11190
+ recomputed_inference = infer_benchmark_study_v2_binary(
11191
+ binary_rows,
11192
+ task_order=task_order,
11193
+ ni_margin=float(plan["noninferiority_margin"]),
10504
11194
  )
10505
- conflicts = [
10506
- name for name, active in (
10507
- ("--task-id", args.task_id is not None),
10508
- ("--variant", args.variant is not None),
10509
- ("--resume", args.resume),
10510
- ("--evidence-jsonl", args.evidence_jsonl is not None),
10511
- ("--dry-run", args.dry_run),
10512
- ("--ledger-jsonl", args.ledger_jsonl is not None),
10513
- ("--report-json", args.report_json is not None),
10514
- ("--dashboard-md", args.dashboard_md is not None),
10515
- ("--csv", args.csv is not None),
10516
- ("--baseline-variant", args.baseline_variant != "baseline"),
10517
- ) if active
10518
- ]
10519
- if conflicts:
10520
- parser.error(f"measurement study mode conflicts with {', '.join(conflicts)}")
10521
- return run_measurement_study_action(args)
10522
- args.csv = args.csv or Path("bench/results.csv")
10523
- validate_distinct_output_paths(args.csv, args.ledger_jsonl, args.report_json, args.dashboard_md)
11195
+ except (AttributeError, TypeError, ValueError):
11196
+ pass
11197
+ binary_ready = bool(
11198
+ recomputed_inference is not None
11199
+ and dict(binary_inference) == recomputed_inference
11200
+ and recomputed_inference["degenerate_all_success"] is False
11201
+ and recomputed_inference["noninferiority_pass"] is True
11202
+ )
10524
11203
 
10525
- variants = parse_variants(args.variants)
10526
- tasks = parse_tasks(args.tasks, variants=variants)
10527
- targets = filter_targets(tasks, variants, args.task_id, args.variant)
10528
- if not targets:
10529
- if args.dry_run and (not tasks or not variants):
10530
- print("completed 0 run(s) (dry-run; no CSV writes)")
10531
- return 0
10532
- print("no (task, variant) targets matched the filters", file=sys.stderr)
10533
- return 1
11204
+ recomputed_effects: dict[str, Any] | None = None
11205
+ if task_order is not None and effect_records is not None:
11206
+ try:
11207
+ recomputed_effects = compute_benchmark_study_v2_effects(
11208
+ effect_records, task_order=task_order,
11209
+ )
11210
+ except (AttributeError, TypeError, ValueError):
11211
+ pass
11212
+ bound_effect_fields = (
11213
+ "primary_contrast", "diagnostic_contrast", "task_ids_sha256",
11214
+ "retained_unfavorable_runs", "token_effect", "diagnostic_token_effect",
11215
+ "correction_effect", "diagnostic_correction_effect",
11216
+ "retrieval_effect", "diagnostic_retrieval_effect",
11217
+ )
11218
+ effects_bound = bool(
11219
+ recomputed_effects is not None
11220
+ and all(
11221
+ effects.get(field) == recomputed_effects.get(field)
11222
+ for field in bound_effect_fields
11223
+ )
11224
+ )
11225
+
11226
+ def interval_gate(field: str, *, strict: bool) -> bool:
11227
+ if recomputed_effects is None:
11228
+ return False
11229
+ interval = recomputed_effects.get(field)
11230
+ if not isinstance(interval, Mapping) or interval.get("method") != "task_cluster_bootstrap_v2":
11231
+ return False
11232
+ lower = interval.get("q025")
11233
+ if isinstance(lower, bool) or not isinstance(lower, (int, float)) or not math.isfinite(float(lower)):
11234
+ return False
11235
+ return bool(lower > 0 if strict else lower >= 0)
11236
+
11237
+ derived_gates = {
11238
+ "quality": binary_ready,
11239
+ "failure": binary_ready,
11240
+ "correction": interval_gate("correction_effect", strict=False),
11241
+ "retrieval": interval_gate("retrieval_effect", strict=False),
11242
+ "shifted_cost": interval_gate("token_effect", strict=True),
11243
+ }
11244
+ effect_ready = bool(effects_bound and all(derived_gates.values()))
11245
+ contamination_ready = provenance.get("contaminated") is False
11246
+ mixed_versions_ready = provenance.get("mixed_versions") is False
11247
+ missing_data_ready = provenance.get("missing_primary_data") is False
11248
+ unmet = []
11249
+ for name, value in (("power", power_ready), ("provider_provenance", provider_ready), ("binary_inference", binary_ready), ("effect_gates", effect_ready), ("contamination", contamination_ready), ("mixed_versions", mixed_versions_ready), ("missing_data", missing_data_ready)):
11250
+ if not value:
11251
+ unmet.append(name)
11252
+ return {
11253
+ "claim_ready": not unmet,
11254
+ "descriptive_only": bool(unmet),
11255
+ "unmet_gates": unmet,
11256
+ "primary_contrast": list(BENCHMARK_STUDY_V2_PRIMARY_CONTRAST),
11257
+ "diagnostic_contrast": list(BENCHMARK_STUDY_V2_DIAGNOSTIC_CONTRAST),
11258
+ "derived_gates": derived_gates,
11259
+ "backend_revision": provenance.get("backend_revision") if provider_ready else "unavailable",
11260
+ "model_revision": provenance.get("model_revision") if provider_ready else "unavailable",
11261
+ }
11262
+
11263
+
11264
+ BENCHMARK_STUDY_V2_EXEC_MANIFEST_SCHEMA_VERSION = "contextguard.bench.study-manifest.v6"
11265
+ BENCHMARK_STUDY_V2_ATTEMPT_SCHEMA_VERSION = "contextguard.bench.study-attempt.v4"
11266
+ BENCHMARK_STUDY_V2_BOUNDED_FAILURE_RESULT_CODES = frozenset({
11267
+ "error_max_turns",
11268
+ })
11269
+ BENCHMARK_STUDY_V2_BOUNDED_FAILURE_CHECKER_STATUS = (
11270
+ "not_run_provider_bounded_failure_v1"
11271
+ )
11272
+ BENCHMARK_STUDY_V2_REPORT_SCHEMA_VERSION = "contextguard.bench.study-report.v4"
11273
+ BENCHMARK_STUDY_V2_INVALID_DECISION_SCHEMA_VERSION = (
11274
+ "contextguard.bench.study-invalid-decision.v1"
11275
+ )
11276
+ BENCHMARK_STUDY_V2_CANDIDATE_SCHEMA_VERSION = "contextguard-npm-candidate-set/v1"
11277
+ BENCHMARK_STUDY_V2_CANDIDATE_NAMES = (
11278
+ "@ictechgy/context-guard-receipt", "@ictechgy/context-guard",
11279
+ )
11280
+ BENCHMARK_STUDY_V2_OVERLAY_NAME = "node_modules"
11281
+ BENCHMARK_STUDY_V2_REWRITE_COMMAND = (
11282
+ "./node_modules/.bin/context-guard-rewrite-bash"
11283
+ )
11284
+ BENCHMARK_STUDY_V2_CLI_BINDING_SCHEMA_VERSION = (
11285
+ "contextguard.bench.cli-binding.v2"
11286
+ )
11287
+ BENCHMARK_STUDY_V2_AUTH_CONTEXT_SCHEMA_VERSION = (
11288
+ "contextguard.bench.auth-context.v1"
11289
+ )
11290
+ BENCHMARK_STUDY_V2_CLI_CAPABILITIES = (
11291
+ "--settings", "--setting-sources", "--include-hook-events",
11292
+ "--no-session-persistence", "stream-json",
11293
+ )
11294
+ BENCHMARK_STUDY_V2_CLI_MAX_BYTES = 1_073_741_824
11295
+ BENCHMARK_STUDY_V2_CANARY_EVENT_SCHEMA_VERSION = (
11296
+ "contextguard.bench.canary-event.v2"
11297
+ )
11298
+ BENCHMARK_STUDY_V2_CANARY_EVIDENCE_SCHEMA_VERSION = (
11299
+ "contextguard.bench.canary-evidence.v3"
11300
+ )
11301
+ BENCHMARK_STUDY_V2_CANARY_ARMS = ("legacy_trim", "bash_reference_v1")
11302
+ BENCHMARK_STUDY_V2_CANARY_TASK_ID = "contextguard-v2-bash-canary"
11303
+ BENCHMARK_STUDY_V2_CANARY_MAX_BUDGET_USD = 0.75
11304
+ BENCHMARK_STUDY_V2_CANARY_MARKER = b"contextguard-v2-host-pretooluse-canary\n"
11305
+ BENCHMARK_STUDY_V2_CANARY_COMMAND = (
11306
+ "python3 -c 'from pathlib import Path;"
11307
+ "Path(\"contextguard-v2-canary.txt\").write_bytes("
11308
+ "b\"contextguard-v2-host-pretooluse-canary\\n\")'"
11309
+ )
11310
+ BENCHMARK_STUDY_V2_CANARY_PROMPT = (
11311
+ "Use the Bash tool exactly once to run this command, then reply done: "
11312
+ + BENCHMARK_STUDY_V2_CANARY_COMMAND
11313
+ )
11314
+ BENCHMARK_STUDY_V2_CANARY_CHECKER = (
11315
+ b"from pathlib import Path\n"
11316
+ b"try:\n"
11317
+ b" value = Path('contextguard-v2-canary.txt').read_bytes()\n"
11318
+ b"except OSError:\n"
11319
+ b" raise SystemExit(1)\n"
11320
+ b"raise SystemExit(0 if value == b'contextguard-v2-host-pretooluse-canary\\n' else 1)\n"
11321
+ )
11322
+
11323
+
11324
+ def _benchmark_study_v2_canary_contract() -> dict[str, Any]:
11325
+ fixture = b"This workspace is used only for the discarded v2 Bash-hook canary.\n"
11326
+ return {
11327
+ "schema_version": "contextguard.bench.canary-contract.v2",
11328
+ "task_id": BENCHMARK_STUDY_V2_CANARY_TASK_ID,
11329
+ "prompt_sha256": _study_sha256_bytes(
11330
+ BENCHMARK_STUDY_V2_CANARY_PROMPT.encode("utf-8")
11331
+ ),
11332
+ "fixture_sha256": _study_sha256_bytes(fixture),
11333
+ "checker_sha256": _study_sha256_bytes(BENCHMARK_STUDY_V2_CANARY_CHECKER),
11334
+ "marker_sha256": _study_sha256_bytes(BENCHMARK_STUDY_V2_CANARY_MARKER),
11335
+ "arms": list(BENCHMARK_STUDY_V2_CANARY_ARMS),
11336
+ "required_event_classes": ["PreToolUse"],
11337
+ "max_budget_usd": BENCHMARK_STUDY_V2_CANARY_MAX_BUDGET_USD,
11338
+ "provider_calls": 2,
11339
+ "discarded": True,
11340
+ "excluded_from_analysis": True,
11341
+ }
11342
+
11343
+
11344
+ def _benchmark_study_v2_canary_task() -> TaskFixture:
11345
+ fixture = b"This workspace is used only for the discarded v2 Bash-hook canary.\n"
11346
+ return TaskFixture(
11347
+ id=BENCHMARK_STUDY_V2_CANARY_TASK_ID,
11348
+ prompt=BENCHMARK_STUDY_V2_CANARY_PROMPT,
11349
+ model="sonnet", max_turns=2,
11350
+ max_budget_usd=BENCHMARK_STUDY_V2_CANARY_MAX_BUDGET_USD,
11351
+ allowed_tools=["Bash"],
11352
+ fixture_tree="inline-canary-fixture",
11353
+ success_checker="inline-canary-checker.py",
11354
+ fixture_tree_entries=(
11355
+ FixtureTreeEntry(path="CANARY.md", data=fixture, executable=False),
11356
+ ),
11357
+ success_checker_bytes=BENCHMARK_STUDY_V2_CANARY_CHECKER,
11358
+ )
11359
+
11360
+
11361
+ def _benchmark_study_v2_output_root(path: Path) -> Path:
11362
+ if path.is_symlink():
11363
+ raise ValueError("v2 output root must not be a symlink")
11364
+ try:
11365
+ return path.resolve(strict=False)
11366
+ except OSError as exc:
11367
+ raise ValueError("v2 output root is unavailable") from exc
11368
+
11369
+
11370
+ @contextmanager
11371
+ def _benchmark_study_v2_action_lock(output_root: Path) -> Iterable[None]:
11372
+ """Serialize every canary/run/resume/analyze mutation for one study root."""
11373
+ root = _benchmark_study_v2_output_root(output_root)
11374
+ try:
11375
+ root_fd = _ensure_directory_no_symlink(root, create=False)
11376
+ except (OSError, SystemExit, ValueError) as exc:
11377
+ raise ValueError(
11378
+ "v2 executable study requires a prepared output root"
11379
+ ) from exc
11380
+ lock_fd = -1
11381
+ locked = False
11382
+ try:
11383
+ if fcntl is None:
11384
+ raise ValueError("v2 lifecycle locking is unavailable")
11385
+ flags = os.O_RDWR | os.O_CREAT
11386
+ if hasattr(os, "O_CLOEXEC"):
11387
+ flags |= os.O_CLOEXEC
11388
+ lock_fd = _open_regular_no_symlink(
11389
+ root / ".study-v2-action.lock", flags, 0o600,
11390
+ )
11391
+ os.fchmod(lock_fd, 0o600)
11392
+ os.fsync(root_fd)
11393
+ try:
11394
+ fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
11395
+ except (BlockingIOError, OSError) as exc:
11396
+ raise ValueError("another v2 lifecycle action is already active") from exc
11397
+ locked = True
11398
+ yield
11399
+ finally:
11400
+ if locked:
11401
+ fcntl.flock(lock_fd, fcntl.LOCK_UN)
11402
+ if lock_fd >= 0:
11403
+ os.close(lock_fd)
11404
+ os.close(root_fd)
11405
+
11406
+
11407
+ def _benchmark_study_v2_validate_cli_binding(binding: Mapping[str, Any]) -> None:
11408
+ required = {
11409
+ "schema_version", "executable", "executable_bytes",
11410
+ "executable_sha256", "bundle", "probe",
11411
+ }
11412
+ probe = binding.get("probe")
11413
+ bundle = binding.get("bundle")
11414
+ if (
11415
+ set(binding) != required
11416
+ or binding.get("schema_version")
11417
+ != BENCHMARK_STUDY_V2_CLI_BINDING_SCHEMA_VERSION
11418
+ or not isinstance(binding.get("executable"), str)
11419
+ or not Path(str(binding.get("executable"))).is_absolute()
11420
+ or isinstance(binding.get("executable_bytes"), bool)
11421
+ or not isinstance(binding.get("executable_bytes"), int)
11422
+ or not 0 < int(binding["executable_bytes"]) <= BENCHMARK_STUDY_V2_CLI_MAX_BYTES
11423
+ or not isinstance(binding.get("executable_sha256"), str)
11424
+ or SHA256_HEX_PATTERN.fullmatch(str(binding["executable_sha256"])) is None
11425
+ or not isinstance(probe, Mapping)
11426
+ or probe.get("schema_version") != MEASUREMENT_CLI_PROBE_SCHEMA_VERSION
11427
+ or probe.get("executable") != binding.get("executable")
11428
+ or probe.get("capabilities") != sorted(BENCHMARK_STUDY_V2_CLI_CAPABILITIES)
11429
+ or "version" in probe
11430
+ or not isinstance(bundle, Mapping)
11431
+ or set(bundle) != {
11432
+ "scope", "root", "file_count", "total_bytes", "sha256",
11433
+ }
11434
+ or bundle.get("scope") != "single-native-executable-v1"
11435
+ or not isinstance(bundle.get("root"), str)
11436
+ or not Path(str(bundle["root"])).is_absolute()
11437
+ or isinstance(bundle.get("file_count"), bool)
11438
+ or not isinstance(bundle.get("file_count"), int)
11439
+ or not 0 < bundle["file_count"] <= 100_000
11440
+ or isinstance(bundle.get("total_bytes"), bool)
11441
+ or not isinstance(bundle.get("total_bytes"), int)
11442
+ or not 0 < bundle["total_bytes"] <= 2_147_483_648
11443
+ or not isinstance(bundle.get("sha256"), str)
11444
+ or SHA256_HEX_PATTERN.fullmatch(bundle["sha256"]) is None
11445
+ or bundle.get("root") != binding.get("executable")
11446
+ or bundle.get("file_count") != 1
11447
+ or bundle.get("total_bytes") != binding.get("executable_bytes")
11448
+ or bundle.get("sha256") != binding.get("executable_sha256")
11449
+ ):
11450
+ raise ValueError("v2 CLI binding schema mismatch")
11451
+
11452
+
11453
+ def _benchmark_study_v2_cli_stat_guard(path: Path) -> dict[str, int | str]:
11454
+ fd = _open_regular_no_symlink(path)
11455
+ try:
11456
+ item = os.fstat(fd)
11457
+ if item.st_size <= 0 or item.st_size > BENCHMARK_STUDY_V2_CLI_MAX_BYTES:
11458
+ raise ValueError("v2 CLI executable size is unsupported")
11459
+ return {
11460
+ "executable": str(path.resolve(strict=True)),
11461
+ "device": int(item.st_dev), "inode": int(item.st_ino),
11462
+ "bytes": int(item.st_size),
11463
+ "mtime_ns": int(item.st_mtime_ns), "ctime_ns": int(item.st_ctime_ns),
11464
+ }
11465
+ finally:
11466
+ os.close(fd)
11467
+
11468
+
11469
+ def _benchmark_study_v2_cli_file_binding(
11470
+ path: Path,
11471
+ ) -> tuple[dict[str, Any], dict[str, int | str]]:
11472
+ """Hash a large native CLI through a bounded no-follow fd without buffering it."""
11473
+ fd = _open_regular_no_symlink(path)
11474
+ try:
11475
+ before = os.fstat(fd)
11476
+ if before.st_size <= 0 or before.st_size > BENCHMARK_STUDY_V2_CLI_MAX_BYTES:
11477
+ raise ValueError("v2 CLI executable size is unsupported")
11478
+ digest = hashlib.sha256()
11479
+ total = 0
11480
+ while True:
11481
+ chunk = os.read(fd, 1024 * 1024)
11482
+ if not chunk:
11483
+ break
11484
+ total += len(chunk)
11485
+ if total > BENCHMARK_STUDY_V2_CLI_MAX_BYTES:
11486
+ raise ValueError("v2 CLI executable size is unsupported")
11487
+ digest.update(chunk)
11488
+ after = os.fstat(fd)
11489
+ stat_fields = ("st_dev", "st_ino", "st_size", "st_mtime_ns", "st_ctime_ns")
11490
+ if total != before.st_size or any(
11491
+ getattr(before, field) != getattr(after, field) for field in stat_fields
11492
+ ):
11493
+ raise ValueError("v2 CLI executable changed while hashing")
11494
+ executable = str(path.resolve(strict=True))
11495
+ return (
11496
+ {
11497
+ "executable": executable,
11498
+ "executable_bytes": total,
11499
+ "executable_sha256": digest.hexdigest(),
11500
+ },
11501
+ {
11502
+ "executable": executable,
11503
+ "device": int(after.st_dev), "inode": int(after.st_ino),
11504
+ "bytes": int(after.st_size),
11505
+ "mtime_ns": int(after.st_mtime_ns),
11506
+ "ctime_ns": int(after.st_ctime_ns),
11507
+ },
11508
+ )
11509
+ finally:
11510
+ os.close(fd)
11511
+
11512
+
11513
+ def _benchmark_study_v2_read_executable_shebang(path: Path) -> str | None:
11514
+ fd = _open_regular_no_symlink(path)
11515
+ try:
11516
+ raw = os.read(fd, 4096)
11517
+ finally:
11518
+ os.close(fd)
11519
+ first = raw.splitlines()[0] if raw else b""
11520
+ if not first.startswith(b"#!"):
11521
+ return None
11522
+ try:
11523
+ value = first[2:].decode("utf-8", "strict").strip()
11524
+ except UnicodeDecodeError:
11525
+ raise ValueError("v2 CLI shebang is not UTF-8") from None
11526
+ if not value or len(value) > 512:
11527
+ raise ValueError("v2 CLI shebang is invalid")
11528
+ return value
11529
+
11530
+
11531
+ def _benchmark_study_v2_cli_bundle_binding(
11532
+ executable: Path, shebang: str | None,
11533
+ ) -> dict[str, Any]:
11534
+ if shebang is not None:
11535
+ raise ValueError("v2 executable study requires a native executable")
11536
+ file_binding, _guard = _benchmark_study_v2_cli_file_binding(executable)
11537
+ return {
11538
+ "scope": "single-native-executable-v1",
11539
+ "root": str(executable), "file_count": 1,
11540
+ "total_bytes": file_binding["executable_bytes"],
11541
+ "sha256": file_binding["executable_sha256"],
11542
+ }
11543
+
11544
+
11545
+ def _benchmark_study_v2_cli_bundle_stat_guard(
11546
+ bundle: Mapping[str, Any],
11547
+ ) -> dict[str, Any]:
11548
+ """Detect a native executable replacement between action hash and launch."""
11549
+ if bundle.get("scope") != "single-native-executable-v1":
11550
+ raise ValueError("v2 CLI bundle scope is invalid")
11551
+ item = _benchmark_study_v2_cli_stat_guard(Path(str(bundle["root"])))
11552
+ return {
11553
+ "entry_count": 1,
11554
+ "sha256": _study_domain_hash(
11555
+ "contextguard.bench.v2.cli-bundle-stat.v1", item,
11556
+ ),
11557
+ }
11558
+
11559
+
11560
+ def _benchmark_study_v2_execution_environment(
11561
+ cli_binding: Mapping[str, Any],
11562
+ ) -> dict[str, Any]:
11563
+ """Freeze the PATH and interpreters used by the CLI and Python hook."""
11564
+ _benchmark_study_v2_validate_cli_binding(cli_binding)
11565
+ names = ["python3"]
11566
+ bindings: list[dict[str, Any]] = []
11567
+ lookup_directories: list[str] = []
11568
+ for name in dict.fromkeys(names):
11569
+ lookup = shutil.which(name)
11570
+ if lookup is None:
11571
+ raise ValueError(f"v2 required runtime unavailable: {name}")
11572
+ lookup_path = Path(lookup).absolute()
11573
+ resolved = lookup_path.resolve(strict=True)
11574
+ file_binding, _guard = _benchmark_study_v2_cli_file_binding(resolved)
11575
+ lookup_directories.append(str(lookup_path.parent))
11576
+ bindings.append({
11577
+ "name": name, "lookup_path": str(lookup_path),
11578
+ **file_binding,
11579
+ })
11580
+ path_value = os.pathsep.join(dict.fromkeys(
11581
+ lookup_directories + ["/usr/bin", "/bin", "/usr/sbin", "/sbin"]
11582
+ ))
11583
+ return {
11584
+ "schema_version": "contextguard.bench.execution-environment.v3",
11585
+ "values": {
11586
+ "PATH": path_value,
11587
+ "LANG": "C",
11588
+ "LC_ALL": "C",
11589
+ "PYTHONDONTWRITEBYTECODE": "1",
11590
+ },
11591
+ "runtime_bindings": bindings,
11592
+ }
11593
+
11594
+
11595
+ def _benchmark_study_v2_validate_execution_environment(
11596
+ binding: Mapping[str, Any],
11597
+ ) -> None:
11598
+ values = binding.get("values")
11599
+ runtimes = binding.get("runtime_bindings")
11600
+ if (
11601
+ set(binding) != {"schema_version", "values", "runtime_bindings"}
11602
+ or binding.get("schema_version")
11603
+ != "contextguard.bench.execution-environment.v3"
11604
+ or not isinstance(values, Mapping)
11605
+ or set(values) != {"PATH", "LANG", "LC_ALL", "PYTHONDONTWRITEBYTECODE"}
11606
+ or values.get("LANG") != "C" or values.get("LC_ALL") != "C"
11607
+ or values.get("PYTHONDONTWRITEBYTECODE") != "1"
11608
+ or not isinstance(values.get("PATH"), str) or not values["PATH"]
11609
+ or any(
11610
+ not part or not Path(part).is_absolute()
11611
+ for part in str(values["PATH"]).split(os.pathsep)
11612
+ )
11613
+ or not isinstance(runtimes, list) or not runtimes
11614
+ ):
11615
+ raise ValueError("v2 execution environment binding schema mismatch")
11616
+ seen: set[str] = set()
11617
+ for runtime in runtimes:
11618
+ if (
11619
+ not isinstance(runtime, Mapping)
11620
+ or set(runtime) != {
11621
+ "name", "lookup_path", "executable", "executable_bytes",
11622
+ "executable_sha256",
11623
+ }
11624
+ or not isinstance(runtime.get("name"), str)
11625
+ or not re.fullmatch(r"[A-Za-z0-9_.+-]+", str(runtime["name"]))
11626
+ or runtime["name"] in seen
11627
+ or not isinstance(runtime.get("lookup_path"), str)
11628
+ or not Path(str(runtime["lookup_path"])).is_absolute()
11629
+ or not isinstance(runtime.get("executable"), str)
11630
+ or not Path(str(runtime["executable"])).is_absolute()
11631
+ or isinstance(runtime.get("executable_bytes"), bool)
11632
+ or not isinstance(runtime.get("executable_bytes"), int)
11633
+ or not 0 < runtime["executable_bytes"] <= BENCHMARK_STUDY_V2_CLI_MAX_BYTES
11634
+ or not isinstance(runtime.get("executable_sha256"), str)
11635
+ or SHA256_HEX_PATTERN.fullmatch(runtime["executable_sha256"]) is None
11636
+ ):
11637
+ raise ValueError("v2 runtime interpreter binding schema mismatch")
11638
+ seen.add(str(runtime["name"]))
11639
+
11640
+
11641
+ def _benchmark_study_v2_assert_execution_environment(
11642
+ binding: Mapping[str, Any],
11643
+ ) -> dict[str, dict[str, int | str]]:
11644
+ _benchmark_study_v2_validate_execution_environment(binding)
11645
+ path_value = str(binding["values"]["PATH"])
11646
+ guards: dict[str, dict[str, int | str]] = {}
11647
+ for runtime in binding["runtime_bindings"]:
11648
+ found = shutil.which(str(runtime["name"]), path=path_value)
11649
+ if found is None or str(Path(found).absolute()) != runtime["lookup_path"]:
11650
+ raise ValueError("v2 runtime interpreter lookup drift")
11651
+ current, _guard = _benchmark_study_v2_cli_file_binding(
11652
+ Path(str(runtime["executable"]))
11653
+ )
11654
+ expected = {
11655
+ key: runtime[key]
11656
+ for key in ("executable", "executable_bytes", "executable_sha256")
11657
+ }
11658
+ if current != expected or Path(found).resolve(strict=True) != Path(
11659
+ str(runtime["executable"])
11660
+ ):
11661
+ raise ValueError("v2 runtime interpreter binding drift")
11662
+ guards[str(runtime["name"])] = _benchmark_study_v2_cli_stat_guard(
11663
+ Path(str(runtime["executable"]))
11664
+ )
11665
+ return guards
11666
+
11667
+
11668
+ def _benchmark_study_v2_assert_runtime_stat_guards(
11669
+ binding: Mapping[str, Any],
11670
+ guards: Mapping[str, Mapping[str, int | str]],
11671
+ ) -> None:
11672
+ _benchmark_study_v2_validate_execution_environment(binding)
11673
+ runtimes = binding["runtime_bindings"]
11674
+ if set(guards) != {str(runtime["name"]) for runtime in runtimes}:
11675
+ raise ValueError("v2 runtime interpreter guard mismatch")
11676
+ for runtime in runtimes:
11677
+ name = str(runtime["name"])
11678
+ guard = guards[name]
11679
+ found = shutil.which(
11680
+ name, path=str(binding["values"]["PATH"]),
11681
+ )
11682
+ if (
11683
+ found is None
11684
+ or str(Path(found).absolute()) != runtime["lookup_path"]
11685
+ or Path(found).resolve(strict=True) != Path(str(runtime["executable"]))
11686
+ or guard.get("executable") != runtime["executable"]
11687
+ or guard.get("bytes") != runtime["executable_bytes"]
11688
+ or _benchmark_study_v2_cli_stat_guard(
11689
+ Path(str(runtime["executable"]))
11690
+ ) != dict(guard)
11691
+ ):
11692
+ raise ValueError("v2 runtime interpreter changed before launch")
11693
+
11694
+
11695
+ def _benchmark_study_v2_auth_home_binding(home: Path) -> tuple[Path, dict[str, Any]]:
11696
+ if "CLAUDE_CONFIG_DIR" in os.environ:
11697
+ raise ValueError("v2 existing-login mode requires CLAUDE_CONFIG_DIR to be unset")
11698
+ if not home.is_absolute() or "\0" in str(home):
11699
+ raise ValueError("v2 existing-login HOME must be an absolute path")
11700
+ resolved = home.resolve(strict=True)
11701
+ directory_fd = _ensure_directory_no_symlink(resolved, create=False)
11702
+ try:
11703
+ item = os.fstat(directory_fd)
11704
+ finally:
11705
+ os.close(directory_fd)
11706
+ mode = stat.S_IMODE(item.st_mode)
11707
+ if (
11708
+ not stat.S_ISDIR(item.st_mode)
11709
+ or item.st_uid != os.geteuid()
11710
+ or mode & 0o022
11711
+ ):
11712
+ raise ValueError("v2 existing-login HOME is not a private owned directory")
11713
+ return resolved, {
11714
+ "device": item.st_dev,
11715
+ "inode": item.st_ino,
11716
+ "uid": item.st_uid,
11717
+ "mode": mode,
11718
+ }
11719
+
11720
+
11721
+ def _benchmark_study_v2_validate_auth_context(binding: Mapping[str, Any]) -> None:
11722
+ if (
11723
+ set(binding) != {
11724
+ "schema_version", "mode", "home_path_sha256", "home_stat",
11725
+ "identity_sha256", "auth_method", "api_provider",
11726
+ "credential_environment",
11727
+ }
11728
+ or binding.get("schema_version")
11729
+ != BENCHMARK_STUDY_V2_AUTH_CONTEXT_SCHEMA_VERSION
11730
+ or binding.get("mode") != "existing_cli_login_v1"
11731
+ or not isinstance(binding.get("home_path_sha256"), str)
11732
+ or SHA256_HEX_PATTERN.fullmatch(str(binding["home_path_sha256"])) is None
11733
+ or not isinstance(binding.get("identity_sha256"), str)
11734
+ or SHA256_HEX_PATTERN.fullmatch(str(binding["identity_sha256"])) is None
11735
+ or binding.get("auth_method") != "claude.ai"
11736
+ or binding.get("api_provider") != "firstParty"
11737
+ or binding.get("credential_environment") != "forbidden"
11738
+ ):
11739
+ raise ValueError("v2 auth context binding schema mismatch")
11740
+ home_stat = binding.get("home_stat")
11741
+ if (
11742
+ not isinstance(home_stat, Mapping)
11743
+ or set(home_stat) != {"device", "inode", "uid", "mode"}
11744
+ or any(isinstance(value, bool) or not isinstance(value, int) or value < 0
11745
+ for value in home_stat.values())
11746
+ or int(home_stat["uid"]) != os.geteuid()
11747
+ or int(home_stat["mode"]) & 0o022
11748
+ ):
11749
+ raise ValueError("v2 auth HOME binding schema mismatch")
11750
+
11751
+
11752
+ def _benchmark_study_v2_auth_context(
11753
+ claude_bin: str,
11754
+ execution_environment: Mapping[str, Any],
11755
+ auth_home: Path,
11756
+ ) -> tuple[Path, dict[str, Any]]:
11757
+ _benchmark_study_v2_validate_execution_environment(execution_environment)
11758
+ resolved_home, home_stat = _benchmark_study_v2_auth_home_binding(auth_home)
11759
+ with tempfile.TemporaryDirectory(prefix="contextguard-v2-auth-probe-") as temporary:
11760
+ root = Path(temporary)
11761
+ os.chmod(root, 0o700)
11762
+ paths = {
11763
+ name: root / name
11764
+ for name in (
11765
+ "cwd", "xdg-config", "xdg-cache", "xdg-data", "xdg-state", "tmp",
11766
+ )
11767
+ }
11768
+ for path in paths.values():
11769
+ path.mkdir(mode=0o700)
11770
+ env = dict(execution_environment["values"])
11771
+ env.update({
11772
+ "HOME": str(resolved_home),
11773
+ "XDG_CONFIG_HOME": str(paths["xdg-config"]),
11774
+ "XDG_CACHE_HOME": str(paths["xdg-cache"]),
11775
+ "XDG_DATA_HOME": str(paths["xdg-data"]),
11776
+ "XDG_STATE_HOME": str(paths["xdg-state"]),
11777
+ "TMPDIR": str(paths["tmp"]),
11778
+ "NO_COLOR": "1",
11779
+ })
11780
+ result = run_bounded_command(
11781
+ [executable_argv0(claude_bin), "auth", "status", "--json"],
11782
+ cwd=paths["cwd"], timeout_seconds=10,
11783
+ max_output_bytes=MEASUREMENT_CLI_PROBE_OUTPUT_MAX_BYTES, env=env,
11784
+ )
11785
+ if (
11786
+ result.returncode != 0 or result.timed_out or result.output_truncated
11787
+ or result.launch_error or result.stderr_bytes
11788
+ or not 0 < len(result.stdout_bytes) <= 4096
11789
+ ):
11790
+ raise ValueError("v2 existing Claude login is unavailable")
11791
+ try:
11792
+ status_payload = json.loads(result.stdout_bytes.decode("utf-8", "strict"))
11793
+ except (UnicodeDecodeError, json.JSONDecodeError):
11794
+ raise ValueError("v2 existing Claude login status is invalid") from None
11795
+ identity_keys = {
11796
+ "loggedIn", "authMethod", "apiProvider", "email", "orgId",
11797
+ "orgName", "subscriptionType",
11798
+ }
11799
+ if (
11800
+ not isinstance(status_payload, Mapping)
11801
+ or set(status_payload) != identity_keys
11802
+ or status_payload.get("loggedIn") is not True
11803
+ or status_payload.get("authMethod") != "claude.ai"
11804
+ or status_payload.get("apiProvider") != "firstParty"
11805
+ or any(
11806
+ not isinstance(status_payload.get(key), str)
11807
+ or len(str(status_payload[key]).encode("utf-8")) > 1024
11808
+ or "\0" in str(status_payload[key])
11809
+ for key in ("email", "orgId", "orgName", "subscriptionType")
11810
+ )
11811
+ ):
11812
+ raise ValueError("v2 existing Claude login status is unsupported")
11813
+ private_identity = {
11814
+ key: status_payload[key]
11815
+ for key in (
11816
+ "authMethod", "apiProvider", "email", "orgId", "orgName",
11817
+ "subscriptionType",
11818
+ )
11819
+ }
11820
+ binding = {
11821
+ "schema_version": BENCHMARK_STUDY_V2_AUTH_CONTEXT_SCHEMA_VERSION,
11822
+ "mode": "existing_cli_login_v1",
11823
+ "home_path_sha256": _study_domain_hash(
11824
+ "contextguard.bench.v2.auth-home-path.v1", str(resolved_home),
11825
+ ),
11826
+ "home_stat": home_stat,
11827
+ "identity_sha256": _study_domain_hash(
11828
+ "contextguard.bench.v2.auth-identity.v1", private_identity,
11829
+ ),
11830
+ "auth_method": "claude.ai",
11831
+ "api_provider": "firstParty",
11832
+ "credential_environment": "forbidden",
11833
+ }
11834
+ _benchmark_study_v2_validate_auth_context(binding)
11835
+ return resolved_home, binding
11836
+
11837
+
11838
+ def _benchmark_study_v2_assert_auth_context(
11839
+ claude_bin: str,
11840
+ execution_environment: Mapping[str, Any],
11841
+ expected: Mapping[str, Any],
11842
+ auth_home: Path,
11843
+ ) -> Path:
11844
+ _benchmark_study_v2_validate_auth_context(expected)
11845
+ try:
11846
+ resolved_home, actual = _benchmark_study_v2_auth_context(
11847
+ claude_bin, execution_environment, auth_home,
11848
+ )
11849
+ except (OSError, SystemExit, TypeError, ValueError) as exc:
11850
+ raise ValueError("v2 existing Claude login binding drift") from exc
11851
+ if actual != dict(expected):
11852
+ raise ValueError("v2 existing Claude login binding drift")
11853
+ return resolved_home
11854
+
11855
+
11856
+ def _benchmark_study_v2_cli_binding(claude_bin: str) -> dict[str, Any]:
11857
+ """Bind the exact executable bytes and isolated version/help capability probes."""
11858
+ executable = Path(executable_argv0(claude_bin))
11859
+ file_binding, stat_guard = _benchmark_study_v2_cli_file_binding(executable)
11860
+ shebang = _benchmark_study_v2_read_executable_shebang(executable)
11861
+ if shebang is not None:
11862
+ raise ValueError(
11863
+ "v2 executable study requires a native executable; script launcher "
11864
+ "dependency closure cannot be proven"
11865
+ )
11866
+ bundle = _benchmark_study_v2_cli_bundle_binding(executable, shebang)
11867
+ probe = run_measurement_cli_probes(
11868
+ str(executable), BENCHMARK_STUDY_V2_CLI_CAPABILITIES,
11869
+ )
11870
+ # Bind exact version bytes by hash/length without persisting arbitrary CLI
11871
+ # display text that could contain a secret-shaped value.
11872
+ probe.pop("version", None)
11873
+ if _benchmark_study_v2_cli_stat_guard(executable) != stat_guard:
11874
+ raise ValueError("v2 CLI executable changed during capability probes")
11875
+ binding = {
11876
+ "schema_version": BENCHMARK_STUDY_V2_CLI_BINDING_SCHEMA_VERSION,
11877
+ **file_binding,
11878
+ "bundle": bundle,
11879
+ "probe": probe,
11880
+ }
11881
+ _benchmark_study_v2_validate_cli_binding(binding)
11882
+ return binding
11883
+
11884
+
11885
+ def _benchmark_study_v2_assert_cli_binding(
11886
+ claude_bin: str, expected: Mapping[str, Any],
11887
+ ) -> dict[str, Any]:
11888
+ """Refuse a changed or incompatible CLI before reserving any identity."""
11889
+ _benchmark_study_v2_validate_cli_binding(expected)
11890
+ try:
11891
+ actual = _benchmark_study_v2_cli_binding(claude_bin)
11892
+ except (OSError, SystemExit, TypeError, ValueError) as exc:
11893
+ raise ValueError("v2 CLI binding drift") from exc
11894
+ if actual != dict(expected):
11895
+ raise ValueError("v2 CLI binding drift")
11896
+ guard: dict[str, Any] = _benchmark_study_v2_cli_stat_guard(
11897
+ Path(str(actual["executable"]))
11898
+ )
11899
+ guard["bundle_stat_guard"] = _benchmark_study_v2_cli_bundle_stat_guard(
11900
+ actual["bundle"]
11901
+ )
11902
+ return guard
11903
+
11904
+
11905
+ def _benchmark_study_v2_assert_cli_executable_bytes(
11906
+ expected: Mapping[str, Any], stat_guard: Mapping[str, Any],
11907
+ ) -> None:
11908
+ """Repeat a cheap post-hash file-identity check before provider reservation."""
11909
+ _benchmark_study_v2_validate_cli_binding(expected)
11910
+ if (
11911
+ stat_guard.get("executable") != expected["executable"]
11912
+ or stat_guard.get("bytes") != expected["executable_bytes"]
11913
+ or _benchmark_study_v2_cli_stat_guard(
11914
+ Path(str(expected["executable"]))
11915
+ ) != {
11916
+ key: stat_guard[key]
11917
+ for key in ("executable", "device", "inode", "bytes", "mtime_ns", "ctime_ns")
11918
+ }
11919
+ or stat_guard.get("bundle_stat_guard")
11920
+ != _benchmark_study_v2_cli_bundle_stat_guard(expected["bundle"])
11921
+ ):
11922
+ raise ValueError("v2 CLI executable bytes drift")
11923
+
11924
+
11925
+ def _benchmark_study_v2_python_binding() -> dict[str, Any]:
11926
+ invoked_python = Path(sys.executable).absolute()
11927
+ resolved_python = invoked_python.resolve(strict=True)
11928
+ python_file, _guard = _benchmark_study_v2_cli_file_binding(resolved_python)
11929
+ return {
11930
+ "invoked_path": str(invoked_python),
11931
+ **python_file,
11932
+ "implementation": sys.implementation.name,
11933
+ "cache_tag": sys.implementation.cache_tag,
11934
+ "version_info": [
11935
+ sys.version_info.major, sys.version_info.minor,
11936
+ sys.version_info.micro, sys.version_info.releaselevel,
11937
+ sys.version_info.serial,
11938
+ ],
11939
+ "version_sha256": _study_sha256_bytes(sys.version.encode("utf-8")),
11940
+ "flags_sha256": _study_domain_hash(
11941
+ "contextguard.bench.v2.python-flags.v1", list(sys.flags),
11942
+ ),
11943
+ "prefix": sys.prefix,
11944
+ "base_prefix": sys.base_prefix,
11945
+ }
11946
+
11947
+
11948
+ def _benchmark_study_v2_runner_binding() -> dict[str, Any]:
11949
+ path = Path(__file__).resolve(strict=True)
11950
+ raw = _read_bytes_no_follow(path, max_bytes=4_000_000)
11951
+ return {
11952
+ "path": str(path), "bytes": len(raw),
11953
+ "sha256": _study_sha256_bytes(raw),
11954
+ "python": _benchmark_study_v2_python_binding(),
11955
+ }
11956
+
11957
+
11958
+ def _benchmark_study_v2_assert_python_binding(
11959
+ binding: Mapping[str, Any], *, require_current: bool,
11960
+ ) -> str:
11961
+ required = {
11962
+ "invoked_path", "executable", "executable_bytes", "executable_sha256",
11963
+ "implementation", "cache_tag", "version_info", "version_sha256",
11964
+ "flags_sha256", "prefix", "base_prefix",
11965
+ }
11966
+ if (
11967
+ set(binding) != required
11968
+ or not isinstance(binding.get("invoked_path"), str)
11969
+ or not Path(str(binding["invoked_path"])).is_absolute()
11970
+ or not isinstance(binding.get("executable"), str)
11971
+ or not Path(str(binding["executable"])).is_absolute()
11972
+ or isinstance(binding.get("executable_bytes"), bool)
11973
+ or not isinstance(binding.get("executable_bytes"), int)
11974
+ or not 0 < int(binding["executable_bytes"]) <= BENCHMARK_STUDY_V2_CLI_MAX_BYTES
11975
+ or not isinstance(binding.get("executable_sha256"), str)
11976
+ or SHA256_HEX_PATTERN.fullmatch(str(binding["executable_sha256"])) is None
11977
+ or not isinstance(binding.get("implementation"), str)
11978
+ or not binding["implementation"]
11979
+ or not isinstance(binding.get("cache_tag"), (str, type(None)))
11980
+ or not isinstance(binding.get("version_info"), list)
11981
+ or len(binding["version_info"]) != 5
11982
+ or any(
11983
+ isinstance(value, bool) or not isinstance(value, (int, str))
11984
+ for value in binding["version_info"]
11985
+ )
11986
+ or not isinstance(binding.get("version_sha256"), str)
11987
+ or SHA256_HEX_PATTERN.fullmatch(str(binding["version_sha256"])) is None
11988
+ or not isinstance(binding.get("flags_sha256"), str)
11989
+ or SHA256_HEX_PATTERN.fullmatch(str(binding["flags_sha256"])) is None
11990
+ or not isinstance(binding.get("prefix"), str)
11991
+ or not isinstance(binding.get("base_prefix"), str)
11992
+ ):
11993
+ raise ValueError("v2 runner Python binding schema mismatch")
11994
+ executable = Path(str(binding["executable"]))
11995
+ current_file, _guard = _benchmark_study_v2_cli_file_binding(executable)
11996
+ expected_file = {
11997
+ key: binding[key]
11998
+ for key in ("executable", "executable_bytes", "executable_sha256")
11999
+ }
12000
+ if current_file != expected_file:
12001
+ raise ValueError("v2 runner Python binding drift")
12002
+ if require_current:
12003
+ invoked = Path(sys.executable).absolute()
12004
+ try:
12005
+ resolved = invoked.resolve(strict=True)
12006
+ except OSError as exc:
12007
+ raise ValueError("v2 runner Python binding drift") from exc
12008
+ if (
12009
+ str(invoked) != binding["invoked_path"]
12010
+ or str(resolved) != binding["executable"]
12011
+ or sys.implementation.name != binding["implementation"]
12012
+ or sys.implementation.cache_tag != binding["cache_tag"]
12013
+ or [
12014
+ sys.version_info.major, sys.version_info.minor,
12015
+ sys.version_info.micro, sys.version_info.releaselevel,
12016
+ sys.version_info.serial,
12017
+ ] != binding["version_info"]
12018
+ or _study_sha256_bytes(sys.version.encode("utf-8"))
12019
+ != binding["version_sha256"]
12020
+ or _study_domain_hash(
12021
+ "contextguard.bench.v2.python-flags.v1", list(sys.flags),
12022
+ ) != binding["flags_sha256"]
12023
+ or sys.prefix != binding["prefix"]
12024
+ or sys.base_prefix != binding["base_prefix"]
12025
+ ):
12026
+ raise ValueError("v2 runner Python binding drift")
12027
+ return str(executable)
12028
+
12029
+
12030
+ def _benchmark_study_v2_read_canonical(path: Path, *, owner: str, maximum: int) -> tuple[dict[str, Any], bytes]:
12031
+ raw = _read_bytes_no_follow(path, max_bytes=maximum)
12032
+ try:
12033
+ value = json.loads(
12034
+ raw.decode("utf-8"), object_pairs_hook=_measurement_object_no_duplicates,
12035
+ parse_constant=_stream_reject_nonfinite,
12036
+ )
12037
+ except (UnicodeDecodeError, json.JSONDecodeError, ValueError) as exc:
12038
+ raise ValueError(f"{owner} is invalid JSON") from exc
12039
+ if not isinstance(value, dict) or raw != _study_canonical_json_bytes(value):
12040
+ raise ValueError(f"{owner} must be exact canonical JSON bytes")
12041
+ return value, raw
12042
+
12043
+
12044
+ def verify_benchmark_study_v2_candidate(
12045
+ manifest_path: Path, *, checksum_path: Path | None = None,
12046
+ expected_manifest_sha256: str | None = None,
12047
+ expected_commit_sha: str | None = None,
12048
+ ) -> dict[str, Any]:
12049
+ """Verify the build-once candidate as inert bytes; never import its code."""
12050
+ manifest, manifest_raw = _benchmark_study_v2_read_canonical(
12051
+ manifest_path, owner="v2 candidate manifest", maximum=1_000_000,
12052
+ )
12053
+ required = {
12054
+ "build_policy", "commit_sha", "exact_dependency", "packages",
12055
+ "policy_sha256", "receipt_package_files_sha256", "protocol",
12056
+ "repository", "schema_version", "tool_versions",
12057
+ }
12058
+ if set(manifest) != required or manifest.get("schema_version") != BENCHMARK_STUDY_V2_CANDIDATE_SCHEMA_VERSION:
12059
+ raise ValueError("v2 candidate manifest schema mismatch")
12060
+ if manifest.get("build_policy") != {
12061
+ "ignore_scripts": True, "lockfiles": [], "network": "offline",
12062
+ "package_build_count": 1,
12063
+ }:
12064
+ raise ValueError("v2 candidate build policy mismatch")
12065
+ if manifest.get("protocol") != {
12066
+ "maximum": 1, "minimum": 1, "name": "bash_reference_v1",
12067
+ }:
12068
+ raise ValueError("v2 candidate protocol mismatch")
12069
+ if (
12070
+ manifest.get("repository") != "ictechgy/context-guard"
12071
+ or not isinstance(manifest.get("commit_sha"), str)
12072
+ or re.fullmatch(r"[0-9a-f]{40}", manifest["commit_sha"]) is None
12073
+ or not isinstance(manifest.get("tool_versions"), dict)
12074
+ or not manifest["tool_versions"]
12075
+ or any(
12076
+ not isinstance(key, str) or not key or len(key) > 64
12077
+ or not isinstance(value, str) or not value or len(value) > 128
12078
+ for key, value in manifest["tool_versions"].items()
12079
+ )
12080
+ or any(
12081
+ not isinstance(manifest.get(key), str)
12082
+ or SHA256_HEX_PATTERN.fullmatch(manifest[key]) is None
12083
+ for key in ("policy_sha256", "receipt_package_files_sha256")
12084
+ )
12085
+ ):
12086
+ raise ValueError("v2 candidate build provenance is invalid")
12087
+ manifest_sha256 = _study_sha256_bytes(manifest_raw)
12088
+ if expected_manifest_sha256 is not None and manifest_sha256 != expected_manifest_sha256:
12089
+ raise ValueError("v2 candidate canonical manifest hash drift")
12090
+ if expected_commit_sha is not None and (
12091
+ re.fullmatch(r"[0-9a-f]{40}", expected_commit_sha) is None
12092
+ or manifest["commit_sha"] != expected_commit_sha
12093
+ ):
12094
+ raise ValueError(
12095
+ "v2 candidate commit does not match approved source revision"
12096
+ )
12097
+ packages = manifest.get("packages")
12098
+ if not isinstance(packages, list) or len(packages) != 2:
12099
+ raise ValueError("v2 candidate must bind exactly two tarballs")
12100
+ records: list[dict[str, Any]] = []
12101
+ checksum_rows: list[str] = []
12102
+ for index, package in enumerate(packages):
12103
+ if not isinstance(package, dict) or set(package) != {
12104
+ "filename", "integrity", "name", "sha256", "size_bytes", "version",
12105
+ }:
12106
+ raise ValueError("v2 candidate tarball record schema mismatch")
12107
+ filename = package.get("filename")
12108
+ digest = package.get("sha256")
12109
+ size = package.get("size_bytes")
12110
+ if (
12111
+ package.get("name") != BENCHMARK_STUDY_V2_CANDIDATE_NAMES[index]
12112
+ or not isinstance(filename, str) or Path(filename).name != filename
12113
+ or not isinstance(digest, str) or SHA256_HEX_PATTERN.fullmatch(digest) is None
12114
+ or isinstance(size, bool) or not isinstance(size, int) or size <= 0
12115
+ or not isinstance(package.get("version"), str) or not package["version"]
12116
+ ):
12117
+ raise ValueError("v2 candidate tarball identity is invalid")
12118
+ tarball_path = manifest_path.parent / filename
12119
+ raw = _read_bytes_no_follow(tarball_path, max_bytes=100_000_000)
12120
+ sri = "sha512-" + base64.b64encode(hashlib.sha512(raw).digest()).decode("ascii")
12121
+ if len(raw) != size or _study_sha256_bytes(raw) != digest or package.get("integrity") != sri:
12122
+ raise ValueError("v2 candidate tarball size, SHA-256, or SRI mismatch")
12123
+ checksum_rows.append(f"{digest} {filename}\n")
12124
+ records.append({**package, "path": str(tarball_path.resolve())})
12125
+ if len({record["filename"] for record in records}) != 2 or manifest.get("exact_dependency") != {
12126
+ "name": BENCHMARK_STUDY_V2_CANDIDATE_NAMES[0],
12127
+ "version": records[0]["version"],
12128
+ }:
12129
+ raise ValueError("v2 candidate exact dependency binding mismatch")
12130
+ checksum = checksum_path or manifest_path.with_name("candidate-sha256sums.txt")
12131
+ checksum_raw = _read_bytes_no_follow(checksum, max_bytes=10_000)
12132
+ expected_checksum = "".join(checksum_rows).encode("ascii")
12133
+ if checksum_raw != expected_checksum:
12134
+ raise ValueError("v2 candidate checksum document mismatch")
12135
+ return {
12136
+ "manifest_path": str(manifest_path.resolve()),
12137
+ "manifest_sha256": manifest_sha256,
12138
+ "manifest_bytes": len(manifest_raw),
12139
+ "checksum_path": str(checksum.resolve()),
12140
+ "checksum_sha256": _study_sha256_bytes(checksum_raw),
12141
+ "commit_sha": manifest["commit_sha"],
12142
+ "packages": records,
12143
+ }
12144
+
12145
+
12146
+ def _benchmark_study_v2_verify_retained_ref(
12147
+ retained_ref: str, expected_commit_sha: str,
12148
+ ) -> dict[str, str]:
12149
+ if (
12150
+ not isinstance(retained_ref, str)
12151
+ or len(retained_ref) > 200
12152
+ or re.fullmatch(
12153
+ r"refs/heads/candidate/[A-Za-z0-9][A-Za-z0-9._/-]*",
12154
+ retained_ref,
12155
+ ) is None
12156
+ or ".." in retained_ref
12157
+ or "//" in retained_ref
12158
+ or retained_ref.endswith(("/", ".lock"))
12159
+ or re.fullmatch(r"[0-9a-f]{40}", expected_commit_sha) is None
12160
+ ):
12161
+ raise ValueError("v2 retained ref binding is invalid")
12162
+ git = Path("/usr/bin/git")
12163
+ if not git.is_file() or not os.access(git, os.X_OK):
12164
+ raise ValueError("v2 retained ref verifier is unavailable")
12165
+ with tempfile.TemporaryDirectory(prefix="contextguard-v2-git-home-") as temp:
12166
+ os.chmod(temp, 0o700)
12167
+ result = run_bounded_command(
12168
+ [
12169
+ str(git), "-c", "credential.helper=", "-c",
12170
+ "core.askPass=", "ls-remote", "--exit-code", "--refs",
12171
+ "https://github.com/ictechgy/context-guard.git", retained_ref,
12172
+ ],
12173
+ cwd=Path(temp), timeout_seconds=30, max_output_bytes=10_000,
12174
+ env={
12175
+ "GIT_ASKPASS": "/usr/bin/false",
12176
+ "GIT_CONFIG_GLOBAL": "/dev/null",
12177
+ "GIT_CONFIG_NOSYSTEM": "1",
12178
+ "GIT_TERMINAL_PROMPT": "0",
12179
+ "HOME": temp,
12180
+ "LC_ALL": "C",
12181
+ "PATH": "/usr/bin:/bin",
12182
+ "SSH_ASKPASS": "/usr/bin/false",
12183
+ },
12184
+ )
12185
+ expected_output = f"{expected_commit_sha}\t{retained_ref}\n"
12186
+ if (
12187
+ result.returncode != 0
12188
+ or result.timed_out
12189
+ or result.output_truncated
12190
+ or result.stderr
12191
+ or result.stdout != expected_output
12192
+ ):
12193
+ raise ValueError("v2 retained ref does not resolve to approved source")
12194
+ return {
12195
+ "commit_sha": expected_commit_sha,
12196
+ "ref": retained_ref,
12197
+ "repository": "ictechgy/context-guard",
12198
+ "verification": "git-ls-remote-v1",
12199
+ }
12200
+
12201
+
12202
+ def _benchmark_study_v2_tarball_inventory(path: Path) -> dict[str, Any]:
12203
+ """Inventory regular npm package members without extracting candidate code."""
12204
+ files: list[dict[str, Any]] = []
12205
+ seen: set[str] = set()
12206
+ total_bytes = 0
12207
+ try:
12208
+ with tarfile.open(path, mode="r:gz") as archive:
12209
+ for index, member in enumerate(archive):
12210
+ if index >= 10_000:
12211
+ raise ValueError("v2 candidate tarball has too many members")
12212
+ raw_parts = member.name.split("/")
12213
+ if (
12214
+ not raw_parts
12215
+ or raw_parts[0] != "package"
12216
+ or any(part in {"", ".", ".."} for part in raw_parts)
12217
+ or any(
12218
+ ord(character) < 0x20 or ord(character) == 0x7F
12219
+ for character in member.name
12220
+ )
12221
+ ):
12222
+ raise ValueError("v2 candidate tarball member path is invalid")
12223
+ if len(raw_parts) == 1:
12224
+ if not member.isdir():
12225
+ raise ValueError("v2 candidate tarball package root is invalid")
12226
+ continue
12227
+ relative = PurePosixPath(*raw_parts[1:]).as_posix()
12228
+ if relative in seen:
12229
+ raise ValueError("v2 candidate tarball has duplicate members")
12230
+ if member.isdir():
12231
+ continue
12232
+ if not member.isreg() or member.size < 0 or member.size > 100_000_000:
12233
+ raise ValueError("v2 candidate tarball member type is unsupported")
12234
+ stream = archive.extractfile(member)
12235
+ if stream is None:
12236
+ raise ValueError("v2 candidate tarball member is unreadable")
12237
+ raw = stream.read(member.size + 1)
12238
+ if len(raw) != member.size:
12239
+ raise ValueError("v2 candidate tarball member size drift")
12240
+ total_bytes += len(raw)
12241
+ if total_bytes > 100_000_000:
12242
+ raise ValueError("v2 candidate tarball expanded size exceeds limit")
12243
+ seen.add(relative)
12244
+ files.append({
12245
+ "path": relative,
12246
+ "bytes": len(raw),
12247
+ "sha256": _study_sha256_bytes(raw),
12248
+ "executable": bool(member.mode & 0o111),
12249
+ "kind": "file",
12250
+ "target": None,
12251
+ })
12252
+ except (OSError, tarfile.TarError) as exc:
12253
+ raise ValueError("v2 candidate tarball is not a readable npm archive") from exc
12254
+ files.sort(key=lambda item: item["path"])
12255
+ if not files:
12256
+ raise ValueError("v2 candidate tarball package is empty")
12257
+ return {
12258
+ "files": files,
12259
+ "file_count": len(files),
12260
+ "sha256": _study_domain_hash("contextguard.bench.v2.inventory.v1", files),
12261
+ }
12262
+
12263
+
12264
+ def _benchmark_study_v2_verify_installed_packages(
12265
+ overlay_root: Path, candidate: Mapping[str, Any],
12266
+ ) -> dict[str, Any]:
12267
+ """Bind the executed package bytes to the two already-verified tarballs."""
12268
+ bindings: list[dict[str, Any]] = []
12269
+ installed_documents: dict[str, dict[str, Any]] = {}
12270
+ allowed_overlay_paths: set[str] = set()
12271
+ for record in candidate["packages"]:
12272
+ name = record["name"]
12273
+ package_root = overlay_root.joinpath(*str(name).split("/"))
12274
+ expected = _benchmark_study_v2_tarball_inventory(Path(record["path"]))
12275
+ actual = _benchmark_study_v2_inventory(package_root)
12276
+ if actual != expected:
12277
+ raise ValueError("v2 installed package bytes differ from candidate tarball")
12278
+ package_raw = _read_bytes_no_follow(
12279
+ package_root / "package.json", max_bytes=128 * 1024,
12280
+ )
12281
+ try:
12282
+ document = json.loads(
12283
+ package_raw.decode("utf-8"),
12284
+ object_pairs_hook=_measurement_object_no_duplicates,
12285
+ )
12286
+ except (UnicodeDecodeError, json.JSONDecodeError, ValueError) as exc:
12287
+ raise ValueError("v2 installed package metadata is invalid") from exc
12288
+ if (
12289
+ not isinstance(document, dict)
12290
+ or document.get("name") != name
12291
+ or document.get("version") != record["version"]
12292
+ ):
12293
+ raise ValueError("v2 installed package identity differs from candidate")
12294
+ installed_documents[str(name)] = document
12295
+ allowed_overlay_paths.update(
12296
+ f"{name}/{item['path']}" for item in expected["files"]
12297
+ )
12298
+ bindings.append({
12299
+ "name": name,
12300
+ "version": record["version"],
12301
+ "inventory_sha256": actual["sha256"],
12302
+ })
12303
+ root_document = installed_documents[BENCHMARK_STUDY_V2_CANDIDATE_NAMES[1]]
12304
+ receipt_record = candidate["packages"][0]
12305
+ if root_document.get("dependencies") != {
12306
+ BENCHMARK_STUDY_V2_CANDIDATE_NAMES[0]: receipt_record["version"],
12307
+ }:
12308
+ raise ValueError("v2 installed root exact dependency binding mismatch")
12309
+ required_bins = ("context-guard", "context-guard-rewrite-bash")
12310
+ root_bin_map = root_document.get("bin")
12311
+ if not isinstance(root_bin_map, dict) or any(
12312
+ not isinstance(root_bin_map.get(name), str) for name in required_bins
12313
+ ):
12314
+ raise ValueError("v2 installed root package lacks required public bins")
12315
+ bin_bindings: list[dict[str, str]] = []
12316
+ seen_bin_names: set[str] = set()
12317
+ for package_name in sorted(installed_documents):
12318
+ bin_map = installed_documents[package_name].get("bin", {})
12319
+ if not isinstance(bin_map, dict):
12320
+ raise ValueError("v2 installed package bin metadata is invalid")
12321
+ package_root = overlay_root.joinpath(*package_name.split("/"))
12322
+ for bin_name, relative_target in sorted(bin_map.items()):
12323
+ if (
12324
+ not isinstance(bin_name, str)
12325
+ or not bin_name
12326
+ or "/" in bin_name
12327
+ or bin_name in {".", ".."}
12328
+ or bin_name in seen_bin_names
12329
+ or not isinstance(relative_target, str)
12330
+ ):
12331
+ raise ValueError("v2 installed package bin metadata is invalid")
12332
+ target_parts = relative_target.split("/")
12333
+ target_path = PurePosixPath(relative_target)
12334
+ if (
12335
+ target_path.is_absolute()
12336
+ or not target_parts
12337
+ or any(part in {"", ".", ".."} for part in target_parts)
12338
+ ):
12339
+ raise ValueError("v2 installed package bin target is unsafe")
12340
+ package_relative_target = (
12341
+ PurePosixPath(package_name) / target_path
12342
+ ).as_posix()
12343
+ if package_relative_target not in allowed_overlay_paths:
12344
+ raise ValueError("v2 installed package bin target is not in its tarball")
12345
+ link_relative = f".bin/{bin_name}"
12346
+ link = overlay_root / ".bin" / bin_name
12347
+ expected_raw_target = (
12348
+ PurePosixPath("..") / package_name / target_path
12349
+ ).as_posix()
12350
+ try:
12351
+ raw_target = os.readlink(link)
12352
+ resolved = link.resolve(strict=True)
12353
+ expected_target = (package_root / Path(*target_parts)).resolve(
12354
+ strict=True
12355
+ )
12356
+ except OSError as exc:
12357
+ raise ValueError("v2 installed public bin link is unavailable") from exc
12358
+ if raw_target != expected_raw_target or resolved != expected_target:
12359
+ raise ValueError(
12360
+ "v2 installed public bin link differs from package metadata"
12361
+ )
12362
+ seen_bin_names.add(bin_name)
12363
+ allowed_overlay_paths.add(link_relative)
12364
+ bin_bindings.append({
12365
+ "name": bin_name,
12366
+ "package": package_name,
12367
+ "target": expected_raw_target,
12368
+ })
12369
+ actual_overlay = _benchmark_study_v2_inventory(overlay_root)
12370
+ actual_overlay_paths = {item["path"] for item in actual_overlay["files"]}
12371
+ if actual_overlay_paths - allowed_overlay_paths:
12372
+ raise ValueError("v2 candidate install contains an unverified overlay path")
12373
+ if actual_overlay_paths != allowed_overlay_paths:
12374
+ raise ValueError("v2 candidate install is missing a verified overlay path")
12375
+ return {
12376
+ "packages": bindings,
12377
+ "bins": bin_bindings,
12378
+ "sha256": _study_domain_hash(
12379
+ "contextguard.bench.v2.installed-packages.v2",
12380
+ {"packages": bindings, "bins": bin_bindings},
12381
+ ),
12382
+ }
12383
+
12384
+
12385
+ def _benchmark_study_v2_inventory(root: Path, *, reject_symlinks: bool = False) -> dict[str, Any]:
12386
+ if root.is_symlink() or not root.is_dir():
12387
+ raise ValueError("v2 inventory root must be a real directory")
12388
+ files: list[dict[str, Any]] = []
12389
+ for path in sorted(root.rglob("*"), key=lambda item: item.relative_to(root).as_posix()):
12390
+ rel = path.relative_to(root).as_posix()
12391
+ mode = os.lstat(path).st_mode
12392
+ if stat.S_ISLNK(mode):
12393
+ if reject_symlinks:
12394
+ raise ValueError("v2 physical overlay must not contain symlinks")
12395
+ raw_target = os.readlink(path)
12396
+ if Path(raw_target).is_absolute():
12397
+ raise ValueError("v2 candidate overlay symlink must be relative")
12398
+ target = path.resolve(strict=True)
12399
+ try:
12400
+ target.relative_to(root.resolve(strict=True))
12401
+ except ValueError:
12402
+ raise ValueError("v2 candidate overlay symlink escapes the install root") from None
12403
+ if not target.is_file():
12404
+ raise ValueError("v2 candidate overlay symlink must resolve to an internal file")
12405
+ raw = _read_bytes_no_follow(target, max_bytes=100_000_000)
12406
+ executable = bool(target.stat().st_mode & 0o111)
12407
+ kind = "symlink"
12408
+ elif stat.S_ISREG(mode):
12409
+ raw = _read_bytes_no_follow(path, max_bytes=100_000_000)
12410
+ executable = bool(mode & 0o111)
12411
+ raw_target = None
12412
+ kind = "file"
12413
+ elif stat.S_ISDIR(mode):
12414
+ continue
12415
+ else:
12416
+ raise ValueError("v2 inventory contains an unsupported filesystem entry")
12417
+ files.append({
12418
+ "path": rel, "bytes": len(raw), "sha256": _study_sha256_bytes(raw),
12419
+ "executable": executable, "kind": kind, "target": raw_target,
12420
+ })
12421
+ if not files:
12422
+ raise ValueError("v2 candidate overlay is empty")
12423
+ return {
12424
+ "files": files,
12425
+ "file_count": len(files),
12426
+ "sha256": _study_domain_hash("contextguard.bench.v2.inventory.v1", files),
12427
+ }
12428
+
12429
+
12430
+ def _benchmark_study_v2_verify_physical_copy(
12431
+ source: Path, destination: Path, *, expected: Mapping[str, Any] | None = None,
12432
+ ) -> dict[str, Any]:
12433
+ source_inventory = _benchmark_study_v2_inventory(source)
12434
+ if expected is not None and source_inventory != dict(expected):
12435
+ raise ValueError("v2 staged candidate changed before the attempt copy")
12436
+ destination_inventory = _benchmark_study_v2_inventory(destination)
12437
+ if destination_inventory != source_inventory:
12438
+ raise ValueError("v2 candidate overlay copy differs from installed candidate")
12439
+ for item in source_inventory["files"]:
12440
+ if item["kind"] != "file":
12441
+ continue
12442
+ source_path = source / item["path"]
12443
+ destination_path = destination / item["path"]
12444
+ if source_path.stat().st_dev == destination_path.stat().st_dev and source_path.stat().st_ino == destination_path.stat().st_ino:
12445
+ raise ValueError("v2 candidate overlay contains a hardlink to the staged install")
12446
+ return destination_inventory
12447
+
12448
+
12449
+ def _benchmark_study_v2_expected_pre_workspace(
12450
+ task: TaskFixture, *, install_root: Path,
12451
+ expected_overlay_inventory: Mapping[str, Any],
12452
+ ) -> dict[str, Any]:
12453
+ """Re-materialize the bound cold workspace without invoking candidate code."""
12454
+ with tempfile.TemporaryDirectory(prefix="contextguard-v2-pre-workspace-") as temp:
12455
+ workspace = Path(temp).resolve()
12456
+ os.chmod(workspace, 0o700)
12457
+ reset_task_fixture_tree(task.fixture_tree_entries or (), workspace)
12458
+ overlay = workspace / BENCHMARK_STUDY_V2_OVERLAY_NAME
12459
+ shutil.copytree(
12460
+ install_root, overlay, symlinks=True, copy_function=shutil.copy2,
12461
+ )
12462
+ _benchmark_study_v2_verify_physical_copy(
12463
+ install_root, overlay, expected=expected_overlay_inventory,
12464
+ )
12465
+ return _benchmark_study_v2_inventory(workspace)
12466
+
12467
+
12468
+ def _benchmark_study_v2_settings() -> dict[str, dict[str, Any]]:
12469
+ common: dict[str, Any] = {
12470
+ "model": "sonnet",
12471
+ "permissions": {"allow": ["Bash", "Edit", "Glob", "Grep", "Read", "Write"]},
12472
+ }
12473
+ result = {"host_unmodified": json.loads(json.dumps(common))}
12474
+ for arm, suffix in (("legacy_trim", ""), ("bash_reference_v1", " --bash-reference-v1")):
12475
+ settings = json.loads(json.dumps(common))
12476
+ settings["hooks"] = {
12477
+ "PreToolUse": [{
12478
+ "matcher": "Bash",
12479
+ "hooks": [{
12480
+ "type": "command",
12481
+ "command": BENCHMARK_STUDY_V2_REWRITE_COMMAND + suffix,
12482
+ }],
12483
+ }],
12484
+ }
12485
+ result[arm] = settings
12486
+ return result
12487
+
12488
+
12489
+ def _benchmark_study_v2_write_settings(output_root: Path) -> dict[str, Any]:
12490
+ settings_root = output_root / "inputs" / "settings-v2"
12491
+ settings_root.mkdir(mode=0o700, parents=True)
12492
+ bindings: dict[str, Any] = {}
12493
+ for arm, value in _benchmark_study_v2_settings().items():
12494
+ path = settings_root / f"{arm}.settings.json"
12495
+ raw = _study_canonical_json_bytes(value)
12496
+ _measurement_write_exclusive(path, raw)
12497
+ bindings[arm] = {
12498
+ "path": str(path.resolve()), "sha256": _study_sha256_bytes(raw),
12499
+ "bytes": len(raw),
12500
+ "bash_hook": arm != "host_unmodified",
12501
+ "bash_reference_v1": arm == "bash_reference_v1",
12502
+ }
12503
+ return bindings
12504
+
12505
+
12506
+ def _benchmark_study_v2_install_candidate(
12507
+ *, npm_bin: str, candidate: Mapping[str, Any], output_root: Path,
12508
+ ) -> tuple[Path, dict[str, Any]]:
12509
+ install_root = output_root / "candidate-install"
12510
+ install_root.mkdir(mode=0o700)
12511
+ isolated = output_root / "npm-isolation"
12512
+ isolated.mkdir(mode=0o700)
12513
+ for name in ("home", "cache", "tmp"):
12514
+ (isolated / name).mkdir(mode=0o700)
12515
+ root_package = next(
12516
+ record for record in candidate["packages"]
12517
+ if record["name"] == "@ictechgy/context-guard"
12518
+ )
12519
+ receipt_package = next(
12520
+ record for record in candidate["packages"]
12521
+ if record["name"] == "@ictechgy/context-guard-receipt"
12522
+ )
12523
+ npm_executable = executable_argv0(npm_bin)
12524
+ runtime_directories = [str(Path(npm_executable).parent)]
12525
+ node_executable = shutil.which("node")
12526
+ if node_executable is not None:
12527
+ runtime_directories.append(str(Path(node_executable).resolve().parent))
12528
+ runtime_directories.extend(os.defpath.split(os.pathsep))
12529
+ runtime_path = os.pathsep.join(dict.fromkeys(runtime_directories))
12530
+ argv = [
12531
+ npm_executable, "install", "--offline", "--ignore-scripts",
12532
+ "--no-audit", "--fund=false", "--package-lock=false",
12533
+ "--prefix", str(install_root.resolve()),
12534
+ str(root_package["path"]), str(receipt_package["path"]),
12535
+ ]
12536
+ env = {
12537
+ "PATH": runtime_path, "HOME": str((isolated / "home").resolve()),
12538
+ "TMPDIR": str((isolated / "tmp").resolve()),
12539
+ "NPM_CONFIG_CACHE": str((isolated / "cache").resolve()),
12540
+ "NPM_CONFIG_OFFLINE": "true", "NPM_CONFIG_IGNORE_SCRIPTS": "true",
12541
+ "NPM_CONFIG_AUDIT": "false", "NPM_CONFIG_FUND": "false",
12542
+ "NPM_CONFIG_PACKAGE_LOCK": "false",
12543
+ "NPM_CONFIG_UPDATE_NOTIFIER": "false",
12544
+ "NPM_CONFIG_REGISTRY": "https://registry.invalid/",
12545
+ }
12546
+ result = run_bounded_command(
12547
+ argv, cwd=output_root, timeout_seconds=180,
12548
+ max_output_bytes=MEASUREMENT_CLI_PROBE_OUTPUT_MAX_BYTES, env=env,
12549
+ )
12550
+ if result.returncode != 0 or result.timed_out or result.output_truncated:
12551
+ raise ValueError("v2 candidate offline npm install failed")
12552
+ overlay_root = install_root / "node_modules"
12553
+ hidden_lock = overlay_root / ".package-lock.json"
12554
+ hidden_lock_removed = False
12555
+ try:
12556
+ hidden_lock_mode = os.lstat(hidden_lock).st_mode
12557
+ except FileNotFoundError:
12558
+ pass
12559
+ else:
12560
+ if not stat.S_ISREG(hidden_lock_mode):
12561
+ raise ValueError("v2 npm hidden lockfile has an unsupported type")
12562
+ hidden_lock.unlink()
12563
+ hidden_lock_removed = True
12564
+ inventory = _benchmark_study_v2_inventory(overlay_root)
12565
+ installed_packages = _benchmark_study_v2_verify_installed_packages(
12566
+ overlay_root, candidate,
12567
+ )
12568
+ receipt = {
12569
+ "schema_version": "contextguard.bench.candidate-install.v2",
12570
+ "install_count": 1,
12571
+ "network": "offline",
12572
+ "ignore_scripts": True,
12573
+ "no_audit": True,
12574
+ "fund": False,
12575
+ "hidden_lockfile_removed": hidden_lock_removed,
12576
+ "candidate_manifest_sha256": candidate["manifest_sha256"],
12577
+ "argv_policy": [
12578
+ "install", "--offline", "--ignore-scripts", "--no-audit",
12579
+ "--fund=false", "--package-lock=false",
12580
+ ],
12581
+ "inventory": inventory,
12582
+ "installed_packages": installed_packages,
12583
+ }
12584
+ _study_write_private(output_root / "candidate-install-receipt.json", receipt)
12585
+ return overlay_root, receipt
12586
+
12587
+
12588
+ def prepare_benchmark_study_v2_executable(
12589
+ *, output_root: Path, plan_path: Path, tasks_path: Path, checkers_dir: Path,
12590
+ candidate_manifest_path: Path, candidate_checksum_path: Path | None,
12591
+ expected_candidate_hash: str, npm_bin: str, claude_bin: str,
12592
+ auth_home: Path, approved_source_commit: str,
12593
+ retained_ref: str | None, offline_rehearsal: bool,
12594
+ ) -> dict[str, Any]:
12595
+ output_root = _benchmark_study_v2_output_root(output_root)
12596
+ if output_root.exists() and (
12597
+ output_root.is_symlink() or not output_root.is_dir() or any(output_root.iterdir())
12598
+ ):
12599
+ raise ValueError("v2 prepare output root must be new or empty")
12600
+ # Candidate validation is deliberately first and performs no candidate import.
12601
+ candidate = verify_benchmark_study_v2_candidate(
12602
+ candidate_manifest_path, checksum_path=candidate_checksum_path,
12603
+ expected_manifest_sha256=expected_candidate_hash,
12604
+ expected_commit_sha=approved_source_commit,
12605
+ )
12606
+ if type(offline_rehearsal) is not bool:
12607
+ raise TypeError("v2 offline rehearsal flag must be boolean")
12608
+ if offline_rehearsal:
12609
+ if retained_ref is not None:
12610
+ raise ValueError("v2 offline rehearsal cannot bind a retained ref")
12611
+ source_ref_binding: dict[str, Any] = {
12612
+ "commit_sha": approved_source_commit,
12613
+ "ref": None,
12614
+ "repository": "ictechgy/context-guard",
12615
+ "verification": "offline-rehearsal-unverified-v1",
12616
+ }
12617
+ else:
12618
+ if retained_ref is None:
12619
+ raise ValueError("v2 live prepare requires a retained ref")
12620
+ source_ref_binding = _benchmark_study_v2_verify_retained_ref(
12621
+ retained_ref, approved_source_commit,
12622
+ )
12623
+ plan = load_benchmark_study_v2_plan(plan_path)
12624
+ corpus_bytes = _read_bytes_no_follow(tasks_path, max_bytes=MAX_FIXTURE_FILE_BYTES)
12625
+ checker_binding = benchmark_study_v2_checker_binding(checkers_dir)
12626
+ validate_benchmark_study_v2_bindings(
12627
+ plan, corpus_bytes=corpus_bytes, checker_binding=checker_binding,
12628
+ )
12629
+ tasks = parse_tasks(tasks_path)
12630
+ load_task_fixture_trees(tasks, task_file_dir=tasks_path.parent)
12631
+ task_definitions = [
12632
+ _study_task_manifest(task, tasks_path.parent) for task in tasks
12633
+ ]
12634
+ task_ids = _benchmark_study_v2_task_ids_from_corpus(corpus_bytes)
12635
+ if [task.id for task in tasks] != task_ids:
12636
+ raise ValueError("v2 parsed task order differs from the bound corpus")
12637
+ cli_binding = _benchmark_study_v2_cli_binding(claude_bin)
12638
+ execution_environment = _benchmark_study_v2_execution_environment(cli_binding)
12639
+ _benchmark_study_v2_assert_execution_environment(execution_environment)
12640
+ _resolved_auth_home, auth_context = _benchmark_study_v2_auth_context(
12641
+ claude_bin, execution_environment, auth_home,
12642
+ )
12643
+ runner_binding = _benchmark_study_v2_runner_binding()
12644
+ schedule = generate_benchmark_study_v2_schedule(
12645
+ task_ids, repetitions=3, schedule_seed=plan["schedule_seed"],
12646
+ )
12647
+ slots = generate_benchmark_study_v2_slots(
12648
+ task_ids, schedule, candidate_hash=candidate["manifest_sha256"],
12649
+ namespace=BENCHMARK_STUDY_V2_NAMESPACE,
12650
+ )
12651
+ output_root.mkdir(mode=0o700, parents=True, exist_ok=True)
12652
+ os.chmod(output_root, 0o700)
12653
+ settings = _benchmark_study_v2_write_settings(output_root)
12654
+ install_root, install_receipt = _benchmark_study_v2_install_candidate(
12655
+ npm_bin=npm_bin, candidate=candidate, output_root=output_root,
12656
+ )
12657
+ manifest = {
12658
+ "schema_version": BENCHMARK_STUDY_V2_EXEC_MANIFEST_SCHEMA_VERSION,
12659
+ "plan": plan,
12660
+ "plan_sha256": _study_sha256_bytes(_study_canonical_json_bytes(plan)),
12661
+ "inputs": {
12662
+ "plan_path": str(plan_path.resolve()),
12663
+ "tasks_path": str(tasks_path.resolve()),
12664
+ "tasks_sha256": _study_sha256_bytes(corpus_bytes),
12665
+ "task_definitions": task_definitions,
12666
+ "checkers_dir": str(checkers_dir.resolve()),
12667
+ "checker_binding": checker_binding,
12668
+ "cli_binding": cli_binding,
12669
+ "execution_environment": execution_environment,
12670
+ "auth_context": auth_context,
12671
+ "runner_binding": runner_binding,
12672
+ "canary_contract": _benchmark_study_v2_canary_contract(),
12673
+ "approved_source_commit": approved_source_commit,
12674
+ "source_ref_binding": source_ref_binding,
12675
+ "candidate": candidate,
12676
+ "candidate_install_root": str(install_root.resolve()),
12677
+ "candidate_install_receipt_sha256": _study_sha256_bytes(
12678
+ _study_canonical_json_bytes(install_receipt)
12679
+ ),
12680
+ "candidate_overlay_inventory": install_receipt["inventory"],
12681
+ "settings": settings,
12682
+ "namespace": BENCHMARK_STUDY_V2_NAMESPACE,
12683
+ "task_ids": task_ids,
12684
+ "task_ids_sha256": _benchmark_study_v2_task_ids_sha256(task_ids),
12685
+ },
12686
+ "schedule": schedule,
12687
+ "slots": slots,
12688
+ "execution": {
12689
+ "identities": 216, "initial_calls": 108,
12690
+ "retry": "exactly_after_valid_initial_failure_v1",
12691
+ "resume": "never_replay_launched_identity_v1",
12692
+ "attempt_schema_version": BENCHMARK_STUDY_V2_ATTEMPT_SCHEMA_VERSION,
12693
+ "invalid_decision_schema_version": (
12694
+ BENCHMARK_STUDY_V2_INVALID_DECISION_SCHEMA_VERSION
12695
+ ),
12696
+ "candidate_imported": False, "candidate_install_count": 1,
12697
+ "overlay_copy": "physical_copy_no_hardlinks_v1",
12698
+ },
12699
+ }
12700
+ _study_write_private(output_root / "study-manifest.json", manifest)
12701
+ return manifest
12702
+
12703
+
12704
+ def load_benchmark_study_v2_executable_manifest(output_root: Path, *, revalidate_external: bool) -> tuple[dict[str, Any], str]:
12705
+ output_root = _benchmark_study_v2_output_root(output_root)
12706
+ manifest_path = output_root / "study-manifest.json"
12707
+ if not manifest_path.exists():
12708
+ raise ValueError("v2 executable study requires a prepared output root")
12709
+ manifest, raw = _benchmark_study_v2_read_canonical(
12710
+ manifest_path, owner="v2 executable manifest", maximum=2_000_000,
12711
+ )
12712
+ if set(manifest) != {"schema_version", "plan", "plan_sha256", "inputs", "schedule", "slots", "execution"} or manifest.get("schema_version") != BENCHMARK_STUDY_V2_EXEC_MANIFEST_SCHEMA_VERSION:
12713
+ raise ValueError("v2 executable manifest schema mismatch")
12714
+ validate_benchmark_study_v2_plan(manifest["plan"])
12715
+ if manifest["plan_sha256"] != _study_sha256_bytes(_study_canonical_json_bytes(manifest["plan"])):
12716
+ raise ValueError("v2 executable plan binding mismatch")
12717
+ inputs = manifest["inputs"]
12718
+ required_inputs = {
12719
+ "plan_path", "tasks_path", "tasks_sha256", "task_definitions",
12720
+ "checkers_dir", "checker_binding", "cli_binding", "execution_environment",
12721
+ "auth_context", "runner_binding", "canary_contract", "candidate",
12722
+ "approved_source_commit",
12723
+ "source_ref_binding",
12724
+ "candidate_install_root", "candidate_install_receipt_sha256",
12725
+ "candidate_overlay_inventory", "settings", "namespace", "task_ids",
12726
+ "task_ids_sha256",
12727
+ }
12728
+ if set(inputs) != required_inputs:
12729
+ raise ValueError("v2 executable input schema mismatch")
12730
+ _benchmark_study_v2_validate_cli_binding(inputs["cli_binding"])
12731
+ _benchmark_study_v2_validate_execution_environment(
12732
+ inputs["execution_environment"]
12733
+ )
12734
+ _benchmark_study_v2_validate_auth_context(inputs["auth_context"])
12735
+ if inputs["runner_binding"] != _benchmark_study_v2_runner_binding():
12736
+ raise ValueError("v2 benchmark runner binding drift")
12737
+ if inputs["canary_contract"] != _benchmark_study_v2_canary_contract():
12738
+ raise ValueError("v2 canary contract binding mismatch")
12739
+ if (
12740
+ not isinstance(inputs["approved_source_commit"], str)
12741
+ or re.fullmatch(r"[0-9a-f]{40}", inputs["approved_source_commit"]) is None
12742
+ or inputs["candidate"].get("commit_sha")
12743
+ != inputs["approved_source_commit"]
12744
+ ):
12745
+ raise ValueError("v2 approved source revision binding mismatch")
12746
+ source_ref_binding = inputs["source_ref_binding"]
12747
+ if (
12748
+ not isinstance(source_ref_binding, Mapping)
12749
+ or set(source_ref_binding) != {
12750
+ "commit_sha", "ref", "repository", "verification",
12751
+ }
12752
+ or source_ref_binding.get("commit_sha")
12753
+ != inputs["approved_source_commit"]
12754
+ or source_ref_binding.get("repository") != "ictechgy/context-guard"
12755
+ or source_ref_binding.get("verification") not in {
12756
+ "git-ls-remote-v1", "offline-rehearsal-unverified-v1",
12757
+ }
12758
+ or (
12759
+ source_ref_binding.get("verification") == "git-ls-remote-v1"
12760
+ and (
12761
+ not isinstance(source_ref_binding.get("ref"), str)
12762
+ or re.fullmatch(
12763
+ r"refs/heads/candidate/[A-Za-z0-9][A-Za-z0-9._/-]*",
12764
+ source_ref_binding["ref"],
12765
+ ) is None
12766
+ )
12767
+ )
12768
+ or (
12769
+ source_ref_binding.get("verification")
12770
+ == "offline-rehearsal-unverified-v1"
12771
+ and source_ref_binding.get("ref") is not None
12772
+ )
12773
+ ):
12774
+ raise ValueError("v2 retained source ref binding mismatch")
12775
+ if (
12776
+ not isinstance(inputs["task_definitions"], list)
12777
+ or len(inputs["task_definitions"]) != 12
12778
+ ):
12779
+ raise ValueError("v2 executable task definition binding mismatch")
12780
+ task_ids = _benchmark_study_v2_task_ids(inputs["task_ids"])
12781
+ task_definition_ids = [
12782
+ item.get("id") if isinstance(item, Mapping) else None
12783
+ for item in inputs["task_definitions"]
12784
+ ]
12785
+ if (
12786
+ inputs.get("namespace") != BENCHMARK_STUDY_V2_NAMESPACE
12787
+ or inputs.get("tasks_sha256") != manifest["plan"]["corpus_sha256"]
12788
+ or not isinstance(inputs.get("checker_binding"), Mapping)
12789
+ or inputs["checker_binding"].get("sha256")
12790
+ != manifest["plan"]["checker_sha256"]
12791
+ or inputs.get("task_ids_sha256") != manifest["plan"]["task_ids_sha256"]
12792
+ or inputs.get("task_ids_sha256")
12793
+ != _benchmark_study_v2_task_ids_sha256(task_ids)
12794
+ or task_definition_ids != task_ids
12795
+ ):
12796
+ raise ValueError("v2 executable task order or namespace binding mismatch")
12797
+ if manifest["execution"] != {
12798
+ "identities": 216, "initial_calls": 108,
12799
+ "retry": "exactly_after_valid_initial_failure_v1",
12800
+ "resume": "never_replay_launched_identity_v1",
12801
+ "attempt_schema_version": BENCHMARK_STUDY_V2_ATTEMPT_SCHEMA_VERSION,
12802
+ "invalid_decision_schema_version": (
12803
+ BENCHMARK_STUDY_V2_INVALID_DECISION_SCHEMA_VERSION
12804
+ ),
12805
+ "candidate_imported": False, "candidate_install_count": 1,
12806
+ "overlay_copy": "physical_copy_no_hardlinks_v1",
12807
+ }:
12808
+ raise ValueError("v2 executable lifecycle contract drift")
12809
+ settings = inputs.get("settings")
12810
+ expected_settings = _benchmark_study_v2_settings()
12811
+ if not isinstance(settings, Mapping) or set(settings) != set(BENCHMARK_STUDY_V2_ARMS):
12812
+ raise ValueError("v2 settings arm binding mismatch")
12813
+ for arm in BENCHMARK_STUDY_V2_ARMS:
12814
+ binding = settings[arm]
12815
+ expected_path = output_root / "inputs" / "settings-v2" / f"{arm}.settings.json"
12816
+ if (
12817
+ not isinstance(binding, Mapping)
12818
+ or set(binding) != {
12819
+ "path", "sha256", "bytes", "bash_hook", "bash_reference_v1",
12820
+ }
12821
+ or binding.get("path") != str(expected_path.resolve())
12822
+ or binding.get("bash_hook") is not (arm != "host_unmodified")
12823
+ or binding.get("bash_reference_v1") is not (arm == "bash_reference_v1")
12824
+ ):
12825
+ raise ValueError(f"v2 {arm} settings binding mismatch")
12826
+ raw_settings = _read_bytes_no_follow(expected_path, max_bytes=100_000)
12827
+ expected_raw = _study_canonical_json_bytes(expected_settings[arm])
12828
+ if (
12829
+ raw_settings != expected_raw
12830
+ or binding.get("sha256") != _study_sha256_bytes(expected_raw)
12831
+ or binding.get("bytes") != len(expected_raw)
12832
+ ):
12833
+ raise ValueError(f"v2 {arm} settings policy drift")
12834
+ install_receipt, install_receipt_raw = _benchmark_study_v2_read_canonical(
12835
+ output_root / "candidate-install-receipt.json",
12836
+ owner="v2 candidate install receipt", maximum=2_000_000,
12837
+ )
12838
+ if (
12839
+ set(install_receipt) != {
12840
+ "schema_version", "install_count", "network", "ignore_scripts",
12841
+ "no_audit", "fund", "hidden_lockfile_removed",
12842
+ "candidate_manifest_sha256", "argv_policy", "inventory",
12843
+ "installed_packages",
12844
+ }
12845
+ or install_receipt.get("schema_version")
12846
+ != "contextguard.bench.candidate-install.v2"
12847
+ or install_receipt.get("install_count") != 1
12848
+ or install_receipt.get("network") != "offline"
12849
+ or install_receipt.get("ignore_scripts") is not True
12850
+ or install_receipt.get("no_audit") is not True
12851
+ or install_receipt.get("fund") is not False
12852
+ or not isinstance(install_receipt.get("hidden_lockfile_removed"), bool)
12853
+ or install_receipt.get("argv_policy") != [
12854
+ "install", "--offline", "--ignore-scripts", "--no-audit",
12855
+ "--fund=false", "--package-lock=false",
12856
+ ]
12857
+ or _study_sha256_bytes(install_receipt_raw)
12858
+ != inputs.get("candidate_install_receipt_sha256")
12859
+ or install_receipt.get("candidate_manifest_sha256")
12860
+ != inputs.get("candidate", {}).get("manifest_sha256")
12861
+ or install_receipt.get("inventory")
12862
+ != inputs.get("candidate_overlay_inventory")
12863
+ ):
12864
+ raise ValueError("v2 candidate install receipt binding mismatch")
12865
+ expected_schedule = generate_benchmark_study_v2_schedule(
12866
+ task_ids, repetitions=3, schedule_seed=manifest["plan"]["schedule_seed"],
12867
+ )
12868
+ expected_slots = generate_benchmark_study_v2_slots(
12869
+ task_ids, expected_schedule,
12870
+ candidate_hash=inputs["candidate"]["manifest_sha256"],
12871
+ namespace=inputs["namespace"],
12872
+ )
12873
+ if manifest["schedule"] != expected_schedule or manifest["slots"] != expected_slots:
12874
+ raise ValueError("v2 executable schedule or identity drift")
12875
+ if revalidate_external:
12876
+ _benchmark_study_v2_assert_execution_environment(
12877
+ inputs["execution_environment"]
12878
+ )
12879
+ external_plan = load_benchmark_study_v2_plan(Path(inputs["plan_path"]))
12880
+ if external_plan != manifest["plan"]:
12881
+ raise ValueError("v2 external study plan binding drift")
12882
+ corpus = _read_bytes_no_follow(Path(inputs["tasks_path"]), max_bytes=MAX_FIXTURE_FILE_BYTES)
12883
+ if _study_sha256_bytes(corpus) != inputs["tasks_sha256"]:
12884
+ raise ValueError("v2 task corpus binding drift")
12885
+ checker = benchmark_study_v2_checker_binding(Path(inputs["checkers_dir"]))
12886
+ if checker != inputs["checker_binding"]:
12887
+ raise ValueError("v2 checker inventory binding drift")
12888
+ validate_benchmark_study_v2_bindings(
12889
+ manifest["plan"], corpus_bytes=corpus, checker_binding=checker,
12890
+ )
12891
+ tasks = parse_tasks(Path(inputs["tasks_path"]))
12892
+ load_task_fixture_trees(
12893
+ tasks, task_file_dir=Path(inputs["tasks_path"]).parent,
12894
+ )
12895
+ task_definitions = [
12896
+ _study_task_manifest(task, Path(inputs["tasks_path"]).parent)
12897
+ for task in tasks
12898
+ ]
12899
+ if task_definitions != inputs["task_definitions"]:
12900
+ raise ValueError("v2 task fixture or checker binding drift")
12901
+ if [task.id for task in tasks] != task_ids:
12902
+ raise ValueError("v2 external task order binding drift")
12903
+ candidate = verify_benchmark_study_v2_candidate(
12904
+ Path(inputs["candidate"]["manifest_path"]),
12905
+ checksum_path=Path(inputs["candidate"]["checksum_path"]),
12906
+ expected_manifest_sha256=inputs["candidate"]["manifest_sha256"],
12907
+ expected_commit_sha=inputs["approved_source_commit"],
12908
+ )
12909
+ if candidate != inputs["candidate"]:
12910
+ raise ValueError("v2 candidate binding drift")
12911
+ if _benchmark_study_v2_verify_installed_packages(
12912
+ Path(inputs["candidate_install_root"]), candidate,
12913
+ ) != install_receipt["installed_packages"]:
12914
+ raise ValueError("v2 installed candidate package binding drift")
12915
+ if _benchmark_study_v2_inventory(Path(inputs["candidate_install_root"])) != inputs["candidate_overlay_inventory"]:
12916
+ raise ValueError("v2 installed candidate inventory drift")
12917
+ for arm, binding in inputs["settings"].items():
12918
+ raw_settings = _read_bytes_no_follow(Path(binding["path"]), max_bytes=100_000)
12919
+ if _study_sha256_bytes(raw_settings) != binding["sha256"] or len(raw_settings) != binding["bytes"]:
12920
+ raise ValueError(f"v2 {arm} settings drift")
12921
+ return manifest, _study_sha256_bytes(raw)
12922
+
12923
+
12924
+ def _benchmark_study_v2_variants(manifest: Mapping[str, Any], output_root: Path) -> dict[str, Variant]:
12925
+ output_root = _benchmark_study_v2_output_root(output_root)
12926
+ result: dict[str, Variant] = {}
12927
+ artifact_root = output_root / "artifacts"
12928
+ for arm in BENCHMARK_STUDY_V2_ARMS:
12929
+ binding = manifest["inputs"]["settings"][arm]
12930
+ settings_path = Path(binding["path"])
12931
+ settings_raw = _read_bytes_no_follow(settings_path, max_bytes=100_000)
12932
+ settings_payload = json.loads(settings_raw)
12933
+ command = BENCHMARK_STUDY_V2_REWRITE_COMMAND + (
12934
+ " --bash-reference-v1" if arm == "bash_reference_v1" else ""
12935
+ )
12936
+ registered = () if arm == "host_unmodified" else (("PreToolUse", command),)
12937
+ identity = MeasurementIdentity(
12938
+ candidate_hash=manifest["inputs"]["candidate"]["manifest_sha256"],
12939
+ repetition=0, arm=arm, attempt=0,
12940
+ namespace=manifest["inputs"]["namespace"],
12941
+ )
12942
+ result[arm] = Variant(
12943
+ name=arm,
12944
+ measurement=MeasurementVariant(
12945
+ settings_file=settings_path,
12946
+ setting_sources=("project",), environment_allow=(),
12947
+ environment_overrides=tuple(
12948
+ (name, str(manifest["inputs"]["execution_environment"]["values"][name]))
12949
+ for name in sorted(
12950
+ manifest["inputs"]["execution_environment"]["values"]
12951
+ )
12952
+ ),
12953
+ workspace_mode="isolated",
12954
+ session_mode="isolated", session_persistence="disabled",
12955
+ hook_events_enabled=True, registered_bindings=registered,
12956
+ required_event_classes=(), pair_registered_bindings=registered,
12957
+ cli_capabilities=(
12958
+ "--settings", "--setting-sources", "--include-hook-events",
12959
+ "--no-session-persistence", "stream-json",
12960
+ ),
12961
+ identity=identity, artifact_root=artifact_root,
12962
+ settings_payload=settings_payload, settings_source_bytes=settings_raw,
12963
+ ),
12964
+ )
12965
+ return result
12966
+
12967
+
12968
+ def _benchmark_study_v2_canary_variants(
12969
+ manifest: Mapping[str, Any], output_root: Path,
12970
+ ) -> dict[str, Variant]:
12971
+ analytic = _benchmark_study_v2_variants(manifest, output_root)
12972
+ result: dict[str, Variant] = {}
12973
+ for arm in BENCHMARK_STUDY_V2_CANARY_ARMS:
12974
+ base = analytic[arm].measurement
12975
+ assert base is not None
12976
+ identity = MeasurementIdentity(
12977
+ candidate_hash=manifest["inputs"]["candidate"]["manifest_sha256"],
12978
+ repetition=0, arm=arm, attempt=0,
12979
+ namespace=f"{manifest['inputs']['namespace']}.canary",
12980
+ )
12981
+ result[arm] = Variant(
12982
+ name=arm,
12983
+ measurement=replace(
12984
+ base,
12985
+ required_event_classes=("PreToolUse",),
12986
+ identity=identity,
12987
+ artifact_root=output_root / "canary-artifacts",
12988
+ ),
12989
+ )
12990
+ return result
12991
+
12992
+
12993
+ def _benchmark_study_v2_canary_base_event(
12994
+ *, arm: str, run_id: str, manifest_sha256: str, state: str,
12995
+ **extra: Any,
12996
+ ) -> dict[str, Any]:
12997
+ event = {
12998
+ "schema_version": BENCHMARK_STUDY_V2_CANARY_EVENT_SCHEMA_VERSION,
12999
+ "manifest_sha256": manifest_sha256,
13000
+ "arm": arm, "run_id": run_id, "state": state,
13001
+ }
13002
+ event.update(extra)
13003
+ return event
13004
+
13005
+
13006
+ def _benchmark_study_v2_read_canary_events(
13007
+ path: Path, *, manifest_sha256: str,
13008
+ variants: Mapping[str, Variant],
13009
+ ) -> list[dict[str, Any]]:
13010
+ if not path.exists():
13011
+ return []
13012
+ raw = _measurement_read_private_file(path, maximum=200_000)
13013
+ expected_run_ids = {}
13014
+ for arm, variant in variants.items():
13015
+ spec = variant.measurement
13016
+ assert spec is not None
13017
+ expected_run_ids[arm] = spec.identity.run_id(BENCHMARK_STUDY_V2_CANARY_TASK_ID)
13018
+ base_keys = {"schema_version", "manifest_sha256", "arm", "run_id", "state"}
13019
+ workspace_keys = base_keys | {
13020
+ "pre_workspace_inventory_sha256", "pre_overlay_inventory_sha256",
13021
+ }
13022
+ terminal_keys = base_keys | {
13023
+ "passed", "measurement_terminal_status", "checker_status",
13024
+ "receipt_sha256", "pre_workspace_inventory_sha256",
13025
+ "post_workspace_inventory_sha256", "pre_overlay_inventory_sha256",
13026
+ "post_overlay_inventory_sha256", "pretooluse_event_count",
13027
+ }
13028
+ states: dict[str, list[str]] = collections.defaultdict(list)
13029
+ rows: list[dict[str, Any]] = []
13030
+ for line in raw.splitlines():
13031
+ try:
13032
+ row = json.loads(
13033
+ line.decode("utf-8"),
13034
+ object_pairs_hook=_measurement_object_no_duplicates,
13035
+ parse_constant=_stream_reject_nonfinite,
13036
+ )
13037
+ except (UnicodeDecodeError, json.JSONDecodeError, ValueError) as exc:
13038
+ raise ValueError("v2 canary event ledger is invalid JSONL") from exc
13039
+ if not isinstance(row, dict) or line + b"\n" != _study_canonical_json_bytes(row):
13040
+ raise ValueError("v2 canary event ledger must be canonical JSONL")
13041
+ arm = row.get("arm")
13042
+ state = row.get("state")
13043
+ if (
13044
+ row.get("schema_version") != BENCHMARK_STUDY_V2_CANARY_EVENT_SCHEMA_VERSION
13045
+ or row.get("manifest_sha256") != manifest_sha256
13046
+ or arm not in BENCHMARK_STUDY_V2_CANARY_ARMS
13047
+ or row.get("run_id") != expected_run_ids.get(str(arm))
13048
+ or state not in {"launch_reserved", "workspace_prepared", "launched", "terminal"}
13049
+ ):
13050
+ raise ValueError("v2 canary event binding mismatch")
13051
+ expected_keys = (
13052
+ workspace_keys if state == "workspace_prepared"
13053
+ else terminal_keys if state == "terminal" else base_keys
13054
+ )
13055
+ if set(row) != expected_keys:
13056
+ raise ValueError("v2 canary event schema mismatch")
13057
+ previous = states[str(arm)]
13058
+ expected_previous = {
13059
+ "launch_reserved": [],
13060
+ "workspace_prepared": ["launch_reserved"],
13061
+ "launched": ["launch_reserved", "workspace_prepared"],
13062
+ "terminal": ["launch_reserved", "workspace_prepared", "launched"],
13063
+ }[str(state)]
13064
+ if previous != expected_previous:
13065
+ raise ValueError("v2 canary event state transition is invalid")
13066
+ if state in {"workspace_prepared", "terminal"}:
13067
+ hash_fields = (
13068
+ ("pre_workspace_inventory_sha256", "pre_overlay_inventory_sha256")
13069
+ if state == "workspace_prepared" else (
13070
+ "receipt_sha256", "pre_workspace_inventory_sha256",
13071
+ "post_workspace_inventory_sha256", "pre_overlay_inventory_sha256",
13072
+ "post_overlay_inventory_sha256",
13073
+ )
13074
+ )
13075
+ if any(
13076
+ not isinstance(row.get(field), str)
13077
+ or SHA256_HEX_PATTERN.fullmatch(str(row[field])) is None
13078
+ for field in hash_fields
13079
+ ):
13080
+ raise ValueError("v2 canary evidence hash is invalid")
13081
+ if state == "terminal" and (
13082
+ not isinstance(row.get("passed"), bool)
13083
+ or row.get("measurement_terminal_status") not in {
13084
+ "success", "raw_byte_limit", "raw_line_limit", "raw_line_byte_limit",
13085
+ "process_timeout", "process_launch_error", "process_error",
13086
+ "terminal_error", "missing_terminal", "invalid_stream",
13087
+ "hook_payload_limit", "hook_lifecycle_limit", "invalid_hook_lifecycle",
13088
+ "unexpected_hook_event_class", "missing_required_hook_event_class",
13089
+ "hook_process_failure",
13090
+ }
13091
+ or row.get("checker_status") not in {
13092
+ "task_success", "valid_task_failure_v1", "success_checker_infra_invalid",
13093
+ "not_run",
13094
+ }
13095
+ or isinstance(row.get("pretooluse_event_count"), bool)
13096
+ or not isinstance(row.get("pretooluse_event_count"), int)
13097
+ or row["pretooluse_event_count"] < 0
13098
+ ):
13099
+ raise ValueError("v2 canary terminal evidence is invalid")
13100
+ previous.append(str(state))
13101
+ rows.append(row)
13102
+ return rows
13103
+
13104
+
13105
+ def _benchmark_study_v2_derive_canary_terminal(
13106
+ *, manifest: Mapping[str, Any], manifest_sha256: str, output_root: Path,
13107
+ arm: str, variant: Variant, task: TaskFixture,
13108
+ workspace_event: Mapping[str, Any],
13109
+ ) -> dict[str, Any]:
13110
+ spec = variant.measurement
13111
+ assert spec is not None
13112
+ run_id = spec.identity.run_id(task.id)
13113
+ receipt = _verify_existing_measurement_run(
13114
+ spec, task.id, run_id, require_index=True,
13115
+ )
13116
+ context = _measurement_existing_context(spec, run_id)
13117
+ receipt_raw = _measurement_read_private_file(context.receipt_path)
13118
+ expected_overlay = manifest["inputs"]["candidate_overlay_inventory"]
13119
+ expected_pre = _benchmark_study_v2_expected_pre_workspace(
13120
+ task,
13121
+ install_root=Path(manifest["inputs"]["candidate_install_root"]),
13122
+ expected_overlay_inventory=expected_overlay,
13123
+ )
13124
+ post_workspace = _benchmark_study_v2_inventory(context.workspace)
13125
+ post_overlay = _benchmark_study_v2_inventory(
13126
+ context.workspace / BENCHMARK_STUDY_V2_OVERLAY_NAME,
13127
+ )
13128
+ checker = (
13129
+ run_task_checker_study(
13130
+ task, context.workspace, env={},
13131
+ interpreter_binding=manifest["inputs"]["runner_binding"]["python"],
13132
+ )
13133
+ if receipt["terminal_status"] == "success" else "not_run"
13134
+ )
13135
+ summary = receipt["hook_summary"]
13136
+ counts = {
13137
+ row["hook_event"]: row["count"]
13138
+ for row in summary["event_class_counts"]
13139
+ }
13140
+ pretooluse_count = int(counts.get("PreToolUse", 0))
13141
+ pre_bound = bool(
13142
+ workspace_event.get("pre_workspace_inventory_sha256") == expected_pre["sha256"]
13143
+ and workspace_event.get("pre_overlay_inventory_sha256") == expected_overlay["sha256"]
13144
+ )
13145
+ hooks_valid = bool(
13146
+ summary["required_event_classes"] == ["PreToolUse"]
13147
+ and pretooluse_count >= 1
13148
+ and set(counts) == {"PreToolUse"}
13149
+ and all(
13150
+ hook["hook_event"] == "PreToolUse"
13151
+ and hook["hook_process_outcome"] == "success"
13152
+ and hook["hook_process_exit_code"] in (None, 0)
13153
+ for hook in receipt["hooks"]
13154
+ )
13155
+ )
13156
+ passed = bool(
13157
+ receipt["terminal_status"] == "success"
13158
+ and checker == "task_success" and hooks_valid and pre_bound
13159
+ and post_overlay == expected_overlay
13160
+ )
13161
+ return _benchmark_study_v2_canary_base_event(
13162
+ arm=arm, run_id=run_id, manifest_sha256=manifest_sha256,
13163
+ state="terminal", passed=passed,
13164
+ measurement_terminal_status=receipt["terminal_status"],
13165
+ checker_status=checker,
13166
+ receipt_sha256=_study_sha256_bytes(receipt_raw),
13167
+ pre_workspace_inventory_sha256=str(
13168
+ workspace_event["pre_workspace_inventory_sha256"]
13169
+ ),
13170
+ post_workspace_inventory_sha256=post_workspace["sha256"],
13171
+ pre_overlay_inventory_sha256=str(
13172
+ workspace_event["pre_overlay_inventory_sha256"]
13173
+ ),
13174
+ post_overlay_inventory_sha256=post_overlay["sha256"],
13175
+ pretooluse_event_count=pretooluse_count,
13176
+ )
13177
+
13178
+
13179
+ def _benchmark_study_v2_expected_canary_evidence(
13180
+ *, manifest: Mapping[str, Any], manifest_sha256: str, output_root: Path,
13181
+ rows: Sequence[Mapping[str, Any]], variants: Mapping[str, Variant],
13182
+ ) -> dict[str, Any]:
13183
+ task = _benchmark_study_v2_canary_task()
13184
+ by_arm: dict[str, list[Mapping[str, Any]]] = {
13185
+ arm: [row for row in rows if row["arm"] == arm]
13186
+ for arm in BENCHMARK_STUDY_V2_CANARY_ARMS
13187
+ }
13188
+ records: list[dict[str, Any]] = []
13189
+ for arm in BENCHMARK_STUDY_V2_CANARY_ARMS:
13190
+ arm_rows = by_arm[arm]
13191
+ if [row["state"] for row in arm_rows] != [
13192
+ "launch_reserved", "workspace_prepared", "launched", "terminal",
13193
+ ]:
13194
+ raise ValueError("v2 canary evidence is incomplete")
13195
+ recomputed = _benchmark_study_v2_derive_canary_terminal(
13196
+ manifest=manifest, manifest_sha256=manifest_sha256,
13197
+ output_root=output_root, arm=arm, variant=variants[arm], task=task,
13198
+ workspace_event=arm_rows[1],
13199
+ )
13200
+ if recomputed != dict(arm_rows[-1]) or recomputed["passed"] is not True:
13201
+ raise ValueError("v2 canary terminal evidence did not pass")
13202
+ spec = variants[arm].measurement
13203
+ assert spec is not None
13204
+ records.append({
13205
+ "arm": arm, "run_id": recomputed["run_id"], "passed": True,
13206
+ "settings_sha256": manifest["inputs"]["settings"][arm]["sha256"],
13207
+ "settings_binding_set_sha256": _measurement_binding_set_sha256(
13208
+ spec.registered_bindings
13209
+ ),
13210
+ "required_event_classes": ["PreToolUse"],
13211
+ "pretooluse_event_count": recomputed["pretooluse_event_count"],
13212
+ "receipt_sha256": recomputed["receipt_sha256"],
13213
+ "pre_overlay_inventory_sha256": recomputed[
13214
+ "pre_overlay_inventory_sha256"
13215
+ ],
13216
+ "post_overlay_inventory_sha256": recomputed[
13217
+ "post_overlay_inventory_sha256"
13218
+ ],
13219
+ })
13220
+ return {
13221
+ "schema_version": BENCHMARK_STUDY_V2_CANARY_EVIDENCE_SCHEMA_VERSION,
13222
+ "manifest_sha256": manifest_sha256,
13223
+ "canary_contract_sha256": _study_domain_hash(
13224
+ "contextguard.bench.v2.canary-contract.v1",
13225
+ manifest["inputs"]["canary_contract"],
13226
+ ),
13227
+ "cli_binding_sha256": _study_domain_hash(
13228
+ "contextguard.bench.v2.cli-binding.v1",
13229
+ manifest["inputs"]["cli_binding"],
13230
+ ),
13231
+ "auth_context_sha256": _study_domain_hash(
13232
+ "contextguard.bench.v2.auth-context.v1",
13233
+ manifest["inputs"]["auth_context"],
13234
+ ),
13235
+ "candidate_manifest_sha256": manifest["inputs"]["candidate"][
13236
+ "manifest_sha256"
13237
+ ],
13238
+ "candidate_overlay_sha256": manifest["inputs"][
13239
+ "candidate_overlay_inventory"
13240
+ ]["sha256"],
13241
+ "discarded": True, "excluded_from_analysis": True,
13242
+ "provider_calls": 2, "arms": records,
13243
+ }
13244
+
13245
+
13246
+ def _benchmark_study_v2_verify_canary_evidence(
13247
+ *, manifest: Mapping[str, Any], manifest_sha256: str, output_root: Path,
13248
+ ) -> tuple[dict[str, Any], str]:
13249
+ variants = _benchmark_study_v2_canary_variants(manifest, output_root)
13250
+ rows = _benchmark_study_v2_read_canary_events(
13251
+ output_root / "canary-events.jsonl",
13252
+ manifest_sha256=manifest_sha256, variants=variants,
13253
+ )
13254
+ expected = _benchmark_study_v2_expected_canary_evidence(
13255
+ manifest=manifest, manifest_sha256=manifest_sha256,
13256
+ output_root=output_root, rows=rows, variants=variants,
13257
+ )
13258
+ observed, raw = _benchmark_study_v2_read_canonical(
13259
+ output_root / "canary-evidence.json",
13260
+ owner="v2 canary evidence", maximum=200_000,
13261
+ )
13262
+ if observed != expected:
13263
+ raise ValueError("v2 canary evidence binding mismatch")
13264
+ return observed, _study_sha256_bytes(raw)
13265
+
13266
+
13267
+ def _benchmark_study_v2_run_canary_arm(
13268
+ *, manifest: Mapping[str, Any], manifest_sha256: str, output_root: Path,
13269
+ arm: str, variant: Variant, task: TaskFixture, claude_bin: str,
13270
+ cli_stat_guard: Mapping[str, Any],
13271
+ runtime_stat_guards: Mapping[str, Mapping[str, int | str]],
13272
+ auth_home: Path,
13273
+ ) -> dict[str, Any]:
13274
+ spec = variant.measurement
13275
+ assert spec is not None
13276
+ run_id = spec.identity.run_id(task.id)
13277
+ ledger_path = output_root / "canary-events.jsonl"
13278
+ _benchmark_study_v2_assert_cli_executable_bytes(
13279
+ manifest["inputs"]["cli_binding"], cli_stat_guard,
13280
+ )
13281
+ _benchmark_study_v2_assert_runtime_stat_guards(
13282
+ manifest["inputs"]["execution_environment"], runtime_stat_guards,
13283
+ )
13284
+ resolved_auth_home = _benchmark_study_v2_assert_auth_context(
13285
+ claude_bin, manifest["inputs"]["execution_environment"],
13286
+ manifest["inputs"]["auth_context"], auth_home,
13287
+ )
13288
+ append_study_attempt_event(
13289
+ ledger_path,
13290
+ _benchmark_study_v2_canary_base_event(
13291
+ arm=arm, run_id=run_id, manifest_sha256=manifest_sha256,
13292
+ state="launch_reserved",
13293
+ ),
13294
+ )
13295
+ pre: dict[str, Any] = {}
13296
+
13297
+ def prepared(workspace: Path) -> None:
13298
+ overlay = workspace / BENCHMARK_STUDY_V2_OVERLAY_NAME
13299
+ overlay_inventory = _benchmark_study_v2_verify_physical_copy(
13300
+ Path(manifest["inputs"]["candidate_install_root"]), overlay,
13301
+ expected=manifest["inputs"]["candidate_overlay_inventory"],
13302
+ )
13303
+ workspace_inventory = _benchmark_study_v2_inventory(workspace)
13304
+ pre.update({"overlay": overlay_inventory, "workspace": workspace_inventory})
13305
+ append_study_attempt_event(
13306
+ ledger_path,
13307
+ _benchmark_study_v2_canary_base_event(
13308
+ arm=arm, run_id=run_id, manifest_sha256=manifest_sha256,
13309
+ state="workspace_prepared",
13310
+ pre_workspace_inventory_sha256=workspace_inventory["sha256"],
13311
+ pre_overlay_inventory_sha256=overlay_inventory["sha256"],
13312
+ ),
13313
+ )
13314
+
13315
+ def launched() -> None:
13316
+ append_study_attempt_event(
13317
+ ledger_path,
13318
+ _benchmark_study_v2_canary_base_event(
13319
+ arm=arm, run_id=run_id, manifest_sha256=manifest_sha256,
13320
+ state="launched",
13321
+ ),
13322
+ )
13323
+
13324
+ root_fd = _ensure_directory_no_symlink(spec.artifact_root, create=True)
13325
+ try:
13326
+ os.fchmod(root_fd, 0o700)
13327
+ if fcntl is not None:
13328
+ fcntl.flock(root_fd, fcntl.LOCK_EX)
13329
+ _run_measurement_fixture_locked(
13330
+ task, variant, claude_bin, Path(manifest_sha256),
13331
+ locked_root_fd=root_fd, on_process_started=launched,
13332
+ measurement_study=True,
13333
+ workspace_overlay=Path(manifest["inputs"]["candidate_install_root"]),
13334
+ on_workspace_prepared=prepared,
13335
+ checker_interpreter_binding=manifest["inputs"]["runner_binding"][
13336
+ "python"
13337
+ ],
13338
+ existing_login_home=resolved_auth_home,
13339
+ )
13340
+ finally:
13341
+ os.close(root_fd)
13342
+ if set(pre) != {"overlay", "workspace"}:
13343
+ raise ValueError("v2 canary workspace was not prepared")
13344
+ workspace_event = _benchmark_study_v2_canary_base_event(
13345
+ arm=arm, run_id=run_id, manifest_sha256=manifest_sha256,
13346
+ state="workspace_prepared",
13347
+ pre_workspace_inventory_sha256=pre["workspace"]["sha256"],
13348
+ pre_overlay_inventory_sha256=pre["overlay"]["sha256"],
13349
+ )
13350
+ terminal = _benchmark_study_v2_derive_canary_terminal(
13351
+ manifest=manifest, manifest_sha256=manifest_sha256,
13352
+ output_root=output_root, arm=arm, variant=variant, task=task,
13353
+ workspace_event=workspace_event,
13354
+ )
13355
+ append_study_attempt_event(ledger_path, terminal)
13356
+ if terminal["passed"] is not True:
13357
+ raise ValueError(f"v2 {arm} host PreToolUse canary failed")
13358
+ return terminal
13359
+
13360
+
13361
+ def execute_benchmark_study_v2_canary(
13362
+ *, output_root: Path, claude_bin: str, auth_home: Path,
13363
+ ) -> dict[str, Any]:
13364
+ with _benchmark_study_v2_action_lock(output_root):
13365
+ return _execute_benchmark_study_v2_canary_unlocked(
13366
+ output_root=output_root, claude_bin=claude_bin,
13367
+ auth_home=auth_home,
13368
+ )
13369
+
13370
+
13371
+ def _execute_benchmark_study_v2_canary_unlocked(
13372
+ *, output_root: Path, claude_bin: str, auth_home: Path,
13373
+ ) -> dict[str, Any]:
13374
+ output_root = _benchmark_study_v2_output_root(output_root)
13375
+ manifest, manifest_sha256 = load_benchmark_study_v2_executable_manifest(
13376
+ output_root, revalidate_external=True,
13377
+ )
13378
+ attempts_path = output_root / "attempts.jsonl"
13379
+ if attempts_path.exists() and attempts_path.stat().st_size:
13380
+ raise ValueError("v2 canary must finish before analytic attempts")
13381
+ cli_stat_guard = _benchmark_study_v2_assert_cli_binding(
13382
+ claude_bin, manifest["inputs"]["cli_binding"],
13383
+ )
13384
+ runtime_stat_guards = _benchmark_study_v2_assert_execution_environment(
13385
+ manifest["inputs"]["execution_environment"]
13386
+ )
13387
+ bound_claude_bin = str(cli_stat_guard["executable"])
13388
+ resolved_auth_home = _benchmark_study_v2_assert_auth_context(
13389
+ bound_claude_bin, manifest["inputs"]["execution_environment"],
13390
+ manifest["inputs"]["auth_context"], auth_home,
13391
+ )
13392
+ variants = _benchmark_study_v2_canary_variants(manifest, output_root)
13393
+ task = _benchmark_study_v2_canary_task()
13394
+ ledger_path = output_root / "canary-events.jsonl"
13395
+ launched_now = 0
13396
+ for arm in BENCHMARK_STUDY_V2_CANARY_ARMS:
13397
+ rows = _benchmark_study_v2_read_canary_events(
13398
+ ledger_path, manifest_sha256=manifest_sha256, variants=variants,
13399
+ )
13400
+ arm_rows = [row for row in rows if row["arm"] == arm]
13401
+ if arm_rows and arm_rows[-1]["state"] == "terminal":
13402
+ if arm_rows[-1]["passed"] is not True:
13403
+ raise ValueError(f"v2 {arm} canary terminal did not pass")
13404
+ continue
13405
+ if arm_rows:
13406
+ if arm_rows[-1]["state"] != "launched":
13407
+ raise ValueError("v2 canary reserved identity cannot be replayed")
13408
+ try:
13409
+ recovered = _benchmark_study_v2_derive_canary_terminal(
13410
+ manifest=manifest, manifest_sha256=manifest_sha256,
13411
+ output_root=output_root, arm=arm, variant=variants[arm],
13412
+ task=task, workspace_event=arm_rows[1],
13413
+ )
13414
+ except (OSError, SystemExit, TypeError, ValueError) as exc:
13415
+ raise ValueError("v2 launched canary cannot be safely recovered") from exc
13416
+ append_study_attempt_event(ledger_path, recovered)
13417
+ if recovered["passed"] is not True:
13418
+ raise ValueError(f"v2 {arm} recovered canary did not pass")
13419
+ continue
13420
+ _benchmark_study_v2_run_canary_arm(
13421
+ manifest=manifest, manifest_sha256=manifest_sha256,
13422
+ output_root=output_root, arm=arm, variant=variants[arm], task=task,
13423
+ claude_bin=bound_claude_bin, cli_stat_guard=cli_stat_guard,
13424
+ runtime_stat_guards=runtime_stat_guards,
13425
+ auth_home=resolved_auth_home,
13426
+ )
13427
+ launched_now += 1
13428
+ rows = _benchmark_study_v2_read_canary_events(
13429
+ ledger_path, manifest_sha256=manifest_sha256, variants=variants,
13430
+ )
13431
+ expected = _benchmark_study_v2_expected_canary_evidence(
13432
+ manifest=manifest, manifest_sha256=manifest_sha256,
13433
+ output_root=output_root, rows=rows, variants=variants,
13434
+ )
13435
+ evidence_path = output_root / "canary-evidence.json"
13436
+ if evidence_path.exists():
13437
+ observed, _raw = _benchmark_study_v2_read_canonical(
13438
+ evidence_path, owner="v2 canary evidence", maximum=200_000,
13439
+ )
13440
+ if observed != expected:
13441
+ raise ValueError("v2 canary evidence binding mismatch")
13442
+ else:
13443
+ _study_write_private(evidence_path, expected)
13444
+ _observed, evidence_sha256 = _benchmark_study_v2_verify_canary_evidence(
13445
+ manifest=manifest, manifest_sha256=manifest_sha256,
13446
+ output_root=output_root,
13447
+ )
13448
+ return {
13449
+ "provider_process_calls": launched_now,
13450
+ "discarded_provider_calls": 2,
13451
+ "canary_evidence_sha256": evidence_sha256,
13452
+ }
13453
+
13454
+
13455
+ def _benchmark_study_v2_bounded_failure_usage(
13456
+ *, receipt: Mapping[str, Any], raw: bytes, arm: str,
13457
+ allowed_event_classes: Sequence[str],
13458
+ required_event_classes: Sequence[str],
13459
+ ) -> dict[str, int] | None:
13460
+ """Return usage only for an exact policy-bounded provider task failure."""
13461
+ if (
13462
+ receipt.get("process_status") != "exited_nonzero"
13463
+ or receipt.get("terminal_status") != "process_error"
13464
+ ):
13465
+ return None
13466
+ parsed = parse_claude_stream_output(
13467
+ raw, max_line_bytes=MEASUREMENT_RAW_MAX_LINE_BYTES,
13468
+ )
13469
+ if (
13470
+ parsed.status != "terminal_error"
13471
+ or parsed.result_code not in BENCHMARK_STUDY_V2_BOUNDED_FAILURE_RESULT_CODES
13472
+ ):
13473
+ return None
13474
+ hooks = _parse_measurement_hook_events(raw)
13475
+ completed_classes = {item["hook_event"] for item in hooks["hooks"]}
13476
+ allowed_classes = set(allowed_event_classes)
13477
+ required_classes = set(required_event_classes)
13478
+ hook_arms = {"legacy_trim", "bash_reference_v1"}
13479
+ if (
13480
+ hooks["classification"] is not None
13481
+ or hooks["failure_flags"]
13482
+ or (arm in hook_arms and completed_classes - allowed_classes)
13483
+ or (arm == "host_unmodified" and hooks["observed"])
13484
+ or required_classes - completed_classes
13485
+ or any(
13486
+ item["hook_process_outcome"] != "success"
13487
+ or item["hook_process_exit_code"] not in (None, 0)
13488
+ for item in hooks["hooks"]
13489
+ )
13490
+ ):
13491
+ return None
13492
+ try:
13493
+ return parse_measurement_terminal_usage(raw)
13494
+ except ValueError:
13495
+ return None
13496
+
13497
+
13498
+ def _benchmark_study_v2_revalidate_terminal_evidence(
13499
+ *, manifest: Mapping[str, Any], output_root: Path,
13500
+ rows: Sequence[Mapping[str, Any]], tasks_by_id: Mapping[str, TaskFixture],
13501
+ variants: Mapping[str, Variant],
13502
+ ) -> None:
13503
+ slots = {slot["run_id"]: slot for slot in manifest["slots"]}
13504
+ install_root = Path(manifest["inputs"]["candidate_install_root"])
13505
+ expected_overlay = manifest["inputs"]["candidate_overlay_inventory"]
13506
+ expected_pre_by_task: dict[str, dict[str, Any]] = {}
13507
+ for row in rows:
13508
+ if row["state"] != "terminal" or row["terminal_status"] == "recovered_process_status_unknown":
13509
+ continue
13510
+ slot = slots[row["run_id"]]
13511
+ task_id = str(slot["task_id"])
13512
+ if task_id not in expected_pre_by_task:
13513
+ expected_pre_by_task[task_id] = _benchmark_study_v2_expected_pre_workspace(
13514
+ tasks_by_id[task_id], install_root=install_root,
13515
+ expected_overlay_inventory=expected_overlay,
13516
+ )
13517
+ if (
13518
+ row["pre_workspace_inventory_sha256"]
13519
+ != expected_pre_by_task[task_id]["sha256"]
13520
+ ):
13521
+ raise ValueError("v2 pre-launch workspace inventory drift")
13522
+ study_variant = _study_variant_for_slot(variants[slot["arm"]], slot)
13523
+ spec = study_variant.measurement
13524
+ assert spec is not None
13525
+ receipt = _verify_existing_measurement_run(
13526
+ spec, task_id, str(slot["run_id"]), require_index=True,
13527
+ )
13528
+ context = _measurement_existing_context(spec, str(slot["run_id"]))
13529
+ receipt_raw = _measurement_read_private_file(context.receipt_path)
13530
+ raw = _measurement_read_private_raw(context.raw_path)
13531
+ if _study_sha256_bytes(receipt_raw) != row["receipt_sha256"]:
13532
+ raise ValueError("v2 attempt receipt hash differs from immutable receipt")
13533
+ if receipt["terminal_status"] != row["provider_terminal_status"]:
13534
+ raise ValueError("v2 attempt provider terminal differs from immutable receipt")
13535
+ bounded_failure_usage = _benchmark_study_v2_bounded_failure_usage(
13536
+ receipt=receipt, raw=raw, arm=str(slot["arm"]),
13537
+ allowed_event_classes=tuple(
13538
+ dict.fromkeys(
13539
+ event for event, _command in spec.pair_registered_bindings
13540
+ )
13541
+ ),
13542
+ required_event_classes=spec.required_event_classes,
13543
+ )
13544
+ if receipt["terminal_status"] == "success":
13545
+ usage = parse_measurement_terminal_usage(raw)
13546
+ expected_buckets = {
13547
+ key: usage[key] for key in MEASUREMENT_STUDY_USAGE_KEYS
13548
+ }
13549
+ checker = run_task_checker_study(
13550
+ tasks_by_id[task_id], context.workspace, env={},
13551
+ interpreter_binding=manifest["inputs"]["runner_binding"][
13552
+ "python"
13553
+ ],
13554
+ )
13555
+ elif bounded_failure_usage is not None:
13556
+ expected_buckets = {
13557
+ key: bounded_failure_usage[key]
13558
+ for key in MEASUREMENT_STUDY_USAGE_KEYS
13559
+ }
13560
+ checker = BENCHMARK_STUDY_V2_BOUNDED_FAILURE_CHECKER_STATUS
13561
+ else:
13562
+ expected_buckets = {key: 0 for key in MEASUREMENT_STUDY_USAGE_KEYS}
13563
+ checker = "not_run"
13564
+ if row["token_buckets"] != expected_buckets or row["primary_tokens"] != sum(expected_buckets.values()):
13565
+ raise ValueError("v2 attempt tokens differ from provider terminal usage")
13566
+ if row["checker_status"] != checker:
13567
+ raise ValueError("v2 attempt checker status differs from the bound checker")
13568
+ derived = (
13569
+ "success" if receipt["terminal_status"] == "success" and checker == "task_success"
13570
+ else "valid_task_failure_v1"
13571
+ if receipt["terminal_status"] == "success" and checker == "valid_task_failure_v1"
13572
+ else "valid_task_failure_v1"
13573
+ if bounded_failure_usage is not None
13574
+ else "study_infra_invalid"
13575
+ )
13576
+ if (
13577
+ row["post_overlay_inventory_sha256"]
13578
+ != row["pre_overlay_inventory_sha256"]
13579
+ ):
13580
+ derived = "study_infra_invalid"
13581
+ if row["terminal_status"] != derived or row["success"] is not (derived == "success"):
13582
+ raise ValueError("v2 attempt outcome was not derived from provider plus checker")
13583
+ workspace_inventory = _benchmark_study_v2_inventory(context.workspace)
13584
+ overlay_inventory = _benchmark_study_v2_inventory(
13585
+ context.workspace / BENCHMARK_STUDY_V2_OVERLAY_NAME,
13586
+ )
13587
+ overlay_drifted = (
13588
+ overlay_inventory != manifest["inputs"]["candidate_overlay_inventory"]
13589
+ )
13590
+ if (
13591
+ workspace_inventory["sha256"] != row["post_workspace_inventory_sha256"]
13592
+ or overlay_inventory["sha256"] != row["post_overlay_inventory_sha256"]
13593
+ or (
13594
+ overlay_drifted
13595
+ and row["terminal_status"] != "study_infra_invalid"
13596
+ )
13597
+ ):
13598
+ raise ValueError("v2 terminal workspace or overlay inventory drift")
13599
+
13600
+
13601
+ def _benchmark_study_v2_read_attempts(path: Path, *, manifest: Mapping[str, Any], manifest_sha256: str) -> list[dict[str, Any]]:
13602
+ if not path.exists():
13603
+ return []
13604
+ raw = _measurement_read_private_file(path, maximum=4_000_000)
13605
+ slots = {slot["run_id"]: slot for slot in manifest["slots"]}
13606
+ rows: list[dict[str, Any]] = []
13607
+ states: dict[str, list[str]] = collections.defaultdict(list)
13608
+ base_keys = {
13609
+ "schema_version", "manifest_sha256", "run_id", "task_id",
13610
+ "repetition", "arm", "attempt", "state",
13611
+ }
13612
+ terminal_keys = base_keys | {
13613
+ "terminal_status", "provider_terminal_status", "checker_status",
13614
+ "success", "token_buckets", "primary_tokens", "correction",
13615
+ "retrieval", "shifted_cost", "pre_workspace_inventory_sha256",
13616
+ "post_workspace_inventory_sha256", "pre_overlay_inventory_sha256",
13617
+ "post_overlay_inventory_sha256", "receipt_sha256",
13618
+ }
13619
+ blocked_keys = base_keys | {"reason"}
13620
+ for line in raw.splitlines():
13621
+ try:
13622
+ row = json.loads(line.decode("utf-8"), object_pairs_hook=_measurement_object_no_duplicates)
13623
+ except (UnicodeDecodeError, json.JSONDecodeError, ValueError) as exc:
13624
+ raise ValueError("v2 attempt index is invalid JSONL") from exc
13625
+ if not isinstance(row, dict) or line + b"\n" != _study_canonical_json_bytes(row):
13626
+ raise ValueError("v2 attempt index must be canonical JSONL")
13627
+ slot = slots.get(row.get("run_id"))
13628
+ if row.get("schema_version") != BENCHMARK_STUDY_V2_ATTEMPT_SCHEMA_VERSION or row.get("manifest_sha256") != manifest_sha256 or slot is None:
13629
+ raise ValueError("v2 attempt index binding mismatch")
13630
+ if any(row.get(key) != slot.get(key) for key in ("task_id", "repetition", "arm", "attempt")):
13631
+ raise ValueError("v2 attempt identity mismatch")
13632
+ state = row.get("state")
13633
+ if state not in {
13634
+ "launch_reserved", "launched", "terminal", "not_needed",
13635
+ "blocked_study_invalid",
13636
+ }:
13637
+ raise ValueError("v2 attempt state is invalid")
13638
+ expected_keys = (
13639
+ terminal_keys if state == "terminal"
13640
+ else blocked_keys if state == "blocked_study_invalid"
13641
+ else base_keys
13642
+ )
13643
+ if set(row) != expected_keys:
13644
+ raise ValueError("v2 attempt state has an inexact key schema")
13645
+ previous = states[row["run_id"]]
13646
+ if (
13647
+ (state == "launch_reserved" and previous)
13648
+ or (state == "launched" and previous != ["launch_reserved"])
13649
+ or (state == "terminal" and previous not in (["launch_reserved"], ["launch_reserved", "launched"]))
13650
+ or (state in {"not_needed", "blocked_study_invalid"} and previous)
13651
+ ):
13652
+ raise ValueError("v2 attempt state transition is invalid")
13653
+ if state in {"not_needed", "blocked_study_invalid"} and row["attempt"] != 1:
13654
+ raise ValueError("v2 non-launched final state is only valid for a retry")
13655
+ if state == "blocked_study_invalid" and row["reason"] != "initial_study_invalid":
13656
+ raise ValueError("v2 blocked retry reason is invalid")
13657
+ previous.append(state)
13658
+ if state == "terminal":
13659
+ status = row["terminal_status"]
13660
+ if status not in {
13661
+ "success", "valid_task_failure_v1", "study_infra_invalid",
13662
+ } or not isinstance(row["success"], bool) or row["success"] is not (status == "success"):
13663
+ raise ValueError("v2 attempt success/classification binding is invalid")
13664
+ buckets = row["token_buckets"]
13665
+ if not isinstance(buckets, dict) or set(buckets) != set(MEASUREMENT_STUDY_USAGE_KEYS):
13666
+ raise ValueError("v2 attempt token bucket schema is invalid")
13667
+ total = 0
13668
+ for key in MEASUREMENT_STUDY_USAGE_KEYS:
13669
+ value = buckets[key]
13670
+ if isinstance(value, bool) or not isinstance(value, int) or not 0 <= value <= MAX_USAGE_TOKEN_COUNT:
13671
+ raise ValueError("v2 attempt token bucket is invalid")
13672
+ total += value
13673
+ if total > MAX_USAGE_TOKEN_COUNT or row["primary_tokens"] != total:
13674
+ raise ValueError("v2 attempt primary token sum is invalid")
13675
+ if any(row[field] is not None for field in ("correction", "retrieval", "shifted_cost")):
13676
+ raise ValueError("v2 absent observer must remain null")
13677
+ hash_fields = (
13678
+ "pre_workspace_inventory_sha256", "post_workspace_inventory_sha256",
13679
+ "pre_overlay_inventory_sha256", "post_overlay_inventory_sha256",
13680
+ "receipt_sha256",
13681
+ )
13682
+ if any(
13683
+ not isinstance(row[field], str)
13684
+ or SHA256_HEX_PATTERN.fullmatch(row[field]) is None
13685
+ for field in hash_fields
13686
+ ):
13687
+ raise ValueError("v2 terminal evidence hash is invalid")
13688
+ expected_overlay_sha256 = manifest["inputs"][
13689
+ "candidate_overlay_inventory"
13690
+ ]["sha256"]
13691
+ if (
13692
+ row["pre_overlay_inventory_sha256"] != expected_overlay_sha256
13693
+ or (
13694
+ row["post_overlay_inventory_sha256"]
13695
+ != row["pre_overlay_inventory_sha256"]
13696
+ and status != "study_infra_invalid"
13697
+ )
13698
+ ):
13699
+ raise ValueError("v2 terminal overlay binding is invalid")
13700
+ if status == "success" and not (
13701
+ row["provider_terminal_status"] == "success"
13702
+ and row["checker_status"] == "task_success"
13703
+ ):
13704
+ raise ValueError("v2 successful outcome lacks provider/checker evidence")
13705
+ if status == "valid_task_failure_v1" and not (
13706
+ (
13707
+ row["provider_terminal_status"] == "success"
13708
+ and row["checker_status"] == "valid_task_failure_v1"
13709
+ )
13710
+ or (
13711
+ row["provider_terminal_status"] == "process_error"
13712
+ and row["checker_status"]
13713
+ == BENCHMARK_STUDY_V2_BOUNDED_FAILURE_CHECKER_STATUS
13714
+ )
13715
+ ):
13716
+ raise ValueError("v2 valid failure lacks provider/checker evidence")
13717
+ rows.append(row)
13718
+ terminal_by_unit = {
13719
+ (row["task_id"], row["repetition"], row["arm"]): row
13720
+ for row in rows
13721
+ if row["attempt"] == 0 and row["state"] == "terminal"
13722
+ }
13723
+ for row in rows:
13724
+ if row["attempt"] != 1:
13725
+ continue
13726
+ initial = terminal_by_unit.get((row["task_id"], row["repetition"], row["arm"]))
13727
+ if row["state"] == "not_needed" and (
13728
+ initial is None or initial["terminal_status"] != "success"
13729
+ ):
13730
+ raise ValueError("v2 not-needed retry lacks a successful initial")
13731
+ if row["state"] == "blocked_study_invalid" and (
13732
+ initial is None
13733
+ or initial["terminal_status"] != "study_infra_invalid"
13734
+ ):
13735
+ raise ValueError("v2 blocked retry lacks an invalid initial")
13736
+ if row["state"] in {"launch_reserved", "launched", "terminal"} and (
13737
+ initial is None or initial["terminal_status"] != "valid_task_failure_v1"
13738
+ ):
13739
+ raise ValueError("v2 launched retry lacks a valid failed initial")
13740
+ return rows
13741
+
13742
+
13743
+ def _benchmark_study_v2_event(slot: Mapping[str, Any], manifest_sha256: str, state: str, **extra: Any) -> dict[str, Any]:
13744
+ event = {
13745
+ "schema_version": BENCHMARK_STUDY_V2_ATTEMPT_SCHEMA_VERSION,
13746
+ "manifest_sha256": manifest_sha256, "run_id": slot["run_id"],
13747
+ "task_id": slot["task_id"], "repetition": slot["repetition"],
13748
+ "arm": slot["arm"], "attempt": slot["attempt"], "state": state,
13749
+ }
13750
+ event.update(extra)
13751
+ return event
13752
+
13753
+
13754
+ def _benchmark_study_v2_run_slot(
13755
+ *, slot: Mapping[str, Any], task: TaskFixture, variant: Variant,
13756
+ claude_bin: str, attempts_path: Path, manifest_sha256: str,
13757
+ install_root: Path, expected_overlay_inventory: Mapping[str, Any],
13758
+ cli_binding: Mapping[str, Any],
13759
+ cli_stat_guard: Mapping[str, Any],
13760
+ execution_environment: Mapping[str, Any],
13761
+ runtime_stat_guards: Mapping[str, Mapping[str, int | str]],
13762
+ checker_interpreter_binding: Mapping[str, Any],
13763
+ auth_context: Mapping[str, Any],
13764
+ auth_home: Path,
13765
+ ) -> str:
13766
+ study_variant = _study_variant_for_slot(variant, slot)
13767
+ spec = study_variant.measurement
13768
+ assert spec is not None
13769
+ pre: dict[str, Any] = {}
13770
+
13771
+ def prepared(workspace: Path) -> None:
13772
+ overlay = workspace / BENCHMARK_STUDY_V2_OVERLAY_NAME
13773
+ pre["overlay"] = _benchmark_study_v2_verify_physical_copy(
13774
+ install_root, overlay, expected=expected_overlay_inventory,
13775
+ )
13776
+ pre["workspace"] = _benchmark_study_v2_inventory(workspace)
13777
+
13778
+ def launched() -> None:
13779
+ append_study_attempt_event(
13780
+ attempts_path,
13781
+ _benchmark_study_v2_event(slot, manifest_sha256, "launched"),
13782
+ )
13783
+
13784
+ # Durable reservation precedes Popen. Resume treats even a reservation-only
13785
+ # identity as consumed/unknown, so a provider that started while launched
13786
+ # accounting failed can never be invoked twice.
13787
+ _benchmark_study_v2_assert_cli_executable_bytes(cli_binding, cli_stat_guard)
13788
+ _benchmark_study_v2_assert_runtime_stat_guards(
13789
+ execution_environment, runtime_stat_guards,
13790
+ )
13791
+ resolved_auth_home = _benchmark_study_v2_assert_auth_context(
13792
+ claude_bin, execution_environment, auth_context, auth_home,
13793
+ )
13794
+ append_study_attempt_event(
13795
+ attempts_path,
13796
+ _benchmark_study_v2_event(slot, manifest_sha256, "launch_reserved"),
13797
+ )
13798
+ root_fd = _ensure_directory_no_symlink(spec.artifact_root, create=True)
13799
+ try:
13800
+ os.fchmod(root_fd, 0o700)
13801
+ if fcntl is not None:
13802
+ fcntl.flock(root_fd, fcntl.LOCK_EX)
13803
+ result = _run_measurement_fixture_locked(
13804
+ task, study_variant, claude_bin, Path(manifest_sha256),
13805
+ locked_root_fd=root_fd, on_process_started=launched,
13806
+ measurement_study=True, workspace_overlay=install_root,
13807
+ on_workspace_prepared=prepared,
13808
+ checker_interpreter_binding=checker_interpreter_binding,
13809
+ existing_login_home=resolved_auth_home,
13810
+ )
13811
+ finally:
13812
+ os.close(root_fd)
13813
+ context = _measurement_existing_context(spec, str(slot["run_id"]))
13814
+ post_workspace = _benchmark_study_v2_inventory(context.workspace)
13815
+ post_overlay = _benchmark_study_v2_inventory(
13816
+ context.workspace / BENCHMARK_STUDY_V2_OVERLAY_NAME,
13817
+ )
13818
+ if _benchmark_study_v2_inventory(install_root) != dict(expected_overlay_inventory):
13819
+ raise ValueError("v2 staged candidate changed during an attempt")
13820
+ receipt_raw = _measurement_read_private_file(context.receipt_path)
13821
+ receipt = _measurement_parse_canonical_json_bytes(receipt_raw, owner="v2 measurement receipt")
13822
+ provider_terminal = str(receipt["terminal_status"])
13823
+ raw = _measurement_read_private_raw(context.raw_path)
13824
+ bounded_failure_usage = _benchmark_study_v2_bounded_failure_usage(
13825
+ receipt=receipt, raw=raw, arm=str(slot["arm"]),
13826
+ allowed_event_classes=tuple(
13827
+ dict.fromkeys(
13828
+ event for event, _command in spec.pair_registered_bindings
13829
+ )
13830
+ ),
13831
+ required_event_classes=spec.required_event_classes,
13832
+ )
13833
+ checker_status = (
13834
+ result.notes if provider_terminal == "success"
13835
+ else BENCHMARK_STUDY_V2_BOUNDED_FAILURE_CHECKER_STATUS
13836
+ if bounded_failure_usage is not None
13837
+ else "not_run"
13838
+ )
13839
+ if provider_terminal == "success" and checker_status == "task_success":
13840
+ classification = "success"
13841
+ elif provider_terminal == "success" and checker_status == "valid_task_failure_v1":
13842
+ classification = "valid_task_failure_v1"
13843
+ elif bounded_failure_usage is not None:
13844
+ classification = "valid_task_failure_v1"
13845
+ else:
13846
+ classification = "study_infra_invalid"
13847
+ if post_overlay != pre.get("overlay"):
13848
+ classification = "study_infra_invalid"
13849
+ token_buckets = (
13850
+ {
13851
+ key: bounded_failure_usage[key]
13852
+ for key in MEASUREMENT_STUDY_USAGE_KEYS
13853
+ }
13854
+ if bounded_failure_usage is not None
13855
+ else {
13856
+ "input_tokens": result.tokens["input_tokens"],
13857
+ "cache_creation_input_tokens": result.tokens["cache_creation"],
13858
+ "cache_read_input_tokens": result.tokens["cache_read"],
13859
+ "output_tokens": result.tokens["output_tokens"],
13860
+ }
13861
+ )
13862
+ append_study_attempt_event(
13863
+ attempts_path,
13864
+ _benchmark_study_v2_event(
13865
+ slot, manifest_sha256, "terminal",
13866
+ terminal_status=classification,
13867
+ provider_terminal_status=provider_terminal,
13868
+ checker_status=checker_status,
13869
+ success=classification == "success",
13870
+ token_buckets=token_buckets,
13871
+ primary_tokens=sum(token_buckets.values()),
13872
+ correction=None, retrieval=None, shifted_cost=None,
13873
+ pre_workspace_inventory_sha256=pre["workspace"]["sha256"],
13874
+ post_workspace_inventory_sha256=post_workspace["sha256"],
13875
+ pre_overlay_inventory_sha256=pre["overlay"]["sha256"],
13876
+ post_overlay_inventory_sha256=post_overlay["sha256"],
13877
+ receipt_sha256=_study_sha256_bytes(receipt_raw),
13878
+ ),
13879
+ )
13880
+ return classification
13881
+
13882
+
13883
+ def execute_benchmark_study_v2(
13884
+ *, output_root: Path, claude_bin: str, resume: bool, auth_home: Path,
13885
+ ) -> dict[str, int]:
13886
+ with _benchmark_study_v2_action_lock(output_root):
13887
+ return _execute_benchmark_study_v2_unlocked(
13888
+ output_root=output_root, claude_bin=claude_bin, resume=resume,
13889
+ auth_home=auth_home,
13890
+ )
13891
+
13892
+
13893
+ def _execute_benchmark_study_v2_unlocked(
13894
+ *, output_root: Path, claude_bin: str, resume: bool, auth_home: Path,
13895
+ ) -> dict[str, int]:
13896
+ output_root = _benchmark_study_v2_output_root(output_root)
13897
+ # Revalidate every inert candidate byte and installed overlay before a provider launch.
13898
+ manifest, manifest_sha256 = load_benchmark_study_v2_executable_manifest(
13899
+ output_root, revalidate_external=True,
13900
+ )
13901
+ cli_stat_guard = _benchmark_study_v2_assert_cli_binding(
13902
+ claude_bin, manifest["inputs"]["cli_binding"],
13903
+ )
13904
+ runtime_stat_guards = _benchmark_study_v2_assert_execution_environment(
13905
+ manifest["inputs"]["execution_environment"]
13906
+ )
13907
+ bound_claude_bin = str(cli_stat_guard["executable"])
13908
+ resolved_auth_home = _benchmark_study_v2_assert_auth_context(
13909
+ bound_claude_bin, manifest["inputs"]["execution_environment"],
13910
+ manifest["inputs"]["auth_context"], auth_home,
13911
+ )
13912
+ attempts_path = output_root / "attempts.jsonl"
13913
+ if not resume and attempts_path.exists() and attempts_path.stat().st_size:
13914
+ raise ValueError("v2 run requires an absent or empty attempt index")
13915
+ if resume and not attempts_path.exists():
13916
+ raise ValueError("v2 resume requires an existing attempt index")
13917
+ _canary_evidence, _canary_evidence_sha256 = (
13918
+ _benchmark_study_v2_verify_canary_evidence(
13919
+ manifest=manifest, manifest_sha256=manifest_sha256,
13920
+ output_root=output_root,
13921
+ )
13922
+ )
13923
+ tasks = parse_tasks(Path(manifest["inputs"]["tasks_path"]))
13924
+ load_task_fixture_trees(tasks, task_file_dir=Path(manifest["inputs"]["tasks_path"]).parent)
13925
+ tasks_by_id = {task.id: task for task in tasks}
13926
+ variants = _benchmark_study_v2_variants(manifest, output_root)
13927
+ rows = _benchmark_study_v2_read_attempts(
13928
+ attempts_path, manifest=manifest, manifest_sha256=manifest_sha256,
13929
+ )
13930
+ _benchmark_study_v2_revalidate_terminal_evidence(
13931
+ manifest=manifest, output_root=output_root, rows=rows,
13932
+ tasks_by_id=tasks_by_id, variants=variants,
13933
+ )
13934
+ launched = {
13935
+ row["run_id"] for row in rows
13936
+ if row["state"] in {"launch_reserved", "launched", "terminal"}
13937
+ }
13938
+ accounted = {row["run_id"] for row in rows}
13939
+ terminal = {row["run_id"]: row for row in rows if row["state"] == "terminal"}
13940
+ ambiguous_run_ids = sorted(launched - set(terminal))
13941
+ if ambiguous_run_ids:
13942
+ raise ValueError(
13943
+ "v2 ambiguous provider process state permanently blocks this study root"
13944
+ )
13945
+ retry_by_unit = {
13946
+ (slot["task_id"], slot["repetition"], slot["arm"]): slot
13947
+ for slot in manifest["slots"] if slot["attempt"] == 1
13948
+ }
13949
+ install_root = Path(manifest["inputs"]["candidate_install_root"])
13950
+ calls_before = len(launched)
13951
+ for initial in (slot for slot in manifest["slots"] if slot["attempt"] == 0):
13952
+ if initial["run_id"] not in launched:
13953
+ _benchmark_study_v2_run_slot(
13954
+ slot=initial, task=tasks_by_id[initial["task_id"]],
13955
+ variant=variants[initial["arm"]], claude_bin=bound_claude_bin,
13956
+ attempts_path=attempts_path, manifest_sha256=manifest_sha256,
13957
+ install_root=install_root,
13958
+ expected_overlay_inventory=manifest["inputs"]["candidate_overlay_inventory"],
13959
+ cli_binding=manifest["inputs"]["cli_binding"],
13960
+ cli_stat_guard=cli_stat_guard,
13961
+ execution_environment=manifest["inputs"]["execution_environment"],
13962
+ runtime_stat_guards=runtime_stat_guards,
13963
+ checker_interpreter_binding=manifest["inputs"]["runner_binding"][
13964
+ "python"
13965
+ ],
13966
+ auth_context=manifest["inputs"]["auth_context"],
13967
+ auth_home=resolved_auth_home,
13968
+ )
13969
+ launched.add(initial["run_id"])
13970
+ terminal = {
13971
+ row["run_id"]: row for row in _benchmark_study_v2_read_attempts(
13972
+ attempts_path, manifest=manifest, manifest_sha256=manifest_sha256,
13973
+ ) if row["state"] == "terminal"
13974
+ }
13975
+ initial_row = terminal.get(initial["run_id"])
13976
+ if initial_row is None:
13977
+ continue
13978
+ retry = retry_by_unit[(initial["task_id"], initial["repetition"], initial["arm"])]
13979
+ if initial_row.get("terminal_status") != "valid_task_failure_v1":
13980
+ state = (
13981
+ "not_needed"
13982
+ if initial_row.get("terminal_status") == "success"
13983
+ else "blocked_study_invalid"
13984
+ )
13985
+ if retry["run_id"] not in accounted:
13986
+ extra = {"reason": "initial_study_invalid"} if state == "blocked_study_invalid" else {}
13987
+ append_study_attempt_event(
13988
+ attempts_path,
13989
+ _benchmark_study_v2_event(
13990
+ retry, manifest_sha256, state, **extra,
13991
+ ),
13992
+ )
13993
+ accounted.add(retry["run_id"])
13994
+ if state == "blocked_study_invalid":
13995
+ raise ValueError(
13996
+ "v2 terminal infrastructure-invalid evidence permanently "
13997
+ "blocks later provider launches"
13998
+ )
13999
+ continue
14000
+ if retry["run_id"] in accounted:
14001
+ continue
14002
+ # A failed retry is retained, but it never stops later scheduled blocks.
14003
+ _benchmark_study_v2_run_slot(
14004
+ slot=retry, task=tasks_by_id[retry["task_id"]],
14005
+ variant=variants[retry["arm"]], claude_bin=bound_claude_bin,
14006
+ attempts_path=attempts_path, manifest_sha256=manifest_sha256,
14007
+ install_root=install_root,
14008
+ expected_overlay_inventory=manifest["inputs"]["candidate_overlay_inventory"],
14009
+ cli_binding=manifest["inputs"]["cli_binding"],
14010
+ cli_stat_guard=cli_stat_guard,
14011
+ execution_environment=manifest["inputs"]["execution_environment"],
14012
+ runtime_stat_guards=runtime_stat_guards,
14013
+ checker_interpreter_binding=manifest["inputs"]["runner_binding"][
14014
+ "python"
14015
+ ],
14016
+ auth_context=manifest["inputs"]["auth_context"],
14017
+ auth_home=resolved_auth_home,
14018
+ )
14019
+ launched.add(retry["run_id"])
14020
+ accounted.add(retry["run_id"])
14021
+ terminal = {
14022
+ row["run_id"]: row for row in _benchmark_study_v2_read_attempts(
14023
+ attempts_path, manifest=manifest, manifest_sha256=manifest_sha256,
14024
+ ) if row["state"] == "terminal"
14025
+ }
14026
+ final_rows = _benchmark_study_v2_read_attempts(
14027
+ attempts_path, manifest=manifest, manifest_sha256=manifest_sha256,
14028
+ )
14029
+ final_states = {row["run_id"]: row["state"] for row in final_rows}
14030
+ return {
14031
+ "provider_process_calls": len(launched) - calls_before,
14032
+ "launched_identities": len(launched),
14033
+ "accounted_identities": len(final_states),
14034
+ }
14035
+
14036
+
14037
+ def analyze_benchmark_study_v2_executable(
14038
+ *, output_root: Path, claude_bin: str,
14039
+ ) -> dict[str, Any]:
14040
+ with _benchmark_study_v2_action_lock(output_root):
14041
+ return _analyze_benchmark_study_v2_executable_unlocked(
14042
+ output_root=output_root, claude_bin=claude_bin,
14043
+ )
14044
+
14045
+
14046
+ def _benchmark_study_v2_persist_analysis(
14047
+ *, output_root: Path, report: Mapping[str, Any],
14048
+ ) -> str:
14049
+ report_name = (
14050
+ "study-invalid-decision.json"
14051
+ if report.get("schema_version")
14052
+ == BENCHMARK_STUDY_V2_INVALID_DECISION_SCHEMA_VERSION
14053
+ else "study-report.json"
14054
+ )
14055
+ path = output_root / report_name
14056
+ expected = _study_canonical_json_bytes(report)
14057
+ if path.exists():
14058
+ observed = _measurement_read_private_file(path, maximum=4_000_000)
14059
+ if observed != expected:
14060
+ raise ValueError("v2 persisted analysis binding mismatch")
14061
+ else:
14062
+ _measurement_write_exclusive(path, expected)
14063
+ return report_name
14064
+
14065
+
14066
+ def _benchmark_study_v2_persist_invalid_after_refusal(
14067
+ *, output_root: Path, claude_bin: str,
14068
+ ) -> None:
14069
+ try:
14070
+ report = analyze_benchmark_study_v2_executable(
14071
+ output_root=output_root, claude_bin=claude_bin,
14072
+ )
14073
+ if (
14074
+ report.get("schema_version")
14075
+ == BENCHMARK_STUDY_V2_INVALID_DECISION_SCHEMA_VERSION
14076
+ ):
14077
+ _benchmark_study_v2_persist_analysis(
14078
+ output_root=output_root, report=report,
14079
+ )
14080
+ except (OSError, SystemExit, TypeError, ValueError):
14081
+ # The original refusal remains authoritative when damaged or incomplete
14082
+ # evidence cannot safely support a canonical P1-X decision.
14083
+ print(
14084
+ "v2 canonical P1-X unavailable after refusal",
14085
+ file=sys.stderr,
14086
+ )
14087
+ return
14088
+
14089
+
14090
+ def _benchmark_study_v2_ledger_binding(
14091
+ path: Path, *, maximum: int,
14092
+ ) -> dict[str, Any]:
14093
+ if not path.exists():
14094
+ return {"bytes": 0, "record_count": 0, "sha256": None}
14095
+ raw = _measurement_read_private_file(path, maximum=maximum)
14096
+ return {
14097
+ "bytes": len(raw), "record_count": len(raw.splitlines()),
14098
+ "sha256": _study_sha256_bytes(raw),
14099
+ }
14100
+
14101
+
14102
+ def _benchmark_study_v2_invalid_canary_decision(
14103
+ *, output_root: Path, manifest: Mapping[str, Any],
14104
+ manifest_sha256: str, rows: Sequence[Mapping[str, Any]],
14105
+ ) -> dict[str, Any] | None:
14106
+ final_by_arm = {str(row["arm"]): row for row in rows}
14107
+ ambiguous = [
14108
+ final_by_arm[arm] for arm in BENCHMARK_STUDY_V2_CANARY_ARMS
14109
+ if arm in final_by_arm and final_by_arm[arm]["state"] != "terminal"
14110
+ ]
14111
+ failed = [
14112
+ final_by_arm[arm] for arm in BENCHMARK_STUDY_V2_CANARY_ARMS
14113
+ if arm in final_by_arm
14114
+ and final_by_arm[arm]["state"] == "terminal"
14115
+ and final_by_arm[arm]["passed"] is not True
14116
+ ]
14117
+ if not ambiguous and not failed:
14118
+ return None
14119
+ return {
14120
+ "schema_version": BENCHMARK_STUDY_V2_INVALID_DECISION_SCHEMA_VERSION,
14121
+ "study_version": "v2", "decision": "P1-X",
14122
+ "stop_reason": (
14123
+ "ambiguous_canary_process_state" if ambiguous
14124
+ else "failed_canary_terminal_evidence"
14125
+ ),
14126
+ "manifest_sha256": manifest_sha256,
14127
+ "attempt_schema_version": BENCHMARK_STUDY_V2_ATTEMPT_SCHEMA_VERSION,
14128
+ "consumed_identity_count": len(final_by_arm),
14129
+ "accounted_identity_count": sum(
14130
+ row["state"] == "terminal" for row in final_by_arm.values()
14131
+ ),
14132
+ "ambiguous_identities": [{
14133
+ "arm": row["arm"], "run_id": row["run_id"],
14134
+ "state": row["state"],
14135
+ } for row in ambiguous],
14136
+ "failed_canary_identities": [{
14137
+ "arm": row["arm"], "run_id": row["run_id"],
14138
+ "state": row["state"],
14139
+ } for row in failed],
14140
+ "ledgers": {
14141
+ "attempts": _benchmark_study_v2_ledger_binding(
14142
+ output_root / "attempts.jsonl", maximum=4_000_000,
14143
+ ),
14144
+ "canary_events": _benchmark_study_v2_ledger_binding(
14145
+ output_root / "canary-events.jsonl", maximum=200_000,
14146
+ ),
14147
+ },
14148
+ "canary_evidence_sha256": None,
14149
+ "descriptive_only": True, "claim_allowed": False, "claim": None,
14150
+ }
14151
+
14152
+
14153
+ def _benchmark_study_v2_invalid_analytic_decision(
14154
+ *, output_root: Path, manifest: Mapping[str, Any],
14155
+ manifest_sha256: str, rows: Sequence[Mapping[str, Any]],
14156
+ canary_evidence_sha256: str,
14157
+ ) -> dict[str, Any] | None:
14158
+ final_by_run = {str(row["run_id"]): row for row in rows}
14159
+ ambiguous = [
14160
+ final_by_run[str(slot["run_id"])]
14161
+ for slot in manifest["slots"]
14162
+ if str(slot["run_id"]) in final_by_run
14163
+ and final_by_run[str(slot["run_id"])]["state"]
14164
+ in {"launch_reserved", "launched"}
14165
+ ]
14166
+ failed = [
14167
+ row for row in final_by_run.values()
14168
+ if row["state"] == "terminal"
14169
+ and row["terminal_status"] == "study_infra_invalid"
14170
+ ]
14171
+ if not ambiguous and not failed:
14172
+ return None
14173
+ attempts_path = output_root / "attempts.jsonl"
14174
+ return {
14175
+ "schema_version": BENCHMARK_STUDY_V2_INVALID_DECISION_SCHEMA_VERSION,
14176
+ "study_version": "v2", "decision": "P1-X",
14177
+ "stop_reason": (
14178
+ "ambiguous_analytic_process_state" if ambiguous
14179
+ else "terminal_analytic_infrastructure_invalid"
14180
+ ),
14181
+ "manifest_sha256": manifest_sha256,
14182
+ "attempt_schema_version": BENCHMARK_STUDY_V2_ATTEMPT_SCHEMA_VERSION,
14183
+ "consumed_identity_count": sum(
14184
+ row["state"] in {"launch_reserved", "launched", "terminal"}
14185
+ for row in final_by_run.values()
14186
+ ),
14187
+ "accounted_identity_count": len(final_by_run),
14188
+ "ambiguous_identities": [{
14189
+ "arm": row["arm"], "attempt": row["attempt"],
14190
+ "repetition": row["repetition"], "run_id": row["run_id"],
14191
+ "state": row["state"], "task_id": row["task_id"],
14192
+ } for row in ambiguous],
14193
+ "failed_analytic_identities": [{
14194
+ "arm": row["arm"], "attempt": row["attempt"],
14195
+ "repetition": row["repetition"], "run_id": row["run_id"],
14196
+ "state": row["state"], "task_id": row["task_id"],
14197
+ "terminal_status": row["terminal_status"],
14198
+ } for row in failed],
14199
+ "ledgers": {
14200
+ "attempts": _benchmark_study_v2_ledger_binding(
14201
+ attempts_path, maximum=4_000_000,
14202
+ ),
14203
+ "canary_events": _benchmark_study_v2_ledger_binding(
14204
+ output_root / "canary-events.jsonl", maximum=200_000,
14205
+ ),
14206
+ },
14207
+ "canary_evidence_sha256": canary_evidence_sha256,
14208
+ "descriptive_only": True, "claim_allowed": False, "claim": None,
14209
+ }
14210
+
14211
+
14212
+ def _analyze_benchmark_study_v2_executable_unlocked(
14213
+ *, output_root: Path, claude_bin: str,
14214
+ ) -> dict[str, Any]:
14215
+ output_root = _benchmark_study_v2_output_root(output_root)
14216
+ manifest, manifest_sha256 = load_benchmark_study_v2_executable_manifest(
14217
+ output_root, revalidate_external=True,
14218
+ )
14219
+ _benchmark_study_v2_assert_cli_binding(
14220
+ claude_bin, manifest["inputs"]["cli_binding"],
14221
+ )
14222
+ canary_variants = _benchmark_study_v2_canary_variants(manifest, output_root)
14223
+ canary_rows = _benchmark_study_v2_read_canary_events(
14224
+ output_root / "canary-events.jsonl",
14225
+ manifest_sha256=manifest_sha256, variants=canary_variants,
14226
+ )
14227
+ invalid_canary_decision = _benchmark_study_v2_invalid_canary_decision(
14228
+ output_root=output_root, manifest=manifest,
14229
+ manifest_sha256=manifest_sha256, rows=canary_rows,
14230
+ )
14231
+ if invalid_canary_decision is not None:
14232
+ return invalid_canary_decision
14233
+ _canary_evidence, canary_evidence_sha256 = (
14234
+ _benchmark_study_v2_verify_canary_evidence(
14235
+ manifest=manifest, manifest_sha256=manifest_sha256,
14236
+ output_root=output_root,
14237
+ )
14238
+ )
14239
+ rows = _benchmark_study_v2_read_attempts(
14240
+ output_root / "attempts.jsonl", manifest=manifest,
14241
+ manifest_sha256=manifest_sha256,
14242
+ )
14243
+ tasks = parse_tasks(Path(manifest["inputs"]["tasks_path"]))
14244
+ load_task_fixture_trees(
14245
+ tasks, task_file_dir=Path(manifest["inputs"]["tasks_path"]).parent,
14246
+ )
14247
+ _benchmark_study_v2_revalidate_terminal_evidence(
14248
+ manifest=manifest, output_root=output_root, rows=rows,
14249
+ tasks_by_id={task.id: task for task in tasks},
14250
+ variants=_benchmark_study_v2_variants(manifest, output_root),
14251
+ )
14252
+ invalid_decision = _benchmark_study_v2_invalid_analytic_decision(
14253
+ output_root=output_root, manifest=manifest,
14254
+ manifest_sha256=manifest_sha256, rows=rows,
14255
+ canary_evidence_sha256=canary_evidence_sha256,
14256
+ )
14257
+ if invalid_decision is not None:
14258
+ return invalid_decision
14259
+ terminal_rows = [row for row in rows if row["state"] == "terminal"]
14260
+ final_states = {row["run_id"]: row["state"] for row in rows}
14261
+ if len(final_states) != 216 or any(
14262
+ state not in {"terminal", "not_needed", "blocked_study_invalid"}
14263
+ for state in final_states.values()
14264
+ ):
14265
+ raise ValueError("v2 analysis requires final accounting for all 216 identities")
14266
+ identity_state_counts = dict(sorted(collections.Counter(final_states.values()).items()))
14267
+ if any(
14268
+ row["terminal_status"] not in {"success", "valid_task_failure_v1"}
14269
+ for row in terminal_rows
14270
+ ):
14271
+ raise ValueError("v2 analysis refuses infrastructure-invalid or recovered attempts")
14272
+ initial = [row for row in terminal_rows if row["attempt"] == 0]
14273
+ if len(initial) != 108:
14274
+ raise ValueError("v2 analysis requires all 108 initial provider calls")
14275
+ by_unit = {(row["task_id"], row["repetition"], row["arm"]): row for row in initial}
14276
+ expected_retries = {
14277
+ key for key, row in by_unit.items()
14278
+ if row["terminal_status"] == "valid_task_failure_v1"
14279
+ }
14280
+ retry_rows = [row for row in terminal_rows if row["attempt"] == 1]
14281
+ if {(row["task_id"], row["repetition"], row["arm"]) for row in retry_rows} != expected_retries:
14282
+ raise ValueError("v2 analysis retry coverage is incomplete or replaced")
14283
+ effect_rows = [{
14284
+ "task_id": row["task_id"], "repetition": row["repetition"],
14285
+ "arm": row["arm"], "attempt": row["attempt"],
14286
+ "terminal_status": row["terminal_status"], "success": row["success"],
14287
+ "tokens": row["primary_tokens"], "correction": row["correction"],
14288
+ "retrieval": row["retrieval"],
14289
+ } for row in terminal_rows]
14290
+ terminal_by_unit = dict(by_unit)
14291
+ for row in retry_rows:
14292
+ terminal_by_unit[(row["task_id"], row["repetition"], row["arm"])] = row
14293
+ binary_rows = [
14294
+ {"task_id": row["task_id"], "repetition": row["repetition"],
14295
+ "arm": row["arm"], "success": row["success"]}
14296
+ for row in terminal_by_unit.values()
14297
+ if row["arm"] in BENCHMARK_STUDY_V2_PRIMARY_CONTRAST
14298
+ ]
14299
+ inference = infer_benchmark_study_v2_binary(
14300
+ binary_rows, task_order=manifest["inputs"]["task_ids"],
14301
+ ni_margin=float(manifest["plan"]["noninferiority_margin"]),
14302
+ )
14303
+ effects = compute_benchmark_study_v2_effects(
14304
+ effect_rows, task_order=manifest["inputs"]["task_ids"],
14305
+ )
14306
+ unavailable = {"available": False, "value": None, "reason": "observer_absent"}
14307
+ return {
14308
+ "schema_version": BENCHMARK_STUDY_V2_REPORT_SCHEMA_VERSION,
14309
+ "study_version": "v2", "decision": "P1-F",
14310
+ "manifest_sha256": manifest_sha256,
14311
+ "record_count": len(terminal_rows), "initial_provider_calls": 108,
14312
+ "retry_provider_calls": len(retry_rows),
14313
+ "discarded_canary_provider_calls": 2,
14314
+ "identity_state_counts": identity_state_counts,
14315
+ "binary_inference": inference, "effects": effects,
14316
+ "observers": {
14317
+ "correction": dict(unavailable), "retrieval": dict(unavailable),
14318
+ "shifted_cost": dict(unavailable),
14319
+ },
14320
+ "descriptive_only": True, "claim_allowed": False, "claim": None,
14321
+ "claim_readiness": {
14322
+ "claim_ready": False, "descriptive_only": True,
14323
+ "claim_allowed": False, "unmet_gates": ["power"],
14324
+ },
14325
+ "provenance": {
14326
+ "source": "direct_cli_plus_bound_checker",
14327
+ "success_source": "provider_terminal_usage_and_bound_checker",
14328
+ "provider_success_boolean_trusted": False,
14329
+ "cli_binding_sha256": _study_domain_hash(
14330
+ "contextguard.bench.v2.cli-binding.v1",
14331
+ manifest["inputs"]["cli_binding"],
14332
+ ),
14333
+ "auth_context_sha256": _study_domain_hash(
14334
+ "contextguard.bench.v2.auth-context.v1",
14335
+ manifest["inputs"]["auth_context"],
14336
+ ),
14337
+ "cli_version_stdout_sha256": manifest["inputs"]["cli_binding"][
14338
+ "probe"
14339
+ ]["version_stdout_sha256"],
14340
+ "backend_revision": "unavailable",
14341
+ "model_revision": "unavailable",
14342
+ "canary_evidence_sha256": canary_evidence_sha256,
14343
+ "canary_discarded_from_analysis": True,
14344
+ },
14345
+ }
14346
+
14347
+
14348
+ BENCHMARK_STUDY_V2_NAMESPACE = "contextguard.bench.v2"
14349
+
14350
+
14351
+ def _benchmark_study_v2_task_ids_from_corpus(corpus_bytes: bytes) -> list[str]:
14352
+ try:
14353
+ corpus = json.loads(corpus_bytes.decode("utf-8"))
14354
+ except (UnicodeDecodeError, json.JSONDecodeError) as exc:
14355
+ raise ValueError("v2 task corpus is invalid JSON") from exc
14356
+ if not isinstance(corpus, list):
14357
+ raise ValueError("v2 task corpus is not a list")
14358
+ return _benchmark_study_v2_task_ids([
14359
+ row.get("id") if isinstance(row, Mapping) else None for row in corpus
14360
+ ])
14361
+
14362
+
14363
+ def _benchmark_study_v2_task_ids_sha256(task_ids: Sequence[str]) -> str:
14364
+ return _study_domain_hash(
14365
+ BENCHMARK_STUDY_V2_CORPUS_TASK_ORDER_DOMAIN,
14366
+ _benchmark_study_v2_task_ids(task_ids),
14367
+ )
14368
+
14369
+
14370
+ def benchmark_study_v2_checker_binding(checkers_dir: Path) -> dict[str, Any]:
14371
+ """Hash an ordered relative filename/size/content-digest checker inventory."""
14372
+ directory_fd = _ensure_directory_no_symlink(checkers_dir, create=False)
14373
+ try:
14374
+ names = sorted(
14375
+ entry.name for entry in checkers_dir.iterdir()
14376
+ if entry.name.endswith(".py") and entry.is_file() and not entry.is_symlink()
14377
+ )
14378
+ finally:
14379
+ os.close(directory_fd)
14380
+ if len(names) != 12:
14381
+ raise ValueError("v2 checker directory must contain exactly 12 regular Python files")
14382
+ files = []
14383
+ for name in names:
14384
+ raw = _read_bytes_no_follow(
14385
+ checkers_dir / name, max_bytes=MAX_FIXTURE_FILE_BYTES,
14386
+ )
14387
+ files.append({
14388
+ "filename": name,
14389
+ "size": len(raw),
14390
+ "sha256": _study_sha256_bytes(raw),
14391
+ })
14392
+ binding = {
14393
+ "domain": BENCHMARK_STUDY_V2_CHECKER_BINDING_DOMAIN,
14394
+ "files": files,
14395
+ "sha256": _study_domain_hash(
14396
+ BENCHMARK_STUDY_V2_CHECKER_BINDING_DOMAIN, files,
14397
+ ),
14398
+ }
14399
+ validate_benchmark_study_v2_checker_binding(binding)
14400
+ return binding
14401
+
14402
+
14403
+ def main(argv: Sequence[str] | None = None) -> int:
14404
+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
14405
+ parser.add_argument("--tasks", default=None, type=Path, help="task fixture JSON")
14406
+ parser.add_argument("--variants", default=None, type=Path, help="variant fixture JSON")
14407
+ parser.add_argument("--csv", default=None, type=Path,
14408
+ help="results CSV path (header is added on first write)")
14409
+ parser.add_argument("--task-id", default=None, help="run only the named task id")
14410
+ parser.add_argument("--variant", default=None, help="run only the named variant")
14411
+ parser.add_argument("--claude-bin", default=os.environ.get("CLAUDE_BIN", "claude"),
14412
+ help="claude CLI executable (default: $CLAUDE_BIN or 'claude')")
14413
+ parser.add_argument("--project-root", default=Path("."), type=Path,
14414
+ help="working directory used for success_command (default: cwd)")
14415
+ parser.add_argument("--dry-run", action="store_true",
14416
+ help="print the claude command without invoking it")
14417
+ parser.add_argument("--resume", action="store_true",
14418
+ help="skip (task_id, variant) rows already present in --csv")
14419
+ parser.add_argument("--ledger-jsonl", default=None, type=Path,
14420
+ help="optional JSONL ledger path for cost-shift accounting per run")
14421
+ parser.add_argument("--report-json", default=None, type=Path,
14422
+ help="optional A/B summary report JSON path generated from --csv after real runs")
14423
+ parser.add_argument("--dashboard-md", default=None, type=Path,
14424
+ help="optional Markdown dashboard path generated from the benchmark report")
14425
+ parser.add_argument("--evidence-jsonl", default=None, type=Path,
14426
+ help="optional validated run-evidence JSONL replay input; skips provider invocation")
14427
+ parser.add_argument("--baseline-variant", default="baseline",
14428
+ help="variant name used as the report baseline (default: baseline)")
14429
+ parser.add_argument("--measurement-study-plan", default=None, type=Path,
14430
+ help="exact S002 measurement study plan JSON")
14431
+ parser.add_argument(
14432
+ "--measurement-study-action",
14433
+ default=None,
14434
+ choices=("prepare", "run", "resume", "analyze"),
14435
+ help="S002 measurement study action",
14436
+ )
14437
+ parser.add_argument("--measurement-study-output-root", default=None, type=Path,
14438
+ help="private S002 measurement study artifact directory")
14439
+ parser.add_argument(
14440
+ "--study-v2-action", default=None,
14441
+ choices=("prepare", "canary", "run", "resume", "analyze"),
14442
+ help="prepare, canary, run, resume, or analyze the executable additive v2 study",
14443
+ )
14444
+ parser.add_argument("--study-v2-plan", default=None, type=Path,
14445
+ help="canonical v2 study plan used only by --study-v2-action prepare")
14446
+ parser.add_argument("--study-v2-tasks", default=None, type=Path,
14447
+ help="frozen v2 task corpus used only by --study-v2-action prepare")
14448
+ parser.add_argument("--study-v2-checkers-dir", default=None, type=Path,
14449
+ help="frozen v2 checker directory used only by --study-v2-action prepare")
14450
+ parser.add_argument("--study-v2-candidate-hash", default=None,
14451
+ help="exact candidate SHA-256 used only by --study-v2-action prepare")
14452
+ parser.add_argument("--study-v2-source-commit", default=None,
14453
+ help="approved 40-hex source commit used only by --study-v2-action prepare")
14454
+ parser.add_argument("--study-v2-retained-ref", default=None,
14455
+ help="retained candidate ref resolved during live v2 prepare")
14456
+ parser.add_argument(
14457
+ "--study-v2-offline-rehearsal", action="store_true",
14458
+ help="mark provider-free fake-host rehearsal; never use for live prepare",
14459
+ )
14460
+ parser.add_argument("--study-v2-output-root", default=None, type=Path,
14461
+ help="private executable v2 lifecycle directory")
14462
+ parser.add_argument("--study-v2-candidate-manifest", default=None, type=Path,
14463
+ help="canonical build-once npm candidate manifest used by v2 prepare")
14464
+ parser.add_argument("--study-v2-candidate-checksums", default=None, type=Path,
14465
+ help="exact candidate checksum document (default: manifest sibling)")
14466
+ parser.add_argument("--study-v2-npm-bin", default="npm",
14467
+ help="npm executable used once for the offline candidate install")
14468
+ parser.add_argument(
14469
+ "--study-v2-use-existing-login", action="store_true",
14470
+ help=(
14471
+ "allow executable v2 provider actions to reuse the exact CLI's "
14472
+ "existing first-party login without importing credential environment variables"
14473
+ ),
14474
+ )
14475
+ args = parser.parse_args(argv)
14476
+
14477
+ require_no_follow_file_ops_supported()
14478
+ v2_values = (
14479
+ args.study_v2_action, args.study_v2_plan, args.study_v2_tasks,
14480
+ args.study_v2_checkers_dir, args.study_v2_candidate_hash,
14481
+ args.study_v2_source_commit, args.study_v2_retained_ref,
14482
+ args.study_v2_output_root, args.study_v2_candidate_manifest,
14483
+ args.study_v2_candidate_checksums,
14484
+ )
14485
+ if args.study_v2_use_existing_login and args.study_v2_action is None:
14486
+ parser.error("--study-v2-use-existing-login requires --study-v2-action")
14487
+ if args.study_v2_offline_rehearsal and args.study_v2_action is None:
14488
+ parser.error("--study-v2-offline-rehearsal requires --study-v2-action prepare")
14489
+ if any(value is not None for value in v2_values):
14490
+ if args.study_v2_action is None:
14491
+ parser.error("--study-v2-action is required when any --study-v2-* option is used")
14492
+ conflicts = [
14493
+ name for name, active in (
14494
+ ("--tasks", args.tasks is not None), ("--variants", args.variants is not None),
14495
+ ("--csv", args.csv is not None), ("--task-id", args.task_id is not None),
14496
+ ("--variant", args.variant is not None), ("--dry-run", args.dry_run),
14497
+ ("--resume", args.resume), ("--ledger-jsonl", args.ledger_jsonl is not None),
14498
+ ("--report-json", args.report_json is not None),
14499
+ ("--dashboard-md", args.dashboard_md is not None),
14500
+ ("--evidence-jsonl", args.evidence_jsonl is not None),
14501
+ ("--measurement-study-plan", args.measurement_study_plan is not None),
14502
+ ("--measurement-study-action", args.measurement_study_action is not None),
14503
+ ("--measurement-study-output-root", args.measurement_study_output_root is not None),
14504
+ ) if active
14505
+ ]
14506
+ if conflicts:
14507
+ parser.error(f"v2 executable mode conflicts with {', '.join(conflicts)}")
14508
+ if args.study_v2_output_root is None:
14509
+ parser.error("v2 executable actions require --study-v2-output-root")
14510
+ if (
14511
+ args.study_v2_action != "analyze"
14512
+ and not args.study_v2_use_existing_login
14513
+ ):
14514
+ parser.error(
14515
+ "v2 prepare/canary/run/resume requires "
14516
+ "--study-v2-use-existing-login"
14517
+ )
14518
+ auth_home = Path(os.environ.get("HOME", ""))
14519
+ if args.study_v2_action == "prepare":
14520
+ required = (
14521
+ args.study_v2_plan, args.study_v2_tasks, args.study_v2_checkers_dir,
14522
+ args.study_v2_candidate_hash, args.study_v2_source_commit,
14523
+ args.study_v2_candidate_manifest,
14524
+ )
14525
+ forbidden = ()
14526
+ if args.study_v2_offline_rehearsal:
14527
+ if args.study_v2_retained_ref is not None:
14528
+ parser.error(
14529
+ "v2 offline rehearsal conflicts with --study-v2-retained-ref"
14530
+ )
14531
+ elif args.study_v2_retained_ref is None:
14532
+ parser.error("v2 live prepare requires --study-v2-retained-ref")
14533
+ else:
14534
+ required = (args.study_v2_output_root,)
14535
+ forbidden = (
14536
+ args.study_v2_plan, args.study_v2_tasks, args.study_v2_checkers_dir,
14537
+ args.study_v2_candidate_hash, args.study_v2_source_commit,
14538
+ args.study_v2_retained_ref, args.study_v2_candidate_manifest,
14539
+ args.study_v2_candidate_checksums,
14540
+ )
14541
+ if args.study_v2_offline_rehearsal:
14542
+ parser.error("--study-v2-offline-rehearsal is prepare-only")
14543
+ if not all(value is not None for value in required) or any(value is not None for value in forbidden):
14544
+ parser.error("v2 executable action has incomplete or conflicting arguments")
14545
+ try:
14546
+ if args.study_v2_action == "prepare":
14547
+ prepare_benchmark_study_v2_executable(
14548
+ output_root=args.study_v2_output_root,
14549
+ plan_path=args.study_v2_plan, tasks_path=args.study_v2_tasks,
14550
+ checkers_dir=args.study_v2_checkers_dir,
14551
+ candidate_manifest_path=args.study_v2_candidate_manifest,
14552
+ candidate_checksum_path=args.study_v2_candidate_checksums,
14553
+ expected_candidate_hash=args.study_v2_candidate_hash,
14554
+ npm_bin=args.study_v2_npm_bin,
14555
+ claude_bin=args.claude_bin,
14556
+ auth_home=auth_home,
14557
+ approved_source_commit=args.study_v2_source_commit,
14558
+ retained_ref=args.study_v2_retained_ref,
14559
+ offline_rehearsal=args.study_v2_offline_rehearsal,
14560
+ )
14561
+ print(f"prepared executable v2 study: {args.study_v2_output_root}")
14562
+ elif args.study_v2_action == "canary":
14563
+ summary = execute_benchmark_study_v2_canary(
14564
+ output_root=args.study_v2_output_root,
14565
+ claude_bin=args.claude_bin,
14566
+ auth_home=auth_home,
14567
+ )
14568
+ print(json.dumps(summary, sort_keys=True, separators=(",", ":")))
14569
+ elif args.study_v2_action in {"run", "resume"}:
14570
+ summary = execute_benchmark_study_v2(
14571
+ output_root=args.study_v2_output_root,
14572
+ claude_bin=args.claude_bin,
14573
+ resume=args.study_v2_action == "resume",
14574
+ auth_home=auth_home,
14575
+ )
14576
+ print(json.dumps(summary, sort_keys=True, separators=(",", ":")))
14577
+ else:
14578
+ report = analyze_benchmark_study_v2_executable(
14579
+ output_root=args.study_v2_output_root,
14580
+ claude_bin=args.claude_bin,
14581
+ )
14582
+ report_name = _benchmark_study_v2_persist_analysis(
14583
+ output_root=args.study_v2_output_root, report=report,
14584
+ )
14585
+ print(
14586
+ "analyzed executable v2 study: "
14587
+ f"{args.study_v2_output_root / report_name}"
14588
+ )
14589
+ if report_name == "study-invalid-decision.json":
14590
+ return 3
14591
+ return 0
14592
+ except (OSError, SystemExit, TypeError, ValueError) as exc:
14593
+ if args.study_v2_action in {"canary", "run", "resume"}:
14594
+ _benchmark_study_v2_persist_invalid_after_refusal(
14595
+ output_root=args.study_v2_output_root,
14596
+ claude_bin=args.claude_bin,
14597
+ )
14598
+ print(f"v2 executable study refused: {exc}", file=sys.stderr)
14599
+ return 2
14600
+ if args.tasks is None or args.variants is None:
14601
+ parser.error("--tasks and --variants are required outside study modes")
14602
+ study_values = (
14603
+ args.measurement_study_plan,
14604
+ args.measurement_study_action,
14605
+ args.measurement_study_output_root,
14606
+ )
14607
+ if any(value is not None for value in study_values):
14608
+ if not all(value is not None for value in study_values):
14609
+ parser.error(
14610
+ "--measurement-study-plan, --measurement-study-action, and "
14611
+ "--measurement-study-output-root are all-or-none"
14612
+ )
14613
+ conflicts = [
14614
+ name for name, active in (
14615
+ ("--task-id", args.task_id is not None),
14616
+ ("--variant", args.variant is not None),
14617
+ ("--resume", args.resume),
14618
+ ("--evidence-jsonl", args.evidence_jsonl is not None),
14619
+ ("--dry-run", args.dry_run),
14620
+ ("--ledger-jsonl", args.ledger_jsonl is not None),
14621
+ ("--report-json", args.report_json is not None),
14622
+ ("--dashboard-md", args.dashboard_md is not None),
14623
+ ("--csv", args.csv is not None),
14624
+ ("--baseline-variant", args.baseline_variant != "baseline"),
14625
+ ) if active
14626
+ ]
14627
+ if conflicts:
14628
+ parser.error(f"measurement study mode conflicts with {', '.join(conflicts)}")
14629
+ return run_measurement_study_action(args)
14630
+ args.csv = args.csv or Path("bench/results.csv")
14631
+ validate_distinct_output_paths(args.csv, args.ledger_jsonl, args.report_json, args.dashboard_md)
14632
+
14633
+ variants = parse_variants(args.variants)
14634
+ tasks = parse_tasks(args.tasks, variants=variants)
14635
+ targets = filter_targets(tasks, variants, args.task_id, args.variant)
14636
+ if not targets:
14637
+ if args.dry_run and (not tasks or not variants):
14638
+ print("completed 0 run(s) (dry-run; no CSV writes)")
14639
+ return 0
14640
+ print("no (task, variant) targets matched the filters", file=sys.stderr)
14641
+ return 1
14642
+ target_task_ids = {task.id for task, _variant in targets}
14643
+ load_task_fixture_trees(
14644
+ [task for task in tasks if task.id in target_task_ids],
14645
+ task_file_dir=args.tasks.parent,
14646
+ )
10534
14647
  preflight_measurement_targets(
10535
14648
  targets,
10536
14649
  claude_bin=args.claude_bin,