@ictechgy/context-guard 0.4.13 → 0.4.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -49,9 +49,10 @@ from __future__ import annotations
49
49
 
50
50
  import argparse
51
51
  import collections
52
- from contextlib import contextmanager
52
+ from contextlib import contextmanager, nullcontext
53
53
  import csv
54
54
  import datetime as _dt
55
+ import hashlib
55
56
  import json
56
57
  import math
57
58
  import os
@@ -65,6 +66,7 @@ import subprocess
65
66
  import sys
66
67
  import time
67
68
  import unicodedata
69
+ from collections.abc import Iterable
68
70
  from dataclasses import dataclass, field
69
71
  from pathlib import Path
70
72
  from typing import Any
@@ -318,6 +320,126 @@ ALLOWED_FIRST_ABSOLUTE_SYMLINKS = {
318
320
  "var": Path("/private/var"),
319
321
  }
320
322
 
323
+ # --- Phase 4/5 optional image-context evaluation profile (evaluation-only) ---
324
+ # 이 profile 은 task fixture 가 명시적으로 opt-in 할 때만 동작한다. profile 이 없는
325
+ # 기존 replay 는 스키마/동작이 그대로 유지된다. profile 이 붙은 report 는 어떤
326
+ # 경우에도 public claim / promotion 권한을 얻지 못하도록 clamp 된다.
327
+ IMAGE_CONTEXT_EVALUATION_PROFILE_ID = "contextguard.bench.image-context-pack-evaluation.v1"
328
+ SUPPORTED_EVALUATION_PROFILE_IDS = frozenset({IMAGE_CONTEXT_EVALUATION_PROFILE_ID})
329
+ IMAGE_CONTEXT_READINESS_SCHEMA_VERSION = "contextguard.bench.image-context-pack-readiness.v1"
330
+ IMAGE_CONTEXT_PROFILE_REPORT_KEY = "image_context_pack"
331
+ EVALUATION_PROFILES_REPORT_KEY = "evaluation_profiles"
332
+ IMAGE_CONTEXT_EVALUATION_ONLY_CLAIM_STATUS = "image_context_pack_evaluation_only_not_public_claim"
333
+ PROFILE_STATUS_BLOCKED = "blocked"
334
+ PROFILE_STATUS_READY_FOR_BOUNDED_PILOT_REVIEW = "ready_for_bounded_pilot_review"
335
+ # 가져온 local proof-verifier 레코드는 "누가 만들었는지" 를 인증하지 않고 artifact 를
336
+ # 다시 읽지도 않는다. 라벨로 그 경계를 명시한다.
337
+ IMPORTED_LOCAL_VERIFIER_ATTESTATION_LABEL = "imported_local_verifier_attestation"
338
+ PROOF_VERIFICATION_SCHEMA_VERSION = "contextguard.experiments.proof-carrying-context-verification.v1"
339
+ PROOF_VERIFICATION_VERIFIED_STATUS = "verified"
340
+ # experimental_registry.PROOF_VERIFICATION_CLAIM_BOUNDARY 와 반드시 같은 문자열이다.
341
+ # 가져온 attestation 은 이 local-only 경계를 그대로 선언할 때만 verified 로 인정한다.
342
+ PROOF_VERIFICATION_CLAIM_BOUNDARY = (
343
+ "Local receipt/hash/range/command binding only; no semantic-safety, protected-zone, freshness, replacement, "
344
+ "omission, or hosted-savings authority."
345
+ )
346
+ # local verifier 는 rehydration 을 절대 실행하지 않는다. 실행했다고 주장하는 레코드는
347
+ # 이 evaluation-only 경계를 벗어나므로 verified 로 받아들이지 않는다.
348
+ PROOF_VERIFICATION_REHYDRATION_EXECUTED = False
349
+ # verified attestation 에서 placeholder 로 취급해 거부할 receipt/command 값이다.
350
+ PROFILE_FALLBACK_PLACEHOLDER_VALUES = frozenset({"", "none", "null", "n/a", "-"})
351
+
352
+ # reject_prewrite 오류 ID. 출력이 하나라도 기록되기 전에 실패해야 하는 구조적 오류다.
353
+ PROFILE_REJECT_CONTROLS_MISSING = "profile_controls_missing"
354
+ PROFILE_REJECT_SCHEMA_INVALID = "profile_schema_invalid"
355
+ PROFILE_REJECT_BINDING_MISMATCH = "profile_binding_mismatch"
356
+ PROFILE_REJECT_BATCH_INCOMPLETE = "profile_batch_incomplete"
357
+ PROFILE_REJECT_FRESH_OUTPUT_REQUIRED = "profile_fresh_output_required"
358
+ PROFILE_REJECT_PROMPT_BINDING_INVALID = "profile_prompt_binding_invalid"
359
+ PROFILE_REJECT_CORRECTION_INCONSISTENT = "profile_correction_inconsistent"
360
+ PROFILE_REJECT_MEASUREMENT_INCONSISTENT = "profile_measurement_inconsistent"
361
+ PROFILE_REJECT_FALLBACK_CLAIM_INCONSISTENT = "profile_fallback_claim_inconsistent"
362
+ # profile 은 evaluation-only replay 전용이다. --evidence-jsonl 없이 profiled task 가
363
+ # 선택되면 provider runtime 을 부르기 전에, 어떤 출력/lock 도 만들기 전에 거부한다.
364
+ PROFILE_REJECT_REPLAY_REQUIRED = "profile_replay_required"
365
+ PROFILE_REJECT_ERROR_IDS = (
366
+ PROFILE_REJECT_CONTROLS_MISSING,
367
+ PROFILE_REJECT_SCHEMA_INVALID,
368
+ PROFILE_REJECT_BINDING_MISMATCH,
369
+ PROFILE_REJECT_BATCH_INCOMPLETE,
370
+ PROFILE_REJECT_FRESH_OUTPUT_REQUIRED,
371
+ PROFILE_REJECT_PROMPT_BINDING_INVALID,
372
+ PROFILE_REJECT_CORRECTION_INCONSISTENT,
373
+ PROFILE_REJECT_MEASUREMENT_INCONSISTENT,
374
+ PROFILE_REJECT_FALLBACK_CLAIM_INCONSISTENT,
375
+ PROFILE_REJECT_REPLAY_REQUIRED,
376
+ )
377
+
378
+ # lane gate ID. 순서는 report/dashboard 출력 순서와 동일하게 고정한다.
379
+ IMAGE_CONTEXT_GATE_PROFILE_AND_PROMPT_BINDING = "profile_and_prompt_binding"
380
+ IMAGE_CONTEXT_GATE_PROTECTED_ZONE_DENY_REVIEW = "protected_zone_deny_review"
381
+ IMAGE_CONTEXT_GATE_EXACT_TEXT_FALLBACK_BINDING = "exact_text_fallback_binding"
382
+ IMAGE_CONTEXT_GATE_MISSED_CONTEXT_REVIEW = "missed_context_review"
383
+ IMAGE_CONTEXT_GATE_HUMAN_CORRECTION_CONSISTENCY = "human_correction_consistency"
384
+ # generic quality gate 결과를 lane 이 직접 소비한다. profile 이 generic regression 을
385
+ # 무시하고 ready 로 올라가는 경로를 막는다.
386
+ IMAGE_CONTEXT_GATE_CORRECTIONS_REGRESSION = "corrections_regression"
387
+ IMAGE_CONTEXT_GATE_FAILURE_RATE_REGRESSION = "failure_rate_regression"
388
+ IMAGE_CONTEXT_GATE_GENERIC_MATCHED_SUCCESS_AND_MEASUREMENT = "generic_matched_success_and_measurement"
389
+ IMAGE_CONTEXT_GATE_EVALUATION_ONLY_PROMOTION_BOUNDARY = "evaluation_only_promotion_boundary"
390
+ IMAGE_CONTEXT_GATE_IDS = (
391
+ IMAGE_CONTEXT_GATE_PROFILE_AND_PROMPT_BINDING,
392
+ IMAGE_CONTEXT_GATE_PROTECTED_ZONE_DENY_REVIEW,
393
+ IMAGE_CONTEXT_GATE_EXACT_TEXT_FALLBACK_BINDING,
394
+ IMAGE_CONTEXT_GATE_MISSED_CONTEXT_REVIEW,
395
+ IMAGE_CONTEXT_GATE_HUMAN_CORRECTION_CONSISTENCY,
396
+ IMAGE_CONTEXT_GATE_CORRECTIONS_REGRESSION,
397
+ IMAGE_CONTEXT_GATE_FAILURE_RATE_REGRESSION,
398
+ IMAGE_CONTEXT_GATE_GENERIC_MATCHED_SUCCESS_AND_MEASUREMENT,
399
+ IMAGE_CONTEXT_GATE_EVALUATION_ONLY_PROMOTION_BOUNDARY,
400
+ )
401
+ # lane gate 를 막는 generic quality_gate 값. summarize_benchmark_rows 가 계산한다.
402
+ GENERIC_QUALITY_GATE_PASS = "pass"
403
+ GENERIC_QUALITY_GATE_CORRECTIONS_REGRESSION = "corrections_regression"
404
+ GENERIC_QUALITY_GATE_FAILURE_RATE_REGRESSION = "failure_rate_regression"
405
+ IMAGE_CONTEXT_PROFILE_BLOCKER_GATE_ID = "image_context_pack_evaluation_only"
406
+ IMAGE_CONTEXT_CLAIM_BOUNDARY = {
407
+ "id": "image_context_pack_evaluation_only_never_promotion_or_public_claim",
408
+ "evaluation_only": True,
409
+ "promotion_authority": False,
410
+ "public_claim_allowed": False,
411
+ "runtime_authority": False,
412
+ "hosted_savings_claim_allowed": False,
413
+ "fallback_attestation_label": IMPORTED_LOCAL_VERIFIER_ATTESTATION_LABEL,
414
+ "fallback_attestation_is_independently_verified": False,
415
+ "protected_zone_evidence_is_review_attestation_not_semantic_proof": True,
416
+ "reason": (
417
+ "The image-context evaluation profile reviews imported evidence only. It does not render, "
418
+ "parse, or reread any image or artifact, does not authenticate who produced an imported "
419
+ "verifier or review record, and can never authorize a public savings claim, a quality "
420
+ "non-inferiority claim, or a runtime promotion."
421
+ ),
422
+ }
423
+ PROFILE_SAMPLE_ADEQUACY_POLICY_STATUS = "not_defined_for_promotion"
424
+
425
+ # profile 중첩 블록의 명시적 byte/count 한계. 타입/한계 검사는 항상 semantic 분류보다
426
+ # 먼저 실행되어 oversize 값이 blocked 분기로 새지 않도록 한다.
427
+ MAX_PROFILE_LABEL_CHARS = 120
428
+ MAX_PROFILE_POLICY_CHARS = 120
429
+ MAX_PROFILE_NOTE_CHARS = 500
430
+ MAX_PROFILE_SUMMARY_CHARS = 500
431
+ MAX_PROFILE_COMMAND_CHARS = 500
432
+ MAX_PROFILE_RECEIPT_ID_CHARS = 200
433
+ MAX_PROFILE_BLOCKER_ITEMS = 20
434
+ MAX_PROFILE_PROTECTED_REGION_COUNT = 10_000
435
+ MAX_PROFILE_CORRECTION_COUNT = 10_000
436
+ SHA256_HEX_PATTERN = re.compile(r"\A[0-9a-f]{64}\Z")
437
+ PROTECTED_ZONE_DENY_POLICY = "deny"
438
+ # 프로파일 진단에 실리는 작성자 통제 라벨은 문자셋이 안전해 보여도 원문을 절대 남기지
439
+ # 않는다. G006 은 regex-safe 값까지 포함한 완전 불투명 표현을 요구한다.
440
+ PROFILE_REDACTED_PLACEHOLDER = "[REDACTED]"
441
+ MAX_PROFILE_ERROR_LABELS = 5
442
+
321
443
 
322
444
  def _base_open_flags() -> int:
323
445
  flags = os.O_RDONLY
@@ -472,6 +594,34 @@ def csv_file_lock(csv_path: Path, *, create_parent: bool) -> Any:
472
594
  os.close(fd)
473
595
 
474
596
 
597
+ @contextmanager
598
+ def csv_parent_directory_lock(csv_path: Path, *, create_parent: bool) -> Any:
599
+ """Serialize a CSV transaction without creating a lock sidecar.
600
+
601
+ Normal writers take this stable directory-inode lock before the historical
602
+ sidecar lock. Profiled replay can therefore hold it across freshness validation
603
+ and its complete batch without leaving a sidecar on rejection. The stable inode
604
+ also avoids an unlink-while-waiters race.
605
+ """
606
+ if fcntl is None:
607
+ raise OSError("platform does not support advisory CSV locks")
608
+ parent = csv_path.parent
609
+ if create_parent:
610
+ parent.mkdir(parents=True, exist_ok=True)
611
+ fd = os.open(parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
612
+ locked = False
613
+ try:
614
+ fcntl.flock(fd, fcntl.LOCK_EX)
615
+ locked = True
616
+ yield
617
+ finally:
618
+ try:
619
+ if locked:
620
+ fcntl.flock(fd, fcntl.LOCK_UN)
621
+ finally:
622
+ os.close(fd)
623
+
624
+
475
625
  # 재현성 우선: fixture 에 명시되지 않은 필드는 argv 로 전달하지 않는다.
476
626
  # 사용자가 baseline 으로 의도한 변형이 implicit default(예: effort="medium")로 인해
477
627
  # 왜곡되지 않도록, 파싱 단계에서 명시 여부를 그대로 보존한다.
@@ -488,6 +638,8 @@ class TaskFixture:
488
638
  success_cwd: str = "."
489
639
  variant_prompt_files: dict[str, str] = field(default_factory=dict)
490
640
  variant_prompt_texts: dict[str, str] = field(default_factory=dict)
641
+ # 선택적 evaluation profile opt-in. None 이면 기존 generic 동작을 그대로 유지한다.
642
+ evaluation_profile: str | None = None
491
643
 
492
644
 
493
645
  @dataclass
@@ -535,6 +687,12 @@ class EvidenceReplayRow:
535
687
  public_claim_eligible: bool
536
688
  explicit_notes: bool
537
689
  line_number: int
690
+ # profile 을 선언하지 않은 row 는 세 필드가 모두 None 이며 generic 경로와 동일하다.
691
+ evaluation_profile: str | None = None
692
+ evaluation_controls: dict[str, Any] | None = None
693
+ # preflight 가 채우는 정규화된 lane 판정. report annotation 은 이 값만 사용하므로
694
+ # 이미 검증된 batch 위에서 절대 실패하지 않는다.
695
+ evaluation_lane: dict[str, Any] | None = None
538
696
 
539
697
  @property
540
698
  def key(self) -> tuple[str, str]:
@@ -639,6 +797,16 @@ def require_argv_safe_prompt(text: str, *, owner: str) -> str:
639
797
 
640
798
  def validate_variant_prompt_file_path(raw_path: str, *, owner: str) -> Path:
641
799
  """Return a safe relative prompt-file path, or fail before any file read."""
800
+ # 결정적으로 거부 가능한 값: 임베디드 NUL 과 로컬 fs 인코딩으로 표현 불가한 문자열.
801
+ # 이 값들은 이후 os.open 에서 ValueError/UnicodeError 로 터질 수 있으므로 미리 막는다.
802
+ if "\x00" in raw_path:
803
+ raise SystemExit(f"{owner} variant_prompt_files path must not contain embedded NUL")
804
+ try:
805
+ os.fsencode(raw_path)
806
+ except UnicodeError:
807
+ raise SystemExit(
808
+ f"{owner} variant_prompt_files path is not representable on the local filesystem"
809
+ ) from None
642
810
  rel_path = Path(raw_path)
643
811
  if rel_path.is_absolute():
644
812
  raise SystemExit(f"{owner} variant_prompt_files path must be relative: {raw_path}")
@@ -658,19 +826,44 @@ def validate_variant_prompt_file_references(
658
826
  Unknown variant keys and unsafe relative paths are rejected before any file
659
827
  read. Missing prompt files are intentionally not checked here so a run
660
828
  narrowed by --task-id/--variant is not blocked by unselected prompt files.
829
+
830
+ Profiled tasks use the redacted profile owner and never echo raw task ids,
831
+ variant labels, mapping keys, or unsafe paths. Unprofiled messages stay
832
+ unchanged for compatibility.
661
833
  """
662
834
  known_variants = {variant.name for variant in variants}
663
835
  for task in tasks:
836
+ profiled = task.evaluation_profile is not None
664
837
  unknown = sorted(set(task.variant_prompt_files) - known_variants)
665
838
  if unknown:
839
+ if profiled:
840
+ # 매핑 키·variant 라벨은 attacker-controlled 이므로 이름 대신 적색 처리한다.
841
+ profile_reject(
842
+ PROFILE_REJECT_PROMPT_BINDING_INVALID,
843
+ profile_owner(task.id),
844
+ "variant_prompt_files references unknown variant(s): "
845
+ f"{redact_profile_labels(unknown)}",
846
+ )
666
847
  raise SystemExit(
667
848
  f"task {task.id} variant_prompt_files references unknown variant(s): {', '.join(unknown)}"
668
849
  )
669
850
  for variant_name, raw_path in task.variant_prompt_files.items():
670
- validate_variant_prompt_file_path(
671
- raw_path,
672
- owner=f"task {task.id} variant {variant_name}",
851
+ owner = (
852
+ profile_owner(task.id, variant_name)
853
+ if profiled
854
+ else f"task {task.id} variant {variant_name}"
673
855
  )
856
+ try:
857
+ validate_variant_prompt_file_path(raw_path, owner=owner)
858
+ except SystemExit:
859
+ if profiled:
860
+ # 원본 경로·라벨이 새어나가지 않도록 안정적인 프로파일 오류로 다시 쓴다.
861
+ profile_reject(
862
+ PROFILE_REJECT_PROMPT_BINDING_INVALID,
863
+ owner,
864
+ "variant_prompt_files path is unsafe or invalid",
865
+ )
866
+ raise
674
867
 
675
868
 
676
869
  def read_variant_prompt_file(path: Path, *, owner: str, display_path: str | None = None) -> str:
@@ -768,41 +961,89 @@ def parse_tasks(path: Path, variants: list["Variant"] | None = None) -> list[Tas
768
961
  for item in raw:
769
962
  if not isinstance(item, dict):
770
963
  raise SystemExit(f"task entry must be a JSON object: {item}")
964
+ # evaluation_profile opt-in 을 필수 라벨(id/prompt)보다 먼저 확정한다.
965
+ # 지원 프로파일의 구조 오류는 raw KeyError 가 아니라 안정적인 프로파일 거부로 끝낸다.
966
+ evaluation_profile = item.get("evaluation_profile")
967
+ profiled = evaluation_profile is not None
968
+ if "id" not in item:
969
+ if profiled:
970
+ profile_reject(
971
+ PROFILE_REJECT_SCHEMA_INVALID,
972
+ profile_owner(None),
973
+ "task fixture fields are invalid",
974
+ )
975
+ raise KeyError("id")
976
+ task_id = str(item["id"])
977
+ owner = profile_owner(task_id) if profiled else f"task {task_id}"
978
+ if profiled and (
979
+ not isinstance(evaluation_profile, str)
980
+ or evaluation_profile not in SUPPORTED_EVALUATION_PROFILE_IDS
981
+ ):
982
+ profile_reject(
983
+ PROFILE_REJECT_SCHEMA_INVALID,
984
+ owner,
985
+ "declares an unsupported evaluation_profile id",
986
+ )
987
+ if "variant_prompts" in item:
988
+ detail = "variant_prompts is not supported; use file-backed variant_prompt_files"
989
+ if profiled:
990
+ profile_reject(PROFILE_REJECT_SCHEMA_INVALID, owner, detail)
991
+ raise SystemExit(f"{owner} {detail}")
771
992
  effort_raw = item.get("effort")
772
993
  budget_raw = item.get("max_budget_usd")
773
994
  if budget_raw is not None:
774
995
  try:
775
996
  budget = float(budget_raw)
776
997
  except (TypeError, ValueError):
777
- raise SystemExit(f"task {item.get('id')} max_budget_usd must be number or null")
998
+ detail = "max_budget_usd must be number or null"
999
+ if profiled:
1000
+ profile_reject(PROFILE_REJECT_SCHEMA_INVALID, owner, detail)
1001
+ raise SystemExit(f"{owner} {detail}")
778
1002
  if not math.isfinite(budget) or budget <= 0:
779
- raise SystemExit(f"task {item.get('id')} max_budget_usd must be finite and > 0 (use null for unlimited)")
1003
+ detail = "max_budget_usd must be finite and > 0 (use null for unlimited)"
1004
+ if profiled:
1005
+ profile_reject(PROFILE_REJECT_SCHEMA_INVALID, owner, detail)
1006
+ raise SystemExit(f"{owner} {detail}")
780
1007
  else:
781
1008
  budget = None
782
- task_id = str(item["id"])
783
- if "variant_prompts" in item:
784
- raise SystemExit(
785
- f"task {task_id} variant_prompts is not supported; use file-backed variant_prompt_files"
1009
+ # profiled 경로에서는 매핑 키 등 attacker-controlled 라벨이 파서 오류에 실릴 수
1010
+ # 있으므로, 구조 필드 오류(파서 SystemExit·필수 prompt 누락 KeyError)를 안정적인
1011
+ # 프로파일 거부로 다시 쓴다. unprofiled 는 원본 예외를 그대로 재발생시킨다.
1012
+ try:
1013
+ max_turns = parse_positive_int(
1014
+ item.get("max_turns", 3), field="max_turns", owner=owner,
1015
+ )
1016
+ allowed_tools = parse_string_list(
1017
+ item.get("allowed_tools", []),
1018
+ field="allowed_tools",
1019
+ owner=owner,
786
1020
  )
1021
+ variant_prompt_files = parse_string_map(
1022
+ item.get("variant_prompt_files"),
1023
+ field="variant_prompt_files",
1024
+ owner=owner,
1025
+ )
1026
+ prompt = str(item["prompt"])
1027
+ except (SystemExit, KeyError):
1028
+ if profiled:
1029
+ profile_reject(
1030
+ PROFILE_REJECT_SCHEMA_INVALID,
1031
+ owner,
1032
+ "task fixture fields are invalid",
1033
+ )
1034
+ raise
787
1035
  fixtures.append(TaskFixture(
1036
+ evaluation_profile=evaluation_profile,
788
1037
  id=task_id,
789
- prompt=str(item["prompt"]),
1038
+ prompt=prompt,
790
1039
  model=str(item.get("model", "sonnet")),
791
1040
  effort=str(effort_raw) if effort_raw is not None else None,
792
- max_turns=parse_positive_int(item.get("max_turns", 3), field="max_turns", owner=f"task {task_id}"),
1041
+ max_turns=max_turns,
793
1042
  max_budget_usd=budget,
794
- allowed_tools=parse_string_list(
795
- item.get("allowed_tools", []),
796
- field="allowed_tools",
797
- owner=f"task {task_id}",
798
- ),
1043
+ allowed_tools=allowed_tools,
799
1044
  success_command=item.get("success_command"),
800
1045
  success_cwd=str(item.get("success_cwd", ".")),
801
- variant_prompt_files=parse_string_map(
802
- item.get("variant_prompt_files"),
803
- field="variant_prompt_files",
804
- owner=f"task {task_id}",
805
- ),
1046
+ variant_prompt_files=variant_prompt_files,
806
1047
  ))
807
1048
  if variants is not None:
808
1049
  validate_variant_prompt_file_references(fixtures, variants)
@@ -1498,72 +1739,93 @@ def append_csv(
1498
1739
  existing_key_cache: set[tuple[str, str]] | None = None,
1499
1740
  existing_key_cache_stamp: dict[str, tuple[int, int, int, int] | None] | None = None,
1500
1741
  ) -> bool:
1501
- with csv_file_lock(csv_path, create_parent=True):
1502
- key = (result.task_id, result.variant)
1503
- if skip_existing:
1504
- if existing_key_cache is not None:
1505
- refresh_existing_key_cache_unlocked(csv_path, existing_key_cache, existing_key_cache_stamp)
1506
- if key in existing_key_cache:
1507
- return False
1508
- elif key in _read_existing_keys_unlocked(csv_path):
1509
- return False
1510
- flags = os.O_CREAT | os.O_APPEND | os.O_WRONLY
1511
- fd = _open_regular_no_symlink(csv_path, flags, 0o600, create_parent=True)
1512
- try:
1513
- new_file = os.fstat(fd).st_size == 0
1514
- if not new_file:
1515
- validate_csv_schema(csv_path, read_csv_header_unlocked(csv_path))
1516
- with os.fdopen(fd, "a", encoding="utf-8", newline="") as f:
1517
- fd = -1
1518
- writer = csv.DictWriter(f, fieldnames=CSV_COLUMNS)
1519
- if new_file:
1520
- writer.writeheader()
1521
- tokens = result.tokens
1522
- total = sum(tokens.values())
1523
- shifted_cost_known = cost_shift_measured(result)
1524
- writer.writerow({
1525
- "date": sanitize_csv_cell(_dt.datetime.now().strftime("%Y-%m-%dT%H:%M:%S")),
1526
- "claude_version": sanitize_csv_cell(claude_ver),
1527
- "task_id": sanitize_csv_cell(result.task_id),
1528
- "variant": sanitize_csv_cell(result.variant),
1529
- "model": sanitize_csv_cell(result.model),
1530
- "effort": sanitize_csv_cell(result.effort),
1531
- "total_tokens": total,
1532
- "input_tokens": tokens.get("input_tokens", 0),
1533
- "output_tokens": tokens.get("output_tokens", 0),
1534
- "cache_read": tokens.get("cache_read", 0),
1535
- "cache_creation": tokens.get("cache_creation", 0),
1536
- "provider_cached_tokens": result.provider_cached_tokens,
1537
- "provider_cached_tokens_measured": (
1538
- "true" if result.provider_cached_tokens_measured else "false"
1539
- ),
1540
- "cost_usd": f"{result.cost_usd:.6f}",
1541
- "cost_measured": "true" if result.cost_measured else "false",
1542
- "wall_time_seconds": f"{result.wall_time_seconds:.6f}",
1543
- "turns": result.turns,
1544
- "hook_triggers": result.hook_triggers,
1545
- "bytes_before": result.bytes_before,
1546
- "bytes_after": result.bytes_after,
1547
- "artifacts_used": result.artifacts_used,
1548
- "external_tokens": result.external_tokens,
1549
- "external_tokens_measured": "true" if result.external_tokens_measured else "false",
1550
- "external_cost_usd": f"{result.external_cost_usd:.6f}",
1551
- "external_cost_measured": "true" if result.external_cost_measured else "false",
1552
- "total_cost_with_shift_usd": (
1553
- f"{(result.cost_usd + result.external_cost_usd):.6f}" if shifted_cost_known else ""
1554
- ),
1555
- "success": "true" if result.success else "false",
1556
- "corrections": result.corrections,
1557
- "notes": sanitize_csv_note(result.notes),
1558
- "primary_tokens_measured": "true" if result.primary_tokens_measured else "false",
1559
- })
1560
- finally:
1561
- if fd != -1:
1562
- os.close(fd)
1742
+ with csv_parent_directory_lock(csv_path, create_parent=True):
1743
+ with csv_file_lock(csv_path, create_parent=False):
1744
+ return append_csv_unlocked(
1745
+ csv_path,
1746
+ claude_ver,
1747
+ result,
1748
+ skip_existing=skip_existing,
1749
+ existing_key_cache=existing_key_cache,
1750
+ existing_key_cache_stamp=existing_key_cache_stamp,
1751
+ )
1752
+
1753
+
1754
+ def append_csv_unlocked(
1755
+ csv_path: Path,
1756
+ claude_ver: str,
1757
+ result: RunResult,
1758
+ *,
1759
+ skip_existing: bool = False,
1760
+ existing_key_cache: set[tuple[str, str]] | None = None,
1761
+ existing_key_cache_stamp: dict[str, tuple[int, int, int, int] | None] | None = None,
1762
+ ) -> bool:
1763
+ """Append one row while the caller holds the CSV transaction lock."""
1764
+ key = (result.task_id, result.variant)
1765
+ if skip_existing:
1563
1766
  if existing_key_cache is not None:
1564
- existing_key_cache.add(key)
1565
- if existing_key_cache_stamp is not None:
1566
- existing_key_cache_stamp["stamp"] = csv_file_stamp_unlocked(csv_path)
1767
+ refresh_existing_key_cache_unlocked(csv_path, existing_key_cache, existing_key_cache_stamp)
1768
+ if key in existing_key_cache:
1769
+ return False
1770
+ elif key in _read_existing_keys_unlocked(csv_path):
1771
+ return False
1772
+ flags = os.O_CREAT | os.O_APPEND | os.O_WRONLY
1773
+ fd = _open_regular_no_symlink(csv_path, flags, 0o600, create_parent=True)
1774
+ try:
1775
+ new_file = os.fstat(fd).st_size == 0
1776
+ if not new_file:
1777
+ validate_csv_schema(csv_path, read_csv_header_unlocked(csv_path))
1778
+ with os.fdopen(fd, "a", encoding="utf-8", newline="") as f:
1779
+ fd = -1
1780
+ writer = csv.DictWriter(f, fieldnames=CSV_COLUMNS)
1781
+ if new_file:
1782
+ writer.writeheader()
1783
+ tokens = result.tokens
1784
+ total = sum(tokens.values())
1785
+ shifted_cost_known = cost_shift_measured(result)
1786
+ writer.writerow({
1787
+ "date": sanitize_csv_cell(_dt.datetime.now().strftime("%Y-%m-%dT%H:%M:%S")),
1788
+ "claude_version": sanitize_csv_cell(claude_ver),
1789
+ "task_id": sanitize_csv_cell(result.task_id),
1790
+ "variant": sanitize_csv_cell(result.variant),
1791
+ "model": sanitize_csv_cell(result.model),
1792
+ "effort": sanitize_csv_cell(result.effort),
1793
+ "total_tokens": total,
1794
+ "input_tokens": tokens.get("input_tokens", 0),
1795
+ "output_tokens": tokens.get("output_tokens", 0),
1796
+ "cache_read": tokens.get("cache_read", 0),
1797
+ "cache_creation": tokens.get("cache_creation", 0),
1798
+ "provider_cached_tokens": result.provider_cached_tokens,
1799
+ "provider_cached_tokens_measured": (
1800
+ "true" if result.provider_cached_tokens_measured else "false"
1801
+ ),
1802
+ "cost_usd": f"{result.cost_usd:.6f}",
1803
+ "cost_measured": "true" if result.cost_measured else "false",
1804
+ "wall_time_seconds": f"{result.wall_time_seconds:.6f}",
1805
+ "turns": result.turns,
1806
+ "hook_triggers": result.hook_triggers,
1807
+ "bytes_before": result.bytes_before,
1808
+ "bytes_after": result.bytes_after,
1809
+ "artifacts_used": result.artifacts_used,
1810
+ "external_tokens": result.external_tokens,
1811
+ "external_tokens_measured": "true" if result.external_tokens_measured else "false",
1812
+ "external_cost_usd": f"{result.external_cost_usd:.6f}",
1813
+ "external_cost_measured": "true" if result.external_cost_measured else "false",
1814
+ "total_cost_with_shift_usd": (
1815
+ f"{(result.cost_usd + result.external_cost_usd):.6f}" if shifted_cost_known else ""
1816
+ ),
1817
+ "success": "true" if result.success else "false",
1818
+ "corrections": result.corrections,
1819
+ "notes": sanitize_csv_note(result.notes),
1820
+ "primary_tokens_measured": "true" if result.primary_tokens_measured else "false",
1821
+ })
1822
+ finally:
1823
+ if fd != -1:
1824
+ os.close(fd)
1825
+ if existing_key_cache is not None:
1826
+ existing_key_cache.add(key)
1827
+ if existing_key_cache_stamp is not None:
1828
+ existing_key_cache_stamp["stamp"] = csv_file_stamp_unlocked(csv_path)
1567
1829
  return True
1568
1830
 
1569
1831
 
@@ -2021,6 +2283,26 @@ def parse_evidence_row(raw_value: Any, *, owner: str, line_number: int) -> Evide
2021
2283
  primary_tokens_measured=primary_tokens_measured,
2022
2284
  self_hosted_metrics=self_hosted_metrics,
2023
2285
  )
2286
+ # Profile metadata 는 추가 필드로만 수집한다. 깊은 검증은 preflight 에서 task/prompt
2287
+ # 문맥과 함께 수행하며, 어떤 출력도 기록되기 전에 끝난다.
2288
+ # profile 오류는 evidence 파일 경로를 절대 echo 하지 않는다. 줄 번호만으로 위치를 준다.
2289
+ profile_row_owner = f"evidence line {line_number}"
2290
+ evaluation_profile = raw.get("evaluation_profile")
2291
+ if evaluation_profile is not None and (
2292
+ not isinstance(evaluation_profile, str)
2293
+ or evaluation_profile not in SUPPORTED_EVALUATION_PROFILE_IDS
2294
+ ):
2295
+ profile_reject(
2296
+ PROFILE_REJECT_SCHEMA_INVALID, profile_row_owner,
2297
+ "evaluation_profile is not a supported profile id",
2298
+ )
2299
+ evaluation_controls = raw.get("evaluation_controls")
2300
+ if evaluation_controls is not None and not isinstance(evaluation_controls, dict):
2301
+ profile_reject(
2302
+ PROFILE_REJECT_SCHEMA_INVALID, profile_row_owner,
2303
+ "evaluation_controls must be a JSON object",
2304
+ )
2305
+
2024
2306
  return EvidenceReplayRow(
2025
2307
  result=result,
2026
2308
  source_type=str(provenance["source_type"]),
@@ -2031,6 +2313,8 @@ def parse_evidence_row(raw_value: Any, *, owner: str, line_number: int) -> Evide
2031
2313
  public_claim_eligible=False,
2032
2314
  explicit_notes=explicit_notes,
2033
2315
  line_number=line_number,
2316
+ evaluation_profile=evaluation_profile,
2317
+ evaluation_controls=evaluation_controls,
2034
2318
  )
2035
2319
 
2036
2320
 
@@ -2236,6 +2520,860 @@ def measurement_baseline_contract() -> dict[str, Any]:
2236
2520
  }
2237
2521
 
2238
2522
 
2523
+ # --- image-context evaluation profile: bounded validation + fail-closed preflight ---
2524
+ #
2525
+ # 검증 순서는 불변이다: 타입/바운드 검사가 항상 semantic 정책 분류보다 먼저 실행된다.
2526
+ # 따라서 oversize 된 non-`deny` 정책 값이 blocked-scorecard 분기로 새어나갈 수 없다.
2527
+ # 구조적으로 해석 불가능한 evidence 는 어떤 출력 바이트도 쓰이기 전에 거부(reject_prewrite)되고,
2528
+ # 형식이 올바른 negative evidence 는 수용되어 blocked lane score 로 보고된다.
2529
+
2530
+ PROFILE_CONTROL_BLOCK_KEYS = (
2531
+ "control_provenance",
2532
+ "exact_text_fallback",
2533
+ "human_correction",
2534
+ "missed_context_review",
2535
+ "prompt_evidence",
2536
+ "protected_zone_review",
2537
+ "provider_usage",
2538
+ "shifted_cost",
2539
+ "source_omission",
2540
+ )
2541
+ PROFILE_NESTED_KEYS: dict[str, tuple[str, ...]] = {
2542
+ "control_provenance": ("review_source", "verifier_label"),
2543
+ "exact_text_fallback": (
2544
+ "available", "verified", "receipt_id", "content_sha256",
2545
+ "retrieval_command", "verifier_projection",
2546
+ ),
2547
+ "human_correction": ("count", "reason"),
2548
+ "missed_context_review": ("correction_required", "present", "review_completed", "summary"),
2549
+ "prompt_evidence": ("sha256", "source_label"),
2550
+ "protected_zone_review": (
2551
+ "included_prompt_like_regions", "included_protected_regions", "policy",
2552
+ "review_completed", "review_note", "reviewer_label",
2553
+ ),
2554
+ "provider_usage": ("primary_cost_measured", "primary_tokens_measured", "provider_called"),
2555
+ "shifted_cost": ("external_cost_measured", "external_tokens_measured", "status"),
2556
+ "source_omission": ("present", "transform"),
2557
+ }
2558
+ PROFILE_PROJECTION_KEYS = (
2559
+ "schema", "status", "blockers", "candidate_replacement", "claim_boundary", "proof_unit",
2560
+ )
2561
+ PROFILE_PROOF_UNIT_KEYS = (
2562
+ "status", "receipt_id", "receipt_verified", "content_hash_declared_value",
2563
+ "content_hash_verified", "rehydration_receipt_bound", "rehydration_syntax_valid",
2564
+ "rehydration_verified", "rehydration_executed", "retrieval_command",
2565
+ )
2566
+ PROFILE_PROOF_UNIT_REQUIRED_FLAGS = (
2567
+ "receipt_verified",
2568
+ "content_hash_verified",
2569
+ "rehydration_receipt_bound",
2570
+ "rehydration_syntax_valid",
2571
+ "rehydration_verified",
2572
+ )
2573
+ PROFILE_SHIFTED_COST_STATUSES = ("measured", "unmeasured")
2574
+
2575
+
2576
+ def redact_profile_label(value: Any) -> str:
2577
+ """Bound one untrusted label so an error message can never carry a payload.
2578
+
2579
+ Task ids, variant names, unknown evidence keys, and prompt-map labels are
2580
+ author-controlled. Opted-in profile diagnostics must use a fully opaque fixed
2581
+ representation for every such value — even regex-safe-looking identifiers —
2582
+ because any preserved attacker text is itself a leak channel. The stable error
2583
+ id and fixed field names carry the meaning; ``value`` is intentionally unused.
2584
+ """
2585
+ # 작성자 통제 값은 형태와 무관하게 절대 진단에 싣지 않는다.
2586
+ return PROFILE_REDACTED_PLACEHOLDER
2587
+
2588
+
2589
+ def redact_profile_labels(values: Iterable[Any]) -> str:
2590
+ """Render a bounded, fully opaque key list for schema errors.
2591
+
2592
+ The count is always truthful; each name collapses to the shared placeholder so
2593
+ no author-controlled text rides the message. The list is truncated so an
2594
+ oversized evidence object cannot flood the diagnostic.
2595
+ """
2596
+ labels = [redact_profile_label(value) for value in values]
2597
+ shown = labels[:MAX_PROFILE_ERROR_LABELS]
2598
+ overflow = len(labels) - len(shown)
2599
+ rendered = ", ".join(shown)
2600
+ if overflow > 0:
2601
+ rendered = f"{rendered}, +{overflow} more"
2602
+ return rendered
2603
+
2604
+
2605
+ def profile_owner(task_id: Any, variant: Any = None) -> str:
2606
+ """Build the redacted owner prefix shared by every profile error."""
2607
+ owner = f"task {redact_profile_label(task_id)}"
2608
+ if variant is not None:
2609
+ owner = f"{owner} variant {redact_profile_label(variant)}"
2610
+ return owner
2611
+
2612
+
2613
+ def profile_reject(error_id: str, owner: str, detail: str) -> "NoReturn":
2614
+ """Fail closed with a stable id and a bounded, redacted message.
2615
+
2616
+ Raw policy text, prompt content, and filesystem paths are never echoed. ``owner``
2617
+ must already come from :func:`profile_owner` and ``detail`` must be a fixed
2618
+ literal or a redacted label list; the final message is sanitized again so no
2619
+ caller can smuggle a secret-shaped value through.
2620
+ """
2621
+ raise SystemExit(sanitize_note_text(f"{error_id}: {owner} {detail}"))
2622
+
2623
+
2624
+ def profile_block(controls: dict[str, Any], key: str, *, owner: str) -> dict[str, Any]:
2625
+ if key not in controls:
2626
+ profile_reject(PROFILE_REJECT_CONTROLS_MISSING, owner, f"evaluation_controls.{key} is required")
2627
+ value = controls[key]
2628
+ if not isinstance(value, dict):
2629
+ profile_reject(PROFILE_REJECT_SCHEMA_INVALID, owner, f"evaluation_controls.{key} must be an object")
2630
+ unknown = sorted(set(value) - set(PROFILE_NESTED_KEYS[key]))
2631
+ if unknown:
2632
+ profile_reject(
2633
+ PROFILE_REJECT_SCHEMA_INVALID, owner,
2634
+ f"evaluation_controls.{key} has unknown v1 key(s): {redact_profile_labels(unknown)}",
2635
+ )
2636
+ return value
2637
+
2638
+
2639
+ def profile_bool(block: dict[str, Any], key: str, *, owner: str, label: str) -> bool:
2640
+ value = block.get(key)
2641
+ if not isinstance(value, bool):
2642
+ profile_reject(PROFILE_REJECT_SCHEMA_INVALID, owner, f"{label}.{key} must be a boolean")
2643
+ return value
2644
+
2645
+
2646
+ def profile_int(block: dict[str, Any], key: str, *, owner: str, label: str, maximum: int) -> int:
2647
+ value = block.get(key)
2648
+ if isinstance(value, bool) or not isinstance(value, int):
2649
+ profile_reject(PROFILE_REJECT_SCHEMA_INVALID, owner, f"{label}.{key} must be an integer")
2650
+ if value < 0 or value > maximum:
2651
+ profile_reject(PROFILE_REJECT_SCHEMA_INVALID, owner, f"{label}.{key} is outside its allowed bounds")
2652
+ return value
2653
+
2654
+
2655
+ def profile_text(block: dict[str, Any], key: str, *, owner: str, label: str, maximum: int) -> str:
2656
+ value = block.get(key)
2657
+ if not isinstance(value, str):
2658
+ profile_reject(PROFILE_REJECT_SCHEMA_INVALID, owner, f"{label}.{key} must be a string")
2659
+ if len(value.encode("utf-8")) > maximum:
2660
+ # 값 자체는 절대 에러에 실지 않는다. 길이 위반 사실만 보고한다.
2661
+ profile_reject(PROFILE_REJECT_SCHEMA_INVALID, owner, f"{label}.{key} exceeds its {maximum}-byte bound")
2662
+ return value
2663
+
2664
+
2665
+ def profile_variant_prompt_sha256(task: TaskFixture, variant_name: str, *, task_file_dir: Path, owner: str) -> str:
2666
+ """SHA-256 of the selected variant prompt file, read with the existing safe reader."""
2667
+ raw_path = task.variant_prompt_files.get(variant_name)
2668
+ if not raw_path:
2669
+ profile_reject(
2670
+ PROFILE_REJECT_PROMPT_BINDING_INVALID, owner,
2671
+ "a profiled task requires a file-backed variant_prompt_files entry for every selected variant",
2672
+ )
2673
+ try:
2674
+ rel_path = validate_variant_prompt_file_path(raw_path, owner=owner)
2675
+ text = read_variant_prompt_file(
2676
+ task_file_dir / rel_path, owner=owner, display_path=str(rel_path),
2677
+ )
2678
+ except (SystemExit, ValueError, UnicodeError):
2679
+ # SystemExit: 경로/내용 누설 방지용 안정 프로파일 오류 재작성.
2680
+ # ValueError/UnicodeError: 조기 검증을 지나친 os.open NUL·인코딩 거부 등만 좁게 정규화.
2681
+ # 프로그래머 버그를 숨기지 않도록 범용 Exception 은 잡지 않는다.
2682
+ profile_reject(
2683
+ PROFILE_REJECT_PROMPT_BINDING_INVALID, owner,
2684
+ "the selected variant prompt file could not be safely read",
2685
+ )
2686
+ return hashlib.sha256(text.encode("utf-8")).hexdigest()
2687
+
2688
+
2689
+ def validate_profile_fallback_projection(fallback: dict[str, Any], *, owner: str) -> None:
2690
+ """Validate one bounded imported local-verifier projection.
2691
+
2692
+ A record that *claims* verification while contradicting its own binding fields is a
2693
+ structural rejection. Replay never authenticates the record's author and never
2694
+ rereads the artifact; this only checks internal consistency.
2695
+ """
2696
+ # 검증을 주장하려면 fallback 자체가 available 이어야 한다. available=false 인데
2697
+ # verified=true 인 레코드는 자기 모순이다.
2698
+ if fallback.get("available") is not True:
2699
+ profile_reject(
2700
+ PROFILE_REJECT_FALLBACK_CLAIM_INCONSISTENT, owner,
2701
+ "exact_text_fallback claims verification while declaring the fallback unavailable",
2702
+ )
2703
+ # placeholder receipt/command 로는 어떤 것도 되찾을 수 없다. 검증 주장에는 실제
2704
+ # 값이 필요하다.
2705
+ for key in ("receipt_id", "retrieval_command"):
2706
+ value = fallback.get(key)
2707
+ if not isinstance(value, str) or value.strip().lower() in PROFILE_FALLBACK_PLACEHOLDER_VALUES:
2708
+ profile_reject(
2709
+ PROFILE_REJECT_FALLBACK_CLAIM_INCONSISTENT, owner,
2710
+ f"exact_text_fallback claims verification without an exact {key}",
2711
+ )
2712
+ projection = fallback.get("verifier_projection")
2713
+ if not isinstance(projection, dict):
2714
+ profile_reject(
2715
+ PROFILE_REJECT_FALLBACK_CLAIM_INCONSISTENT, owner,
2716
+ "exact_text_fallback claims verification without a bounded verifier projection",
2717
+ )
2718
+ unknown = sorted(set(projection) - set(PROFILE_PROJECTION_KEYS))
2719
+ if unknown:
2720
+ profile_reject(
2721
+ PROFILE_REJECT_SCHEMA_INVALID, owner,
2722
+ f"exact_text_fallback.verifier_projection has unknown v1 key(s): {redact_profile_labels(unknown)}",
2723
+ )
2724
+ if projection.get("schema") != PROOF_VERIFICATION_SCHEMA_VERSION:
2725
+ profile_reject(
2726
+ PROFILE_REJECT_FALLBACK_CLAIM_INCONSISTENT, owner,
2727
+ "verifier projection schema does not match the local proof-verification contract",
2728
+ )
2729
+ if projection.get("status") != PROOF_VERIFICATION_VERIFIED_STATUS:
2730
+ profile_reject(
2731
+ PROFILE_REJECT_FALLBACK_CLAIM_INCONSISTENT, owner,
2732
+ "exact_text_fallback claims verification but the verifier status is not verified",
2733
+ )
2734
+ blockers = projection.get("blockers")
2735
+ if not isinstance(blockers, list) or len(blockers) > MAX_PROFILE_BLOCKER_ITEMS:
2736
+ profile_reject(PROFILE_REJECT_SCHEMA_INVALID, owner, "verifier projection blockers must be a bounded list")
2737
+ if blockers:
2738
+ profile_reject(
2739
+ PROFILE_REJECT_FALLBACK_CLAIM_INCONSISTENT, owner,
2740
+ "exact_text_fallback claims verification but the verifier reports blockers",
2741
+ )
2742
+ if projection.get("candidate_replacement") is not None:
2743
+ profile_reject(
2744
+ PROFILE_REJECT_FALLBACK_CLAIM_INCONSISTENT, owner,
2745
+ "exact_text_fallback claims verification but the verifier proposes a candidate replacement",
2746
+ )
2747
+ # 가져온 레코드는 local-only 경계를 그대로 선언해야 한다. 경계가 다르면 이 lane 이
2748
+ # 인정할 수 있는 권한 범위를 벗어난 주장이다.
2749
+ if projection.get("claim_boundary") != PROOF_VERIFICATION_CLAIM_BOUNDARY:
2750
+ profile_reject(
2751
+ PROFILE_REJECT_FALLBACK_CLAIM_INCONSISTENT, owner,
2752
+ "exact_text_fallback claims verification without the expected local-only claim boundary",
2753
+ )
2754
+ unit = projection.get("proof_unit")
2755
+ if not isinstance(unit, dict):
2756
+ profile_reject(
2757
+ PROFILE_REJECT_FALLBACK_CLAIM_INCONSISTENT, owner,
2758
+ "exact_text_fallback claims verification without exactly one verified proof unit",
2759
+ )
2760
+ unknown_unit = sorted(set(unit) - set(PROFILE_PROOF_UNIT_KEYS))
2761
+ if unknown_unit:
2762
+ profile_reject(
2763
+ PROFILE_REJECT_SCHEMA_INVALID, owner,
2764
+ f"verifier proof_unit has unknown v1 key(s): {redact_profile_labels(unknown_unit)}",
2765
+ )
2766
+ if unit.get("status") != PROOF_VERIFICATION_VERIFIED_STATUS:
2767
+ profile_reject(
2768
+ PROFILE_REJECT_FALLBACK_CLAIM_INCONSISTENT, owner,
2769
+ "exact_text_fallback claims verification but its proof unit is not verified",
2770
+ )
2771
+ for flag in PROFILE_PROOF_UNIT_REQUIRED_FLAGS:
2772
+ if unit.get(flag) is not True:
2773
+ profile_reject(
2774
+ PROFILE_REJECT_FALLBACK_CLAIM_INCONSISTENT, owner,
2775
+ f"exact_text_fallback claims verification but proof_unit.{flag} does not confirm it",
2776
+ )
2777
+ # local verifier 는 rehydration 을 실행하지 않는다. 실행했다고 주장하면 이 레코드는
2778
+ # 우리가 검증할 수 있는 evaluation-only 경계 밖의 산출물이다.
2779
+ if unit.get("rehydration_executed") is not PROOF_VERIFICATION_REHYDRATION_EXECUTED:
2780
+ profile_reject(
2781
+ PROFILE_REJECT_FALLBACK_CLAIM_INCONSISTENT, owner,
2782
+ "exact_text_fallback claims verification but proof_unit.rehydration_executed leaves the local-only boundary",
2783
+ )
2784
+ if unit.get("receipt_id") != fallback["receipt_id"]:
2785
+ profile_reject(
2786
+ PROFILE_REJECT_FALLBACK_CLAIM_INCONSISTENT, owner,
2787
+ "exact_text_fallback receipt id is not bound to the verified proof unit",
2788
+ )
2789
+ if unit.get("content_hash_declared_value") != fallback["content_sha256"]:
2790
+ profile_reject(
2791
+ PROFILE_REJECT_FALLBACK_CLAIM_INCONSISTENT, owner,
2792
+ "exact_text_fallback content hash is not bound to the verified proof unit",
2793
+ )
2794
+ if unit.get("retrieval_command") != fallback["retrieval_command"]:
2795
+ profile_reject(
2796
+ PROFILE_REJECT_FALLBACK_CLAIM_INCONSISTENT, owner,
2797
+ "exact_text_fallback retrieval command is not bound to the verified proof unit",
2798
+ )
2799
+
2800
+
2801
+ def validate_profile_row_controls(
2802
+ row: EvidenceReplayRow,
2803
+ task: TaskFixture,
2804
+ *,
2805
+ task_file_dir: Path,
2806
+ ) -> dict[str, Any]:
2807
+ """Validate one profiled evidence row and return its normalized lane record."""
2808
+ owner = profile_owner(task.id, row.result.variant)
2809
+ controls = row.evaluation_controls
2810
+ if controls is None:
2811
+ profile_reject(PROFILE_REJECT_CONTROLS_MISSING, owner, "evaluation_controls is required for a profiled row")
2812
+ unknown = sorted(set(controls) - set(PROFILE_CONTROL_BLOCK_KEYS))
2813
+ if unknown:
2814
+ profile_reject(
2815
+ PROFILE_REJECT_SCHEMA_INVALID, owner,
2816
+ f"evaluation_controls has unknown v1 key(s): {redact_profile_labels(unknown)}",
2817
+ )
2818
+
2819
+ # --- prompt binding -------------------------------------------------
2820
+ prompt_evidence = profile_block(controls, "prompt_evidence", owner=owner)
2821
+ declared_sha = profile_text(prompt_evidence, "sha256", owner=owner, label="prompt_evidence", maximum=64)
2822
+ profile_text(prompt_evidence, "source_label", owner=owner, label="prompt_evidence", maximum=MAX_PROFILE_LABEL_CHARS)
2823
+ if not SHA256_HEX_PATTERN.match(declared_sha):
2824
+ profile_reject(PROFILE_REJECT_SCHEMA_INVALID, owner, "prompt_evidence.sha256 must be lowercase hex SHA-256")
2825
+ actual_sha = profile_variant_prompt_sha256(task, row.result.variant, task_file_dir=task_file_dir, owner=owner)
2826
+ if declared_sha != actual_sha:
2827
+ profile_reject(
2828
+ PROFILE_REJECT_PROMPT_BINDING_INVALID, owner,
2829
+ "prompt_evidence.sha256 does not match the locally recomputed prompt hash",
2830
+ )
2831
+
2832
+ # --- omission + exact-text fallback ---------------------------------
2833
+ source_omission = profile_block(controls, "source_omission", owner=owner)
2834
+ omission_present = profile_bool(source_omission, "present", owner=owner, label="source_omission")
2835
+ profile_text(source_omission, "transform", owner=owner, label="source_omission", maximum=MAX_PROFILE_LABEL_CHARS)
2836
+
2837
+ fallback = profile_block(controls, "exact_text_fallback", owner=owner)
2838
+ profile_bool(fallback, "available", owner=owner, label="exact_text_fallback")
2839
+ fallback_verified = profile_bool(fallback, "verified", owner=owner, label="exact_text_fallback")
2840
+ profile_text(fallback, "receipt_id", owner=owner, label="exact_text_fallback", maximum=MAX_PROFILE_RECEIPT_ID_CHARS)
2841
+ fallback_sha = profile_text(
2842
+ fallback, "content_sha256", owner=owner, label="exact_text_fallback", maximum=MAX_PROFILE_RECEIPT_ID_CHARS,
2843
+ )
2844
+ profile_text(
2845
+ fallback, "retrieval_command", owner=owner, label="exact_text_fallback", maximum=MAX_PROFILE_COMMAND_CHARS,
2846
+ )
2847
+ if fallback_verified:
2848
+ if not SHA256_HEX_PATTERN.match(fallback_sha):
2849
+ profile_reject(
2850
+ PROFILE_REJECT_SCHEMA_INVALID, owner,
2851
+ "exact_text_fallback.content_sha256 must be lowercase hex SHA-256 when verification is claimed",
2852
+ )
2853
+ validate_profile_fallback_projection(fallback, owner=owner)
2854
+ # 생략된 원문이 있는데 통과한 attestation 이 없으면 lane 을 막는다(거부가 아니다).
2855
+ fallback_bound = (not omission_present) or fallback_verified
2856
+
2857
+ # --- protected zone review ------------------------------------------
2858
+ protection = profile_block(controls, "protected_zone_review", owner=owner)
2859
+ # 바운드 검사가 정책 의미 분류보다 먼저다: oversize 값은 blocked 가 아니라 reject 다.
2860
+ policy = profile_text(
2861
+ protection, "policy", owner=owner, label="protected_zone_review", maximum=MAX_PROFILE_POLICY_CHARS,
2862
+ )
2863
+ protection_completed = profile_bool(protection, "review_completed", owner=owner, label="protected_zone_review")
2864
+ included_protected = profile_int(
2865
+ protection, "included_protected_regions", owner=owner,
2866
+ label="protected_zone_review", maximum=MAX_PROFILE_PROTECTED_REGION_COUNT,
2867
+ )
2868
+ included_prompt_like = profile_int(
2869
+ protection, "included_prompt_like_regions", owner=owner,
2870
+ label="protected_zone_review", maximum=MAX_PROFILE_PROTECTED_REGION_COUNT,
2871
+ )
2872
+ profile_text(
2873
+ protection, "reviewer_label", owner=owner, label="protected_zone_review", maximum=MAX_PROFILE_LABEL_CHARS,
2874
+ )
2875
+ profile_text(
2876
+ protection, "review_note", owner=owner, label="protected_zone_review", maximum=MAX_PROFILE_NOTE_CHARS,
2877
+ )
2878
+ protection_attested = (
2879
+ policy == PROTECTED_ZONE_DENY_POLICY
2880
+ and protection_completed
2881
+ and included_protected == 0
2882
+ and included_prompt_like == 0
2883
+ )
2884
+
2885
+ # --- missed context review ------------------------------------------
2886
+ missed = profile_block(controls, "missed_context_review", owner=owner)
2887
+ missed_completed = profile_bool(missed, "review_completed", owner=owner, label="missed_context_review")
2888
+ missed_present = profile_bool(missed, "present", owner=owner, label="missed_context_review")
2889
+ profile_bool(missed, "correction_required", owner=owner, label="missed_context_review")
2890
+ profile_text(missed, "summary", owner=owner, label="missed_context_review", maximum=MAX_PROFILE_SUMMARY_CHARS)
2891
+ missed_reviewed = missed_completed and not missed_present
2892
+
2893
+ # --- human correction consistency -----------------------------------
2894
+ correction = profile_block(controls, "human_correction", owner=owner)
2895
+ correction_count = profile_int(
2896
+ correction, "count", owner=owner, label="human_correction", maximum=MAX_PROFILE_CORRECTION_COUNT,
2897
+ )
2898
+ correction_reason = profile_text(
2899
+ correction, "reason", owner=owner, label="human_correction", maximum=MAX_PROFILE_NOTE_CHARS,
2900
+ )
2901
+ if correction_count != row.result.corrections:
2902
+ profile_reject(
2903
+ PROFILE_REJECT_CORRECTION_INCONSISTENT, owner,
2904
+ "human_correction.count does not equal the row's top-level corrections field",
2905
+ )
2906
+ if correction_count > 0 and (not correction_reason.strip() or correction_reason.strip().lower() == "none"):
2907
+ profile_reject(
2908
+ PROFILE_REJECT_CORRECTION_INCONSISTENT, owner,
2909
+ "a positive human_correction.count requires an explicit bounded reason",
2910
+ )
2911
+
2912
+ # --- measurement flags may never upgrade the generic normalized fields
2913
+ provider_usage = profile_block(controls, "provider_usage", owner=owner)
2914
+ lane_tokens_measured = profile_bool(provider_usage, "primary_tokens_measured", owner=owner, label="provider_usage")
2915
+ lane_cost_measured = profile_bool(provider_usage, "primary_cost_measured", owner=owner, label="provider_usage")
2916
+ profile_bool(provider_usage, "provider_called", owner=owner, label="provider_usage")
2917
+ if lane_tokens_measured != row.result.primary_tokens_measured or lane_cost_measured != row.result.cost_measured:
2918
+ profile_reject(
2919
+ PROFILE_REJECT_MEASUREMENT_INCONSISTENT, owner,
2920
+ "provider_usage measurement flags contradict the generic normalized provider fields",
2921
+ )
2922
+
2923
+ shifted = profile_block(controls, "shifted_cost", owner=owner)
2924
+ lane_external_tokens = profile_bool(shifted, "external_tokens_measured", owner=owner, label="shifted_cost")
2925
+ lane_external_cost = profile_bool(shifted, "external_cost_measured", owner=owner, label="shifted_cost")
2926
+ shifted_status = profile_text(
2927
+ shifted, "status", owner=owner, label="shifted_cost", maximum=MAX_PROFILE_LABEL_CHARS,
2928
+ )
2929
+ if shifted_status not in PROFILE_SHIFTED_COST_STATUSES:
2930
+ profile_reject(PROFILE_REJECT_SCHEMA_INVALID, owner, "shifted_cost.status must be measured or unmeasured")
2931
+ if (
2932
+ lane_external_tokens != row.result.external_tokens_measured
2933
+ or lane_external_cost != row.result.external_cost_measured
2934
+ ):
2935
+ profile_reject(
2936
+ PROFILE_REJECT_MEASUREMENT_INCONSISTENT, owner,
2937
+ "shifted_cost measurement flags contradict the generic normalized shifted-cost fields",
2938
+ )
2939
+ if (shifted_status == "measured") != (lane_external_tokens and lane_external_cost):
2940
+ profile_reject(
2941
+ PROFILE_REJECT_MEASUREMENT_INCONSISTENT, owner,
2942
+ "shifted_cost.status contradicts its own measurement flags",
2943
+ )
2944
+
2945
+ provenance_block = profile_block(controls, "control_provenance", owner=owner)
2946
+ profile_text(
2947
+ provenance_block, "review_source", owner=owner, label="control_provenance", maximum=MAX_PROFILE_LABEL_CHARS,
2948
+ )
2949
+ profile_text(
2950
+ provenance_block, "verifier_label", owner=owner, label="control_provenance", maximum=MAX_PROFILE_LABEL_CHARS,
2951
+ )
2952
+
2953
+ # 정규화된 lane 판정만 보고에 전달한다. 자유 텍스트(정책/노트/요약/라벨)는 절대
2954
+ # 포함하지 않으므로 secret-shaped 값이 report/dashboard 로 새어나갈 수 없다.
2955
+ return {
2956
+ "task_id": task.id,
2957
+ "variant": row.result.variant,
2958
+ "success": bool(row.result.success),
2959
+ "source_omission_present": omission_present,
2960
+ "fallback_bound": fallback_bound,
2961
+ "fallback_verified": fallback_verified,
2962
+ # 아무 verifier 레코드도 제출되지 않은 경우("missing")와 제출되었으나 실패를
2963
+ # 보고하는 경우("failed")는 서로 다른 증거 수준이다.
2964
+ "fallback_projection_supplied": fallback.get("verifier_projection") is not None,
2965
+ "protected_zone_attested": protection_attested,
2966
+ "missed_context_reviewed": missed_reviewed,
2967
+ # lane 이 correction 판정을 lane 데이터에서 직접 유도할 수 있도록 정규화해 넘긴다.
2968
+ "human_correction_consistent": correction_count == row.result.corrections,
2969
+ "provider_measured": bool(row.result.primary_tokens_measured and row.result.cost_measured),
2970
+ "shifted_cost_measured": bool(lane_external_tokens and lane_external_cost),
2971
+ }
2972
+
2973
+
2974
+ def selected_profiled_task_ids(
2975
+ tasks: list[TaskFixture],
2976
+ targets: list[tuple[TaskFixture, Variant]],
2977
+ ) -> list[str]:
2978
+ """Profiled task ids that this invocation actually selected, in stable order."""
2979
+ profiled = {task.id for task in tasks if task.evaluation_profile is not None}
2980
+ selected = {task.id for task, _ in targets}
2981
+ return sorted(profiled & selected)
2982
+
2983
+
2984
+ def preflight_profile_replay_mode(
2985
+ tasks: list[TaskFixture],
2986
+ targets: list[tuple[TaskFixture, Variant]],
2987
+ *,
2988
+ evidence_replay_active: bool,
2989
+ ) -> None:
2990
+ """Refuse a profiled task outside evidence replay, before any provider call.
2991
+
2992
+ The profile is evaluation-only: it validates imported evidence and clamps every
2993
+ authority surface. Those checks live entirely on the replay path, so running a
2994
+ profiled task through the provider would execute a real run whose report never
2995
+ sees profile validation or the evaluation-only clamp. Fail closed instead, before
2996
+ the provider runtime, the lock sidecar, and the first output byte.
2997
+
2998
+ This also refuses ``--dry-run``. A dry run writes nothing and spawns no provider,
2999
+ so it is not itself dangerous, but keeping the invariant absolute — a profiled task
3000
+ never enters the provider path — is what makes the boundary auditable. The useful
3001
+ preview, ``--evidence-jsonl --dry-run``, is unaffected.
3002
+ """
3003
+ if evidence_replay_active:
3004
+ return
3005
+ for task_id in selected_profiled_task_ids(tasks, targets):
3006
+ profile_reject(
3007
+ PROFILE_REJECT_REPLAY_REQUIRED, profile_owner(task_id),
3008
+ "a profiled task is evaluation-only and runs only under --evidence-jsonl replay; "
3009
+ "provider execution is refused",
3010
+ )
3011
+
3012
+
3013
+ def preflight_profile_fresh_output(
3014
+ tasks: list[TaskFixture],
3015
+ targets: list[tuple[TaskFixture, Variant]],
3016
+ *,
3017
+ resume: bool,
3018
+ csv_has_preexisting_content: bool,
3019
+ ) -> None:
3020
+ """Refuse a resumed or pre-existing profiled batch before any lock/read helper.
3021
+
3022
+ v1 gives up incremental replay so profile context cannot silently vanish from a
3023
+ resumed report. This must run before the resume key snapshot, which acquires the
3024
+ CSV lock and therefore creates a ``.lock`` sidecar: rejecting afterwards would
3025
+ leave a byte on disk for a run we refused.
3026
+ """
3027
+ if not (resume or csv_has_preexisting_content):
3028
+ return
3029
+ for task_id in selected_profiled_task_ids(tasks, targets):
3030
+ profile_reject(
3031
+ PROFILE_REJECT_FRESH_OUTPUT_REQUIRED, profile_owner(task_id),
3032
+ "v1 profiled replay requires a fresh empty results CSV and forbids --resume",
3033
+ )
3034
+
3035
+
3036
+ def profile_batch_freshness_gate_unlocked(
3037
+ tasks: list[TaskFixture],
3038
+ targets: list[tuple[TaskFixture, Variant]],
3039
+ csv_path: Path,
3040
+ ) -> None:
3041
+ """Recheck output freshness while the caller holds the full-batch lock.
3042
+
3043
+ The pre-lock gate reads the CSV without the lock, so a concurrent writer could
3044
+ still land a row between that check and the first append. One recheck under the
3045
+ caller's held parent-directory lock closes that window for the whole batch.
3046
+ """
3047
+ profiled_task_ids = selected_profiled_task_ids(tasks, targets)
3048
+ if not profiled_task_ids:
3049
+ return
3050
+ if file_has_content_no_follow(csv_path):
3051
+ profile_reject(
3052
+ PROFILE_REJECT_FRESH_OUTPUT_REQUIRED,
3053
+ profile_owner(profiled_task_ids[0]),
3054
+ "the results CSV gained content after the profiled batch was validated",
3055
+ )
3056
+
3057
+
3058
+ def preflight_evaluation_profiles(
3059
+ tasks: list[TaskFixture],
3060
+ variants: list[Variant],
3061
+ targets: list[tuple[TaskFixture, Variant]],
3062
+ evidence_rows: list[EvidenceReplayRow],
3063
+ *,
3064
+ task_file_dir: Path,
3065
+ resume: bool,
3066
+ csv_has_preexisting_content: bool,
3067
+ baseline_variant: str = "baseline",
3068
+ ) -> None:
3069
+ """Validate the complete profiled batch before the first output byte is written.
3070
+
3071
+ Runs before any CSV/ledger/report/dashboard write and before any lock sidecar is
3072
+ created, so a rejection leaves the filesystem byte-unchanged. Attaches the
3073
+ normalized lane record to each row, which makes report annotation infallible over
3074
+ an already-validated batch.
3075
+ """
3076
+ # 같은 gate 를 main 이 lock helper 이전에 이미 호출한다. 여기서 다시 부르는 것은
3077
+ # 이 함수를 직접 쓰는 호출자도 같은 boundary 를 얻게 하기 위한 이중 방어다.
3078
+ preflight_profile_fresh_output(
3079
+ tasks, targets, resume=resume, csv_has_preexisting_content=csv_has_preexisting_content,
3080
+ )
3081
+ profiled_tasks = {task.id: task for task in tasks if task.evaluation_profile is not None}
3082
+ rows_by_task: dict[str, list[EvidenceReplayRow]] = collections.defaultdict(list)
3083
+ for row in evidence_rows:
3084
+ rows_by_task[row.result.task_id].append(row)
3085
+
3086
+ # profiled row 가 unprofiled task 에 붙는 경우도 binding 위반이다.
3087
+ for row in evidence_rows:
3088
+ task = profiled_tasks.get(row.result.task_id)
3089
+ if task is None and (row.evaluation_profile is not None or row.evaluation_controls is not None):
3090
+ profile_reject(
3091
+ PROFILE_REJECT_BINDING_MISMATCH,
3092
+ profile_owner(row.result.task_id, row.result.variant),
3093
+ "a profiled evidence row cannot be replayed against a task that does not declare the profile",
3094
+ )
3095
+ if not profiled_tasks:
3096
+ return
3097
+
3098
+ variant_names = {variant.name for variant in variants}
3099
+ if baseline_variant not in variant_names or len(variant_names - {baseline_variant}) < 1:
3100
+ profile_reject(
3101
+ PROFILE_REJECT_BATCH_INCOMPLETE,
3102
+ profile_owner(next(iter(sorted(profiled_tasks)))),
3103
+ "v1 profiled replay requires the configured baseline and at least one candidate variant",
3104
+ )
3105
+
3106
+ selected_by_task: dict[str, set[str]] = collections.defaultdict(set)
3107
+ for task, variant in targets:
3108
+ selected_by_task[task.id].add(variant.name)
3109
+
3110
+ for task_id, task in sorted(profiled_tasks.items()):
3111
+ selected = selected_by_task.get(task_id, set())
3112
+ if not selected:
3113
+ continue
3114
+ # v1 은 부분 배치를 허용하지 않는다: 선택된 배치가 모든 variant 를 덮어야 한다.
3115
+ expected = {variant.name for variant in variants}
3116
+ if selected != expected:
3117
+ profile_reject(
3118
+ PROFILE_REJECT_BATCH_INCOMPLETE, profile_owner(task_id),
3119
+ "v1 profiled replay requires the complete baseline/candidate batch; "
3120
+ "partial variant selection is not supported",
3121
+ )
3122
+ task_rows = rows_by_task.get(task_id, [])
3123
+ # 중복/여분 row 는 coverage 집합 계산에 흡수되어 조용히 통과했다. 배치가
3124
+ # 모호하면 어떤 lane 판정도 신뢰할 수 없으므로 안정 ID 로 먼저 거부한다.
3125
+ seen_variants: set[str] = set()
3126
+ for row in task_rows:
3127
+ owner = profile_owner(task_id, row.result.variant)
3128
+ if row.result.variant in seen_variants:
3129
+ profile_reject(
3130
+ PROFILE_REJECT_BATCH_INCOMPLETE, owner,
3131
+ "a profiled task cannot carry duplicate evidence rows for one variant",
3132
+ )
3133
+ seen_variants.add(row.result.variant)
3134
+ if row.result.variant not in expected:
3135
+ profile_reject(
3136
+ PROFILE_REJECT_BATCH_INCOMPLETE, owner,
3137
+ "a profiled batch cannot carry an evidence row for an unknown variant",
3138
+ )
3139
+ for row in task_rows:
3140
+ owner = profile_owner(task_id, row.result.variant)
3141
+ if row.evaluation_profile is None and row.evaluation_controls is None:
3142
+ profile_reject(
3143
+ PROFILE_REJECT_BATCH_INCOMPLETE, owner,
3144
+ "a profiled task cannot mix profiled and unprofiled evidence rows",
3145
+ )
3146
+ if row.evaluation_profile != task.evaluation_profile:
3147
+ profile_reject(
3148
+ PROFILE_REJECT_BINDING_MISMATCH, owner,
3149
+ "the evidence row profile does not equal the task profile",
3150
+ )
3151
+
3152
+ covered = {row.result.variant for row in task_rows}
3153
+ if not expected.issubset(covered):
3154
+ profile_reject(
3155
+ PROFILE_REJECT_BATCH_INCOMPLETE, profile_owner(task_id),
3156
+ "profiled replay requires complete baseline and candidate evidence coverage",
3157
+ )
3158
+ for row in task_rows:
3159
+ row.evaluation_lane = validate_profile_row_controls(row, task, task_file_dir=task_file_dir)
3160
+
3161
+
3162
+ def build_image_context_evaluation_lane(
3163
+ replay_rows: list[EvidenceReplayRow],
3164
+ report: dict[str, Any],
3165
+ ) -> dict[str, Any] | None:
3166
+ """Aggregate the validated lane records into the additive report block."""
3167
+ lanes = [row.evaluation_lane for row in replay_rows if row.evaluation_lane is not None]
3168
+ if not lanes:
3169
+ return None
3170
+
3171
+ provider_measured = all(lane["provider_measured"] for lane in lanes)
3172
+ shifted_measured = all(lane["shifted_cost_measured"] for lane in lanes)
3173
+ all_success = all(lane["success"] for lane in lanes)
3174
+ omission_lanes = [lane for lane in lanes if lane["source_omission_present"]]
3175
+
3176
+ # generic quality gate 결과를 lane 이 그대로 소비한다. profile 이 자기만의 판정으로
3177
+ # generic regression 을 덮어쓰지 못하게 하는 것이 목적이므로, 어떤 matched pair 가
3178
+ # regression 을 보고하면 lane 도 막힌다(보수적으로 fail-closed).
3179
+ #
3180
+ # 이름 붙은 두 gate 만 매핑하면 구멍이 남는다: quality_gate 는 corrections_regression /
3181
+ # failure_rate_regression 외에도 insufficient_corrections_data 같은 값을 낼 수 있고,
3182
+ # 그때 lane 이 ready 로 올라가면 안 된다. 따라서 불변식은 "pass 가 아니면 막는다" 이며,
3183
+ # 두 이름은 이유를 드러내는 구체 blocker 로 함께 유지한다.
3184
+ generic_quality_gates: set[str] = set()
3185
+ pairs = report.get("matched_pair_evidence")
3186
+ pair_keys: set[tuple[str, str]] = set()
3187
+ if isinstance(pairs, list):
3188
+ for pair in pairs:
3189
+ if isinstance(pair, dict) and isinstance(pair.get("quality_gate"), str):
3190
+ generic_quality_gates.add(pair["quality_gate"])
3191
+ if isinstance(pair.get("task_id"), str) and isinstance(pair.get("variant"), str):
3192
+ pair_keys.add((pair["task_id"], pair["variant"]))
3193
+ baseline_variant = report.get("baseline_variant")
3194
+ expected_pair_keys = {
3195
+ (lane["task_id"], lane["variant"])
3196
+ for lane in lanes
3197
+ if isinstance(baseline_variant, str) and lane["variant"] != baseline_variant
3198
+ }
3199
+ comparisons = report.get("comparisons")
3200
+ comparison_rows = [item for item in comparisons if isinstance(item, dict)] if isinstance(comparisons, list) else []
3201
+ expected_candidate_variants = {variant for _, variant in expected_pair_keys}
3202
+ comparison_variants = {
3203
+ item["variant"] for item in comparison_rows if isinstance(item.get("variant"), str)
3204
+ }
3205
+ all_comparisons_pass = (
3206
+ bool(comparison_rows)
3207
+ and comparison_variants == expected_candidate_variants
3208
+ and all(
3209
+ item.get("quality_gate") == GENERIC_QUALITY_GATE_PASS
3210
+ and isinstance(item.get("matched_successful_task_count"), int)
3211
+ and item["matched_successful_task_count"] > 0
3212
+ for item in comparison_rows
3213
+ )
3214
+ )
3215
+ complete_matched_pairs = bool(expected_pair_keys) and pair_keys == expected_pair_keys
3216
+ generic_quality_pass = (
3217
+ complete_matched_pairs
3218
+ and all_comparisons_pass
3219
+ and generic_quality_gates == {GENERIC_QUALITY_GATE_PASS}
3220
+ )
3221
+
3222
+ gate_results = {
3223
+ # preflight 를 통과했다면 profile/prompt binding 은 이미 증명되었다.
3224
+ IMAGE_CONTEXT_GATE_PROFILE_AND_PROMPT_BINDING: True,
3225
+ IMAGE_CONTEXT_GATE_PROTECTED_ZONE_DENY_REVIEW: all(lane["protected_zone_attested"] for lane in lanes),
3226
+ IMAGE_CONTEXT_GATE_EXACT_TEXT_FALLBACK_BINDING: all(lane["fallback_bound"] for lane in lanes),
3227
+ IMAGE_CONTEXT_GATE_MISSED_CONTEXT_REVIEW: all(lane["missed_context_reviewed"] for lane in lanes),
3228
+ # count/reason 모순은 거부되지만, 판정은 lane 레코드에서 직접 유도한다.
3229
+ IMAGE_CONTEXT_GATE_HUMAN_CORRECTION_CONSISTENCY: all(
3230
+ lane["human_correction_consistent"] for lane in lanes
3231
+ ),
3232
+ IMAGE_CONTEXT_GATE_CORRECTIONS_REGRESSION: (
3233
+ GENERIC_QUALITY_GATE_CORRECTIONS_REGRESSION not in generic_quality_gates
3234
+ ),
3235
+ IMAGE_CONTEXT_GATE_FAILURE_RATE_REGRESSION: (
3236
+ GENERIC_QUALITY_GATE_FAILURE_RATE_REGRESSION not in generic_quality_gates
3237
+ ),
3238
+ # generic quality gate 가 pass 가 아니면 어떤 값이든 여기서 막힌다.
3239
+ IMAGE_CONTEXT_GATE_GENERIC_MATCHED_SUCCESS_AND_MEASUREMENT: (
3240
+ all_success and provider_measured and shifted_measured and generic_quality_pass
3241
+ ),
3242
+ # 이 gate 는 통과해도 권한을 주지 않는다. 경계 자체가 불변이므로 항상 참이다.
3243
+ IMAGE_CONTEXT_GATE_EVALUATION_ONLY_PROMOTION_BOUNDARY: True,
3244
+ }
3245
+ blocking_gate_ids = [gate_id for gate_id in IMAGE_CONTEXT_GATE_IDS if not gate_results[gate_id]]
3246
+
3247
+ if not omission_lanes:
3248
+ fallback_level = "missing"
3249
+ elif all(lane["fallback_verified"] for lane in omission_lanes):
3250
+ fallback_level = IMPORTED_LOCAL_VERIFIER_ATTESTATION_LABEL
3251
+ elif any(
3252
+ lane["fallback_projection_supplied"] and not lane["fallback_verified"]
3253
+ for lane in omission_lanes
3254
+ ):
3255
+ # 검증 레코드가 제출되었지만 성공을 증명하지 못했다.
3256
+ fallback_level = "failed"
3257
+ else:
3258
+ # 생략은 선언되었으나 어떤 verifier attestation 도 제출되지 않았다.
3259
+ fallback_level = "missing"
3260
+
3261
+ matched_task_ids = sorted({lane["task_id"] for lane in lanes})
3262
+ return {
3263
+ "schema_version": IMAGE_CONTEXT_READINESS_SCHEMA_VERSION,
3264
+ "status": PROFILE_STATUS_BLOCKED if blocking_gate_ids else PROFILE_STATUS_READY_FOR_BOUNDED_PILOT_REVIEW,
3265
+ "evaluation_only": True,
3266
+ "promotion_authority": False,
3267
+ "public_claim_allowed": False,
3268
+ "gate_ids": list(IMAGE_CONTEXT_GATE_IDS),
3269
+ "blocking_gate_ids": blocking_gate_ids,
3270
+ "matched_task_count": len(matched_task_ids),
3271
+ "evidence_levels": {
3272
+ "provider_measurement": "measured" if provider_measured else "unmeasured",
3273
+ "fallback_binding": fallback_level,
3274
+ "protected_zone": (
3275
+ "review_attested"
3276
+ if gate_results[IMAGE_CONTEXT_GATE_PROTECTED_ZONE_DENY_REVIEW] else "failed"
3277
+ ),
3278
+ "missed_context": (
3279
+ "reviewed" if gate_results[IMAGE_CONTEXT_GATE_MISSED_CONTEXT_REVIEW] else "missing"
3280
+ ),
3281
+ },
3282
+ # 표본 관측치는 어떤 승격 임계값도 정의하지 않는다.
3283
+ "sample_adequacy": {
3284
+ "matched_task_count": len(matched_task_ids),
3285
+ "profiled_row_count": len(lanes),
3286
+ "task_class_labels": matched_task_ids,
3287
+ "policy_status": PROFILE_SAMPLE_ADEQUACY_POLICY_STATUS,
3288
+ },
3289
+ "claim_boundary": IMAGE_CONTEXT_CLAIM_BOUNDARY,
3290
+ }
3291
+
3292
+
3293
+ def clamp_report_for_evaluation_profile(report: dict[str, Any], lane: dict[str, Any]) -> None:
3294
+ """Force every public-authority surface to a non-candidate, evaluation-only value.
3295
+
3296
+ Complete lane evidence may reach ``ready_for_bounded_pilot_review``; it may never
3297
+ reach promotion authority or a public claim. Pre-clamp measurements survive only in
3298
+ explicitly non-authoritative fields such as ``raw_metric_claim_status``.
3299
+ """
3300
+ report[EVALUATION_PROFILES_REPORT_KEY] = {IMAGE_CONTEXT_PROFILE_REPORT_KEY: lane}
3301
+ report["public_claim_status"] = IMAGE_CONTEXT_EVALUATION_ONLY_CLAIM_STATUS
3302
+ # 콘솔이 report['claim_status'] 를 그대로 출력하므로 legacy 필드도 함께 clamp 한다.
3303
+ report["claim_status"] = IMAGE_CONTEXT_EVALUATION_ONLY_CLAIM_STATUS
3304
+ report["public_claim_eligible"] = False
3305
+
3306
+ # replay_evidence 는 top-level 과 같은 이름의 권한 필드를 복사해 들고 있다. 여기를
3307
+ # 함께 clamp 하지 않으면 report.json 소비자가 중첩 사본에서 candidate/eligible=true 를
3308
+ # 그대로 읽어 evaluation-only 경계를 우회한다.
3309
+ replay_evidence = report.get("replay_evidence")
3310
+ if isinstance(replay_evidence, dict):
3311
+ replay_evidence["public_claim_status"] = IMAGE_CONTEXT_EVALUATION_ONLY_CLAIM_STATUS
3312
+ replay_evidence["public_claim_eligible"] = False
3313
+ replay_evidence["report_claim_gates_allow_public_claim"] = False
3314
+
3315
+ readiness = report.get("public_claim_readiness")
3316
+ if isinstance(readiness, dict):
3317
+ readiness["claim_allowed"] = False
3318
+ readiness["status"] = IMAGE_CONTEXT_EVALUATION_ONLY_CLAIM_STATUS
3319
+ readiness["reason"] = IMAGE_CONTEXT_PROFILE_BLOCKER_GATE_ID
3320
+ # `_observed` 라는 이름이 붙었어도 값 자체가 "..._public_claim_candidate" 라는
3321
+ # 권한 문자열이다. 그대로 두면 downstream 이 이 필드를 읽고 candidate 로 오해할
3322
+ # 수 있으므로, profiled report 에서는 관측치도 evaluation-only 로 clamp 한다.
3323
+ readiness["public_claim_status_observed"] = IMAGE_CONTEXT_EVALUATION_ONLY_CLAIM_STATUS
3324
+ readiness["public_claim_eligible_observed"] = False
3325
+ blocking = readiness.get("blocking_gate_ids")
3326
+ if isinstance(blocking, list) and IMAGE_CONTEXT_PROFILE_BLOCKER_GATE_ID not in blocking:
3327
+ blocking.append(IMAGE_CONTEXT_PROFILE_BLOCKER_GATE_ID)
3328
+
3329
+ pairs = report.get("matched_pair_evidence")
3330
+ if isinstance(pairs, list):
3331
+ for pair in pairs:
3332
+ if not isinstance(pair, dict):
3333
+ continue
3334
+ boundary = pair.get("claim_boundary")
3335
+ if isinstance(boundary, dict):
3336
+ boundary["token_savings_claim_allowed"] = False
3337
+ boundary["shifted_cost_claim_allowed"] = False
3338
+ boundary["evaluation_profile"] = IMAGE_CONTEXT_EVALUATION_PROFILE_ID
3339
+
3340
+
3341
+ def render_image_context_evaluation_section(report: dict[str, Any]) -> list[str]:
3342
+ """Bounded dashboard section: statuses and ids only, never raw evidence text."""
3343
+ profiles = report.get(EVALUATION_PROFILES_REPORT_KEY)
3344
+ if not isinstance(profiles, dict):
3345
+ return []
3346
+ lane = profiles.get(IMAGE_CONTEXT_PROFILE_REPORT_KEY)
3347
+ if not isinstance(lane, dict):
3348
+ return []
3349
+ levels = lane.get("evidence_levels") if isinstance(lane.get("evidence_levels"), dict) else {}
3350
+ blockers = lane.get("blocking_gate_ids") or []
3351
+ sample = lane.get("sample_adequacy") if isinstance(lane.get("sample_adequacy"), dict) else {}
3352
+ return [
3353
+ "## Image-context evaluation",
3354
+ "",
3355
+ f"- Schema: `{markdown_value(lane.get('schema_version'))}`",
3356
+ f"- Status: `{markdown_value(lane.get('status'))}`",
3357
+ f"- Matched tasks: {markdown_value(lane.get('matched_task_count'))}",
3358
+ f"- Evaluation only: `{markdown_value(lane.get('evaluation_only'))}`",
3359
+ f"- Promotion authority: `{markdown_value(lane.get('promotion_authority'))}`",
3360
+ f"- Public claim allowed: `{markdown_value(lane.get('public_claim_allowed'))}`",
3361
+ f"- Provider measurement: `{markdown_value(levels.get('provider_measurement'))}`",
3362
+ f"- Fallback binding: `{markdown_value(levels.get('fallback_binding'))}`",
3363
+ f"- Protected zone: `{markdown_value(levels.get('protected_zone'))}`",
3364
+ f"- Missed context: `{markdown_value(levels.get('missed_context'))}`",
3365
+ f"- Blocking gates: `{markdown_value(', '.join(str(item) for item in blockers) if blockers else 'none')}`",
3366
+ f"- Sample policy: `{markdown_value(sample.get('policy_status'))}`",
3367
+ "",
3368
+ "> Claim boundary: this lane is evaluation-only. `ready_for_bounded_pilot_review` authorizes a "
3369
+ "bounded human pilot review of imported evidence; it is not promotion, not runtime authority, "
3370
+ "not quality proof, and not a hosted API token/cost savings claim. The fallback record is an "
3371
+ "imported local-verifier attestation: replay does not authenticate its author and does not "
3372
+ "reread the artifact.",
3373
+ "",
3374
+ ]
3375
+
3376
+
2239
3377
  def summarize_benchmark_rows(rows: list[dict[str, str]], baseline_variant: str) -> dict[str, Any]:
2240
3378
  by_variant: dict[str, dict[str, Any]] = {}
2241
3379
  successful_rows_by_variant_task: dict[str, dict[str, list[dict[str, str]]]] = {}
@@ -2967,6 +4105,11 @@ def annotate_replay_report(
2967
4105
  replay_rows=replay_rows,
2968
4106
  mixed_csv=mixed_csv,
2969
4107
  )
4108
+ # Additive lane block. 이미 preflight 로 검증된 batch 위에서만 동작하므로 실패하지 않는다.
4109
+ # profile 이 없는 report 는 이 블록도, clamp 도 얻지 않는다(기존 동작 그대로).
4110
+ lane = build_image_context_evaluation_lane(replay_rows, report)
4111
+ if lane is not None:
4112
+ clamp_report_for_evaluation_profile(report, lane)
2970
4113
  report["default_matrix"] = build_default_matrix(report)
2971
4114
  return report
2972
4115
 
@@ -3552,6 +4695,8 @@ def render_dashboard_markdown(report: dict[str, Any]) -> str:
3552
4695
  "allow it and public-claim provenance is complete. Proxy byte reductions are diagnostic "
3553
4696
  "and are not hosted API token savings.",
3554
4697
  "",
4698
+ # profile 이 선언된 report 에만 추가되는 bounded 섹션. 원문/정책/영수증 내용은 넣지 않는다.
4699
+ *render_image_context_evaluation_section(report),
3555
4700
  "## Variant summary",
3556
4701
  "",
3557
4702
  "| Variant | Runs | Successes | Failure rate | Tokens/success | Bytes saved | Token proxy saved | Quality notes |",
@@ -3869,6 +5014,25 @@ def main() -> int:
3869
5014
  print("no (task, variant) targets matched the filters", file=sys.stderr)
3870
5015
  return 1
3871
5016
 
5017
+ # profile gate 는 어떤 lock/read helper 보다 먼저 끝난다. existing_keys_snapshot 은
5018
+ # CSV lock sidecar 를 만들기 때문에, 그 뒤에서 거부하면 우리가 거절한 실행이 이미
5019
+ # 파일 시스템에 바이트를 남긴 뒤가 된다.
5020
+ #
5021
+ # replay 경계는 --dry-run 에도 적용한다. dry-run 이 provider 를 부르지 않는 것은
5022
+ # 맞지만, 불변식을 "profiled task 는 provider 경로에 진입하지 않는다" 로 단순하게
5023
+ # 유지하는 편이 감사 가능하고 fail-closed 다. 의미 있는 미리보기인
5024
+ # `--evidence-jsonl --dry-run` 은 그대로 동작한다.
5025
+ preflight_profile_replay_mode(tasks, targets, evidence_replay_active=args.evidence_jsonl is not None)
5026
+ # 반대로 freshness 는 출력을 쓰는 실행에만 의미가 있다. dry-run 은 CSV/ledger/report 를
5027
+ # 하나도 쓰지 않으므로 기존 CSV 가 있어도 잃을 profile 문맥이 없다.
5028
+ if not args.dry_run:
5029
+ preflight_profile_fresh_output(
5030
+ tasks,
5031
+ targets,
5032
+ resume=args.resume,
5033
+ csv_has_preexisting_content=file_has_content_no_follow(args.csv),
5034
+ )
5035
+
3872
5036
  if args.resume:
3873
5037
  skip_keys, skip_keys_loaded_stamp = existing_keys_snapshot(args.csv)
3874
5038
  skip_keys_stamp = {"stamp": skip_keys_loaded_stamp}
@@ -3893,6 +5057,18 @@ def main() -> int:
3893
5057
  return 0
3894
5058
  csv_had_preexisting_content = file_has_content_no_follow(args.csv)
3895
5059
  evidence_rows = read_evidence_jsonl(args.evidence_jsonl)
5060
+ # 완전한 profile preflight 는 첫 append_csv 이전, 그리고 어떤 lock sidecar 도
5061
+ # 만들어지기 전에 끝난다. 실패 시 파일 시스템은 바이트 단위로 그대로 남는다.
5062
+ preflight_evaluation_profiles(
5063
+ tasks,
5064
+ variants,
5065
+ targets,
5066
+ evidence_rows,
5067
+ task_file_dir=args.tasks.parent,
5068
+ resume=args.resume,
5069
+ csv_has_preexisting_content=csv_had_preexisting_content,
5070
+ baseline_variant=args.baseline_variant,
5071
+ )
3896
5072
  runnable_targets = resume_runnable_targets(
3897
5073
  args.csv,
3898
5074
  targets,
@@ -3901,40 +5077,54 @@ def main() -> int:
3901
5077
  existing_key_cache_stamp=skip_keys_stamp,
3902
5078
  )
3903
5079
  evidence_by_key = validate_evidence_coverage(evidence_rows, runnable_targets)
5080
+ profiled_batch = bool(selected_profiled_task_ids(tasks, targets))
3904
5081
  runnable_keys = {(task.id, variant.name) for task, variant in runnable_targets}
3905
5082
  claude_ver = "evidence-replay"
3906
5083
  completed = 0
3907
5084
  replay_rows_written: list[EvidenceReplayRow] = []
3908
- for task, variant in targets:
3909
- if args.resume and (task.id, variant.name) not in runnable_keys:
3910
- print(f"skip {task.id}/{variant.name} (already in {args.csv})")
3911
- continue
3912
- evidence = evidence_by_key[(task.id, variant.name)]
3913
- print(f"replay {task.id}/{variant.name} ...", flush=True)
3914
- result = run_evidence_fixture(task, variant, evidence)
3915
- wrote = append_csv(
3916
- args.csv,
5085
+ pending_ledger_rows: list[tuple[EvidenceReplayRow, RunResult]] = []
5086
+ batch_lock = csv_parent_directory_lock(args.csv, create_parent=True) if profiled_batch else nullcontext()
5087
+ with batch_lock:
5088
+ if profiled_batch:
5089
+ # The same stable lock covers the raced freshness recheck and every
5090
+ # row. No sidecar is created, and no foreign append can land between
5091
+ # profiled rows because append_csv takes this lock first.
5092
+ profile_batch_freshness_gate_unlocked(tasks, targets, args.csv)
5093
+ for task, variant in targets:
5094
+ if args.resume and (task.id, variant.name) not in runnable_keys:
5095
+ print(f"skip {task.id}/{variant.name} (already in {args.csv})")
5096
+ continue
5097
+ evidence = evidence_by_key[(task.id, variant.name)]
5098
+ print(f"replay {task.id}/{variant.name} ...", flush=True)
5099
+ result = run_evidence_fixture(task, variant, evidence)
5100
+ writer = append_csv_unlocked if profiled_batch else append_csv
5101
+ wrote = writer(
5102
+ args.csv,
5103
+ claude_ver,
5104
+ result,
5105
+ skip_existing=args.resume,
5106
+ existing_key_cache=skip_keys if args.resume else None,
5107
+ existing_key_cache_stamp=skip_keys_stamp,
5108
+ )
5109
+ if wrote:
5110
+ replay_rows_written.append(evidence)
5111
+ if args.ledger_jsonl is not None:
5112
+ pending_ledger_rows.append((evidence, result))
5113
+ completed += 1
5114
+ status = "ok" if result.success else "FAIL"
5115
+ suffix = "" if wrote else " (CSV not updated; row already present)"
5116
+ print(
5117
+ f" {status} tokens={sum(result.tokens.values())} cost=${result.cost_usd:.4f} "
5118
+ f"wall_time={result.wall_time_seconds:.3f}s {sanitize_note_text(result.notes)}{suffix}"
5119
+ )
5120
+ # Ledger/report/dashboard writes happen after the CSV batch lock so distinct
5121
+ # outputs in the same directory cannot deadlock on the directory inode.
5122
+ for evidence, result in pending_ledger_rows:
5123
+ append_cost_shift_ledger(
5124
+ args.ledger_jsonl,
3917
5125
  claude_ver,
3918
5126
  result,
3919
- skip_existing=args.resume,
3920
- existing_key_cache=skip_keys if args.resume else None,
3921
- existing_key_cache_stamp=skip_keys_stamp,
3922
- )
3923
- if wrote:
3924
- replay_rows_written.append(evidence)
3925
- if args.ledger_jsonl is not None:
3926
- append_cost_shift_ledger(
3927
- args.ledger_jsonl,
3928
- claude_ver,
3929
- result,
3930
- replay_provenance=evidence.provenance_payload(),
3931
- )
3932
- completed += 1
3933
- status = "ok" if result.success else "FAIL"
3934
- suffix = "" if wrote else " (CSV not updated; row already present)"
3935
- print(
3936
- f" {status} tokens={sum(result.tokens.values())} cost=${result.cost_usd:.4f} "
3937
- f"wall_time={result.wall_time_seconds:.3f}s {sanitize_note_text(result.notes)}{suffix}"
5127
+ replay_provenance=evidence.provenance_payload(),
3938
5128
  )
3939
5129
  if args.report_json is not None or args.dashboard_md is not None:
3940
5130
  report = write_report_outputs(