@ictechgy/context-guard 0.4.13 → 0.4.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/README.ko.md +36 -3
- package/README.md +40 -3
- package/docs/benchmark-fixtures/image-context-pack-full-evidence.prompt.example.md +28 -0
- package/docs/benchmark-fixtures/image-context-pack-packed-evidence.prompt.example.md +31 -0
- package/docs/benchmark-fixtures/image-context-pack.evidence.example.jsonl +2 -0
- package/docs/benchmark-fixtures/image-context-pack.tasks.example.json +18 -0
- package/docs/benchmark-fixtures/image-context-pack.variants.example.json +10 -0
- package/docs/benchmark-workflow-examples.md +16 -0
- package/docs/experimental-benchmark-fixtures.md +52 -1
- package/package.json +2 -1
- package/plugins/context-guard/.claude-plugin/plugin.json +1 -1
- package/plugins/context-guard/README.ko.md +30 -1
- package/plugins/context-guard/README.md +33 -2
- package/plugins/context-guard/bin/context-guard-bench +1305 -115
- package/plugins/context-guard/bin/context-guard-experiments +3548 -146
- package/plugins/context-guard/bin/context-guard-mcp +999 -0
- package/plugins/context-guard/bin/context-guard-pack +634 -9
- package/plugins/context-guard/lib/context_guard_commands.py +8 -0
|
@@ -49,9 +49,10 @@ from __future__ import annotations
|
|
|
49
49
|
|
|
50
50
|
import argparse
|
|
51
51
|
import collections
|
|
52
|
-
from contextlib import contextmanager
|
|
52
|
+
from contextlib import contextmanager, nullcontext
|
|
53
53
|
import csv
|
|
54
54
|
import datetime as _dt
|
|
55
|
+
import hashlib
|
|
55
56
|
import json
|
|
56
57
|
import math
|
|
57
58
|
import os
|
|
@@ -65,6 +66,7 @@ import subprocess
|
|
|
65
66
|
import sys
|
|
66
67
|
import time
|
|
67
68
|
import unicodedata
|
|
69
|
+
from collections.abc import Iterable
|
|
68
70
|
from dataclasses import dataclass, field
|
|
69
71
|
from pathlib import Path
|
|
70
72
|
from typing import Any
|
|
@@ -318,6 +320,126 @@ ALLOWED_FIRST_ABSOLUTE_SYMLINKS = {
|
|
|
318
320
|
"var": Path("/private/var"),
|
|
319
321
|
}
|
|
320
322
|
|
|
323
|
+
# --- Phase 4/5 optional image-context evaluation profile (evaluation-only) ---
|
|
324
|
+
# 이 profile 은 task fixture 가 명시적으로 opt-in 할 때만 동작한다. profile 이 없는
|
|
325
|
+
# 기존 replay 는 스키마/동작이 그대로 유지된다. profile 이 붙은 report 는 어떤
|
|
326
|
+
# 경우에도 public claim / promotion 권한을 얻지 못하도록 clamp 된다.
|
|
327
|
+
IMAGE_CONTEXT_EVALUATION_PROFILE_ID = "contextguard.bench.image-context-pack-evaluation.v1"
|
|
328
|
+
SUPPORTED_EVALUATION_PROFILE_IDS = frozenset({IMAGE_CONTEXT_EVALUATION_PROFILE_ID})
|
|
329
|
+
IMAGE_CONTEXT_READINESS_SCHEMA_VERSION = "contextguard.bench.image-context-pack-readiness.v1"
|
|
330
|
+
IMAGE_CONTEXT_PROFILE_REPORT_KEY = "image_context_pack"
|
|
331
|
+
EVALUATION_PROFILES_REPORT_KEY = "evaluation_profiles"
|
|
332
|
+
IMAGE_CONTEXT_EVALUATION_ONLY_CLAIM_STATUS = "image_context_pack_evaluation_only_not_public_claim"
|
|
333
|
+
PROFILE_STATUS_BLOCKED = "blocked"
|
|
334
|
+
PROFILE_STATUS_READY_FOR_BOUNDED_PILOT_REVIEW = "ready_for_bounded_pilot_review"
|
|
335
|
+
# 가져온 local proof-verifier 레코드는 "누가 만들었는지" 를 인증하지 않고 artifact 를
|
|
336
|
+
# 다시 읽지도 않는다. 라벨로 그 경계를 명시한다.
|
|
337
|
+
IMPORTED_LOCAL_VERIFIER_ATTESTATION_LABEL = "imported_local_verifier_attestation"
|
|
338
|
+
PROOF_VERIFICATION_SCHEMA_VERSION = "contextguard.experiments.proof-carrying-context-verification.v1"
|
|
339
|
+
PROOF_VERIFICATION_VERIFIED_STATUS = "verified"
|
|
340
|
+
# experimental_registry.PROOF_VERIFICATION_CLAIM_BOUNDARY 와 반드시 같은 문자열이다.
|
|
341
|
+
# 가져온 attestation 은 이 local-only 경계를 그대로 선언할 때만 verified 로 인정한다.
|
|
342
|
+
PROOF_VERIFICATION_CLAIM_BOUNDARY = (
|
|
343
|
+
"Local receipt/hash/range/command binding only; no semantic-safety, protected-zone, freshness, replacement, "
|
|
344
|
+
"omission, or hosted-savings authority."
|
|
345
|
+
)
|
|
346
|
+
# local verifier 는 rehydration 을 절대 실행하지 않는다. 실행했다고 주장하는 레코드는
|
|
347
|
+
# 이 evaluation-only 경계를 벗어나므로 verified 로 받아들이지 않는다.
|
|
348
|
+
PROOF_VERIFICATION_REHYDRATION_EXECUTED = False
|
|
349
|
+
# verified attestation 에서 placeholder 로 취급해 거부할 receipt/command 값이다.
|
|
350
|
+
PROFILE_FALLBACK_PLACEHOLDER_VALUES = frozenset({"", "none", "null", "n/a", "-"})
|
|
351
|
+
|
|
352
|
+
# reject_prewrite 오류 ID. 출력이 하나라도 기록되기 전에 실패해야 하는 구조적 오류다.
|
|
353
|
+
PROFILE_REJECT_CONTROLS_MISSING = "profile_controls_missing"
|
|
354
|
+
PROFILE_REJECT_SCHEMA_INVALID = "profile_schema_invalid"
|
|
355
|
+
PROFILE_REJECT_BINDING_MISMATCH = "profile_binding_mismatch"
|
|
356
|
+
PROFILE_REJECT_BATCH_INCOMPLETE = "profile_batch_incomplete"
|
|
357
|
+
PROFILE_REJECT_FRESH_OUTPUT_REQUIRED = "profile_fresh_output_required"
|
|
358
|
+
PROFILE_REJECT_PROMPT_BINDING_INVALID = "profile_prompt_binding_invalid"
|
|
359
|
+
PROFILE_REJECT_CORRECTION_INCONSISTENT = "profile_correction_inconsistent"
|
|
360
|
+
PROFILE_REJECT_MEASUREMENT_INCONSISTENT = "profile_measurement_inconsistent"
|
|
361
|
+
PROFILE_REJECT_FALLBACK_CLAIM_INCONSISTENT = "profile_fallback_claim_inconsistent"
|
|
362
|
+
# profile 은 evaluation-only replay 전용이다. --evidence-jsonl 없이 profiled task 가
|
|
363
|
+
# 선택되면 provider runtime 을 부르기 전에, 어떤 출력/lock 도 만들기 전에 거부한다.
|
|
364
|
+
PROFILE_REJECT_REPLAY_REQUIRED = "profile_replay_required"
|
|
365
|
+
PROFILE_REJECT_ERROR_IDS = (
|
|
366
|
+
PROFILE_REJECT_CONTROLS_MISSING,
|
|
367
|
+
PROFILE_REJECT_SCHEMA_INVALID,
|
|
368
|
+
PROFILE_REJECT_BINDING_MISMATCH,
|
|
369
|
+
PROFILE_REJECT_BATCH_INCOMPLETE,
|
|
370
|
+
PROFILE_REJECT_FRESH_OUTPUT_REQUIRED,
|
|
371
|
+
PROFILE_REJECT_PROMPT_BINDING_INVALID,
|
|
372
|
+
PROFILE_REJECT_CORRECTION_INCONSISTENT,
|
|
373
|
+
PROFILE_REJECT_MEASUREMENT_INCONSISTENT,
|
|
374
|
+
PROFILE_REJECT_FALLBACK_CLAIM_INCONSISTENT,
|
|
375
|
+
PROFILE_REJECT_REPLAY_REQUIRED,
|
|
376
|
+
)
|
|
377
|
+
|
|
378
|
+
# lane gate ID. 순서는 report/dashboard 출력 순서와 동일하게 고정한다.
|
|
379
|
+
IMAGE_CONTEXT_GATE_PROFILE_AND_PROMPT_BINDING = "profile_and_prompt_binding"
|
|
380
|
+
IMAGE_CONTEXT_GATE_PROTECTED_ZONE_DENY_REVIEW = "protected_zone_deny_review"
|
|
381
|
+
IMAGE_CONTEXT_GATE_EXACT_TEXT_FALLBACK_BINDING = "exact_text_fallback_binding"
|
|
382
|
+
IMAGE_CONTEXT_GATE_MISSED_CONTEXT_REVIEW = "missed_context_review"
|
|
383
|
+
IMAGE_CONTEXT_GATE_HUMAN_CORRECTION_CONSISTENCY = "human_correction_consistency"
|
|
384
|
+
# generic quality gate 결과를 lane 이 직접 소비한다. profile 이 generic regression 을
|
|
385
|
+
# 무시하고 ready 로 올라가는 경로를 막는다.
|
|
386
|
+
IMAGE_CONTEXT_GATE_CORRECTIONS_REGRESSION = "corrections_regression"
|
|
387
|
+
IMAGE_CONTEXT_GATE_FAILURE_RATE_REGRESSION = "failure_rate_regression"
|
|
388
|
+
IMAGE_CONTEXT_GATE_GENERIC_MATCHED_SUCCESS_AND_MEASUREMENT = "generic_matched_success_and_measurement"
|
|
389
|
+
IMAGE_CONTEXT_GATE_EVALUATION_ONLY_PROMOTION_BOUNDARY = "evaluation_only_promotion_boundary"
|
|
390
|
+
IMAGE_CONTEXT_GATE_IDS = (
|
|
391
|
+
IMAGE_CONTEXT_GATE_PROFILE_AND_PROMPT_BINDING,
|
|
392
|
+
IMAGE_CONTEXT_GATE_PROTECTED_ZONE_DENY_REVIEW,
|
|
393
|
+
IMAGE_CONTEXT_GATE_EXACT_TEXT_FALLBACK_BINDING,
|
|
394
|
+
IMAGE_CONTEXT_GATE_MISSED_CONTEXT_REVIEW,
|
|
395
|
+
IMAGE_CONTEXT_GATE_HUMAN_CORRECTION_CONSISTENCY,
|
|
396
|
+
IMAGE_CONTEXT_GATE_CORRECTIONS_REGRESSION,
|
|
397
|
+
IMAGE_CONTEXT_GATE_FAILURE_RATE_REGRESSION,
|
|
398
|
+
IMAGE_CONTEXT_GATE_GENERIC_MATCHED_SUCCESS_AND_MEASUREMENT,
|
|
399
|
+
IMAGE_CONTEXT_GATE_EVALUATION_ONLY_PROMOTION_BOUNDARY,
|
|
400
|
+
)
|
|
401
|
+
# lane gate 를 막는 generic quality_gate 값. summarize_benchmark_rows 가 계산한다.
|
|
402
|
+
GENERIC_QUALITY_GATE_PASS = "pass"
|
|
403
|
+
GENERIC_QUALITY_GATE_CORRECTIONS_REGRESSION = "corrections_regression"
|
|
404
|
+
GENERIC_QUALITY_GATE_FAILURE_RATE_REGRESSION = "failure_rate_regression"
|
|
405
|
+
IMAGE_CONTEXT_PROFILE_BLOCKER_GATE_ID = "image_context_pack_evaluation_only"
|
|
406
|
+
IMAGE_CONTEXT_CLAIM_BOUNDARY = {
|
|
407
|
+
"id": "image_context_pack_evaluation_only_never_promotion_or_public_claim",
|
|
408
|
+
"evaluation_only": True,
|
|
409
|
+
"promotion_authority": False,
|
|
410
|
+
"public_claim_allowed": False,
|
|
411
|
+
"runtime_authority": False,
|
|
412
|
+
"hosted_savings_claim_allowed": False,
|
|
413
|
+
"fallback_attestation_label": IMPORTED_LOCAL_VERIFIER_ATTESTATION_LABEL,
|
|
414
|
+
"fallback_attestation_is_independently_verified": False,
|
|
415
|
+
"protected_zone_evidence_is_review_attestation_not_semantic_proof": True,
|
|
416
|
+
"reason": (
|
|
417
|
+
"The image-context evaluation profile reviews imported evidence only. It does not render, "
|
|
418
|
+
"parse, or reread any image or artifact, does not authenticate who produced an imported "
|
|
419
|
+
"verifier or review record, and can never authorize a public savings claim, a quality "
|
|
420
|
+
"non-inferiority claim, or a runtime promotion."
|
|
421
|
+
),
|
|
422
|
+
}
|
|
423
|
+
PROFILE_SAMPLE_ADEQUACY_POLICY_STATUS = "not_defined_for_promotion"
|
|
424
|
+
|
|
425
|
+
# profile 중첩 블록의 명시적 byte/count 한계. 타입/한계 검사는 항상 semantic 분류보다
|
|
426
|
+
# 먼저 실행되어 oversize 값이 blocked 분기로 새지 않도록 한다.
|
|
427
|
+
MAX_PROFILE_LABEL_CHARS = 120
|
|
428
|
+
MAX_PROFILE_POLICY_CHARS = 120
|
|
429
|
+
MAX_PROFILE_NOTE_CHARS = 500
|
|
430
|
+
MAX_PROFILE_SUMMARY_CHARS = 500
|
|
431
|
+
MAX_PROFILE_COMMAND_CHARS = 500
|
|
432
|
+
MAX_PROFILE_RECEIPT_ID_CHARS = 200
|
|
433
|
+
MAX_PROFILE_BLOCKER_ITEMS = 20
|
|
434
|
+
MAX_PROFILE_PROTECTED_REGION_COUNT = 10_000
|
|
435
|
+
MAX_PROFILE_CORRECTION_COUNT = 10_000
|
|
436
|
+
SHA256_HEX_PATTERN = re.compile(r"\A[0-9a-f]{64}\Z")
|
|
437
|
+
PROTECTED_ZONE_DENY_POLICY = "deny"
|
|
438
|
+
# 프로파일 진단에 실리는 작성자 통제 라벨은 문자셋이 안전해 보여도 원문을 절대 남기지
|
|
439
|
+
# 않는다. G006 은 regex-safe 값까지 포함한 완전 불투명 표현을 요구한다.
|
|
440
|
+
PROFILE_REDACTED_PLACEHOLDER = "[REDACTED]"
|
|
441
|
+
MAX_PROFILE_ERROR_LABELS = 5
|
|
442
|
+
|
|
321
443
|
|
|
322
444
|
def _base_open_flags() -> int:
|
|
323
445
|
flags = os.O_RDONLY
|
|
@@ -472,6 +594,34 @@ def csv_file_lock(csv_path: Path, *, create_parent: bool) -> Any:
|
|
|
472
594
|
os.close(fd)
|
|
473
595
|
|
|
474
596
|
|
|
597
|
+
@contextmanager
|
|
598
|
+
def csv_parent_directory_lock(csv_path: Path, *, create_parent: bool) -> Any:
|
|
599
|
+
"""Serialize a CSV transaction without creating a lock sidecar.
|
|
600
|
+
|
|
601
|
+
Normal writers take this stable directory-inode lock before the historical
|
|
602
|
+
sidecar lock. Profiled replay can therefore hold it across freshness validation
|
|
603
|
+
and its complete batch without leaving a sidecar on rejection. The stable inode
|
|
604
|
+
also avoids an unlink-while-waiters race.
|
|
605
|
+
"""
|
|
606
|
+
if fcntl is None:
|
|
607
|
+
raise OSError("platform does not support advisory CSV locks")
|
|
608
|
+
parent = csv_path.parent
|
|
609
|
+
if create_parent:
|
|
610
|
+
parent.mkdir(parents=True, exist_ok=True)
|
|
611
|
+
fd = os.open(parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
|
|
612
|
+
locked = False
|
|
613
|
+
try:
|
|
614
|
+
fcntl.flock(fd, fcntl.LOCK_EX)
|
|
615
|
+
locked = True
|
|
616
|
+
yield
|
|
617
|
+
finally:
|
|
618
|
+
try:
|
|
619
|
+
if locked:
|
|
620
|
+
fcntl.flock(fd, fcntl.LOCK_UN)
|
|
621
|
+
finally:
|
|
622
|
+
os.close(fd)
|
|
623
|
+
|
|
624
|
+
|
|
475
625
|
# 재현성 우선: fixture 에 명시되지 않은 필드는 argv 로 전달하지 않는다.
|
|
476
626
|
# 사용자가 baseline 으로 의도한 변형이 implicit default(예: effort="medium")로 인해
|
|
477
627
|
# 왜곡되지 않도록, 파싱 단계에서 명시 여부를 그대로 보존한다.
|
|
@@ -488,6 +638,8 @@ class TaskFixture:
|
|
|
488
638
|
success_cwd: str = "."
|
|
489
639
|
variant_prompt_files: dict[str, str] = field(default_factory=dict)
|
|
490
640
|
variant_prompt_texts: dict[str, str] = field(default_factory=dict)
|
|
641
|
+
# 선택적 evaluation profile opt-in. None 이면 기존 generic 동작을 그대로 유지한다.
|
|
642
|
+
evaluation_profile: str | None = None
|
|
491
643
|
|
|
492
644
|
|
|
493
645
|
@dataclass
|
|
@@ -535,6 +687,12 @@ class EvidenceReplayRow:
|
|
|
535
687
|
public_claim_eligible: bool
|
|
536
688
|
explicit_notes: bool
|
|
537
689
|
line_number: int
|
|
690
|
+
# profile 을 선언하지 않은 row 는 세 필드가 모두 None 이며 generic 경로와 동일하다.
|
|
691
|
+
evaluation_profile: str | None = None
|
|
692
|
+
evaluation_controls: dict[str, Any] | None = None
|
|
693
|
+
# preflight 가 채우는 정규화된 lane 판정. report annotation 은 이 값만 사용하므로
|
|
694
|
+
# 이미 검증된 batch 위에서 절대 실패하지 않는다.
|
|
695
|
+
evaluation_lane: dict[str, Any] | None = None
|
|
538
696
|
|
|
539
697
|
@property
|
|
540
698
|
def key(self) -> tuple[str, str]:
|
|
@@ -639,6 +797,16 @@ def require_argv_safe_prompt(text: str, *, owner: str) -> str:
|
|
|
639
797
|
|
|
640
798
|
def validate_variant_prompt_file_path(raw_path: str, *, owner: str) -> Path:
|
|
641
799
|
"""Return a safe relative prompt-file path, or fail before any file read."""
|
|
800
|
+
# 결정적으로 거부 가능한 값: 임베디드 NUL 과 로컬 fs 인코딩으로 표현 불가한 문자열.
|
|
801
|
+
# 이 값들은 이후 os.open 에서 ValueError/UnicodeError 로 터질 수 있으므로 미리 막는다.
|
|
802
|
+
if "\x00" in raw_path:
|
|
803
|
+
raise SystemExit(f"{owner} variant_prompt_files path must not contain embedded NUL")
|
|
804
|
+
try:
|
|
805
|
+
os.fsencode(raw_path)
|
|
806
|
+
except UnicodeError:
|
|
807
|
+
raise SystemExit(
|
|
808
|
+
f"{owner} variant_prompt_files path is not representable on the local filesystem"
|
|
809
|
+
) from None
|
|
642
810
|
rel_path = Path(raw_path)
|
|
643
811
|
if rel_path.is_absolute():
|
|
644
812
|
raise SystemExit(f"{owner} variant_prompt_files path must be relative: {raw_path}")
|
|
@@ -658,19 +826,44 @@ def validate_variant_prompt_file_references(
|
|
|
658
826
|
Unknown variant keys and unsafe relative paths are rejected before any file
|
|
659
827
|
read. Missing prompt files are intentionally not checked here so a run
|
|
660
828
|
narrowed by --task-id/--variant is not blocked by unselected prompt files.
|
|
829
|
+
|
|
830
|
+
Profiled tasks use the redacted profile owner and never echo raw task ids,
|
|
831
|
+
variant labels, mapping keys, or unsafe paths. Unprofiled messages stay
|
|
832
|
+
unchanged for compatibility.
|
|
661
833
|
"""
|
|
662
834
|
known_variants = {variant.name for variant in variants}
|
|
663
835
|
for task in tasks:
|
|
836
|
+
profiled = task.evaluation_profile is not None
|
|
664
837
|
unknown = sorted(set(task.variant_prompt_files) - known_variants)
|
|
665
838
|
if unknown:
|
|
839
|
+
if profiled:
|
|
840
|
+
# 매핑 키·variant 라벨은 attacker-controlled 이므로 이름 대신 적색 처리한다.
|
|
841
|
+
profile_reject(
|
|
842
|
+
PROFILE_REJECT_PROMPT_BINDING_INVALID,
|
|
843
|
+
profile_owner(task.id),
|
|
844
|
+
"variant_prompt_files references unknown variant(s): "
|
|
845
|
+
f"{redact_profile_labels(unknown)}",
|
|
846
|
+
)
|
|
666
847
|
raise SystemExit(
|
|
667
848
|
f"task {task.id} variant_prompt_files references unknown variant(s): {', '.join(unknown)}"
|
|
668
849
|
)
|
|
669
850
|
for variant_name, raw_path in task.variant_prompt_files.items():
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
851
|
+
owner = (
|
|
852
|
+
profile_owner(task.id, variant_name)
|
|
853
|
+
if profiled
|
|
854
|
+
else f"task {task.id} variant {variant_name}"
|
|
673
855
|
)
|
|
856
|
+
try:
|
|
857
|
+
validate_variant_prompt_file_path(raw_path, owner=owner)
|
|
858
|
+
except SystemExit:
|
|
859
|
+
if profiled:
|
|
860
|
+
# 원본 경로·라벨이 새어나가지 않도록 안정적인 프로파일 오류로 다시 쓴다.
|
|
861
|
+
profile_reject(
|
|
862
|
+
PROFILE_REJECT_PROMPT_BINDING_INVALID,
|
|
863
|
+
owner,
|
|
864
|
+
"variant_prompt_files path is unsafe or invalid",
|
|
865
|
+
)
|
|
866
|
+
raise
|
|
674
867
|
|
|
675
868
|
|
|
676
869
|
def read_variant_prompt_file(path: Path, *, owner: str, display_path: str | None = None) -> str:
|
|
@@ -768,41 +961,89 @@ def parse_tasks(path: Path, variants: list["Variant"] | None = None) -> list[Tas
|
|
|
768
961
|
for item in raw:
|
|
769
962
|
if not isinstance(item, dict):
|
|
770
963
|
raise SystemExit(f"task entry must be a JSON object: {item}")
|
|
964
|
+
# evaluation_profile opt-in 을 필수 라벨(id/prompt)보다 먼저 확정한다.
|
|
965
|
+
# 지원 프로파일의 구조 오류는 raw KeyError 가 아니라 안정적인 프로파일 거부로 끝낸다.
|
|
966
|
+
evaluation_profile = item.get("evaluation_profile")
|
|
967
|
+
profiled = evaluation_profile is not None
|
|
968
|
+
if "id" not in item:
|
|
969
|
+
if profiled:
|
|
970
|
+
profile_reject(
|
|
971
|
+
PROFILE_REJECT_SCHEMA_INVALID,
|
|
972
|
+
profile_owner(None),
|
|
973
|
+
"task fixture fields are invalid",
|
|
974
|
+
)
|
|
975
|
+
raise KeyError("id")
|
|
976
|
+
task_id = str(item["id"])
|
|
977
|
+
owner = profile_owner(task_id) if profiled else f"task {task_id}"
|
|
978
|
+
if profiled and (
|
|
979
|
+
not isinstance(evaluation_profile, str)
|
|
980
|
+
or evaluation_profile not in SUPPORTED_EVALUATION_PROFILE_IDS
|
|
981
|
+
):
|
|
982
|
+
profile_reject(
|
|
983
|
+
PROFILE_REJECT_SCHEMA_INVALID,
|
|
984
|
+
owner,
|
|
985
|
+
"declares an unsupported evaluation_profile id",
|
|
986
|
+
)
|
|
987
|
+
if "variant_prompts" in item:
|
|
988
|
+
detail = "variant_prompts is not supported; use file-backed variant_prompt_files"
|
|
989
|
+
if profiled:
|
|
990
|
+
profile_reject(PROFILE_REJECT_SCHEMA_INVALID, owner, detail)
|
|
991
|
+
raise SystemExit(f"{owner} {detail}")
|
|
771
992
|
effort_raw = item.get("effort")
|
|
772
993
|
budget_raw = item.get("max_budget_usd")
|
|
773
994
|
if budget_raw is not None:
|
|
774
995
|
try:
|
|
775
996
|
budget = float(budget_raw)
|
|
776
997
|
except (TypeError, ValueError):
|
|
777
|
-
|
|
998
|
+
detail = "max_budget_usd must be number or null"
|
|
999
|
+
if profiled:
|
|
1000
|
+
profile_reject(PROFILE_REJECT_SCHEMA_INVALID, owner, detail)
|
|
1001
|
+
raise SystemExit(f"{owner} {detail}")
|
|
778
1002
|
if not math.isfinite(budget) or budget <= 0:
|
|
779
|
-
|
|
1003
|
+
detail = "max_budget_usd must be finite and > 0 (use null for unlimited)"
|
|
1004
|
+
if profiled:
|
|
1005
|
+
profile_reject(PROFILE_REJECT_SCHEMA_INVALID, owner, detail)
|
|
1006
|
+
raise SystemExit(f"{owner} {detail}")
|
|
780
1007
|
else:
|
|
781
1008
|
budget = None
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
1009
|
+
# profiled 경로에서는 매핑 키 등 attacker-controlled 라벨이 파서 오류에 실릴 수
|
|
1010
|
+
# 있으므로, 구조 필드 오류(파서 SystemExit·필수 prompt 누락 KeyError)를 안정적인
|
|
1011
|
+
# 프로파일 거부로 다시 쓴다. unprofiled 는 원본 예외를 그대로 재발생시킨다.
|
|
1012
|
+
try:
|
|
1013
|
+
max_turns = parse_positive_int(
|
|
1014
|
+
item.get("max_turns", 3), field="max_turns", owner=owner,
|
|
1015
|
+
)
|
|
1016
|
+
allowed_tools = parse_string_list(
|
|
1017
|
+
item.get("allowed_tools", []),
|
|
1018
|
+
field="allowed_tools",
|
|
1019
|
+
owner=owner,
|
|
786
1020
|
)
|
|
1021
|
+
variant_prompt_files = parse_string_map(
|
|
1022
|
+
item.get("variant_prompt_files"),
|
|
1023
|
+
field="variant_prompt_files",
|
|
1024
|
+
owner=owner,
|
|
1025
|
+
)
|
|
1026
|
+
prompt = str(item["prompt"])
|
|
1027
|
+
except (SystemExit, KeyError):
|
|
1028
|
+
if profiled:
|
|
1029
|
+
profile_reject(
|
|
1030
|
+
PROFILE_REJECT_SCHEMA_INVALID,
|
|
1031
|
+
owner,
|
|
1032
|
+
"task fixture fields are invalid",
|
|
1033
|
+
)
|
|
1034
|
+
raise
|
|
787
1035
|
fixtures.append(TaskFixture(
|
|
1036
|
+
evaluation_profile=evaluation_profile,
|
|
788
1037
|
id=task_id,
|
|
789
|
-
prompt=
|
|
1038
|
+
prompt=prompt,
|
|
790
1039
|
model=str(item.get("model", "sonnet")),
|
|
791
1040
|
effort=str(effort_raw) if effort_raw is not None else None,
|
|
792
|
-
max_turns=
|
|
1041
|
+
max_turns=max_turns,
|
|
793
1042
|
max_budget_usd=budget,
|
|
794
|
-
allowed_tools=
|
|
795
|
-
item.get("allowed_tools", []),
|
|
796
|
-
field="allowed_tools",
|
|
797
|
-
owner=f"task {task_id}",
|
|
798
|
-
),
|
|
1043
|
+
allowed_tools=allowed_tools,
|
|
799
1044
|
success_command=item.get("success_command"),
|
|
800
1045
|
success_cwd=str(item.get("success_cwd", ".")),
|
|
801
|
-
variant_prompt_files=
|
|
802
|
-
item.get("variant_prompt_files"),
|
|
803
|
-
field="variant_prompt_files",
|
|
804
|
-
owner=f"task {task_id}",
|
|
805
|
-
),
|
|
1046
|
+
variant_prompt_files=variant_prompt_files,
|
|
806
1047
|
))
|
|
807
1048
|
if variants is not None:
|
|
808
1049
|
validate_variant_prompt_file_references(fixtures, variants)
|
|
@@ -1498,72 +1739,93 @@ def append_csv(
|
|
|
1498
1739
|
existing_key_cache: set[tuple[str, str]] | None = None,
|
|
1499
1740
|
existing_key_cache_stamp: dict[str, tuple[int, int, int, int] | None] | None = None,
|
|
1500
1741
|
) -> bool:
|
|
1501
|
-
with
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
"date": sanitize_csv_cell(_dt.datetime.now().strftime("%Y-%m-%dT%H:%M:%S")),
|
|
1526
|
-
"claude_version": sanitize_csv_cell(claude_ver),
|
|
1527
|
-
"task_id": sanitize_csv_cell(result.task_id),
|
|
1528
|
-
"variant": sanitize_csv_cell(result.variant),
|
|
1529
|
-
"model": sanitize_csv_cell(result.model),
|
|
1530
|
-
"effort": sanitize_csv_cell(result.effort),
|
|
1531
|
-
"total_tokens": total,
|
|
1532
|
-
"input_tokens": tokens.get("input_tokens", 0),
|
|
1533
|
-
"output_tokens": tokens.get("output_tokens", 0),
|
|
1534
|
-
"cache_read": tokens.get("cache_read", 0),
|
|
1535
|
-
"cache_creation": tokens.get("cache_creation", 0),
|
|
1536
|
-
"provider_cached_tokens": result.provider_cached_tokens,
|
|
1537
|
-
"provider_cached_tokens_measured": (
|
|
1538
|
-
"true" if result.provider_cached_tokens_measured else "false"
|
|
1539
|
-
),
|
|
1540
|
-
"cost_usd": f"{result.cost_usd:.6f}",
|
|
1541
|
-
"cost_measured": "true" if result.cost_measured else "false",
|
|
1542
|
-
"wall_time_seconds": f"{result.wall_time_seconds:.6f}",
|
|
1543
|
-
"turns": result.turns,
|
|
1544
|
-
"hook_triggers": result.hook_triggers,
|
|
1545
|
-
"bytes_before": result.bytes_before,
|
|
1546
|
-
"bytes_after": result.bytes_after,
|
|
1547
|
-
"artifacts_used": result.artifacts_used,
|
|
1548
|
-
"external_tokens": result.external_tokens,
|
|
1549
|
-
"external_tokens_measured": "true" if result.external_tokens_measured else "false",
|
|
1550
|
-
"external_cost_usd": f"{result.external_cost_usd:.6f}",
|
|
1551
|
-
"external_cost_measured": "true" if result.external_cost_measured else "false",
|
|
1552
|
-
"total_cost_with_shift_usd": (
|
|
1553
|
-
f"{(result.cost_usd + result.external_cost_usd):.6f}" if shifted_cost_known else ""
|
|
1554
|
-
),
|
|
1555
|
-
"success": "true" if result.success else "false",
|
|
1556
|
-
"corrections": result.corrections,
|
|
1557
|
-
"notes": sanitize_csv_note(result.notes),
|
|
1558
|
-
"primary_tokens_measured": "true" if result.primary_tokens_measured else "false",
|
|
1559
|
-
})
|
|
1560
|
-
finally:
|
|
1561
|
-
if fd != -1:
|
|
1562
|
-
os.close(fd)
|
|
1742
|
+
with csv_parent_directory_lock(csv_path, create_parent=True):
|
|
1743
|
+
with csv_file_lock(csv_path, create_parent=False):
|
|
1744
|
+
return append_csv_unlocked(
|
|
1745
|
+
csv_path,
|
|
1746
|
+
claude_ver,
|
|
1747
|
+
result,
|
|
1748
|
+
skip_existing=skip_existing,
|
|
1749
|
+
existing_key_cache=existing_key_cache,
|
|
1750
|
+
existing_key_cache_stamp=existing_key_cache_stamp,
|
|
1751
|
+
)
|
|
1752
|
+
|
|
1753
|
+
|
|
1754
|
+
def append_csv_unlocked(
|
|
1755
|
+
csv_path: Path,
|
|
1756
|
+
claude_ver: str,
|
|
1757
|
+
result: RunResult,
|
|
1758
|
+
*,
|
|
1759
|
+
skip_existing: bool = False,
|
|
1760
|
+
existing_key_cache: set[tuple[str, str]] | None = None,
|
|
1761
|
+
existing_key_cache_stamp: dict[str, tuple[int, int, int, int] | None] | None = None,
|
|
1762
|
+
) -> bool:
|
|
1763
|
+
"""Append one row while the caller holds the CSV transaction lock."""
|
|
1764
|
+
key = (result.task_id, result.variant)
|
|
1765
|
+
if skip_existing:
|
|
1563
1766
|
if existing_key_cache is not None:
|
|
1564
|
-
existing_key_cache
|
|
1565
|
-
|
|
1566
|
-
|
|
1767
|
+
refresh_existing_key_cache_unlocked(csv_path, existing_key_cache, existing_key_cache_stamp)
|
|
1768
|
+
if key in existing_key_cache:
|
|
1769
|
+
return False
|
|
1770
|
+
elif key in _read_existing_keys_unlocked(csv_path):
|
|
1771
|
+
return False
|
|
1772
|
+
flags = os.O_CREAT | os.O_APPEND | os.O_WRONLY
|
|
1773
|
+
fd = _open_regular_no_symlink(csv_path, flags, 0o600, create_parent=True)
|
|
1774
|
+
try:
|
|
1775
|
+
new_file = os.fstat(fd).st_size == 0
|
|
1776
|
+
if not new_file:
|
|
1777
|
+
validate_csv_schema(csv_path, read_csv_header_unlocked(csv_path))
|
|
1778
|
+
with os.fdopen(fd, "a", encoding="utf-8", newline="") as f:
|
|
1779
|
+
fd = -1
|
|
1780
|
+
writer = csv.DictWriter(f, fieldnames=CSV_COLUMNS)
|
|
1781
|
+
if new_file:
|
|
1782
|
+
writer.writeheader()
|
|
1783
|
+
tokens = result.tokens
|
|
1784
|
+
total = sum(tokens.values())
|
|
1785
|
+
shifted_cost_known = cost_shift_measured(result)
|
|
1786
|
+
writer.writerow({
|
|
1787
|
+
"date": sanitize_csv_cell(_dt.datetime.now().strftime("%Y-%m-%dT%H:%M:%S")),
|
|
1788
|
+
"claude_version": sanitize_csv_cell(claude_ver),
|
|
1789
|
+
"task_id": sanitize_csv_cell(result.task_id),
|
|
1790
|
+
"variant": sanitize_csv_cell(result.variant),
|
|
1791
|
+
"model": sanitize_csv_cell(result.model),
|
|
1792
|
+
"effort": sanitize_csv_cell(result.effort),
|
|
1793
|
+
"total_tokens": total,
|
|
1794
|
+
"input_tokens": tokens.get("input_tokens", 0),
|
|
1795
|
+
"output_tokens": tokens.get("output_tokens", 0),
|
|
1796
|
+
"cache_read": tokens.get("cache_read", 0),
|
|
1797
|
+
"cache_creation": tokens.get("cache_creation", 0),
|
|
1798
|
+
"provider_cached_tokens": result.provider_cached_tokens,
|
|
1799
|
+
"provider_cached_tokens_measured": (
|
|
1800
|
+
"true" if result.provider_cached_tokens_measured else "false"
|
|
1801
|
+
),
|
|
1802
|
+
"cost_usd": f"{result.cost_usd:.6f}",
|
|
1803
|
+
"cost_measured": "true" if result.cost_measured else "false",
|
|
1804
|
+
"wall_time_seconds": f"{result.wall_time_seconds:.6f}",
|
|
1805
|
+
"turns": result.turns,
|
|
1806
|
+
"hook_triggers": result.hook_triggers,
|
|
1807
|
+
"bytes_before": result.bytes_before,
|
|
1808
|
+
"bytes_after": result.bytes_after,
|
|
1809
|
+
"artifacts_used": result.artifacts_used,
|
|
1810
|
+
"external_tokens": result.external_tokens,
|
|
1811
|
+
"external_tokens_measured": "true" if result.external_tokens_measured else "false",
|
|
1812
|
+
"external_cost_usd": f"{result.external_cost_usd:.6f}",
|
|
1813
|
+
"external_cost_measured": "true" if result.external_cost_measured else "false",
|
|
1814
|
+
"total_cost_with_shift_usd": (
|
|
1815
|
+
f"{(result.cost_usd + result.external_cost_usd):.6f}" if shifted_cost_known else ""
|
|
1816
|
+
),
|
|
1817
|
+
"success": "true" if result.success else "false",
|
|
1818
|
+
"corrections": result.corrections,
|
|
1819
|
+
"notes": sanitize_csv_note(result.notes),
|
|
1820
|
+
"primary_tokens_measured": "true" if result.primary_tokens_measured else "false",
|
|
1821
|
+
})
|
|
1822
|
+
finally:
|
|
1823
|
+
if fd != -1:
|
|
1824
|
+
os.close(fd)
|
|
1825
|
+
if existing_key_cache is not None:
|
|
1826
|
+
existing_key_cache.add(key)
|
|
1827
|
+
if existing_key_cache_stamp is not None:
|
|
1828
|
+
existing_key_cache_stamp["stamp"] = csv_file_stamp_unlocked(csv_path)
|
|
1567
1829
|
return True
|
|
1568
1830
|
|
|
1569
1831
|
|
|
@@ -2021,6 +2283,26 @@ def parse_evidence_row(raw_value: Any, *, owner: str, line_number: int) -> Evide
|
|
|
2021
2283
|
primary_tokens_measured=primary_tokens_measured,
|
|
2022
2284
|
self_hosted_metrics=self_hosted_metrics,
|
|
2023
2285
|
)
|
|
2286
|
+
# Profile metadata 는 추가 필드로만 수집한다. 깊은 검증은 preflight 에서 task/prompt
|
|
2287
|
+
# 문맥과 함께 수행하며, 어떤 출력도 기록되기 전에 끝난다.
|
|
2288
|
+
# profile 오류는 evidence 파일 경로를 절대 echo 하지 않는다. 줄 번호만으로 위치를 준다.
|
|
2289
|
+
profile_row_owner = f"evidence line {line_number}"
|
|
2290
|
+
evaluation_profile = raw.get("evaluation_profile")
|
|
2291
|
+
if evaluation_profile is not None and (
|
|
2292
|
+
not isinstance(evaluation_profile, str)
|
|
2293
|
+
or evaluation_profile not in SUPPORTED_EVALUATION_PROFILE_IDS
|
|
2294
|
+
):
|
|
2295
|
+
profile_reject(
|
|
2296
|
+
PROFILE_REJECT_SCHEMA_INVALID, profile_row_owner,
|
|
2297
|
+
"evaluation_profile is not a supported profile id",
|
|
2298
|
+
)
|
|
2299
|
+
evaluation_controls = raw.get("evaluation_controls")
|
|
2300
|
+
if evaluation_controls is not None and not isinstance(evaluation_controls, dict):
|
|
2301
|
+
profile_reject(
|
|
2302
|
+
PROFILE_REJECT_SCHEMA_INVALID, profile_row_owner,
|
|
2303
|
+
"evaluation_controls must be a JSON object",
|
|
2304
|
+
)
|
|
2305
|
+
|
|
2024
2306
|
return EvidenceReplayRow(
|
|
2025
2307
|
result=result,
|
|
2026
2308
|
source_type=str(provenance["source_type"]),
|
|
@@ -2031,6 +2313,8 @@ def parse_evidence_row(raw_value: Any, *, owner: str, line_number: int) -> Evide
|
|
|
2031
2313
|
public_claim_eligible=False,
|
|
2032
2314
|
explicit_notes=explicit_notes,
|
|
2033
2315
|
line_number=line_number,
|
|
2316
|
+
evaluation_profile=evaluation_profile,
|
|
2317
|
+
evaluation_controls=evaluation_controls,
|
|
2034
2318
|
)
|
|
2035
2319
|
|
|
2036
2320
|
|
|
@@ -2236,6 +2520,860 @@ def measurement_baseline_contract() -> dict[str, Any]:
|
|
|
2236
2520
|
}
|
|
2237
2521
|
|
|
2238
2522
|
|
|
2523
|
+
# --- image-context evaluation profile: bounded validation + fail-closed preflight ---
|
|
2524
|
+
#
|
|
2525
|
+
# 검증 순서는 불변이다: 타입/바운드 검사가 항상 semantic 정책 분류보다 먼저 실행된다.
|
|
2526
|
+
# 따라서 oversize 된 non-`deny` 정책 값이 blocked-scorecard 분기로 새어나갈 수 없다.
|
|
2527
|
+
# 구조적으로 해석 불가능한 evidence 는 어떤 출력 바이트도 쓰이기 전에 거부(reject_prewrite)되고,
|
|
2528
|
+
# 형식이 올바른 negative evidence 는 수용되어 blocked lane score 로 보고된다.
|
|
2529
|
+
|
|
2530
|
+
PROFILE_CONTROL_BLOCK_KEYS = (
|
|
2531
|
+
"control_provenance",
|
|
2532
|
+
"exact_text_fallback",
|
|
2533
|
+
"human_correction",
|
|
2534
|
+
"missed_context_review",
|
|
2535
|
+
"prompt_evidence",
|
|
2536
|
+
"protected_zone_review",
|
|
2537
|
+
"provider_usage",
|
|
2538
|
+
"shifted_cost",
|
|
2539
|
+
"source_omission",
|
|
2540
|
+
)
|
|
2541
|
+
PROFILE_NESTED_KEYS: dict[str, tuple[str, ...]] = {
|
|
2542
|
+
"control_provenance": ("review_source", "verifier_label"),
|
|
2543
|
+
"exact_text_fallback": (
|
|
2544
|
+
"available", "verified", "receipt_id", "content_sha256",
|
|
2545
|
+
"retrieval_command", "verifier_projection",
|
|
2546
|
+
),
|
|
2547
|
+
"human_correction": ("count", "reason"),
|
|
2548
|
+
"missed_context_review": ("correction_required", "present", "review_completed", "summary"),
|
|
2549
|
+
"prompt_evidence": ("sha256", "source_label"),
|
|
2550
|
+
"protected_zone_review": (
|
|
2551
|
+
"included_prompt_like_regions", "included_protected_regions", "policy",
|
|
2552
|
+
"review_completed", "review_note", "reviewer_label",
|
|
2553
|
+
),
|
|
2554
|
+
"provider_usage": ("primary_cost_measured", "primary_tokens_measured", "provider_called"),
|
|
2555
|
+
"shifted_cost": ("external_cost_measured", "external_tokens_measured", "status"),
|
|
2556
|
+
"source_omission": ("present", "transform"),
|
|
2557
|
+
}
|
|
2558
|
+
PROFILE_PROJECTION_KEYS = (
|
|
2559
|
+
"schema", "status", "blockers", "candidate_replacement", "claim_boundary", "proof_unit",
|
|
2560
|
+
)
|
|
2561
|
+
PROFILE_PROOF_UNIT_KEYS = (
|
|
2562
|
+
"status", "receipt_id", "receipt_verified", "content_hash_declared_value",
|
|
2563
|
+
"content_hash_verified", "rehydration_receipt_bound", "rehydration_syntax_valid",
|
|
2564
|
+
"rehydration_verified", "rehydration_executed", "retrieval_command",
|
|
2565
|
+
)
|
|
2566
|
+
PROFILE_PROOF_UNIT_REQUIRED_FLAGS = (
|
|
2567
|
+
"receipt_verified",
|
|
2568
|
+
"content_hash_verified",
|
|
2569
|
+
"rehydration_receipt_bound",
|
|
2570
|
+
"rehydration_syntax_valid",
|
|
2571
|
+
"rehydration_verified",
|
|
2572
|
+
)
|
|
2573
|
+
PROFILE_SHIFTED_COST_STATUSES = ("measured", "unmeasured")
|
|
2574
|
+
|
|
2575
|
+
|
|
2576
|
+
def redact_profile_label(value: Any) -> str:
|
|
2577
|
+
"""Bound one untrusted label so an error message can never carry a payload.
|
|
2578
|
+
|
|
2579
|
+
Task ids, variant names, unknown evidence keys, and prompt-map labels are
|
|
2580
|
+
author-controlled. Opted-in profile diagnostics must use a fully opaque fixed
|
|
2581
|
+
representation for every such value — even regex-safe-looking identifiers —
|
|
2582
|
+
because any preserved attacker text is itself a leak channel. The stable error
|
|
2583
|
+
id and fixed field names carry the meaning; ``value`` is intentionally unused.
|
|
2584
|
+
"""
|
|
2585
|
+
# 작성자 통제 값은 형태와 무관하게 절대 진단에 싣지 않는다.
|
|
2586
|
+
return PROFILE_REDACTED_PLACEHOLDER
|
|
2587
|
+
|
|
2588
|
+
|
|
2589
|
+
def redact_profile_labels(values: Iterable[Any]) -> str:
|
|
2590
|
+
"""Render a bounded, fully opaque key list for schema errors.
|
|
2591
|
+
|
|
2592
|
+
The count is always truthful; each name collapses to the shared placeholder so
|
|
2593
|
+
no author-controlled text rides the message. The list is truncated so an
|
|
2594
|
+
oversized evidence object cannot flood the diagnostic.
|
|
2595
|
+
"""
|
|
2596
|
+
labels = [redact_profile_label(value) for value in values]
|
|
2597
|
+
shown = labels[:MAX_PROFILE_ERROR_LABELS]
|
|
2598
|
+
overflow = len(labels) - len(shown)
|
|
2599
|
+
rendered = ", ".join(shown)
|
|
2600
|
+
if overflow > 0:
|
|
2601
|
+
rendered = f"{rendered}, +{overflow} more"
|
|
2602
|
+
return rendered
|
|
2603
|
+
|
|
2604
|
+
|
|
2605
|
+
def profile_owner(task_id: Any, variant: Any = None) -> str:
|
|
2606
|
+
"""Build the redacted owner prefix shared by every profile error."""
|
|
2607
|
+
owner = f"task {redact_profile_label(task_id)}"
|
|
2608
|
+
if variant is not None:
|
|
2609
|
+
owner = f"{owner} variant {redact_profile_label(variant)}"
|
|
2610
|
+
return owner
|
|
2611
|
+
|
|
2612
|
+
|
|
2613
|
+
def profile_reject(error_id: str, owner: str, detail: str) -> "NoReturn":
|
|
2614
|
+
"""Fail closed with a stable id and a bounded, redacted message.
|
|
2615
|
+
|
|
2616
|
+
Raw policy text, prompt content, and filesystem paths are never echoed. ``owner``
|
|
2617
|
+
must already come from :func:`profile_owner` and ``detail`` must be a fixed
|
|
2618
|
+
literal or a redacted label list; the final message is sanitized again so no
|
|
2619
|
+
caller can smuggle a secret-shaped value through.
|
|
2620
|
+
"""
|
|
2621
|
+
raise SystemExit(sanitize_note_text(f"{error_id}: {owner} {detail}"))
|
|
2622
|
+
|
|
2623
|
+
|
|
2624
|
+
def profile_block(controls: dict[str, Any], key: str, *, owner: str) -> dict[str, Any]:
|
|
2625
|
+
if key not in controls:
|
|
2626
|
+
profile_reject(PROFILE_REJECT_CONTROLS_MISSING, owner, f"evaluation_controls.{key} is required")
|
|
2627
|
+
value = controls[key]
|
|
2628
|
+
if not isinstance(value, dict):
|
|
2629
|
+
profile_reject(PROFILE_REJECT_SCHEMA_INVALID, owner, f"evaluation_controls.{key} must be an object")
|
|
2630
|
+
unknown = sorted(set(value) - set(PROFILE_NESTED_KEYS[key]))
|
|
2631
|
+
if unknown:
|
|
2632
|
+
profile_reject(
|
|
2633
|
+
PROFILE_REJECT_SCHEMA_INVALID, owner,
|
|
2634
|
+
f"evaluation_controls.{key} has unknown v1 key(s): {redact_profile_labels(unknown)}",
|
|
2635
|
+
)
|
|
2636
|
+
return value
|
|
2637
|
+
|
|
2638
|
+
|
|
2639
|
+
def profile_bool(block: dict[str, Any], key: str, *, owner: str, label: str) -> bool:
|
|
2640
|
+
value = block.get(key)
|
|
2641
|
+
if not isinstance(value, bool):
|
|
2642
|
+
profile_reject(PROFILE_REJECT_SCHEMA_INVALID, owner, f"{label}.{key} must be a boolean")
|
|
2643
|
+
return value
|
|
2644
|
+
|
|
2645
|
+
|
|
2646
|
+
def profile_int(block: dict[str, Any], key: str, *, owner: str, label: str, maximum: int) -> int:
|
|
2647
|
+
value = block.get(key)
|
|
2648
|
+
if isinstance(value, bool) or not isinstance(value, int):
|
|
2649
|
+
profile_reject(PROFILE_REJECT_SCHEMA_INVALID, owner, f"{label}.{key} must be an integer")
|
|
2650
|
+
if value < 0 or value > maximum:
|
|
2651
|
+
profile_reject(PROFILE_REJECT_SCHEMA_INVALID, owner, f"{label}.{key} is outside its allowed bounds")
|
|
2652
|
+
return value
|
|
2653
|
+
|
|
2654
|
+
|
|
2655
|
+
def profile_text(block: dict[str, Any], key: str, *, owner: str, label: str, maximum: int) -> str:
|
|
2656
|
+
value = block.get(key)
|
|
2657
|
+
if not isinstance(value, str):
|
|
2658
|
+
profile_reject(PROFILE_REJECT_SCHEMA_INVALID, owner, f"{label}.{key} must be a string")
|
|
2659
|
+
if len(value.encode("utf-8")) > maximum:
|
|
2660
|
+
# 값 자체는 절대 에러에 실지 않는다. 길이 위반 사실만 보고한다.
|
|
2661
|
+
profile_reject(PROFILE_REJECT_SCHEMA_INVALID, owner, f"{label}.{key} exceeds its {maximum}-byte bound")
|
|
2662
|
+
return value
|
|
2663
|
+
|
|
2664
|
+
|
|
2665
|
+
def profile_variant_prompt_sha256(task: TaskFixture, variant_name: str, *, task_file_dir: Path, owner: str) -> str:
|
|
2666
|
+
"""SHA-256 of the selected variant prompt file, read with the existing safe reader."""
|
|
2667
|
+
raw_path = task.variant_prompt_files.get(variant_name)
|
|
2668
|
+
if not raw_path:
|
|
2669
|
+
profile_reject(
|
|
2670
|
+
PROFILE_REJECT_PROMPT_BINDING_INVALID, owner,
|
|
2671
|
+
"a profiled task requires a file-backed variant_prompt_files entry for every selected variant",
|
|
2672
|
+
)
|
|
2673
|
+
try:
|
|
2674
|
+
rel_path = validate_variant_prompt_file_path(raw_path, owner=owner)
|
|
2675
|
+
text = read_variant_prompt_file(
|
|
2676
|
+
task_file_dir / rel_path, owner=owner, display_path=str(rel_path),
|
|
2677
|
+
)
|
|
2678
|
+
except (SystemExit, ValueError, UnicodeError):
|
|
2679
|
+
# SystemExit: 경로/내용 누설 방지용 안정 프로파일 오류 재작성.
|
|
2680
|
+
# ValueError/UnicodeError: 조기 검증을 지나친 os.open NUL·인코딩 거부 등만 좁게 정규화.
|
|
2681
|
+
# 프로그래머 버그를 숨기지 않도록 범용 Exception 은 잡지 않는다.
|
|
2682
|
+
profile_reject(
|
|
2683
|
+
PROFILE_REJECT_PROMPT_BINDING_INVALID, owner,
|
|
2684
|
+
"the selected variant prompt file could not be safely read",
|
|
2685
|
+
)
|
|
2686
|
+
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
|
2687
|
+
|
|
2688
|
+
|
|
2689
|
+
def validate_profile_fallback_projection(fallback: dict[str, Any], *, owner: str) -> None:
|
|
2690
|
+
"""Validate one bounded imported local-verifier projection.
|
|
2691
|
+
|
|
2692
|
+
A record that *claims* verification while contradicting its own binding fields is a
|
|
2693
|
+
structural rejection. Replay never authenticates the record's author and never
|
|
2694
|
+
rereads the artifact; this only checks internal consistency.
|
|
2695
|
+
"""
|
|
2696
|
+
# 검증을 주장하려면 fallback 자체가 available 이어야 한다. available=false 인데
|
|
2697
|
+
# verified=true 인 레코드는 자기 모순이다.
|
|
2698
|
+
if fallback.get("available") is not True:
|
|
2699
|
+
profile_reject(
|
|
2700
|
+
PROFILE_REJECT_FALLBACK_CLAIM_INCONSISTENT, owner,
|
|
2701
|
+
"exact_text_fallback claims verification while declaring the fallback unavailable",
|
|
2702
|
+
)
|
|
2703
|
+
# placeholder receipt/command 로는 어떤 것도 되찾을 수 없다. 검증 주장에는 실제
|
|
2704
|
+
# 값이 필요하다.
|
|
2705
|
+
for key in ("receipt_id", "retrieval_command"):
|
|
2706
|
+
value = fallback.get(key)
|
|
2707
|
+
if not isinstance(value, str) or value.strip().lower() in PROFILE_FALLBACK_PLACEHOLDER_VALUES:
|
|
2708
|
+
profile_reject(
|
|
2709
|
+
PROFILE_REJECT_FALLBACK_CLAIM_INCONSISTENT, owner,
|
|
2710
|
+
f"exact_text_fallback claims verification without an exact {key}",
|
|
2711
|
+
)
|
|
2712
|
+
projection = fallback.get("verifier_projection")
|
|
2713
|
+
if not isinstance(projection, dict):
|
|
2714
|
+
profile_reject(
|
|
2715
|
+
PROFILE_REJECT_FALLBACK_CLAIM_INCONSISTENT, owner,
|
|
2716
|
+
"exact_text_fallback claims verification without a bounded verifier projection",
|
|
2717
|
+
)
|
|
2718
|
+
unknown = sorted(set(projection) - set(PROFILE_PROJECTION_KEYS))
|
|
2719
|
+
if unknown:
|
|
2720
|
+
profile_reject(
|
|
2721
|
+
PROFILE_REJECT_SCHEMA_INVALID, owner,
|
|
2722
|
+
f"exact_text_fallback.verifier_projection has unknown v1 key(s): {redact_profile_labels(unknown)}",
|
|
2723
|
+
)
|
|
2724
|
+
if projection.get("schema") != PROOF_VERIFICATION_SCHEMA_VERSION:
|
|
2725
|
+
profile_reject(
|
|
2726
|
+
PROFILE_REJECT_FALLBACK_CLAIM_INCONSISTENT, owner,
|
|
2727
|
+
"verifier projection schema does not match the local proof-verification contract",
|
|
2728
|
+
)
|
|
2729
|
+
if projection.get("status") != PROOF_VERIFICATION_VERIFIED_STATUS:
|
|
2730
|
+
profile_reject(
|
|
2731
|
+
PROFILE_REJECT_FALLBACK_CLAIM_INCONSISTENT, owner,
|
|
2732
|
+
"exact_text_fallback claims verification but the verifier status is not verified",
|
|
2733
|
+
)
|
|
2734
|
+
blockers = projection.get("blockers")
|
|
2735
|
+
if not isinstance(blockers, list) or len(blockers) > MAX_PROFILE_BLOCKER_ITEMS:
|
|
2736
|
+
profile_reject(PROFILE_REJECT_SCHEMA_INVALID, owner, "verifier projection blockers must be a bounded list")
|
|
2737
|
+
if blockers:
|
|
2738
|
+
profile_reject(
|
|
2739
|
+
PROFILE_REJECT_FALLBACK_CLAIM_INCONSISTENT, owner,
|
|
2740
|
+
"exact_text_fallback claims verification but the verifier reports blockers",
|
|
2741
|
+
)
|
|
2742
|
+
if projection.get("candidate_replacement") is not None:
|
|
2743
|
+
profile_reject(
|
|
2744
|
+
PROFILE_REJECT_FALLBACK_CLAIM_INCONSISTENT, owner,
|
|
2745
|
+
"exact_text_fallback claims verification but the verifier proposes a candidate replacement",
|
|
2746
|
+
)
|
|
2747
|
+
# 가져온 레코드는 local-only 경계를 그대로 선언해야 한다. 경계가 다르면 이 lane 이
|
|
2748
|
+
# 인정할 수 있는 권한 범위를 벗어난 주장이다.
|
|
2749
|
+
if projection.get("claim_boundary") != PROOF_VERIFICATION_CLAIM_BOUNDARY:
|
|
2750
|
+
profile_reject(
|
|
2751
|
+
PROFILE_REJECT_FALLBACK_CLAIM_INCONSISTENT, owner,
|
|
2752
|
+
"exact_text_fallback claims verification without the expected local-only claim boundary",
|
|
2753
|
+
)
|
|
2754
|
+
unit = projection.get("proof_unit")
|
|
2755
|
+
if not isinstance(unit, dict):
|
|
2756
|
+
profile_reject(
|
|
2757
|
+
PROFILE_REJECT_FALLBACK_CLAIM_INCONSISTENT, owner,
|
|
2758
|
+
"exact_text_fallback claims verification without exactly one verified proof unit",
|
|
2759
|
+
)
|
|
2760
|
+
unknown_unit = sorted(set(unit) - set(PROFILE_PROOF_UNIT_KEYS))
|
|
2761
|
+
if unknown_unit:
|
|
2762
|
+
profile_reject(
|
|
2763
|
+
PROFILE_REJECT_SCHEMA_INVALID, owner,
|
|
2764
|
+
f"verifier proof_unit has unknown v1 key(s): {redact_profile_labels(unknown_unit)}",
|
|
2765
|
+
)
|
|
2766
|
+
if unit.get("status") != PROOF_VERIFICATION_VERIFIED_STATUS:
|
|
2767
|
+
profile_reject(
|
|
2768
|
+
PROFILE_REJECT_FALLBACK_CLAIM_INCONSISTENT, owner,
|
|
2769
|
+
"exact_text_fallback claims verification but its proof unit is not verified",
|
|
2770
|
+
)
|
|
2771
|
+
for flag in PROFILE_PROOF_UNIT_REQUIRED_FLAGS:
|
|
2772
|
+
if unit.get(flag) is not True:
|
|
2773
|
+
profile_reject(
|
|
2774
|
+
PROFILE_REJECT_FALLBACK_CLAIM_INCONSISTENT, owner,
|
|
2775
|
+
f"exact_text_fallback claims verification but proof_unit.{flag} does not confirm it",
|
|
2776
|
+
)
|
|
2777
|
+
# local verifier 는 rehydration 을 실행하지 않는다. 실행했다고 주장하면 이 레코드는
|
|
2778
|
+
# 우리가 검증할 수 있는 evaluation-only 경계 밖의 산출물이다.
|
|
2779
|
+
if unit.get("rehydration_executed") is not PROOF_VERIFICATION_REHYDRATION_EXECUTED:
|
|
2780
|
+
profile_reject(
|
|
2781
|
+
PROFILE_REJECT_FALLBACK_CLAIM_INCONSISTENT, owner,
|
|
2782
|
+
"exact_text_fallback claims verification but proof_unit.rehydration_executed leaves the local-only boundary",
|
|
2783
|
+
)
|
|
2784
|
+
if unit.get("receipt_id") != fallback["receipt_id"]:
|
|
2785
|
+
profile_reject(
|
|
2786
|
+
PROFILE_REJECT_FALLBACK_CLAIM_INCONSISTENT, owner,
|
|
2787
|
+
"exact_text_fallback receipt id is not bound to the verified proof unit",
|
|
2788
|
+
)
|
|
2789
|
+
if unit.get("content_hash_declared_value") != fallback["content_sha256"]:
|
|
2790
|
+
profile_reject(
|
|
2791
|
+
PROFILE_REJECT_FALLBACK_CLAIM_INCONSISTENT, owner,
|
|
2792
|
+
"exact_text_fallback content hash is not bound to the verified proof unit",
|
|
2793
|
+
)
|
|
2794
|
+
if unit.get("retrieval_command") != fallback["retrieval_command"]:
|
|
2795
|
+
profile_reject(
|
|
2796
|
+
PROFILE_REJECT_FALLBACK_CLAIM_INCONSISTENT, owner,
|
|
2797
|
+
"exact_text_fallback retrieval command is not bound to the verified proof unit",
|
|
2798
|
+
)
|
|
2799
|
+
|
|
2800
|
+
|
|
2801
|
+
def validate_profile_row_controls(
|
|
2802
|
+
row: EvidenceReplayRow,
|
|
2803
|
+
task: TaskFixture,
|
|
2804
|
+
*,
|
|
2805
|
+
task_file_dir: Path,
|
|
2806
|
+
) -> dict[str, Any]:
|
|
2807
|
+
"""Validate one profiled evidence row and return its normalized lane record."""
|
|
2808
|
+
owner = profile_owner(task.id, row.result.variant)
|
|
2809
|
+
controls = row.evaluation_controls
|
|
2810
|
+
if controls is None:
|
|
2811
|
+
profile_reject(PROFILE_REJECT_CONTROLS_MISSING, owner, "evaluation_controls is required for a profiled row")
|
|
2812
|
+
unknown = sorted(set(controls) - set(PROFILE_CONTROL_BLOCK_KEYS))
|
|
2813
|
+
if unknown:
|
|
2814
|
+
profile_reject(
|
|
2815
|
+
PROFILE_REJECT_SCHEMA_INVALID, owner,
|
|
2816
|
+
f"evaluation_controls has unknown v1 key(s): {redact_profile_labels(unknown)}",
|
|
2817
|
+
)
|
|
2818
|
+
|
|
2819
|
+
# --- prompt binding -------------------------------------------------
|
|
2820
|
+
prompt_evidence = profile_block(controls, "prompt_evidence", owner=owner)
|
|
2821
|
+
declared_sha = profile_text(prompt_evidence, "sha256", owner=owner, label="prompt_evidence", maximum=64)
|
|
2822
|
+
profile_text(prompt_evidence, "source_label", owner=owner, label="prompt_evidence", maximum=MAX_PROFILE_LABEL_CHARS)
|
|
2823
|
+
if not SHA256_HEX_PATTERN.match(declared_sha):
|
|
2824
|
+
profile_reject(PROFILE_REJECT_SCHEMA_INVALID, owner, "prompt_evidence.sha256 must be lowercase hex SHA-256")
|
|
2825
|
+
actual_sha = profile_variant_prompt_sha256(task, row.result.variant, task_file_dir=task_file_dir, owner=owner)
|
|
2826
|
+
if declared_sha != actual_sha:
|
|
2827
|
+
profile_reject(
|
|
2828
|
+
PROFILE_REJECT_PROMPT_BINDING_INVALID, owner,
|
|
2829
|
+
"prompt_evidence.sha256 does not match the locally recomputed prompt hash",
|
|
2830
|
+
)
|
|
2831
|
+
|
|
2832
|
+
# --- omission + exact-text fallback ---------------------------------
|
|
2833
|
+
source_omission = profile_block(controls, "source_omission", owner=owner)
|
|
2834
|
+
omission_present = profile_bool(source_omission, "present", owner=owner, label="source_omission")
|
|
2835
|
+
profile_text(source_omission, "transform", owner=owner, label="source_omission", maximum=MAX_PROFILE_LABEL_CHARS)
|
|
2836
|
+
|
|
2837
|
+
fallback = profile_block(controls, "exact_text_fallback", owner=owner)
|
|
2838
|
+
profile_bool(fallback, "available", owner=owner, label="exact_text_fallback")
|
|
2839
|
+
fallback_verified = profile_bool(fallback, "verified", owner=owner, label="exact_text_fallback")
|
|
2840
|
+
profile_text(fallback, "receipt_id", owner=owner, label="exact_text_fallback", maximum=MAX_PROFILE_RECEIPT_ID_CHARS)
|
|
2841
|
+
fallback_sha = profile_text(
|
|
2842
|
+
fallback, "content_sha256", owner=owner, label="exact_text_fallback", maximum=MAX_PROFILE_RECEIPT_ID_CHARS,
|
|
2843
|
+
)
|
|
2844
|
+
profile_text(
|
|
2845
|
+
fallback, "retrieval_command", owner=owner, label="exact_text_fallback", maximum=MAX_PROFILE_COMMAND_CHARS,
|
|
2846
|
+
)
|
|
2847
|
+
if fallback_verified:
|
|
2848
|
+
if not SHA256_HEX_PATTERN.match(fallback_sha):
|
|
2849
|
+
profile_reject(
|
|
2850
|
+
PROFILE_REJECT_SCHEMA_INVALID, owner,
|
|
2851
|
+
"exact_text_fallback.content_sha256 must be lowercase hex SHA-256 when verification is claimed",
|
|
2852
|
+
)
|
|
2853
|
+
validate_profile_fallback_projection(fallback, owner=owner)
|
|
2854
|
+
# 생략된 원문이 있는데 통과한 attestation 이 없으면 lane 을 막는다(거부가 아니다).
|
|
2855
|
+
fallback_bound = (not omission_present) or fallback_verified
|
|
2856
|
+
|
|
2857
|
+
# --- protected zone review ------------------------------------------
|
|
2858
|
+
protection = profile_block(controls, "protected_zone_review", owner=owner)
|
|
2859
|
+
# 바운드 검사가 정책 의미 분류보다 먼저다: oversize 값은 blocked 가 아니라 reject 다.
|
|
2860
|
+
policy = profile_text(
|
|
2861
|
+
protection, "policy", owner=owner, label="protected_zone_review", maximum=MAX_PROFILE_POLICY_CHARS,
|
|
2862
|
+
)
|
|
2863
|
+
protection_completed = profile_bool(protection, "review_completed", owner=owner, label="protected_zone_review")
|
|
2864
|
+
included_protected = profile_int(
|
|
2865
|
+
protection, "included_protected_regions", owner=owner,
|
|
2866
|
+
label="protected_zone_review", maximum=MAX_PROFILE_PROTECTED_REGION_COUNT,
|
|
2867
|
+
)
|
|
2868
|
+
included_prompt_like = profile_int(
|
|
2869
|
+
protection, "included_prompt_like_regions", owner=owner,
|
|
2870
|
+
label="protected_zone_review", maximum=MAX_PROFILE_PROTECTED_REGION_COUNT,
|
|
2871
|
+
)
|
|
2872
|
+
profile_text(
|
|
2873
|
+
protection, "reviewer_label", owner=owner, label="protected_zone_review", maximum=MAX_PROFILE_LABEL_CHARS,
|
|
2874
|
+
)
|
|
2875
|
+
profile_text(
|
|
2876
|
+
protection, "review_note", owner=owner, label="protected_zone_review", maximum=MAX_PROFILE_NOTE_CHARS,
|
|
2877
|
+
)
|
|
2878
|
+
protection_attested = (
|
|
2879
|
+
policy == PROTECTED_ZONE_DENY_POLICY
|
|
2880
|
+
and protection_completed
|
|
2881
|
+
and included_protected == 0
|
|
2882
|
+
and included_prompt_like == 0
|
|
2883
|
+
)
|
|
2884
|
+
|
|
2885
|
+
# --- missed context review ------------------------------------------
|
|
2886
|
+
missed = profile_block(controls, "missed_context_review", owner=owner)
|
|
2887
|
+
missed_completed = profile_bool(missed, "review_completed", owner=owner, label="missed_context_review")
|
|
2888
|
+
missed_present = profile_bool(missed, "present", owner=owner, label="missed_context_review")
|
|
2889
|
+
profile_bool(missed, "correction_required", owner=owner, label="missed_context_review")
|
|
2890
|
+
profile_text(missed, "summary", owner=owner, label="missed_context_review", maximum=MAX_PROFILE_SUMMARY_CHARS)
|
|
2891
|
+
missed_reviewed = missed_completed and not missed_present
|
|
2892
|
+
|
|
2893
|
+
# --- human correction consistency -----------------------------------
|
|
2894
|
+
correction = profile_block(controls, "human_correction", owner=owner)
|
|
2895
|
+
correction_count = profile_int(
|
|
2896
|
+
correction, "count", owner=owner, label="human_correction", maximum=MAX_PROFILE_CORRECTION_COUNT,
|
|
2897
|
+
)
|
|
2898
|
+
correction_reason = profile_text(
|
|
2899
|
+
correction, "reason", owner=owner, label="human_correction", maximum=MAX_PROFILE_NOTE_CHARS,
|
|
2900
|
+
)
|
|
2901
|
+
if correction_count != row.result.corrections:
|
|
2902
|
+
profile_reject(
|
|
2903
|
+
PROFILE_REJECT_CORRECTION_INCONSISTENT, owner,
|
|
2904
|
+
"human_correction.count does not equal the row's top-level corrections field",
|
|
2905
|
+
)
|
|
2906
|
+
if correction_count > 0 and (not correction_reason.strip() or correction_reason.strip().lower() == "none"):
|
|
2907
|
+
profile_reject(
|
|
2908
|
+
PROFILE_REJECT_CORRECTION_INCONSISTENT, owner,
|
|
2909
|
+
"a positive human_correction.count requires an explicit bounded reason",
|
|
2910
|
+
)
|
|
2911
|
+
|
|
2912
|
+
# --- measurement flags may never upgrade the generic normalized fields
|
|
2913
|
+
provider_usage = profile_block(controls, "provider_usage", owner=owner)
|
|
2914
|
+
lane_tokens_measured = profile_bool(provider_usage, "primary_tokens_measured", owner=owner, label="provider_usage")
|
|
2915
|
+
lane_cost_measured = profile_bool(provider_usage, "primary_cost_measured", owner=owner, label="provider_usage")
|
|
2916
|
+
profile_bool(provider_usage, "provider_called", owner=owner, label="provider_usage")
|
|
2917
|
+
if lane_tokens_measured != row.result.primary_tokens_measured or lane_cost_measured != row.result.cost_measured:
|
|
2918
|
+
profile_reject(
|
|
2919
|
+
PROFILE_REJECT_MEASUREMENT_INCONSISTENT, owner,
|
|
2920
|
+
"provider_usage measurement flags contradict the generic normalized provider fields",
|
|
2921
|
+
)
|
|
2922
|
+
|
|
2923
|
+
shifted = profile_block(controls, "shifted_cost", owner=owner)
|
|
2924
|
+
lane_external_tokens = profile_bool(shifted, "external_tokens_measured", owner=owner, label="shifted_cost")
|
|
2925
|
+
lane_external_cost = profile_bool(shifted, "external_cost_measured", owner=owner, label="shifted_cost")
|
|
2926
|
+
shifted_status = profile_text(
|
|
2927
|
+
shifted, "status", owner=owner, label="shifted_cost", maximum=MAX_PROFILE_LABEL_CHARS,
|
|
2928
|
+
)
|
|
2929
|
+
if shifted_status not in PROFILE_SHIFTED_COST_STATUSES:
|
|
2930
|
+
profile_reject(PROFILE_REJECT_SCHEMA_INVALID, owner, "shifted_cost.status must be measured or unmeasured")
|
|
2931
|
+
if (
|
|
2932
|
+
lane_external_tokens != row.result.external_tokens_measured
|
|
2933
|
+
or lane_external_cost != row.result.external_cost_measured
|
|
2934
|
+
):
|
|
2935
|
+
profile_reject(
|
|
2936
|
+
PROFILE_REJECT_MEASUREMENT_INCONSISTENT, owner,
|
|
2937
|
+
"shifted_cost measurement flags contradict the generic normalized shifted-cost fields",
|
|
2938
|
+
)
|
|
2939
|
+
if (shifted_status == "measured") != (lane_external_tokens and lane_external_cost):
|
|
2940
|
+
profile_reject(
|
|
2941
|
+
PROFILE_REJECT_MEASUREMENT_INCONSISTENT, owner,
|
|
2942
|
+
"shifted_cost.status contradicts its own measurement flags",
|
|
2943
|
+
)
|
|
2944
|
+
|
|
2945
|
+
provenance_block = profile_block(controls, "control_provenance", owner=owner)
|
|
2946
|
+
profile_text(
|
|
2947
|
+
provenance_block, "review_source", owner=owner, label="control_provenance", maximum=MAX_PROFILE_LABEL_CHARS,
|
|
2948
|
+
)
|
|
2949
|
+
profile_text(
|
|
2950
|
+
provenance_block, "verifier_label", owner=owner, label="control_provenance", maximum=MAX_PROFILE_LABEL_CHARS,
|
|
2951
|
+
)
|
|
2952
|
+
|
|
2953
|
+
# 정규화된 lane 판정만 보고에 전달한다. 자유 텍스트(정책/노트/요약/라벨)는 절대
|
|
2954
|
+
# 포함하지 않으므로 secret-shaped 값이 report/dashboard 로 새어나갈 수 없다.
|
|
2955
|
+
return {
|
|
2956
|
+
"task_id": task.id,
|
|
2957
|
+
"variant": row.result.variant,
|
|
2958
|
+
"success": bool(row.result.success),
|
|
2959
|
+
"source_omission_present": omission_present,
|
|
2960
|
+
"fallback_bound": fallback_bound,
|
|
2961
|
+
"fallback_verified": fallback_verified,
|
|
2962
|
+
# 아무 verifier 레코드도 제출되지 않은 경우("missing")와 제출되었으나 실패를
|
|
2963
|
+
# 보고하는 경우("failed")는 서로 다른 증거 수준이다.
|
|
2964
|
+
"fallback_projection_supplied": fallback.get("verifier_projection") is not None,
|
|
2965
|
+
"protected_zone_attested": protection_attested,
|
|
2966
|
+
"missed_context_reviewed": missed_reviewed,
|
|
2967
|
+
# lane 이 correction 판정을 lane 데이터에서 직접 유도할 수 있도록 정규화해 넘긴다.
|
|
2968
|
+
"human_correction_consistent": correction_count == row.result.corrections,
|
|
2969
|
+
"provider_measured": bool(row.result.primary_tokens_measured and row.result.cost_measured),
|
|
2970
|
+
"shifted_cost_measured": bool(lane_external_tokens and lane_external_cost),
|
|
2971
|
+
}
|
|
2972
|
+
|
|
2973
|
+
|
|
2974
|
+
def selected_profiled_task_ids(
|
|
2975
|
+
tasks: list[TaskFixture],
|
|
2976
|
+
targets: list[tuple[TaskFixture, Variant]],
|
|
2977
|
+
) -> list[str]:
|
|
2978
|
+
"""Profiled task ids that this invocation actually selected, in stable order."""
|
|
2979
|
+
profiled = {task.id for task in tasks if task.evaluation_profile is not None}
|
|
2980
|
+
selected = {task.id for task, _ in targets}
|
|
2981
|
+
return sorted(profiled & selected)
|
|
2982
|
+
|
|
2983
|
+
|
|
2984
|
+
def preflight_profile_replay_mode(
|
|
2985
|
+
tasks: list[TaskFixture],
|
|
2986
|
+
targets: list[tuple[TaskFixture, Variant]],
|
|
2987
|
+
*,
|
|
2988
|
+
evidence_replay_active: bool,
|
|
2989
|
+
) -> None:
|
|
2990
|
+
"""Refuse a profiled task outside evidence replay, before any provider call.
|
|
2991
|
+
|
|
2992
|
+
The profile is evaluation-only: it validates imported evidence and clamps every
|
|
2993
|
+
authority surface. Those checks live entirely on the replay path, so running a
|
|
2994
|
+
profiled task through the provider would execute a real run whose report never
|
|
2995
|
+
sees profile validation or the evaluation-only clamp. Fail closed instead, before
|
|
2996
|
+
the provider runtime, the lock sidecar, and the first output byte.
|
|
2997
|
+
|
|
2998
|
+
This also refuses ``--dry-run``. A dry run writes nothing and spawns no provider,
|
|
2999
|
+
so it is not itself dangerous, but keeping the invariant absolute — a profiled task
|
|
3000
|
+
never enters the provider path — is what makes the boundary auditable. The useful
|
|
3001
|
+
preview, ``--evidence-jsonl --dry-run``, is unaffected.
|
|
3002
|
+
"""
|
|
3003
|
+
if evidence_replay_active:
|
|
3004
|
+
return
|
|
3005
|
+
for task_id in selected_profiled_task_ids(tasks, targets):
|
|
3006
|
+
profile_reject(
|
|
3007
|
+
PROFILE_REJECT_REPLAY_REQUIRED, profile_owner(task_id),
|
|
3008
|
+
"a profiled task is evaluation-only and runs only under --evidence-jsonl replay; "
|
|
3009
|
+
"provider execution is refused",
|
|
3010
|
+
)
|
|
3011
|
+
|
|
3012
|
+
|
|
3013
|
+
def preflight_profile_fresh_output(
|
|
3014
|
+
tasks: list[TaskFixture],
|
|
3015
|
+
targets: list[tuple[TaskFixture, Variant]],
|
|
3016
|
+
*,
|
|
3017
|
+
resume: bool,
|
|
3018
|
+
csv_has_preexisting_content: bool,
|
|
3019
|
+
) -> None:
|
|
3020
|
+
"""Refuse a resumed or pre-existing profiled batch before any lock/read helper.
|
|
3021
|
+
|
|
3022
|
+
v1 gives up incremental replay so profile context cannot silently vanish from a
|
|
3023
|
+
resumed report. This must run before the resume key snapshot, which acquires the
|
|
3024
|
+
CSV lock and therefore creates a ``.lock`` sidecar: rejecting afterwards would
|
|
3025
|
+
leave a byte on disk for a run we refused.
|
|
3026
|
+
"""
|
|
3027
|
+
if not (resume or csv_has_preexisting_content):
|
|
3028
|
+
return
|
|
3029
|
+
for task_id in selected_profiled_task_ids(tasks, targets):
|
|
3030
|
+
profile_reject(
|
|
3031
|
+
PROFILE_REJECT_FRESH_OUTPUT_REQUIRED, profile_owner(task_id),
|
|
3032
|
+
"v1 profiled replay requires a fresh empty results CSV and forbids --resume",
|
|
3033
|
+
)
|
|
3034
|
+
|
|
3035
|
+
|
|
3036
|
+
def profile_batch_freshness_gate_unlocked(
|
|
3037
|
+
tasks: list[TaskFixture],
|
|
3038
|
+
targets: list[tuple[TaskFixture, Variant]],
|
|
3039
|
+
csv_path: Path,
|
|
3040
|
+
) -> None:
|
|
3041
|
+
"""Recheck output freshness while the caller holds the full-batch lock.
|
|
3042
|
+
|
|
3043
|
+
The pre-lock gate reads the CSV without the lock, so a concurrent writer could
|
|
3044
|
+
still land a row between that check and the first append. One recheck under the
|
|
3045
|
+
caller's held parent-directory lock closes that window for the whole batch.
|
|
3046
|
+
"""
|
|
3047
|
+
profiled_task_ids = selected_profiled_task_ids(tasks, targets)
|
|
3048
|
+
if not profiled_task_ids:
|
|
3049
|
+
return
|
|
3050
|
+
if file_has_content_no_follow(csv_path):
|
|
3051
|
+
profile_reject(
|
|
3052
|
+
PROFILE_REJECT_FRESH_OUTPUT_REQUIRED,
|
|
3053
|
+
profile_owner(profiled_task_ids[0]),
|
|
3054
|
+
"the results CSV gained content after the profiled batch was validated",
|
|
3055
|
+
)
|
|
3056
|
+
|
|
3057
|
+
|
|
3058
|
+
def preflight_evaluation_profiles(
|
|
3059
|
+
tasks: list[TaskFixture],
|
|
3060
|
+
variants: list[Variant],
|
|
3061
|
+
targets: list[tuple[TaskFixture, Variant]],
|
|
3062
|
+
evidence_rows: list[EvidenceReplayRow],
|
|
3063
|
+
*,
|
|
3064
|
+
task_file_dir: Path,
|
|
3065
|
+
resume: bool,
|
|
3066
|
+
csv_has_preexisting_content: bool,
|
|
3067
|
+
baseline_variant: str = "baseline",
|
|
3068
|
+
) -> None:
|
|
3069
|
+
"""Validate the complete profiled batch before the first output byte is written.
|
|
3070
|
+
|
|
3071
|
+
Runs before any CSV/ledger/report/dashboard write and before any lock sidecar is
|
|
3072
|
+
created, so a rejection leaves the filesystem byte-unchanged. Attaches the
|
|
3073
|
+
normalized lane record to each row, which makes report annotation infallible over
|
|
3074
|
+
an already-validated batch.
|
|
3075
|
+
"""
|
|
3076
|
+
# 같은 gate 를 main 이 lock helper 이전에 이미 호출한다. 여기서 다시 부르는 것은
|
|
3077
|
+
# 이 함수를 직접 쓰는 호출자도 같은 boundary 를 얻게 하기 위한 이중 방어다.
|
|
3078
|
+
preflight_profile_fresh_output(
|
|
3079
|
+
tasks, targets, resume=resume, csv_has_preexisting_content=csv_has_preexisting_content,
|
|
3080
|
+
)
|
|
3081
|
+
profiled_tasks = {task.id: task for task in tasks if task.evaluation_profile is not None}
|
|
3082
|
+
rows_by_task: dict[str, list[EvidenceReplayRow]] = collections.defaultdict(list)
|
|
3083
|
+
for row in evidence_rows:
|
|
3084
|
+
rows_by_task[row.result.task_id].append(row)
|
|
3085
|
+
|
|
3086
|
+
# profiled row 가 unprofiled task 에 붙는 경우도 binding 위반이다.
|
|
3087
|
+
for row in evidence_rows:
|
|
3088
|
+
task = profiled_tasks.get(row.result.task_id)
|
|
3089
|
+
if task is None and (row.evaluation_profile is not None or row.evaluation_controls is not None):
|
|
3090
|
+
profile_reject(
|
|
3091
|
+
PROFILE_REJECT_BINDING_MISMATCH,
|
|
3092
|
+
profile_owner(row.result.task_id, row.result.variant),
|
|
3093
|
+
"a profiled evidence row cannot be replayed against a task that does not declare the profile",
|
|
3094
|
+
)
|
|
3095
|
+
if not profiled_tasks:
|
|
3096
|
+
return
|
|
3097
|
+
|
|
3098
|
+
variant_names = {variant.name for variant in variants}
|
|
3099
|
+
if baseline_variant not in variant_names or len(variant_names - {baseline_variant}) < 1:
|
|
3100
|
+
profile_reject(
|
|
3101
|
+
PROFILE_REJECT_BATCH_INCOMPLETE,
|
|
3102
|
+
profile_owner(next(iter(sorted(profiled_tasks)))),
|
|
3103
|
+
"v1 profiled replay requires the configured baseline and at least one candidate variant",
|
|
3104
|
+
)
|
|
3105
|
+
|
|
3106
|
+
selected_by_task: dict[str, set[str]] = collections.defaultdict(set)
|
|
3107
|
+
for task, variant in targets:
|
|
3108
|
+
selected_by_task[task.id].add(variant.name)
|
|
3109
|
+
|
|
3110
|
+
for task_id, task in sorted(profiled_tasks.items()):
|
|
3111
|
+
selected = selected_by_task.get(task_id, set())
|
|
3112
|
+
if not selected:
|
|
3113
|
+
continue
|
|
3114
|
+
# v1 은 부분 배치를 허용하지 않는다: 선택된 배치가 모든 variant 를 덮어야 한다.
|
|
3115
|
+
expected = {variant.name for variant in variants}
|
|
3116
|
+
if selected != expected:
|
|
3117
|
+
profile_reject(
|
|
3118
|
+
PROFILE_REJECT_BATCH_INCOMPLETE, profile_owner(task_id),
|
|
3119
|
+
"v1 profiled replay requires the complete baseline/candidate batch; "
|
|
3120
|
+
"partial variant selection is not supported",
|
|
3121
|
+
)
|
|
3122
|
+
task_rows = rows_by_task.get(task_id, [])
|
|
3123
|
+
# 중복/여분 row 는 coverage 집합 계산에 흡수되어 조용히 통과했다. 배치가
|
|
3124
|
+
# 모호하면 어떤 lane 판정도 신뢰할 수 없으므로 안정 ID 로 먼저 거부한다.
|
|
3125
|
+
seen_variants: set[str] = set()
|
|
3126
|
+
for row in task_rows:
|
|
3127
|
+
owner = profile_owner(task_id, row.result.variant)
|
|
3128
|
+
if row.result.variant in seen_variants:
|
|
3129
|
+
profile_reject(
|
|
3130
|
+
PROFILE_REJECT_BATCH_INCOMPLETE, owner,
|
|
3131
|
+
"a profiled task cannot carry duplicate evidence rows for one variant",
|
|
3132
|
+
)
|
|
3133
|
+
seen_variants.add(row.result.variant)
|
|
3134
|
+
if row.result.variant not in expected:
|
|
3135
|
+
profile_reject(
|
|
3136
|
+
PROFILE_REJECT_BATCH_INCOMPLETE, owner,
|
|
3137
|
+
"a profiled batch cannot carry an evidence row for an unknown variant",
|
|
3138
|
+
)
|
|
3139
|
+
for row in task_rows:
|
|
3140
|
+
owner = profile_owner(task_id, row.result.variant)
|
|
3141
|
+
if row.evaluation_profile is None and row.evaluation_controls is None:
|
|
3142
|
+
profile_reject(
|
|
3143
|
+
PROFILE_REJECT_BATCH_INCOMPLETE, owner,
|
|
3144
|
+
"a profiled task cannot mix profiled and unprofiled evidence rows",
|
|
3145
|
+
)
|
|
3146
|
+
if row.evaluation_profile != task.evaluation_profile:
|
|
3147
|
+
profile_reject(
|
|
3148
|
+
PROFILE_REJECT_BINDING_MISMATCH, owner,
|
|
3149
|
+
"the evidence row profile does not equal the task profile",
|
|
3150
|
+
)
|
|
3151
|
+
|
|
3152
|
+
covered = {row.result.variant for row in task_rows}
|
|
3153
|
+
if not expected.issubset(covered):
|
|
3154
|
+
profile_reject(
|
|
3155
|
+
PROFILE_REJECT_BATCH_INCOMPLETE, profile_owner(task_id),
|
|
3156
|
+
"profiled replay requires complete baseline and candidate evidence coverage",
|
|
3157
|
+
)
|
|
3158
|
+
for row in task_rows:
|
|
3159
|
+
row.evaluation_lane = validate_profile_row_controls(row, task, task_file_dir=task_file_dir)
|
|
3160
|
+
|
|
3161
|
+
|
|
3162
|
+
def build_image_context_evaluation_lane(
|
|
3163
|
+
replay_rows: list[EvidenceReplayRow],
|
|
3164
|
+
report: dict[str, Any],
|
|
3165
|
+
) -> dict[str, Any] | None:
|
|
3166
|
+
"""Aggregate the validated lane records into the additive report block."""
|
|
3167
|
+
lanes = [row.evaluation_lane for row in replay_rows if row.evaluation_lane is not None]
|
|
3168
|
+
if not lanes:
|
|
3169
|
+
return None
|
|
3170
|
+
|
|
3171
|
+
provider_measured = all(lane["provider_measured"] for lane in lanes)
|
|
3172
|
+
shifted_measured = all(lane["shifted_cost_measured"] for lane in lanes)
|
|
3173
|
+
all_success = all(lane["success"] for lane in lanes)
|
|
3174
|
+
omission_lanes = [lane for lane in lanes if lane["source_omission_present"]]
|
|
3175
|
+
|
|
3176
|
+
# generic quality gate 결과를 lane 이 그대로 소비한다. profile 이 자기만의 판정으로
|
|
3177
|
+
# generic regression 을 덮어쓰지 못하게 하는 것이 목적이므로, 어떤 matched pair 가
|
|
3178
|
+
# regression 을 보고하면 lane 도 막힌다(보수적으로 fail-closed).
|
|
3179
|
+
#
|
|
3180
|
+
# 이름 붙은 두 gate 만 매핑하면 구멍이 남는다: quality_gate 는 corrections_regression /
|
|
3181
|
+
# failure_rate_regression 외에도 insufficient_corrections_data 같은 값을 낼 수 있고,
|
|
3182
|
+
# 그때 lane 이 ready 로 올라가면 안 된다. 따라서 불변식은 "pass 가 아니면 막는다" 이며,
|
|
3183
|
+
# 두 이름은 이유를 드러내는 구체 blocker 로 함께 유지한다.
|
|
3184
|
+
generic_quality_gates: set[str] = set()
|
|
3185
|
+
pairs = report.get("matched_pair_evidence")
|
|
3186
|
+
pair_keys: set[tuple[str, str]] = set()
|
|
3187
|
+
if isinstance(pairs, list):
|
|
3188
|
+
for pair in pairs:
|
|
3189
|
+
if isinstance(pair, dict) and isinstance(pair.get("quality_gate"), str):
|
|
3190
|
+
generic_quality_gates.add(pair["quality_gate"])
|
|
3191
|
+
if isinstance(pair.get("task_id"), str) and isinstance(pair.get("variant"), str):
|
|
3192
|
+
pair_keys.add((pair["task_id"], pair["variant"]))
|
|
3193
|
+
baseline_variant = report.get("baseline_variant")
|
|
3194
|
+
expected_pair_keys = {
|
|
3195
|
+
(lane["task_id"], lane["variant"])
|
|
3196
|
+
for lane in lanes
|
|
3197
|
+
if isinstance(baseline_variant, str) and lane["variant"] != baseline_variant
|
|
3198
|
+
}
|
|
3199
|
+
comparisons = report.get("comparisons")
|
|
3200
|
+
comparison_rows = [item for item in comparisons if isinstance(item, dict)] if isinstance(comparisons, list) else []
|
|
3201
|
+
expected_candidate_variants = {variant for _, variant in expected_pair_keys}
|
|
3202
|
+
comparison_variants = {
|
|
3203
|
+
item["variant"] for item in comparison_rows if isinstance(item.get("variant"), str)
|
|
3204
|
+
}
|
|
3205
|
+
all_comparisons_pass = (
|
|
3206
|
+
bool(comparison_rows)
|
|
3207
|
+
and comparison_variants == expected_candidate_variants
|
|
3208
|
+
and all(
|
|
3209
|
+
item.get("quality_gate") == GENERIC_QUALITY_GATE_PASS
|
|
3210
|
+
and isinstance(item.get("matched_successful_task_count"), int)
|
|
3211
|
+
and item["matched_successful_task_count"] > 0
|
|
3212
|
+
for item in comparison_rows
|
|
3213
|
+
)
|
|
3214
|
+
)
|
|
3215
|
+
complete_matched_pairs = bool(expected_pair_keys) and pair_keys == expected_pair_keys
|
|
3216
|
+
generic_quality_pass = (
|
|
3217
|
+
complete_matched_pairs
|
|
3218
|
+
and all_comparisons_pass
|
|
3219
|
+
and generic_quality_gates == {GENERIC_QUALITY_GATE_PASS}
|
|
3220
|
+
)
|
|
3221
|
+
|
|
3222
|
+
gate_results = {
|
|
3223
|
+
# preflight 를 통과했다면 profile/prompt binding 은 이미 증명되었다.
|
|
3224
|
+
IMAGE_CONTEXT_GATE_PROFILE_AND_PROMPT_BINDING: True,
|
|
3225
|
+
IMAGE_CONTEXT_GATE_PROTECTED_ZONE_DENY_REVIEW: all(lane["protected_zone_attested"] for lane in lanes),
|
|
3226
|
+
IMAGE_CONTEXT_GATE_EXACT_TEXT_FALLBACK_BINDING: all(lane["fallback_bound"] for lane in lanes),
|
|
3227
|
+
IMAGE_CONTEXT_GATE_MISSED_CONTEXT_REVIEW: all(lane["missed_context_reviewed"] for lane in lanes),
|
|
3228
|
+
# count/reason 모순은 거부되지만, 판정은 lane 레코드에서 직접 유도한다.
|
|
3229
|
+
IMAGE_CONTEXT_GATE_HUMAN_CORRECTION_CONSISTENCY: all(
|
|
3230
|
+
lane["human_correction_consistent"] for lane in lanes
|
|
3231
|
+
),
|
|
3232
|
+
IMAGE_CONTEXT_GATE_CORRECTIONS_REGRESSION: (
|
|
3233
|
+
GENERIC_QUALITY_GATE_CORRECTIONS_REGRESSION not in generic_quality_gates
|
|
3234
|
+
),
|
|
3235
|
+
IMAGE_CONTEXT_GATE_FAILURE_RATE_REGRESSION: (
|
|
3236
|
+
GENERIC_QUALITY_GATE_FAILURE_RATE_REGRESSION not in generic_quality_gates
|
|
3237
|
+
),
|
|
3238
|
+
# generic quality gate 가 pass 가 아니면 어떤 값이든 여기서 막힌다.
|
|
3239
|
+
IMAGE_CONTEXT_GATE_GENERIC_MATCHED_SUCCESS_AND_MEASUREMENT: (
|
|
3240
|
+
all_success and provider_measured and shifted_measured and generic_quality_pass
|
|
3241
|
+
),
|
|
3242
|
+
# 이 gate 는 통과해도 권한을 주지 않는다. 경계 자체가 불변이므로 항상 참이다.
|
|
3243
|
+
IMAGE_CONTEXT_GATE_EVALUATION_ONLY_PROMOTION_BOUNDARY: True,
|
|
3244
|
+
}
|
|
3245
|
+
blocking_gate_ids = [gate_id for gate_id in IMAGE_CONTEXT_GATE_IDS if not gate_results[gate_id]]
|
|
3246
|
+
|
|
3247
|
+
if not omission_lanes:
|
|
3248
|
+
fallback_level = "missing"
|
|
3249
|
+
elif all(lane["fallback_verified"] for lane in omission_lanes):
|
|
3250
|
+
fallback_level = IMPORTED_LOCAL_VERIFIER_ATTESTATION_LABEL
|
|
3251
|
+
elif any(
|
|
3252
|
+
lane["fallback_projection_supplied"] and not lane["fallback_verified"]
|
|
3253
|
+
for lane in omission_lanes
|
|
3254
|
+
):
|
|
3255
|
+
# 검증 레코드가 제출되었지만 성공을 증명하지 못했다.
|
|
3256
|
+
fallback_level = "failed"
|
|
3257
|
+
else:
|
|
3258
|
+
# 생략은 선언되었으나 어떤 verifier attestation 도 제출되지 않았다.
|
|
3259
|
+
fallback_level = "missing"
|
|
3260
|
+
|
|
3261
|
+
matched_task_ids = sorted({lane["task_id"] for lane in lanes})
|
|
3262
|
+
return {
|
|
3263
|
+
"schema_version": IMAGE_CONTEXT_READINESS_SCHEMA_VERSION,
|
|
3264
|
+
"status": PROFILE_STATUS_BLOCKED if blocking_gate_ids else PROFILE_STATUS_READY_FOR_BOUNDED_PILOT_REVIEW,
|
|
3265
|
+
"evaluation_only": True,
|
|
3266
|
+
"promotion_authority": False,
|
|
3267
|
+
"public_claim_allowed": False,
|
|
3268
|
+
"gate_ids": list(IMAGE_CONTEXT_GATE_IDS),
|
|
3269
|
+
"blocking_gate_ids": blocking_gate_ids,
|
|
3270
|
+
"matched_task_count": len(matched_task_ids),
|
|
3271
|
+
"evidence_levels": {
|
|
3272
|
+
"provider_measurement": "measured" if provider_measured else "unmeasured",
|
|
3273
|
+
"fallback_binding": fallback_level,
|
|
3274
|
+
"protected_zone": (
|
|
3275
|
+
"review_attested"
|
|
3276
|
+
if gate_results[IMAGE_CONTEXT_GATE_PROTECTED_ZONE_DENY_REVIEW] else "failed"
|
|
3277
|
+
),
|
|
3278
|
+
"missed_context": (
|
|
3279
|
+
"reviewed" if gate_results[IMAGE_CONTEXT_GATE_MISSED_CONTEXT_REVIEW] else "missing"
|
|
3280
|
+
),
|
|
3281
|
+
},
|
|
3282
|
+
# 표본 관측치는 어떤 승격 임계값도 정의하지 않는다.
|
|
3283
|
+
"sample_adequacy": {
|
|
3284
|
+
"matched_task_count": len(matched_task_ids),
|
|
3285
|
+
"profiled_row_count": len(lanes),
|
|
3286
|
+
"task_class_labels": matched_task_ids,
|
|
3287
|
+
"policy_status": PROFILE_SAMPLE_ADEQUACY_POLICY_STATUS,
|
|
3288
|
+
},
|
|
3289
|
+
"claim_boundary": IMAGE_CONTEXT_CLAIM_BOUNDARY,
|
|
3290
|
+
}
|
|
3291
|
+
|
|
3292
|
+
|
|
3293
|
+
def clamp_report_for_evaluation_profile(report: dict[str, Any], lane: dict[str, Any]) -> None:
|
|
3294
|
+
"""Force every public-authority surface to a non-candidate, evaluation-only value.
|
|
3295
|
+
|
|
3296
|
+
Complete lane evidence may reach ``ready_for_bounded_pilot_review``; it may never
|
|
3297
|
+
reach promotion authority or a public claim. Pre-clamp measurements survive only in
|
|
3298
|
+
explicitly non-authoritative fields such as ``raw_metric_claim_status``.
|
|
3299
|
+
"""
|
|
3300
|
+
report[EVALUATION_PROFILES_REPORT_KEY] = {IMAGE_CONTEXT_PROFILE_REPORT_KEY: lane}
|
|
3301
|
+
report["public_claim_status"] = IMAGE_CONTEXT_EVALUATION_ONLY_CLAIM_STATUS
|
|
3302
|
+
# 콘솔이 report['claim_status'] 를 그대로 출력하므로 legacy 필드도 함께 clamp 한다.
|
|
3303
|
+
report["claim_status"] = IMAGE_CONTEXT_EVALUATION_ONLY_CLAIM_STATUS
|
|
3304
|
+
report["public_claim_eligible"] = False
|
|
3305
|
+
|
|
3306
|
+
# replay_evidence 는 top-level 과 같은 이름의 권한 필드를 복사해 들고 있다. 여기를
|
|
3307
|
+
# 함께 clamp 하지 않으면 report.json 소비자가 중첩 사본에서 candidate/eligible=true 를
|
|
3308
|
+
# 그대로 읽어 evaluation-only 경계를 우회한다.
|
|
3309
|
+
replay_evidence = report.get("replay_evidence")
|
|
3310
|
+
if isinstance(replay_evidence, dict):
|
|
3311
|
+
replay_evidence["public_claim_status"] = IMAGE_CONTEXT_EVALUATION_ONLY_CLAIM_STATUS
|
|
3312
|
+
replay_evidence["public_claim_eligible"] = False
|
|
3313
|
+
replay_evidence["report_claim_gates_allow_public_claim"] = False
|
|
3314
|
+
|
|
3315
|
+
readiness = report.get("public_claim_readiness")
|
|
3316
|
+
if isinstance(readiness, dict):
|
|
3317
|
+
readiness["claim_allowed"] = False
|
|
3318
|
+
readiness["status"] = IMAGE_CONTEXT_EVALUATION_ONLY_CLAIM_STATUS
|
|
3319
|
+
readiness["reason"] = IMAGE_CONTEXT_PROFILE_BLOCKER_GATE_ID
|
|
3320
|
+
# `_observed` 라는 이름이 붙었어도 값 자체가 "..._public_claim_candidate" 라는
|
|
3321
|
+
# 권한 문자열이다. 그대로 두면 downstream 이 이 필드를 읽고 candidate 로 오해할
|
|
3322
|
+
# 수 있으므로, profiled report 에서는 관측치도 evaluation-only 로 clamp 한다.
|
|
3323
|
+
readiness["public_claim_status_observed"] = IMAGE_CONTEXT_EVALUATION_ONLY_CLAIM_STATUS
|
|
3324
|
+
readiness["public_claim_eligible_observed"] = False
|
|
3325
|
+
blocking = readiness.get("blocking_gate_ids")
|
|
3326
|
+
if isinstance(blocking, list) and IMAGE_CONTEXT_PROFILE_BLOCKER_GATE_ID not in blocking:
|
|
3327
|
+
blocking.append(IMAGE_CONTEXT_PROFILE_BLOCKER_GATE_ID)
|
|
3328
|
+
|
|
3329
|
+
pairs = report.get("matched_pair_evidence")
|
|
3330
|
+
if isinstance(pairs, list):
|
|
3331
|
+
for pair in pairs:
|
|
3332
|
+
if not isinstance(pair, dict):
|
|
3333
|
+
continue
|
|
3334
|
+
boundary = pair.get("claim_boundary")
|
|
3335
|
+
if isinstance(boundary, dict):
|
|
3336
|
+
boundary["token_savings_claim_allowed"] = False
|
|
3337
|
+
boundary["shifted_cost_claim_allowed"] = False
|
|
3338
|
+
boundary["evaluation_profile"] = IMAGE_CONTEXT_EVALUATION_PROFILE_ID
|
|
3339
|
+
|
|
3340
|
+
|
|
3341
|
+
def render_image_context_evaluation_section(report: dict[str, Any]) -> list[str]:
|
|
3342
|
+
"""Bounded dashboard section: statuses and ids only, never raw evidence text."""
|
|
3343
|
+
profiles = report.get(EVALUATION_PROFILES_REPORT_KEY)
|
|
3344
|
+
if not isinstance(profiles, dict):
|
|
3345
|
+
return []
|
|
3346
|
+
lane = profiles.get(IMAGE_CONTEXT_PROFILE_REPORT_KEY)
|
|
3347
|
+
if not isinstance(lane, dict):
|
|
3348
|
+
return []
|
|
3349
|
+
levels = lane.get("evidence_levels") if isinstance(lane.get("evidence_levels"), dict) else {}
|
|
3350
|
+
blockers = lane.get("blocking_gate_ids") or []
|
|
3351
|
+
sample = lane.get("sample_adequacy") if isinstance(lane.get("sample_adequacy"), dict) else {}
|
|
3352
|
+
return [
|
|
3353
|
+
"## Image-context evaluation",
|
|
3354
|
+
"",
|
|
3355
|
+
f"- Schema: `{markdown_value(lane.get('schema_version'))}`",
|
|
3356
|
+
f"- Status: `{markdown_value(lane.get('status'))}`",
|
|
3357
|
+
f"- Matched tasks: {markdown_value(lane.get('matched_task_count'))}",
|
|
3358
|
+
f"- Evaluation only: `{markdown_value(lane.get('evaluation_only'))}`",
|
|
3359
|
+
f"- Promotion authority: `{markdown_value(lane.get('promotion_authority'))}`",
|
|
3360
|
+
f"- Public claim allowed: `{markdown_value(lane.get('public_claim_allowed'))}`",
|
|
3361
|
+
f"- Provider measurement: `{markdown_value(levels.get('provider_measurement'))}`",
|
|
3362
|
+
f"- Fallback binding: `{markdown_value(levels.get('fallback_binding'))}`",
|
|
3363
|
+
f"- Protected zone: `{markdown_value(levels.get('protected_zone'))}`",
|
|
3364
|
+
f"- Missed context: `{markdown_value(levels.get('missed_context'))}`",
|
|
3365
|
+
f"- Blocking gates: `{markdown_value(', '.join(str(item) for item in blockers) if blockers else 'none')}`",
|
|
3366
|
+
f"- Sample policy: `{markdown_value(sample.get('policy_status'))}`",
|
|
3367
|
+
"",
|
|
3368
|
+
"> Claim boundary: this lane is evaluation-only. `ready_for_bounded_pilot_review` authorizes a "
|
|
3369
|
+
"bounded human pilot review of imported evidence; it is not promotion, not runtime authority, "
|
|
3370
|
+
"not quality proof, and not a hosted API token/cost savings claim. The fallback record is an "
|
|
3371
|
+
"imported local-verifier attestation: replay does not authenticate its author and does not "
|
|
3372
|
+
"reread the artifact.",
|
|
3373
|
+
"",
|
|
3374
|
+
]
|
|
3375
|
+
|
|
3376
|
+
|
|
2239
3377
|
def summarize_benchmark_rows(rows: list[dict[str, str]], baseline_variant: str) -> dict[str, Any]:
|
|
2240
3378
|
by_variant: dict[str, dict[str, Any]] = {}
|
|
2241
3379
|
successful_rows_by_variant_task: dict[str, dict[str, list[dict[str, str]]]] = {}
|
|
@@ -2967,6 +4105,11 @@ def annotate_replay_report(
|
|
|
2967
4105
|
replay_rows=replay_rows,
|
|
2968
4106
|
mixed_csv=mixed_csv,
|
|
2969
4107
|
)
|
|
4108
|
+
# Additive lane block. 이미 preflight 로 검증된 batch 위에서만 동작하므로 실패하지 않는다.
|
|
4109
|
+
# profile 이 없는 report 는 이 블록도, clamp 도 얻지 않는다(기존 동작 그대로).
|
|
4110
|
+
lane = build_image_context_evaluation_lane(replay_rows, report)
|
|
4111
|
+
if lane is not None:
|
|
4112
|
+
clamp_report_for_evaluation_profile(report, lane)
|
|
2970
4113
|
report["default_matrix"] = build_default_matrix(report)
|
|
2971
4114
|
return report
|
|
2972
4115
|
|
|
@@ -3552,6 +4695,8 @@ def render_dashboard_markdown(report: dict[str, Any]) -> str:
|
|
|
3552
4695
|
"allow it and public-claim provenance is complete. Proxy byte reductions are diagnostic "
|
|
3553
4696
|
"and are not hosted API token savings.",
|
|
3554
4697
|
"",
|
|
4698
|
+
# profile 이 선언된 report 에만 추가되는 bounded 섹션. 원문/정책/영수증 내용은 넣지 않는다.
|
|
4699
|
+
*render_image_context_evaluation_section(report),
|
|
3555
4700
|
"## Variant summary",
|
|
3556
4701
|
"",
|
|
3557
4702
|
"| Variant | Runs | Successes | Failure rate | Tokens/success | Bytes saved | Token proxy saved | Quality notes |",
|
|
@@ -3869,6 +5014,25 @@ def main() -> int:
|
|
|
3869
5014
|
print("no (task, variant) targets matched the filters", file=sys.stderr)
|
|
3870
5015
|
return 1
|
|
3871
5016
|
|
|
5017
|
+
# profile gate 는 어떤 lock/read helper 보다 먼저 끝난다. existing_keys_snapshot 은
|
|
5018
|
+
# CSV lock sidecar 를 만들기 때문에, 그 뒤에서 거부하면 우리가 거절한 실행이 이미
|
|
5019
|
+
# 파일 시스템에 바이트를 남긴 뒤가 된다.
|
|
5020
|
+
#
|
|
5021
|
+
# replay 경계는 --dry-run 에도 적용한다. dry-run 이 provider 를 부르지 않는 것은
|
|
5022
|
+
# 맞지만, 불변식을 "profiled task 는 provider 경로에 진입하지 않는다" 로 단순하게
|
|
5023
|
+
# 유지하는 편이 감사 가능하고 fail-closed 다. 의미 있는 미리보기인
|
|
5024
|
+
# `--evidence-jsonl --dry-run` 은 그대로 동작한다.
|
|
5025
|
+
preflight_profile_replay_mode(tasks, targets, evidence_replay_active=args.evidence_jsonl is not None)
|
|
5026
|
+
# 반대로 freshness 는 출력을 쓰는 실행에만 의미가 있다. dry-run 은 CSV/ledger/report 를
|
|
5027
|
+
# 하나도 쓰지 않으므로 기존 CSV 가 있어도 잃을 profile 문맥이 없다.
|
|
5028
|
+
if not args.dry_run:
|
|
5029
|
+
preflight_profile_fresh_output(
|
|
5030
|
+
tasks,
|
|
5031
|
+
targets,
|
|
5032
|
+
resume=args.resume,
|
|
5033
|
+
csv_has_preexisting_content=file_has_content_no_follow(args.csv),
|
|
5034
|
+
)
|
|
5035
|
+
|
|
3872
5036
|
if args.resume:
|
|
3873
5037
|
skip_keys, skip_keys_loaded_stamp = existing_keys_snapshot(args.csv)
|
|
3874
5038
|
skip_keys_stamp = {"stamp": skip_keys_loaded_stamp}
|
|
@@ -3893,6 +5057,18 @@ def main() -> int:
|
|
|
3893
5057
|
return 0
|
|
3894
5058
|
csv_had_preexisting_content = file_has_content_no_follow(args.csv)
|
|
3895
5059
|
evidence_rows = read_evidence_jsonl(args.evidence_jsonl)
|
|
5060
|
+
# 완전한 profile preflight 는 첫 append_csv 이전, 그리고 어떤 lock sidecar 도
|
|
5061
|
+
# 만들어지기 전에 끝난다. 실패 시 파일 시스템은 바이트 단위로 그대로 남는다.
|
|
5062
|
+
preflight_evaluation_profiles(
|
|
5063
|
+
tasks,
|
|
5064
|
+
variants,
|
|
5065
|
+
targets,
|
|
5066
|
+
evidence_rows,
|
|
5067
|
+
task_file_dir=args.tasks.parent,
|
|
5068
|
+
resume=args.resume,
|
|
5069
|
+
csv_has_preexisting_content=csv_had_preexisting_content,
|
|
5070
|
+
baseline_variant=args.baseline_variant,
|
|
5071
|
+
)
|
|
3896
5072
|
runnable_targets = resume_runnable_targets(
|
|
3897
5073
|
args.csv,
|
|
3898
5074
|
targets,
|
|
@@ -3901,40 +5077,54 @@ def main() -> int:
|
|
|
3901
5077
|
existing_key_cache_stamp=skip_keys_stamp,
|
|
3902
5078
|
)
|
|
3903
5079
|
evidence_by_key = validate_evidence_coverage(evidence_rows, runnable_targets)
|
|
5080
|
+
profiled_batch = bool(selected_profiled_task_ids(tasks, targets))
|
|
3904
5081
|
runnable_keys = {(task.id, variant.name) for task, variant in runnable_targets}
|
|
3905
5082
|
claude_ver = "evidence-replay"
|
|
3906
5083
|
completed = 0
|
|
3907
5084
|
replay_rows_written: list[EvidenceReplayRow] = []
|
|
3908
|
-
|
|
3909
|
-
|
|
3910
|
-
|
|
3911
|
-
|
|
3912
|
-
|
|
3913
|
-
|
|
3914
|
-
|
|
3915
|
-
|
|
3916
|
-
|
|
5085
|
+
pending_ledger_rows: list[tuple[EvidenceReplayRow, RunResult]] = []
|
|
5086
|
+
batch_lock = csv_parent_directory_lock(args.csv, create_parent=True) if profiled_batch else nullcontext()
|
|
5087
|
+
with batch_lock:
|
|
5088
|
+
if profiled_batch:
|
|
5089
|
+
# The same stable lock covers the raced freshness recheck and every
|
|
5090
|
+
# row. No sidecar is created, and no foreign append can land between
|
|
5091
|
+
# profiled rows because append_csv takes this lock first.
|
|
5092
|
+
profile_batch_freshness_gate_unlocked(tasks, targets, args.csv)
|
|
5093
|
+
for task, variant in targets:
|
|
5094
|
+
if args.resume and (task.id, variant.name) not in runnable_keys:
|
|
5095
|
+
print(f"skip {task.id}/{variant.name} (already in {args.csv})")
|
|
5096
|
+
continue
|
|
5097
|
+
evidence = evidence_by_key[(task.id, variant.name)]
|
|
5098
|
+
print(f"replay {task.id}/{variant.name} ...", flush=True)
|
|
5099
|
+
result = run_evidence_fixture(task, variant, evidence)
|
|
5100
|
+
writer = append_csv_unlocked if profiled_batch else append_csv
|
|
5101
|
+
wrote = writer(
|
|
5102
|
+
args.csv,
|
|
5103
|
+
claude_ver,
|
|
5104
|
+
result,
|
|
5105
|
+
skip_existing=args.resume,
|
|
5106
|
+
existing_key_cache=skip_keys if args.resume else None,
|
|
5107
|
+
existing_key_cache_stamp=skip_keys_stamp,
|
|
5108
|
+
)
|
|
5109
|
+
if wrote:
|
|
5110
|
+
replay_rows_written.append(evidence)
|
|
5111
|
+
if args.ledger_jsonl is not None:
|
|
5112
|
+
pending_ledger_rows.append((evidence, result))
|
|
5113
|
+
completed += 1
|
|
5114
|
+
status = "ok" if result.success else "FAIL"
|
|
5115
|
+
suffix = "" if wrote else " (CSV not updated; row already present)"
|
|
5116
|
+
print(
|
|
5117
|
+
f" {status} tokens={sum(result.tokens.values())} cost=${result.cost_usd:.4f} "
|
|
5118
|
+
f"wall_time={result.wall_time_seconds:.3f}s {sanitize_note_text(result.notes)}{suffix}"
|
|
5119
|
+
)
|
|
5120
|
+
# Ledger/report/dashboard writes happen after the CSV batch lock so distinct
|
|
5121
|
+
# outputs in the same directory cannot deadlock on the directory inode.
|
|
5122
|
+
for evidence, result in pending_ledger_rows:
|
|
5123
|
+
append_cost_shift_ledger(
|
|
5124
|
+
args.ledger_jsonl,
|
|
3917
5125
|
claude_ver,
|
|
3918
5126
|
result,
|
|
3919
|
-
|
|
3920
|
-
existing_key_cache=skip_keys if args.resume else None,
|
|
3921
|
-
existing_key_cache_stamp=skip_keys_stamp,
|
|
3922
|
-
)
|
|
3923
|
-
if wrote:
|
|
3924
|
-
replay_rows_written.append(evidence)
|
|
3925
|
-
if args.ledger_jsonl is not None:
|
|
3926
|
-
append_cost_shift_ledger(
|
|
3927
|
-
args.ledger_jsonl,
|
|
3928
|
-
claude_ver,
|
|
3929
|
-
result,
|
|
3930
|
-
replay_provenance=evidence.provenance_payload(),
|
|
3931
|
-
)
|
|
3932
|
-
completed += 1
|
|
3933
|
-
status = "ok" if result.success else "FAIL"
|
|
3934
|
-
suffix = "" if wrote else " (CSV not updated; row already present)"
|
|
3935
|
-
print(
|
|
3936
|
-
f" {status} tokens={sum(result.tokens.values())} cost=${result.cost_usd:.4f} "
|
|
3937
|
-
f"wall_time={result.wall_time_seconds:.3f}s {sanitize_note_text(result.notes)}{suffix}"
|
|
5127
|
+
replay_provenance=evidence.provenance_payload(),
|
|
3938
5128
|
)
|
|
3939
5129
|
if args.report_json is not None or args.dashboard_md is not None:
|
|
3940
5130
|
report = write_report_outputs(
|