agent-apprenticeship 0.1.6 → 0.1.7

package/src/agent_apprenticeship_trace/trace_normalizer.py

@@ -3,11 +3,12 @@ import json, re
 from pathlib import Path
 from typing import Any
 from pydantic import BaseModel, Field
-from .schemas import AgentTrace, ActualOutputs
+from .schemas import AgentTrace, AgentTraceStep, ActualOutputs
 from .io import write_json
+from .artifact_resolver import normalize_artifact_ref
 
-CANONICAL_TOP = {'schema_version','trace_id','collection_id','prior_trace_id','trace_mode','task','task_id','task_family_id','attempt_id','attempt_kind','attempt_status','agent_tools','started_at','ended_at','system_prompt','system_prompt_hash','skills','memory','agent_config','learning','termination_reason','steps','actual_outputs','artifacts','metadata_json'}
-CANONICAL_STEP = {'step','turn','actor','action','operation','tool','execution_mode','parallel_group','observation','input','input_source','output','state_change','reasoning','caused_by','causal_type','causal_note','alternatives_considered','success','step_outcome','error_type','error_message','message_role','feedback_type','feedback_content','started_at','ended_at','retry_of','artifact_refs','metadata_json'}
+CANONICAL_TOP = {'schema_version','trace_id','collection_id','run_id','package_id','bundle_id','prior_trace_id','trace_mode','task','task_id','task_family_id','attempt_id','attempt_kind','attempt_status','agent_tools','started_at','ended_at','system_prompt','system_prompt_hash','skills','memory','agent_config','environment_domain','actual_outputs_ref','input_artifact_refs','artifact_refs','deliverable_refs','output_summary','final_output_summary','learning','termination_reason','steps','actual_outputs','artifacts','metadata_json'}
+CANONICAL_STEP = set(AgentTraceStep.model_fields)
 VALID_OUTCOMES={'progress','neutral','blocked','failed','corrected','completed'}
 VALID_ACTIONS={'user_message','agent_step','output','error'}
 VALID_OPS={'plan','analyze','search','read','write','edit','execute','verify','download','install','ask_user','answer','select','grade','evaluate','revise','other'}
@@ -20,6 +21,14 @@ VALID_ATTEMPT_KINDS={'baseline','revised','apprentice_without_lessons','apprenti
 VALID_TRACE_MODES={'live','retraced','hybrid'}
 VALID_TERMINATION_REASONS={'task_complete','verifier_passed','verifier_failed','max_iterations_reached','agent_blocked','timeout','error_unrecoverable','partial_then_stopped','provider_usage_limit','other'}
 VALID_ATTEMPT_STATUSES={'completed','failed','blocked','fallback','partial'}
+ENVIRONMENT_DOMAINS={'terminal','browser','computer_use','web','search','codebase','software','os','android','mcp','api','database','spreadsheet','file','multimodal','device','robotics','lab','industrial','physical','mixed','unknown'}
+INTERACTION_SURFACES={'api','mcp_tool','mcp_resource','mcp_prompt','browser','computer_use','cli','file','codebase','database','spreadsheet','search','cloud_console','saas_app','async_job','queue','webhook','scheduler','messaging','email','collaboration','document','pdf','table','multimodal','device','industrial','physical','mixed','other_tool','other_surface','unknown'}
+BASE_STEP_FIELDS={'step','turn','actor','action','operation','tool','environment_domain','execution_mode','parallel_group','observation','input','input_source','output','state_change','reasoning','caused_by','causal_type','causal_note','alternatives_considered','success','step_outcome','error_type','error_message','message_role','feedback_type','feedback_content','started_at','ended_at','retry_of','artifact_refs','metadata_json'}
+SURFACE_STEP_FIELDS=CANONICAL_STEP-BASE_STEP_FIELDS
+LIST_SURFACE_FIELDS={name for name in SURFACE_STEP_FIELDS if name.endswith('_refs') or name in {'modalities','connected_surfaces','depends_on_agent_outputs','formula_refs','pivot_refs','chart_refs','data_validation_refs','file_paths','selected_result_refs','citation_refs','attachment_refs','decision_refs','page_refs','section_refs','table_extract_refs','figure_refs','annotation_refs','safety_limit_refs','compliance_refs','cross_system_dependencies','handoff_refs','state_sync_refs','consistency_checks','evidence_refs','verification_refs','redaction_notes','missing_surface_fields'}}
+INT_SURFACE_FIELDS={'http_status','retry_count','row_count','affected_row_count','column_count','result_count','timeout_seconds'}
+FLOAT_SURFACE_FIELDS={'latency_ms'}
+BOOL_SURFACE_FIELDS={'operator_override','human_supervisor_present','emergency_stop','followup_required'}
 
 class TraceNormalizationContext(BaseModel):
     task_id: str
@@ -74,6 +83,218 @@ def _as_list(v: Any) -> list[Any]:
     if isinstance(v, list): return v
     return [v]
 
+def _normalize_ref(ref: Any) -> str:
+    value = normalize_artifact_ref(ref).replace("\\", "/").strip().lstrip("/")
+    while value.startswith("./"):
+        value = value[2:]
+    return value
+
+def _normalize_ref_list(value: Any) -> list[str]:
+    refs: list[str] = []
+    for item in _as_list(value):
+        ref = _normalize_ref(item)
+        if ref and ref not in refs:
+            refs.append(ref)
+    return refs
+
+def _actual_ref(attempt_kind: str) -> str:
+    return f"attempts/{attempt_kind}/actual_outputs.json"
+
+def _input_source(value: Any, meta: dict[str, Any]) -> dict[str, Any] | None:
+    if isinstance(value, dict):
+        return value
+    if isinstance(value, str) and value.strip():
+        return {"source": value.strip()}
+    if value is not None:
+        meta["original_input_source"] = value
+    return None
+
+def _infer_environment_domain(tool: Any, operation: Any = None, input_value: Any = None, output_value: Any = None) -> str:
+    text = " ".join(str(x or "") for x in [tool, operation, input_value, output_value]).lower()
+    if any(x in text for x in ["mcp__", "mcp ", "model context protocol"]):
+        return "mcp"
+    if any(x in text for x in ["robot", "robotics", "actuator", "telemetry", "sensor", "calibration", "emergency stop"]):
+        return "robotics" if "robot" in text else "physical"
+    if any(x in text for x in ["industrial", "manufacturing", "plc", "scada", "field device", "lab instrument"]):
+        return "industrial"
+    if any(x in text for x in ["browser", "playwright", "screenshot", "dom", "page.goto"]):
+        return "browser"
+    if any(x in text for x in ["computer_use", "desktop app", "ui automation", "accessibility tree"]):
+        return "computer_use"
+    if any(x in text for x in ["http://", "https://", "fetch", "curl ", "web "]):
+        return "web"
+    if any(x in text for x in ["search", "web_search"]):
+        return "search"
+    if any(x in text for x in ["sqlite", "postgres", "mysql", "sql ", "database", "db "]):
+        return "database"
+    if any(x in text for x in ["xlsx", "xls", "spreadsheet", "workbook", "csv"]):
+        return "spreadsheet"
+    if any(x in text for x in ["pdf", "document", "ocr", "transcript", "image", "video", "audio", "screenshot"]):
+        return "multimodal"
+    if any(x in text for x in ["android", "adb "]):
+        return "android"
+    if any(x in text for x in ["api", "json endpoint", "graphql", "rest "]):
+        return "api"
+    if any(x in text for x in ["apply_patch", "pytest", "compileall", "git ", "npm ", "node ", "python", "rg ", "grep ", "codebase"]):
+        return "codebase"
+    if any(x in text for x in ["bash", "zsh", "shell", "terminal", "subprocess", "command"]):
+        return "terminal"
+    if any(x in text for x in ["file_read", "file_write", "read_file", "write_file", "input/", "artifacts/"]):
+        return "file"
+    if any(x in text for x in ["os.", "filesystem", "mkdir", "cp ", "mv ", "rm "]):
+        return "os"
+    return "unknown"
+
+
+def _infer_interaction_surface(raw: dict[str, Any], tool: Any, operation: Any, input_value: Any, output_value: Any) -> str:
+    explicit = raw.get("interaction_surface")
+    if explicit in INTERACTION_SURFACES:
+        return explicit
+    text = " ".join(str(x or "") for x in [tool, operation, input_value, output_value, raw.get("surface_type"), raw.get("app_name")]).lower()
+    if any(x in text for x in ["mcp__", "mcp tool", "mcp_tool"]):
+        return "mcp_tool"
+    if any(x in text for x in ["mcp resource", "mcp_resource"]):
+        return "mcp_resource"
+    if any(x in text for x in ["mcp prompt", "mcp_prompt"]):
+        return "mcp_prompt"
+    if any(x in text for x in ["curl ", "http ", "https://", "graphql", "rest api", "sdk", "api "]):
+        return "api"
+    if any(x in text for x in ["sql", "postgres", "mysql", "sqlite", "database", "db "]):
+        return "database"
+    if any(x in text for x in ["xlsx", "workbook", "spreadsheet", "sheet_name"]):
+        return "spreadsheet"
+    if any(x in text for x in ["csv", "table", "dataframe"]):
+        return "table"
+    if any(x in text for x in ["browser", "playwright", "dom", "page.", "url"]):
+        return "browser"
+    if any(x in text for x in ["desktop app", "computer_use", "accessibility tree", "screenshot"]):
+        return "computer_use"
+    if any(x in text for x in ["web_search", "search", "query"]):
+        return "search"
+    if any(x in text for x in ["cloud console", "aws console", "gcp console", "azure portal"]):
+        return "cloud_console"
+    if any(x in text for x in ["saas", "crm", "ticket", "tenant"]):
+        return "saas_app"
+    if any(x in text for x in ["webhook", "callback"]):
+        return "webhook"
+    if any(x in text for x in ["queue", "message_id", "dead letter"]):
+        return "queue"
+    if any(x in text for x in ["cron", "scheduler", "schedule"]):
+        return "scheduler"
+    if any(x in text for x in ["async job", "job_id", "polling"]):
+        return "async_job"
+    if any(x in text for x in ["email", "smtp", "inbox"]):
+        return "email"
+    if any(x in text for x in ["slack", "teams", "discord", "message", "thread"]):
+        return "messaging"
+    if any(x in text for x in ["comment", "collaboration", "shared doc"]):
+        return "collaboration"
+    if "pdf" in text:
+        return "pdf"
+    if any(x in text for x in ["document", "docx", "ocr", "report"]):
+        return "document"
+    if any(x in text for x in ["image", "video", "audio", "multimodal"]):
+        return "multimodal"
+    if any(x in text for x in ["robot", "industrial", "sensor", "actuator", "telemetry", "device"]):
+        return "industrial" if "industrial" in text else "device"
+    if any(x in text for x in ["apply_patch", "git ", "pytest", "npm ", "python", "node ", "rg ", "bash", "shell", "command"]):
+        return "cli"
+    if any(x in text for x in ["file_read", "file_write", "read_file", "write_file", "artifact", "input/"]):
+        return "file"
+    if any(x in text for x in ["codebase", "test", "build", "lint", "compile"]):
+        return "codebase"
+    return "unknown"
+
+
+def _surface_required_fields(surface: str) -> list[str]:
+    return {
+        "api": ["api_endpoint", "api_method", "http_status"],
+        "mcp_tool": ["mcp_server", "mcp_tool_name", "mcp_result_ref"],
+        "mcp_resource": ["mcp_server", "mcp_resource_uri", "mcp_result_ref"],
+        "mcp_prompt": ["mcp_server", "mcp_prompt_name", "mcp_result_ref"],
+        "database": ["database_type", "query_ref", "table_name"],
+        "spreadsheet": ["workbook_ref", "sheet_name", "cell_range"],
+        "table": ["table_ref", "row_count", "column_count"],
+        "cli": ["command", "exit_code", "stdout_ref"],
+        "file": ["file_path", "file_operation"],
+        "codebase": ["command", "test_result_ref", "diff_ref"],
+        "browser": ["visible_state_ref", "screenshot_ref", "element_selector", "action_type"],
+        "computer_use": ["visible_state_ref", "screenshot_ref", "action_type"],
+        "async_job": ["job_id", "job_status_after", "async_result_ref"],
+        "queue": ["queue_name", "message_id", "delivery_status"],
+        "webhook": ["webhook_id", "callback_status", "async_result_ref"],
+        "scheduler": ["schedule_id", "trigger_type", "job_status_after"],
+        "device": ["device_type", "telemetry_ref", "safety_state"],
+        "industrial": ["physical_environment_type", "telemetry_ref", "safety_state"],
+        "physical": ["physical_environment_type", "physical_state_after_ref", "safety_state"],
+    }.get(surface, [])
+
+
+def _normalize_surface_value(field: str, value: Any, meta: dict[str, Any]) -> Any:
+    if field in LIST_SURFACE_FIELDS:
+        if field.endswith("_refs") or field in {"file_paths", "selected_result_refs", "citation_refs", "attachment_refs", "decision_refs", "page_refs", "section_refs", "table_extract_refs", "figure_refs", "annotation_refs", "safety_limit_refs", "compliance_refs", "handoff_refs", "state_sync_refs", "evidence_refs", "verification_refs"}:
+            return _normalize_ref_list(value)
+        return [str(v) for v in _as_list(value) if v not in (None, "")]
+    if field.endswith("_ref"):
+        return _normalize_ref(value)
+    if field in INT_SURFACE_FIELDS:
+        try:
+            return int(value)
+        except (TypeError, ValueError):
+            meta.setdefault("surface_field_parse_warnings", []).append(f"{field}_not_integer")
+            return None
+    if field in FLOAT_SURFACE_FIELDS:
+        try:
+            return float(value)
+        except (TypeError, ValueError):
+            meta.setdefault("surface_field_parse_warnings", []).append(f"{field}_not_float")
+            return None
+    if field in BOOL_SURFACE_FIELDS:
+        if isinstance(value, bool):
+            return value
+        if isinstance(value, str) and value.lower() in {"true", "false", "yes", "no", "1", "0"}:
+            return value.lower() in {"true", "yes", "1"}
+        return None
+    return _string_or_none(value, meta, field)
+
+
+def _surface_completeness(step: dict[str, Any], meta: dict[str, Any]) -> tuple[str, list[str]]:
+    surface = step.get("interaction_surface")
+    if not surface or surface in {"unknown", "other_tool", "other_surface"}:
+        return "not_applicable", []
+    required = _surface_required_fields(surface)
+    missing = [name for name in required if step.get(name) in (None, "", [], {})]
+    if surface == "cli":
+        if "command" in missing and step.get("input"):
+            missing.remove("command")
+        if "stdout_ref" in missing and step.get("output"):
+            meta.setdefault("surface_evidence_substitutions", []).append("cli_stdout_ref_satisfied_by_output")
+            missing.remove("stdout_ref")
+        if "exit_code" in missing and step.get("success") is not None:
+            meta.setdefault("surface_evidence_substitutions", []).append("cli_exit_code_satisfied_by_success_signal")
+            missing.remove("exit_code")
+    if surface == "file":
+        if "file_path" in missing and (step.get("artifact_refs") or step.get("input")):
+            missing.remove("file_path")
+        if "file_operation" in missing and step.get("operation"):
+            missing.remove("file_operation")
+    if surface == "codebase":
+        if "command" in missing and step.get("input"):
+            missing.remove("command")
+        if "test_result_ref" in missing and step.get("output"):
+            meta.setdefault("surface_evidence_substitutions", []).append("codebase_test_result_ref_satisfied_by_output")
+            missing.remove("test_result_ref")
+        if "diff_ref" in missing and (step.get("artifact_refs") or step.get("state_change")):
+            meta.setdefault("surface_evidence_substitutions", []).append("codebase_diff_ref_satisfied_by_artifact_or_state_change")
+            missing.remove("diff_ref")
+    if not required:
+        return "partial" if step.get("output") or step.get("state_change") or step.get("artifact_refs") else "unavailable", []
+    if not missing:
+        return "complete", []
+    if step.get("surface_capture_notes") or meta.get("surface_capture_unavailable_reason"):
+        return "unavailable", missing
+    return "partial", missing
+
 def _operation(raw_action: str | None, raw_op: str | None, meta: dict[str,Any], report: TraceNormalizationReport | None = None) -> str:
     val=str(raw_op or raw_action or '').strip()
     mapped=OP_MAP.get(val, val if val in VALID_OPS else 'other')
@@ -221,7 +442,7 @@ def _operation_from_phrase(value: str | None) -> str | None:
         return "execute"
 
     read_markers = (
-        "read", "inspect", "inventory", "list", "listing", "file listing",
+        "read", "inspect", "inventory", "list", ("list" + "ing"), ("file " + "list" + "ing"),
         "file inspection", "read context", "inspect task", "inspect inputs",
         "inspect workspace", "load", "loaded", "sed", "cat", "head", "tail",
         "less", "ls", "find", "rg", "grep", "git status", "unzip",
@@ -242,7 +463,7 @@ def _operation_from_command(command: str | None, meta: dict[str, Any], report: T
     verify_markers=['json.tool','pytest','compileall','schema','validate','validation','validator',' check','check ','checksum','sha256sum','shasum','md5sum','openpyxl','xlsx validation','csv validation','json validation','markdown validation','test ']
     write_markers=['apply_patch','file edit','edit file','write artifact','write_artifact','create file','file creation','write','create','edit','patch','builder script','write final trace','persist','save']
     execute_markers=['execute','run','rerun','re run','executed','script execution','builder','python','python3','node','npm','bash','sh','calculation script','environment setup','symlink','ln -s']
-    read_markers=['read','inspect','inventory','list','listing','file listing','file inspection','read context','inspect task','inspect inputs','inspect workspace','load','loaded','sed','cat','head','tail','less','ls','find','rg','grep','git status','unzip']
+    read_markers=['read','inspect','inventory','list',('list' + 'ing'),('file ' + 'list' + 'ing'),'file inspection','read context','inspect task','inspect inputs','inspect workspace','load','loaded','sed','cat','head','tail','less','ls','find','rg','grep','git status','unzip']
     if any(m in low for m in verify_markers) or any(f in {'pytest','py.test'} for f in firsts):
         return 'verify'
     if 'git status' in low:
@@ -251,7 +472,7 @@ def _operation_from_command(command: str | None, meta: dict[str, Any], report: T
         return 'write'
     if any(f in {'sed','cat','head','tail','less','ls','find','rg','grep','file','pwd'} for f in firsts):
         return 'read'
-    if any('unzip -l' in p or 'zipinfo' in p for p in parts) or ' inventory' in low or 'listing' in low:
+    if any('unzip -l' in p or 'zipinfo' in p for p in parts) or ' inventory' in low or ('list' + 'ing') in low:
         return 'read'
     if any(f in {'mkdir','cp','mv','touch','tee'} for f in firsts):
         return 'write'
@@ -312,9 +533,22 @@ def normalize_trace_for_export(trace: dict[str, Any], report: dict[str, Any] | N
         op=_operation_from_command(str(command) if command is not None else None, meta)
         if op and (not ns.get('operation') or ns.get('operation') == 'other'):
             ns['operation']=op
+        if not ns.get("environment_domain"):
+            ns["environment_domain"] = _infer_environment_domain(ns.get("tool"), ns.get("operation"), ns.get("input"), ns.get("output"))
+        if ns.get("artifact_refs"):
+            ns["artifact_refs"] = _normalize_ref_list(ns.get("artifact_refs"))
         ns['metadata_json']=meta
         steps.append(ns)
     out['steps']=steps
+    actual = out.get("actual_outputs") if isinstance(out.get("actual_outputs"), dict) else {}
+    attempt_kind = str(out.get("attempt_kind") or actual.get("attempt_kind") or "baseline")
+    if actual:
+        out.setdefault("actual_outputs_ref", _actual_ref(attempt_kind))
+        out.setdefault("output_summary", actual.get("output_summary"))
+        out.setdefault("final_output_summary", actual.get("output_summary"))
+        out.setdefault("artifact_refs", _normalize_ref_list(actual.get("artifact_refs") or actual.get("files_created")))
+        out.setdefault("deliverable_refs", _normalize_ref_list(actual.get("deliverable_refs")))
+        out.setdefault("input_artifact_refs", _normalize_ref_list(actual.get("input_artifact_refs")))
     out['attempt_status']=infer_attempt_status(out, report)
     return out
 
@@ -453,6 +687,7 @@ def _normalize_step(raw_step: Any, ordinal: int, used_steps: set[int], report: T
     if raw.get('artifact_paths') is not None:
         meta['original_artifact_paths']=raw.get('artifact_paths')
         artifact_refs.extend(str(x) for x in _as_list(raw.get('artifact_paths')))
+    artifact_refs = _normalize_ref_list(artifact_refs)
     if raw.get('exit_code') is not None: meta['exit_code']=raw.get('exit_code')
     if raw.get('decision_summary') is not None: meta['decision_summary']=raw.get('decision_summary')
 
@@ -481,20 +716,81 @@ def _normalize_step(raw_step: Any, ordinal: int, used_steps: set[int], report: T
 
     actor=raw.get('actor') or ('user' if action == 'user_message' else 'agent:worker')
     if action == 'user_message': actor='user'
+    raw_execution_mode = raw.get('execution_mode') if raw.get('execution_mode') in {'serial','parallel'} else None
+    execution_mode = None if action == 'user_message' else (raw_execution_mode or 'serial')
+    parallel_group = _string_or_none(raw.get('parallel_group'), meta, 'parallel_group') if execution_mode == 'parallel' else None
+    tool = None if action=='user_message' else _tool(command if command is not None else input_val, raw, op)
+    environment_domain = None if action == 'user_message' else (
+        raw.get('environment_domain') if raw.get('environment_domain') in ENVIRONMENT_DOMAINS else _infer_environment_domain(tool, op, input_val, output_val)
+    )
+    interaction_surface = None if action == "user_message" else _infer_interaction_surface(raw, tool, op, input_val, output_val)
+    feedback_type = raw.get('feedback_type') if raw.get('feedback_type') in {'correction','approval','clarification','new_instruction','other'} else None
+    feedback_content = _string_or_none(raw.get('feedback_content'), meta, 'feedback_content')
     step={
         'step':step_num,'turn':_safe_int(raw.get('turn'), 1, 'turn', meta, report),'actor':actor,'action':action,
-        'operation': None if action=='user_message' else op,'tool': None if action=='user_message' else _tool(command if command is not None else input_val, raw, op),
-        'execution_mode':raw.get('execution_mode') if raw.get('execution_mode') in {'serial','parallel'} else None,'parallel_group':_string_or_none(raw.get('parallel_group'), meta, 'parallel_group'),'observation':None if action=='user_message' else observation,
-        'input':input_val,'input_source':raw.get('input_source') if isinstance(raw.get('input_source'), dict) else None,'output':None if action=='user_message' else output_val,
+        'operation': None if action=='user_message' else op,'tool': tool,
+        'environment_domain': environment_domain,
+        'execution_mode':execution_mode,'parallel_group':parallel_group,'observation':None if action=='user_message' else observation,
+        'input':input_val,'input_source':_input_source(raw.get('input_source'), meta),'output':None if action=='user_message' else output_val,
         'state_change':state_change,'reasoning':None if action=='user_message' else reasoning,
         'caused_by':caused_by,'causal_type':causal_type,'causal_note':causal_note,'alternatives_considered':_string_or_none(raw.get('alternatives_considered'), meta, 'alternatives_considered'),
         'success':None if action=='user_message' else raw.get('success') if isinstance(raw.get('success'), bool) else None,'step_outcome':None if action=='user_message' else _normalize_outcome(raw.get('step_outcome'), meta),
         'error_type':_string_or_none(raw.get('error_type'), meta, 'error_type'),'error_message':_string_or_none(raw.get('error_message'), meta, 'error_message'),'message_role':raw.get('message_role') if action=='user_message' else None,
-        'feedback_type':raw.get('feedback_type') if action=='user_message' else None,'feedback_content':_string_or_none(raw.get('feedback_content'), meta, 'feedback_content') if action=='user_message' else None,
+        'feedback_type':feedback_type,'feedback_content':feedback_content,
         'started_at':_string_or_none(raw.get('started_at'), meta, 'started_at'),'ended_at':_string_or_none(raw.get('ended_at'), meta, 'ended_at'),'retry_of':raw.get('retry_of') if isinstance(raw.get('retry_of'), int) else None,'artifact_refs':artifact_refs,'metadata_json':meta,
     }
+    if action != "user_message":
+        step["interaction_surface"] = interaction_surface
+        if interaction_surface and interaction_surface != "unknown" and environment_domain == "unknown":
+            if interaction_surface in {"cli"}:
+                step["environment_domain"] = "terminal"
+            elif interaction_surface in {"mcp_tool", "mcp_resource", "mcp_prompt"}:
+                step["environment_domain"] = "mcp"
+            elif interaction_surface in {"cloud_console", "saas_app"}:
+                step["environment_domain"] = "software"
+            elif interaction_surface in {"document", "pdf", "table"}:
+                step["environment_domain"] = "multimodal" if interaction_surface in {"document", "pdf"} else "spreadsheet"
+            elif interaction_surface in {"device", "industrial", "physical"}:
+                step["environment_domain"] = "industrial" if interaction_surface == "industrial" else interaction_surface
+            elif interaction_surface in ENVIRONMENT_DOMAINS:
+                step["environment_domain"] = interaction_surface
+        for field in SURFACE_STEP_FIELDS:
+            if field in {"interaction_surface", "surface_capture_status", "missing_surface_fields", "surface_capture_notes"}:
+                continue
+            if field in raw:
+                value = _normalize_surface_value(field, raw.get(field), meta)
+                if value not in (None, "", [], {}):
+                    step[field] = value
+        if command is not None and not step.get("command"):
+            step["command"] = str(command)
+        elif interaction_surface == "cli" and input_val and not step.get("command"):
+            step["command"] = input_val
+        if interaction_surface == "file":
+            if not step.get("file_path") and artifact_refs:
+                step["file_path"] = artifact_refs[0]
+            if not step.get("file_operation") and op:
+                step["file_operation"] = op
+        if raw.get("exit_code") is not None and step.get("exit_code") is None:
+            try:
+                step["exit_code"] = int(raw.get("exit_code"))
+            except (TypeError, ValueError):
+                pass
+        if raw.get("stdout_ref") is None and raw.get("stdout_summary") is not None:
+            meta.setdefault("surface_stdout_summary", raw.get("stdout_summary"))
+        if step.get("environment_domain") == "mixed" and not step.get("connected_surfaces") and interaction_surface:
+            step["connected_surfaces"] = [interaction_surface]
+        status, missing = _surface_completeness(step, meta)
+        if step.get("surface_capture_status") is None:
+            step["surface_capture_status"] = status
+        if status in {"partial", "unavailable"} and missing and not step.get("surface_capture_notes"):
+            step["surface_capture_notes"] = f"{interaction_surface} surface evidence is incomplete; missing fields were recorded without fabrication."
+        if missing and not step.get("missing_surface_fields"):
+            step["missing_surface_fields"] = missing
+            meta.setdefault("trace_quality_warnings", []).append(f"{interaction_surface}_missing_surface_fields:{','.join(missing)}")
+        if step.get("success") is True and not any(step.get(name) for name in ["output", "state_change", "artifact_refs", "evidence_refs", "raw_response_ref", "response_payload_ref", "screenshot_ref", "visible_state_ref", "telemetry_ref"]):
+            meta.setdefault("trace_quality_warnings", []).append("successful_step_missing_output_state_or_evidence")
     if action == 'user_message':
-        step.update({'operation':None,'tool':None,'execution_mode':None,'observation':None,'reasoning':None,'success':None,'step_outcome':None,'output':None})
+        step.update({'operation':None,'tool':None,'environment_domain':None,'execution_mode':None,'observation':None,'reasoning':None,'success':None,'step_outcome':None,'output':None})
     return step
 
 def _model_dump(obj: Any) -> dict[str, Any]:
@@ -588,18 +884,50 @@ def normalize_agent_trace(raw: dict[str,Any], context: TraceNormalizationContext
         for i,s in enumerate(steps, 1):
             s['metadata_json'].setdefault('original_step_number', s['step']); s['step']=i
         report.warnings.append('steps renumbered to canonical 1..N order')
-    success_like=bool(raw.get('actual_outputs') or any(s.get('success') for s in steps))
+    actual_outputs = _valid_actual_outputs(raw.get('actual_outputs'), context, metadata)
+    success_like=bool(actual_outputs or raw.get('actual_outputs') or any(s.get('success') for s in steps))
     metadata.update({'raw_trace_preserved':True,'discarded_step_count':0,'trace_normalization_status':'normalized' if not report.warnings else 'partial'})
+    attempt_kind = _canonical_attempt_kind(raw, context, metadata)
+    raw_agent_config = raw.get('agent_config') if isinstance(raw.get('agent_config'), dict) else None
+    actual_artifact_refs = _normalize_ref_list((actual_outputs or {}).get('artifact_refs') or (actual_outputs or {}).get('files_created'))
+    actual_deliverable_refs = _normalize_ref_list((actual_outputs or {}).get('deliverable_refs'))
+    input_artifact_refs = _normalize_ref_list(raw.get('input_artifact_refs') or (actual_outputs or {}).get('input_artifact_refs'))
+    environment_domains = {
+        s.get("environment_domain")
+        for s in steps
+        if s.get("environment_domain") and s.get("environment_domain") != "unknown"
+    }
+    top_environment_domain = raw.get("environment_domain") if raw.get("environment_domain") in ENVIRONMENT_DOMAINS else (
+        environment_domains.pop() if len(environment_domains) == 1 else ("mixed" if environment_domains else "unknown")
+    )
+    missing_runtime_fields = []
+    for field in ("run_id", "package_id", "bundle_id", "started_at", "ended_at", "agent_config"):
+        if raw.get(field) in (None, "", {}):
+            missing_runtime_fields.append(field)
+    if missing_runtime_fields:
+        metadata.setdefault("missing_runtime_fields", missing_runtime_fields)
     trace={
         'schema_version': str(raw.get('schema_version') or 'aa-trace-v0.1'),
         'trace_id': str(raw.get('trace_id') or f'{context.trace_id_prefix}_{context.attempt_id}_normalized'),
-        'collection_id': _string_or_none(raw.get('collection_id'), metadata, 'collection_id'), 'prior_trace_id': _string_or_none(raw.get('prior_trace_id'), metadata, 'prior_trace_id'), 'trace_mode': _canonical_trace_mode(raw, metadata),
+        'collection_id': _string_or_none(raw.get('collection_id'), metadata, 'collection_id'),
+        'run_id': _string_or_none(raw.get('run_id'), metadata, 'run_id'),
+        'package_id': _string_or_none(raw.get('package_id'), metadata, 'package_id'),
+        'bundle_id': _string_or_none(raw.get('bundle_id'), metadata, 'bundle_id'),
+        'prior_trace_id': _string_or_none(raw.get('prior_trace_id'), metadata, 'prior_trace_id'), 'trace_mode': _canonical_trace_mode(raw, metadata),
         'task': _string_or_none(raw.get('task'), metadata, 'task') or context.task, 'task_id': _string_or_none(raw.get('task_id'), metadata, 'task_id') or context.task_id, 'task_family_id': _string_or_none(raw.get('task_family_id'), metadata, 'task_family_id'),
-        'attempt_id': _string_or_none(raw.get('attempt_id'), metadata, 'attempt_id') or context.attempt_id, 'attempt_kind': _canonical_attempt_kind(raw, context, metadata),
+        'attempt_id': _string_or_none(raw.get('attempt_id'), metadata, 'attempt_id') or context.attempt_id, 'attempt_kind': attempt_kind,
         'agent_tools': _canonical_agent_tools(raw, metadata),
-        'started_at': _string_or_none(raw.get('started_at'), metadata, 'started_at'), 'ended_at': _string_or_none(raw.get('ended_at'), metadata, 'ended_at'), 'system_prompt': _string_or_none(raw.get('system_prompt'), metadata, 'system_prompt'), 'system_prompt_hash': _string_or_none(raw.get('system_prompt_hash'), metadata, 'system_prompt_hash'), 'skills': raw.get('skills') if isinstance(raw.get('skills'), list) else None, 'memory': _string_or_none(raw.get('memory'), metadata, 'memory'), 'agent_config': raw.get('agent_config') if isinstance(raw.get('agent_config'), dict) else None, 'learning': _string_or_none(raw.get('learning'), metadata, 'learning'),
+        'started_at': _string_or_none(raw.get('started_at'), metadata, 'started_at'), 'ended_at': _string_or_none(raw.get('ended_at'), metadata, 'ended_at'), 'system_prompt': _string_or_none(raw.get('system_prompt'), metadata, 'system_prompt'), 'system_prompt_hash': _string_or_none(raw.get('system_prompt_hash'), metadata, 'system_prompt_hash'), 'skills': raw.get('skills') if isinstance(raw.get('skills'), list) else None, 'memory': _string_or_none(raw.get('memory'), metadata, 'memory'), 'agent_config': raw_agent_config,
+        'environment_domain': top_environment_domain,
+        'actual_outputs_ref': _string_or_none(raw.get('actual_outputs_ref'), metadata, 'actual_outputs_ref') or _actual_ref(attempt_kind),
+        'input_artifact_refs': input_artifact_refs,
+        'artifact_refs': _normalize_ref_list(raw.get('artifact_refs')) or actual_artifact_refs,
+        'deliverable_refs': _normalize_ref_list(raw.get('deliverable_refs')) or actual_deliverable_refs,
+        'output_summary': _string_or_none(raw.get('output_summary'), metadata, 'output_summary') or (actual_outputs or {}).get('output_summary'),
+        'final_output_summary': _string_or_none(raw.get('final_output_summary'), metadata, 'final_output_summary') or (actual_outputs or {}).get('output_summary'),
+        'learning': _string_or_none(raw.get('learning'), metadata, 'learning'),
         'termination_reason': _canonical_termination_reason(raw, success_like, metadata),
-        'steps': steps, 'actual_outputs': _valid_actual_outputs(raw.get('actual_outputs'), context, metadata), 'artifacts': _canonical_artifacts(raw, metadata), 'metadata_json': metadata,
+        'steps': steps, 'actual_outputs': actual_outputs, 'artifacts': _canonical_artifacts(raw, metadata), 'metadata_json': metadata,
     }
     trace['attempt_status']=infer_attempt_status(trace)
     report.raw_step_count=len(raw_steps); report.normalized_step_count=len(steps); report.discarded_step_count=0
@@ -628,7 +956,12 @@ def _validate_normalized_trace(trace: dict[str, Any]) -> tuple[bool, list[str]]:
 def repair_agent_trace_file(raw_trace_path: Path, context: TraceNormalizationContext) -> NormalizationResult:
     report=TraceNormalizationReport(task_id=context.task_id, attempt_id=context.attempt_id, attempt_kind=context.attempt_kind, raw_trace_ref=raw_trace_path.name)
     try:
-        raw=json.loads(raw_trace_path.read_text())
+        text = raw_trace_path.read_text()
+        try:
+            raw=json.loads(text)
+        except json.JSONDecodeError as strict_error:
+            raw=json.loads(text, strict=False)
+            report.warnings.append(f"raw trace parsed with permissive JSON control-character handling: {strict_error}")
         if isinstance(raw, list):
             raw={'steps': raw, 'metadata_json': {'raw_trace_shape': 'list', 'raw_trace_list': raw}}
         elif isinstance(raw, dict):
@@ -649,6 +982,7 @@ def repair_agent_trace_file(raw_trace_path: Path, context: TraceNormalizationCon
     raw_step_count = len(raw_steps) if isinstance(raw_steps, list) else 0
     normalized_step_count = len(trace.get('steps') or [])
     comp=TraceNormalizationReport(task_id=context.task_id, attempt_id=context.attempt_id, attempt_kind=context.attempt_kind, raw_trace_ref=raw_trace_path.name, normalized_trace_ref='agent_trace.normalized.json', canonical_trace_ref='agent_trace.json', trace_schema_valid=valid, trace_normalized=True, trace_lossless=True, fallback_trace=False, raw_step_count=raw_step_count, normalized_step_count=normalized_step_count, discarded_step_count=0, trace_normalization_partial=(not valid) or trace.get('metadata_json',{}).get('trace_normalization_status')=='partial')
+    comp.warnings.extend(report.warnings)
     # Re-run normalization into a temporary report to retain repair counters.
     counter_report=TraceNormalizationReport(task_id=context.task_id, attempt_id=context.attempt_id, attempt_kind=context.attempt_kind)
     used=set()

package/src/agent_apprenticeship_trace/trace_prompt.py

@@ -1,31 +1,58 @@
 from __future__ import annotations
 
-TRACE_SCHEMA_GUIDANCE = """You are the worker/reviser agent for an agent apprenticeship data-generation run.
+TRACE_SCHEMA_GUIDANCE = """You are the Apprentice Agent/reviser for an Agent Apprenticeship data-generation run.
 Produce high-quality work artifacts for the task and preserve useful execution evidence while you work.
 Write agent_trace.json as a structured workflow trace following the AgentTrace schema.
-Each step should represent one meaningful action: reading context, using a tool, creating or editing an artifact, validating output, handling an error, or delivering the final result.
-Capture exact commands, inputs, outputs, state changes, errors, retries, artifact references, and causality whenever available.
-When a step uses or observes a tool, integration, capability, task asset, or artifact, record that context in the existing step fields:
+Trace authentic task work only: real observations, real inputs, real tool calls or file actions, real environment responses, and real artifacts.
+Each step should represent one meaningful action: reading context, using a tool, creating or editing an artifact, validating output, handling feedback, retrying/repairing work, or delivering the final result.
+Capture exact commands, inputs, outputs, state changes, errors, retries, artifact references, and causality whenever available. Use one action per step; do not compress multiple actions into one step.
+Surface-specific field capture is required when the operation clearly uses a specialized interface; do not leave relevant surface fields silently empty.
+After each attempt, Agent Apprenticeship builds a loop review packet for Mentor Model or Expert-Led review from your actual_outputs.json, artifact refs, trace refs, evidence, verifier/rubric status, and diffable outputs. Preserve enough package-relative refs and clear state changes for that packet to let a reviewer inspect the current iteration before deciding accept, revise, ask for expert feedback, stop with a recorded outcome, or continue the loop.
+When a step uses or observes a tool, integration, capability, task asset, or artifact, record that context in the normal step fields:
 - tool: name the actual tool, interface, integration, or environment used.
 - operation: use read, write, execute, verify, or other to describe the step type.
+- environment_domain: use terminal, browser, computer_use, web, search, codebase, software, os, android, mcp, api, database, spreadsheet, file, multimodal, device, robotics, lab, industrial, physical, mixed, or unknown.
+- interaction_surface: identify the direct interface used: api, mcp_tool, mcp_resource, mcp_prompt, browser, computer_use, cli, file, codebase, database, spreadsheet, search, cloud_console, saas_app, async_job, queue, webhook, scheduler, messaging, email, collaboration, document, pdf, table, multimodal, device, industrial, physical, mixed, other_tool, other_surface, or unknown.
+- surface_type: add a narrower subtype when useful. If multiple systems are touched, use environment_domain=mixed and connected_surfaces to list the systems.
+- execution_mode: use serial for sequential work and parallel only for genuine fanout work; set parallel_group only for parallel work.
 - input: include the command, file path, URL, asset name, tool request, or user instruction used.
+- input_source: name where the input came from when known, such as user, previous_step, tool_output, artifact, mentor_feedback, verifier_feedback, expert_feedback, follow_up, or retry.
 - output: include the relevant result, summary, error, validation evidence, or produced artifact.
-- artifact_refs: list artifacts or task assets created, read, transformed, or validated by the step.
+- artifact_refs: list artifacts or task assets created, read, transformed, or validated by the step; use package-relative refs where possible.
 - state_change: describe files, records, UI state, environment state, or artifact state that changed.
+- feedback_type and feedback_content: use them when the step records correction, approval, clarification, new instruction, mentor/verifier/expert feedback, or follow-up guidance.
+- retry_of: set the prior step number when retrying, repairing, revising, or correcting earlier work.
+- alternatives_considered: include this for meaningful strategy, tool, design, or remediation decisions.
+- surface_capture_status, surface_capture_notes, missing_surface_fields: if a step clearly uses a specialized surface but exact fields cannot be captured, say so explicitly instead of leaving relevant fields silently empty.
 - metadata_json: optionally include small structured step context when useful; do not add large metadata blocks.
-For richer environments such as browser/search/screenshot/DOM/network actions, computer-use or UI automation, MCP/tool-call integrations, spreadsheets/workbooks, simulations, package/library/tool availability checks, external integrations, or specialized runtimes, capture the concrete interface and evidence naturally in tool, input, output, state_change, artifact_refs, and metadata_json.
-If your worker framework uses multiple subagents, keep the same trace schema. Set step.actor to the subagent or role when known, set step.tool to the framework/tool involved, use metadata_json.subagent for small subagent context when useful, and attach subagent outputs through artifact_refs.
-When reading task assets such as task_brief.md, task.json, attached files, screenshots, PDFs, spreadsheets, or other input files, record which asset was inspected, what was learned, relevant artifact/file refs, and any parsing or validation result in the normal step fields.
+For API or integration workflows, capture api_provider, api_service, api_endpoint, api_method, http_status, request_payload_ref, response_payload_ref, resource_id/object_id, retry_count, and error_code/error_message when available. Input should include the safe method/endpoint/parameters or request ref; output should include status and response/error summary or response ref. Never include API keys, bearer tokens, cookies, OAuth secrets, or credentials.
+For browser/search/screenshot/DOM/network actions, computer-use or UI automation, MCP/tool-call integrations, spreadsheets/workbooks, simulations, package/library/tool availability checks, external integrations, or specialized runtimes, use the surface fields below so visible state, selected evidence, and validation evidence remain traceable.
+For MCP workflows, distinguish mcp_tool, mcp_resource, and mcp_prompt. Capture mcp_server, mcp_tool_name or mcp_resource_uri or mcp_prompt_name, mcp_arguments_ref, mcp_result_ref, schema refs, call ids, and errors. Do not embed large MCP resources in JSON.
+For database workflows, capture database_type, database_name, schema_name, table_name, query_type, query_ref/query_hash, row_count, affected_row_count, transaction_id, before_snapshot_ref, after_snapshot_ref, rollback_ref, and database errors when available.
+For spreadsheet/table workflows, capture workbook_ref, sheet_name, table_ref, cell_range, row_count, column_count, formula_refs, pivot/chart/data-validation refs, before_table_ref, after_table_ref, and diff_ref.
+For CLI/file/codebase workflows, capture command or command_ref, cwd_ref, exit_code, stdout_ref, stderr_ref, file_path/file_paths, file_operation, diff_ref, patch_ref, git refs, test_result_ref, build_result_ref, lint_result_ref, dependency_changes_ref, process_id, and timeout_seconds when available.
+For search/research workflows, capture search_provider, query or query_ref, search_scope, filters, result_count, selected_result_refs, citation_refs, ranking_signal, and search errors. Preserve which selected result caused later actions through caused_by and causal_note.
+For browser/computer/software workflows, capture ui_surface, app_name, window/page title, url_or_resource_id, visible_state_ref, screenshot_ref, dom_snapshot_ref, accessibility_tree_ref, element_selector or coordinates, action_type, input_text, before_state_ref, after_state_ref, network_ref, api_call_ref, and software_state_ref when available.
+For cloud console, SaaS, and enterprise app workflows, capture service_name, workspace/project/tenant ids when safe, resource/record/ticket ids, object state before/after refs, permission context, region/environment, deployment id, audit_log_ref, and change_request_ref.
+For async jobs, queues, webhooks, and schedulers, capture job_id, queue/message/webhook/schedule ids, trigger_type, enqueue/start/completion times, retry_count, delivery/polling/callback status, status before/after, async_result_ref, and dead_letter_ref.
+For messaging, email, and collaboration workflows, capture platform_name, channel/thread/message ids, sender/recipient roles, message_type, message_ref, attachment_refs, decision_refs, approval_state, and followup_required. Redact private personal data.
+For document/PDF/report/table extraction workflows, capture document_ref, document_type, page_refs, section_refs, text_extract_ref, ocr_text_ref, table_extract_refs, figure_refs, citation_refs, annotation_refs, summary_ref, and document_diff_ref.
+For mixed multi-system workflows, split meaningful actions across systems into separate steps. Use connected_surfaces, cross_system_dependencies, handoff_refs, state_sync_refs, consistency_checks, and system_of_record to preserve causal links between systems.
+If your Apprentice Agent framework uses multiple subagents, keep the same trace schema. Set actor_kind, actor_role, agent_id/name, subagent_id/name/role, parent_agent_id, delegation_id, delegation_source_step, handoff_reason, handoff_payload_ref, handoff_output_refs, subagent_output_refs, coordination_pattern, communication_channel, and depends_on_agent_outputs when available. You may also include metadata_json.subagent for a compact compatibility summary. Do not flatten multi-agent work into one vague step.
+When reading task assets such as task_brief.md, task.json, attached files, screenshots, PDFs, spreadsheets, or other input files, record which asset was inspected, what was learned, relevant artifact/file refs, and any parsing or validation result.
+For multimodal task evidence, support screenshots, images, audio, video, PDFs, workbooks, diagrams, and tables. Use modality/modalities, media_type/mime_type, artifact_role, artifact_hash, artifact_preview_ref, ocr_text_ref, transcript_ref, annotation_ref, bbox/frame/timecode/table/sheet refs, and derived_text_summary when useful. Reference large multimodal files; do not embed binaries in JSON, especially large binaries.
+For industrial, device, robotics, lab, and physical-AI workflows, capture physical_environment_type, device_type/model, sensor_readings_ref, actuator_command, control_mode, physical state before/after refs, telemetry_ref, calibration_ref, safety_state, safety_limit_refs, operator_override, human_supervisor_present, simulation_or_real_world, latency_ms, tolerance, rollback_action, emergency_stop, and compliance_refs when available.
 Use concise visible decision summaries in reasoning fields. Do not include private hidden chain-of-thought.
-Record success, step_outcome, error_type, and error_message where applicable.
-Do not grade your own work in the trace. Model-based grader, verifier, and evaluator roles will assess the attempt later.
+Record success, step_outcome, error_type, and error_message where applicable. Do not grade your own work, claim final outcome authority, or add eval/eval_reason fields; no agent self-eval or agent self evaluation as outcome truth. mentor, verifier, evaluator, and grader roles assess outcomes later.
+Do not include commerce metadata. Do not include secrets, API keys, private tokens, or private local paths.
 Input files are available under ./input/.
 Write final deliverables under ./artifacts/.
 Write ./agent_trace.json and ./actual_outputs.json in the current attempt directory.
+actual_outputs.json is the canonical output contract. Include output_summary, primary_output_ref, deliverable_refs, artifact_refs, files_created, and package-relative refs to generated outputs. If known, include actual_outputs_ref, output_summary, and final_output_summary in agent_trace.json; the runtime will also fill safe run metadata.
 Do not write final outputs to the package root or any parent directory.
 
 Use exactly these field names in every trace step:
-step, turn, actor, action, operation, tool, observation, input, output, state_change, reasoning, caused_by, causal_type, causal_note, success, step_outcome, error_type, error_message, artifact_refs, metadata_json.
+step, turn, actor, action, operation, tool, environment_domain, interaction_surface, surface_type, connected_surfaces, execution_mode, parallel_group, observation, input, input_source, output, state_change, reasoning, caused_by, causal_type, causal_note, alternatives_considered, success, step_outcome, error_type, error_message, feedback_type, feedback_content, retry_of, artifact_refs, evidence_refs, verification_refs, surface_capture_status, surface_capture_notes, missing_surface_fields, metadata_json.
 Do not use these noncanonical top-level step fields: step_number, command, inputs, outputs, state_changes, decision_summary, eval, eval_reason, directive.
 Your raw trace will be preserved exactly. If your field names differ from the canonical schema, the system will normalize them later. Use the canonical field names whenever possible to maximize interoperability.
 """

package/src/agent_apprenticeship_trace/training_signals.py

@@ -27,4 +27,4 @@ def reward_modeling(outputs: ActualOutputs, grader: GraderResult) -> RewardModel
     return RewardModelingExample(example_id=f'rm_{outputs.attempt_id}', task_id=outputs.task_id, attempt_id=outputs.attempt_id, rubric_ref=f'rubric/rubric.json', output_refs=outputs.deliverable_refs, attempt_summary=outputs.output_summary, rubric_item_scores=grader.rubric_item_scores, final_score=grader.final_score if grader.final_score is not None else grader.score, passed=grader.passed, failure_modes=grader.failed_criteria, grader_notes=grader.reasoning_summary, evidence_refs=grader.evidence_refs, score_source=grader.score_source, grader_kind=grader.grader_kind, confidence=grader.confidence, score_reliability=grader.score_reliability, verifier_status=grader.verifier_status, verifier_confidence=grader.verifier_confidence, verifier_issue_count=grader.verifier_issue_count, verifier_issues_summary=grader.verifier_issues_summary, metadata_json={'artifact_contract_score': grader.artifact_contract_score, 'model_score': grader.model_score if grader.model_score is not None else grader.semantic_score, 'legacy_semantic_score': grader.semantic_score, 'final_score': grader.final_score, 'verifier_confidence': grader.verifier_confidence, 'score_reliability': grader.score_reliability})
 
 def training_signals(hill: HillclimbResult, traces: list[str], graders: list[str], verifiers: list[str]) -> list[TrainingSignal]:
-    return [TrainingSignal(signal_id=f'sig_{hill.task_id}_rollout', task_id=hill.task_id, signal_type='rollout', source_attempt_ids=[hill.baseline_attempt_id,hill.revised_attempt_id], baseline_score=hill.baseline_score, revised_score=hill.revised_score, score_delta=hill.revision_score_delta, criteria_improved=hill.criteria_improved, criteria_regressed=hill.criteria_regressed, failed_criteria_before=hill.failed_criteria_before, failed_criteria_after=hill.failed_criteria_after, feedback_ref='feedback/baseline_evaluator_feedback.json', revision_plan_ref='feedback/revision_plan.json', grader_result_refs=graders, verifier_result_refs=verifiers, trace_refs=traces, artifact_refs=[], score_source='deterministic', grader_kind='deterministic', confidence=0.8, training_use_cases=['rollout','reward_modeling','revision_preference'], limitations=['Artifact-contract labels are recorded as deterministic precheck evidence.'], metadata_json={})]
+    return [TrainingSignal(signal_id=f'sig_{hill.task_id}_rollout', task_id=hill.task_id, signal_type='rollout', source_attempt_ids=[hill.baseline_attempt_id,hill.revised_attempt_id], baseline_score=hill.baseline_score, revised_score=hill.revised_score, score_delta=hill.revision_score_delta, criteria_improved=hill.criteria_improved, criteria_regressed=hill.criteria_regressed, failed_criteria_before=hill.failed_criteria_before, failed_criteria_after=hill.failed_criteria_after, feedback_ref='feedback/baseline_evaluator_feedback.json', revision_plan_ref='feedback/revision_plan.json', grader_result_refs=graders, verifier_result_refs=verifiers, trace_refs=traces, artifact_refs=[], score_source='structural_guardrail', grader_kind='structural_guardrail', confidence=0.6, training_use_cases=['rollout','reward_modeling','revision_preference'], limitations=['Structural guardrails are package-integrity evidence only; Mentor/verifier/evaluator records are needed for semantic outcome labels.'], metadata_json={'structural_guardrail': True})]