agent-apprenticeship 0.1.6 → 0.1.7

package/src/agent_apprenticeship_trace/codex_runner.py

@@ -13,12 +13,57 @@ from .contract_diagnostics import build_contract_diagnostics, diagnostics_text
 from .trace_normalizer import TraceNormalizationContext, repair_agent_trace_file, TraceNormalizationReport
 from .actual_outputs_normalizer import ActualOutputsNormalizationContext, repair_actual_outputs_file, write_actual_outputs_normalization
 from .public_sanitizer import classify_provider_failure, public_error_summary
+from .progress import utc_now
 
 class AttemptResult(dict): pass
 
 CODEX_TRUST_RETRY_MESSAGE = "Codex refused to run because the workspace is not a trusted Git directory. Retrying with --skip-git-repo-check if supported."
 
 
+def _safe_agent_config(settings, runner_kind: str, attempt_kind: str) -> dict:
+    return {
+        "apprentice_agent_id": settings.worker_agent,
+        "apprentice_agent_name": settings.custom_worker_display_name if settings.worker_agent == "custom" else settings.worker_agent,
+        "apprenticeship_mode": getattr(settings, "apprenticeship_mode", None),
+        "mentor_mode": settings.mentor_mode,
+        "mentor_model_provider": settings.model_provider,
+        "max_improvement_loops": settings.max_improvement_loops,
+        "attempt_kind": attempt_kind,
+        "runner_kind": runner_kind,
+        "sensitive_info_masking": settings.sensitive_info_masking,
+    }
+
+
+def _enrich_runtime_trace(
+    trace: AgentTrace,
+    actual: ActualOutputs,
+    package_root: Path,
+    attempt_kind: str,
+    runner_kind: str,
+    started_at: str | None,
+    ended_at: str | None,
+) -> AgentTrace:
+    settings = get_settings()
+    trace.run_id = package_root.parent.parent.name if package_root.parent.parent.name else None
+    trace.package_id = package_root.name
+    trace.started_at = trace.started_at or started_at
+    trace.ended_at = trace.ended_at or ended_at
+    trace.agent_config = trace.agent_config or _safe_agent_config(settings, runner_kind, attempt_kind)
+    trace.actual_outputs_ref = f"attempts/{attempt_kind}/actual_outputs.json"
+    trace.input_artifact_refs = list(dict.fromkeys(actual.input_artifact_refs or trace.input_artifact_refs or []))
+    trace.artifact_refs = list(dict.fromkeys((actual.artifact_refs or actual.files_created or []) + (trace.artifact_refs or [])))
+    trace.deliverable_refs = list(dict.fromkeys((actual.deliverable_refs or []) + (trace.deliverable_refs or [])))
+    trace.output_summary = trace.output_summary or actual.output_summary
+    trace.final_output_summary = trace.final_output_summary or actual.output_summary
+    trace.environment_domain = trace.environment_domain or (
+        "mixed"
+        if len({s.environment_domain for s in trace.steps if s.environment_domain and s.environment_domain != "unknown"}) > 1
+        else next(iter({s.environment_domain for s in trace.steps if s.environment_domain and s.environment_domain != "unknown"}), "unknown")
+    )
+    trace.metadata_json.setdefault("runtime_metadata_populated", True)
+    return trace
+
+
 def _run_with_process_group_timeout(command, *, cwd: Path, timeout: int | None, shell: bool = False) -> subprocess.CompletedProcess:
     process = subprocess.Popen(
         command,
@@ -53,6 +98,19 @@ def _attempt_dir(package_root: Path, attempt_kind: str) -> Path:
     (p/'input').mkdir(parents=True, exist_ok=True)
     return p
 
+
+def _recover_misplaced_contract_files(d: Path) -> dict[str, str]:
+    """Recover contract files agents sometimes place under ./artifacts/."""
+    recovered: dict[str, str] = {}
+    for name in ("agent_trace.json", "actual_outputs.json"):
+        dst = d / name
+        src = d / "artifacts" / name
+        if dst.exists() or not src.exists() or not src.is_file():
+            continue
+        shutil.copy2(src, dst)
+        recovered[name] = f"artifacts/{name}"
+    return recovered
+
 def _copy_attempt_inputs(package_root: Path, attempt_dir: Path) -> list[str]:
     src=package_root/'input'; dst=attempt_dir/'input'
     dst.mkdir(parents=True, exist_ok=True)
@@ -101,9 +159,11 @@ def _fallback_actual(spec: TaskIntakeSpec, attempt_kind: str, error_type: str, e
 
 def _apprentice_operational_error(error: object | None, stdout: str = "", stderr: str = "", returncode: int | None = None) -> str | None:
     text = f"{error or ''}\n{stdout or ''}\n{stderr or ''}".lower()
+    if error is None and returncode == 0:
+        return None
     if isinstance(error, FileNotFoundError) or "no such file or directory: 'codex'" in text or "no such file or directory: codex" in text:
         return "Apprentice Agent command not found: codex"
-    if isinstance(error, subprocess.TimeoutExpired) or "timed out" in text or "timeout" in text:
+    if isinstance(error, subprocess.TimeoutExpired) or "timed out" in text or "timeout expired" in text or "command timed out" in text:
         return "Apprentice Agent timed out while running Codex."
     if _is_codex_workspace_trust_error(stdout, stderr, error):
         return "Codex workspace trust error: Codex refused to run because the workspace is not a trusted Git directory. Use a Codex CLI with --skip-git-repo-check support or run from a trusted Git repository."
@@ -191,7 +251,7 @@ def _fallback_trace(spec: TaskIntakeSpec, attempt_kind: str, prompt: str, actual
     error_type=cls.get('error_type') or error_type
     steps=[
         AgentTraceStep(step=1, turn=1, actor='user', action='user_message', input=spec.normalized_instruction, message_role='direct_request'),
-        AgentTraceStep(step=2, turn=1, actor=actor, action='error', operation='other', tool='codex_cli', observation='Codex attempt ended without a valid agent_trace.json or actual_outputs.json.', input='Validate required output contract for ./agent_trace.json and ./actual_outputs.json.', output=safe, state_change='A minimal failure trace and failed actual_outputs.json were written by the runner.', reasoning='The package must preserve failure evidence without fabricating a successful detailed trace.', caused_by=[1], causal_type='dependency_on_tool_result', success=False, step_outcome='failed', error_type=error_type, error_message=safe, artifact_refs=[f'attempts/{attempt_kind}/prompt.md', f'attempts/{attempt_kind}/stdout.txt', f'attempts/{attempt_kind}/stderr.txt', f'attempts/{attempt_kind}/final_message.txt']),
+        AgentTraceStep(step=2, turn=1, actor=actor, action='error', operation='other', tool='codex_cli', environment_domain='terminal', execution_mode='serial', observation='Codex attempt ended without a valid agent_trace.json or actual_outputs.json.', input='Validate required output contract for ./agent_trace.json and ./actual_outputs.json.', input_source={'source':'runtime_contract_check'}, output=safe, state_change='A minimal failure trace and failed actual_outputs.json were written by the runner.', reasoning='The package must preserve failure evidence without fabricating a successful detailed trace.', caused_by=[1], causal_type='dependency_on_tool_result', success=False, step_outcome='failed', error_type=error_type, error_message=safe, artifact_refs=[f'attempts/{attempt_kind}/prompt.md', f'attempts/{attempt_kind}/stdout.txt', f'attempts/{attempt_kind}/stderr.txt', f'attempts/{attempt_kind}/final_message.txt']),
     ]
     return AgentTrace(trace_id=f'trace_{attempt_id}_fallback', collection_id=None, trace_mode='live', task=spec.normalized_instruction, task_id=spec.task_id, attempt_id=attempt_id, attempt_kind=attempt_kind if attempt_kind in ['baseline','revised'] else 'other', agent_tools=['codex_cli','Bash','python','file_read','file_write'], system_prompt=prompt, system_prompt_hash=None, skills=['agent_trace_skill'], learning='When a live agent cannot write outputs, preserve logs and create a minimal failure trace.', termination_reason='agent_blocked', steps=steps, actual_outputs=actual, artifacts=[], metadata_json={'trace_valid':False,'fallback_trace_created':True,'codex_sandbox':codex_sandbox, **cls})
 
@@ -202,6 +262,7 @@ def _write_fallback_report(d: Path, spec: TaskIntakeSpec, attempt_kind: str, rea
 
 def ensure_attempt_outputs(package_root: Path, spec: TaskIntakeSpec, attempt_kind: str, prompt: str, codex_sandbox: str, validation_error: Exception | None=None) -> tuple[AgentTrace, ActualOutputs, bool]:
     d=_attempt_dir(package_root, attempt_kind)
+    recovered_contract_files = _recover_misplaced_contract_files(d)
     actual_path=d/'actual_outputs.json'; trace_path=d/'agent_trace.json'; raw_path=d/'agent_trace.raw.json'
     err = validation_error or FileNotFoundError('agent_trace.json missing or invalid')
     required_artifacts=_deliverables(RawTaskRecord(raw_task_id=spec.task_id, source_kind='normalized_spec', raw_title=spec.normalized_title, raw_description=spec.normalized_instruction, raw_payload={'expected_deliverable': spec.expected_agent_deliverable}), spec)
@@ -225,7 +286,11 @@ def ensure_attempt_outputs(package_root: Path, spec: TaskIntakeSpec, attempt_kin
             write_json(d/'agent_trace.normalized.json', result.normalized_trace)
             write_json(d/'agent_trace.json', result.normalized_trace)
             write_json(d/'trace_normalization_report.json', result.report)
-            return AgentTrace.model_validate(result.normalized_trace), actual, bool(result.report.trace_schema_valid)
+            trace = AgentTrace.model_validate(result.normalized_trace)
+            if recovered_contract_files:
+                trace.metadata_json["contract_files_recovered_from"] = recovered_contract_files
+                actual.metadata_json.setdefault("contract_files_recovered_from", recovered_contract_files)
+            return trace, actual, bool(result.report.trace_schema_valid)
         reason=result.parse_error or 'parseable trace contained no steps'
         final=(d/'final_message.txt').read_text(errors='ignore') if (d/'final_message.txt').exists() else ''
         stderr=(d/'stderr.txt').read_text(errors='ignore') if (d/'stderr.txt').exists() else ''
@@ -243,6 +308,7 @@ def ensure_attempt_outputs(package_root: Path, spec: TaskIntakeSpec, attempt_kin
     return trace, actual, False
 
 def deterministic_attempt(package_root: Path, raw: RawTaskRecord, spec: TaskIntakeSpec, attempt_kind='baseline', feedback: str | None=None) -> AttemptResult:
+    started_at = utc_now()
     attempt_id=f'{spec.task_id}_{attempt_kind}'
     d=_attempt_dir(package_root, attempt_kind)
     input_files=_copy_attempt_inputs(package_root, d)
@@ -258,18 +324,22 @@ def deterministic_attempt(package_root: Path, raw: RawTaskRecord, spec: TaskInta
         fname=name if '.' in name and '/' not in name else name.lower().replace(' ','_') + '.txt'
         (d/'artifacts'/fname).write_text(f'{attempt_kind} deterministic artifact for {name}\n')
         files.append(f'attempts/{attempt_kind}/artifacts/{fname}')
-    actual=ActualOutputs(task_id=spec.task_id, attempt_id=attempt_id, attempt_kind=attempt_kind, status='success', output_summary=f'Deterministic {attempt_kind} outputs for {spec.normalized_title}', primary_output_ref=files[0] if files else None, deliverable_refs=files, final_message_ref=f'attempts/{attempt_kind}/final_message.txt', artifact_refs=files, files_created=files, files_modified=[], files_deleted=[], stdout_ref=f'attempts/{attempt_kind}/stdout.txt', stderr_ref=f'attempts/{attempt_kind}/stderr.txt', raw_log_refs=[f'attempts/{attempt_kind}/stdout.txt', f'attempts/{attempt_kind}/stderr.txt'], error_type=None, error_message=None, metadata_json={'runner':'deterministic','expected_deliverable_items':deliverables,'produced_deliverable_items':[Path(f).name for f in files]})
+    actual=ActualOutputs(task_id=spec.task_id, attempt_id=attempt_id, attempt_kind=attempt_kind, status='success', output_summary=f'Deterministic {attempt_kind} outputs for {spec.normalized_title}', primary_output_ref=files[0] if files else None, input_artifact_refs=[f'input/{name}' for name in input_files], deliverable_refs=files, final_message_ref=f'attempts/{attempt_kind}/final_message.txt', artifact_refs=files, files_created=files, files_modified=[], files_deleted=[], stdout_ref=f'attempts/{attempt_kind}/stdout.txt', stderr_ref=f'attempts/{attempt_kind}/stderr.txt', raw_log_refs=[f'attempts/{attempt_kind}/stdout.txt', f'attempts/{attempt_kind}/stderr.txt'], error_type=None, error_message=None, metadata_json={'runner':'deterministic','expected_deliverable_items':deliverables,'produced_deliverable_items':[Path(f).name for f in files]})
     steps=[
         AgentTraceStep(step=1, turn=1, actor='user', action='user_message', input=spec.normalized_instruction, message_role='direct_request'),
-        AgentTraceStep(step=2, turn=1, actor=f'agent:{"reviser" if attempt_kind=="revised" else "worker"}', action='agent_step', operation='plan', tool=None, observation='Task instructions, ./input files, and worker-visible rubric are available.', input='Review task and required deliverables.', output='Plan created.', state_change='The attempt plan identified required artifacts.', reasoning='A short plan reduces missed deliverables.', caused_by=[1], causal_type='user_request', success=True, step_outcome='progress'),
-        AgentTraceStep(step=3, turn=1, actor=f'agent:{"reviser" if attempt_kind=="revised" else "worker"}', action='agent_step', operation='write', tool='file_write', observation='Deliverable names are known.', input=', '.join(files), output='Artifacts written.', state_change='Required deterministic artifacts were created under ./artifacts.', reasoning='Writing explicit files satisfies the artifact contract.', caused_by=[2], causal_type='execution_of_plan', success=True, step_outcome='completed', artifact_refs=files),
+        AgentTraceStep(step=2, turn=1, actor=f'agent:{"reviser" if attempt_kind=="revised" else "worker"}', action='agent_step', operation='plan', tool=None, environment_domain='file', execution_mode='serial', observation='Task instructions, ./input files, and worker-visible rubric are available.', input='Review task and required deliverables.', input_source={'source':'task_packet'}, output='Plan created.', state_change='The attempt plan identified required artifacts.', reasoning='A short plan reduces missed deliverables.', caused_by=[1], causal_type='user_request', success=True, step_outcome='progress', artifact_refs=[f'input/{name}' for name in input_files]),
+        AgentTraceStep(step=3, turn=1, actor=f'agent:{"reviser" if attempt_kind=="revised" else "worker"}', action='agent_step', operation='write', tool='file_write', environment_domain='file', execution_mode='serial', observation='Deliverable names are known.', input=', '.join(files), input_source={'source':'planned_deliverables'}, output='Artifacts written.', state_change='Required deterministic artifacts were created under ./artifacts.', reasoning='Writing explicit files satisfies the artifact contract.', caused_by=[2], causal_type='execution_of_plan', success=True, step_outcome='completed', artifact_refs=files),
     ]
+    ended_at = utc_now()
     trace=AgentTrace(trace_id=f'trace_{attempt_id}', collection_id=None, trace_mode='live', task=spec.normalized_instruction, task_id=spec.task_id, attempt_id=attempt_id, attempt_kind=attempt_kind if attempt_kind in ['baseline','revised'] else 'other', agent_tools=['deterministic_runner','file_write'], system_prompt=prompt, system_prompt_hash=None, skills=['agent_trace_skill'], learning='Materialize required artifacts and keep evaluation outside the trace.', termination_reason='task_complete', steps=steps, actual_outputs=actual, artifacts=[], metadata_json={'trace_valid': True})
+    trace = _enrich_runtime_trace(trace, actual, package_root, attempt_kind, 'deterministic', started_at, ended_at)
     write_json(d/'actual_outputs.json', actual)
     actual_ctx=ActualOutputsNormalizationContext(task_id=spec.task_id, attempt_id=attempt_id, attempt_kind=attempt_kind, package_root=package_root, required_artifacts=deliverables)
     actual_result=repair_actual_outputs_file(d/'actual_outputs.json', actual_ctx)
     write_actual_outputs_normalization(d, actual_result)
     actual=ActualOutputs.model_validate(actual_result.actual_outputs)
+    trace.actual_outputs = actual
+    trace = _enrich_runtime_trace(trace, actual, package_root, attempt_kind, 'deterministic', started_at, ended_at)
     write_json(d/'agent_trace.raw.json', trace); write_json(d/'agent_trace.normalized.json', trace); write_json(d/'agent_trace.json', trace)
     report=TraceNormalizationReport(task_id=spec.task_id, attempt_id=attempt_id, attempt_kind=attempt_kind, raw_trace_ref='agent_trace.raw.json', normalized_trace_ref='agent_trace.normalized.json', canonical_trace_ref='agent_trace.json', trace_schema_valid=True, trace_normalized=False, trace_lossless=True, fallback_trace=False, raw_step_count=len(steps), normalized_step_count=len(steps), discarded_step_count=0)
     write_json(d/'trace_normalization_report.json', report)
@@ -300,6 +370,7 @@ def codex_command(
     return cmd
 
 def run_codex_attempt(package_root: Path, raw: RawTaskRecord, spec: TaskIntakeSpec, attempt_kind='baseline', timeout=900) -> AttemptResult:
+    started_at = utc_now()
     d=_attempt_dir(package_root, attempt_kind)
     input_files=_copy_attempt_inputs(package_root, d)
     deliverables=_deliverables(raw, spec)
@@ -350,22 +421,29 @@ def run_codex_attempt(package_root: Path, raw: RawTaskRecord, spec: TaskIntakeSp
     except Exception as e:
         run_error=e
         (d/'stdout.txt').write_text(''); (d/'stderr.txt').write_text(redact_secrets(str(e))); (d/'final_message.txt').write_text('Codex run failed before producing validated trace.')
+    recovered_contract_files = _recover_misplaced_contract_files(d)
     contract_diagnostics = None
     if not (d/'agent_trace.json').exists() or not (d/'actual_outputs.json').exists():
         contract_diagnostics = build_contract_diagnostics(d, command=cmd, working_directory=d, agent_display_name='Codex', prompt=prompt)
         with (d/'final_message.txt').open('a') as f:
             f.write('\n\n' + diagnostics_text(contract_diagnostics))
     trace, actual, trace_valid = ensure_attempt_outputs(package_root, spec, attempt_kind, prompt, sandbox, run_error)
+    ended_at = utc_now()
+    trace = _enrich_runtime_trace(trace, actual, package_root, attempt_kind, 'codex', started_at, ended_at)
     trace.metadata_json['codex_sandbox']=sandbox
     trace.metadata_json['codex_command']=[part if part != prompt else '<prompt>' for part in cmd]
     trace.metadata_json['codex_skip_git_repo_check_supported']=skip_supported
     trace.metadata_json['codex_skip_git_repo_check_used']='--skip-git-repo-check' in cmd
     trace.metadata_json['trace_valid']=trace_valid
+    if recovered_contract_files:
+        trace.metadata_json['contract_files_recovered_from'] = recovered_contract_files
     if contract_diagnostics:
         trace.metadata_json['apprentice_agent_contract_diagnostics']=contract_diagnostics
     write_json(d/'agent_trace.json', trace)
     if actual.metadata_json is None: actual.metadata_json={}
     actual.metadata_json['codex_sandbox']=sandbox
+    if recovered_contract_files:
+        actual.metadata_json['contract_files_recovered_from'] = recovered_contract_files
     if contract_diagnostics:
         actual.metadata_json['apprentice_agent_contract_diagnostics']=contract_diagnostics
     op_error = _apprentice_operational_error(run_error, stdout, stderr, returncode)
@@ -393,6 +471,7 @@ def run_codex_attempt(package_root: Path, raw: RawTaskRecord, spec: TaskIntakeSp
 
 
 def run_custom_attempt(package_root: Path, raw: RawTaskRecord, spec: TaskIntakeSpec, attempt_kind='baseline', timeout: int | None=None) -> AttemptResult:
+    started_at = utc_now()
     settings=get_settings()
     d=_attempt_dir(package_root, attempt_kind)
     input_files=_copy_attempt_inputs(package_root, d)
@@ -411,6 +490,8 @@ def run_custom_attempt(package_root: Path, raw: RawTaskRecord, spec: TaskIntakeS
         with (d/'final_message.txt').open('a') as f:
             f.write('\n\n' + diagnostics_text(contract_diagnostics))
         trace, actual, trace_valid=ensure_attempt_outputs(package_root, spec, attempt_kind, prompt, 'custom', err)
+        ended_at = utc_now()
+        trace = _enrich_runtime_trace(trace, actual, package_root, attempt_kind, 'custom', started_at, ended_at)
         if actual.metadata_json is None:
             actual.metadata_json = {}
         actual.metadata_json['apprentice_agent_contract_diagnostics'] = contract_diagnostics
@@ -449,6 +530,8 @@ def run_custom_attempt(package_root: Path, raw: RawTaskRecord, spec: TaskIntakeS
         with (d/'final_message.txt').open('a') as f:
             f.write('\n\n' + diagnostics_text(contract_diagnostics))
     trace, actual, trace_valid=ensure_attempt_outputs(package_root, spec, attempt_kind, prompt, 'custom', run_error)
+    ended_at = utc_now()
+    trace = _enrich_runtime_trace(trace, actual, package_root, attempt_kind, 'custom', started_at, ended_at)
     trace.metadata_json['custom_worker_display_name']=settings.custom_worker_display_name
     trace.metadata_json['custom_worker_command_template']=template
     trace.metadata_json['trace_valid']=trace_valid