agent-apprenticeship 0.1.0

package/src/agent_apprenticeship_trace/env.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+import os, re
+from pathlib import Path
+
+def parse_env_file(path: Path) -> dict[str,str]:
+    vals={}
+    if not path.exists(): return vals
+    for line in path.read_text().splitlines():
+        line=line.strip()
+        if not line or line.startswith('#') or '=' not in line: continue
+        k,v=line.split('=',1); vals[k.strip()]=v.strip().strip('"').strip("'")
+    return vals
+
+def load_local_env(root: Path | None=None) -> dict[str,str]:
+    if os.getenv("AA_DISABLE_LOCAL_ENV") == "1":
+        return {}
+    root = root or Path.cwd()
+    loaded={}
+    for name in ['.env.local','.env']:
+        for k,v in parse_env_file(root/name).items():
+            if k not in os.environ:
+                os.environ[k]=v; loaded[k]=v
+    return loaded
+
+SECRET_PATTERNS=[
+    re.compile(r"sk-proj-[A-Za-z0-9_-]{20,}"),
+    re.compile(r"sk-ant-[A-Za-z0-9_-]{20,}"),
+    re.compile(r"sk-or-v1-[A-Za-z0-9_-]{20,}"),
+    re.compile(r"sk-[A-Za-z0-9_-]{20,}"),
+    re.compile(r"AIza[A-Za-z0-9_-]{20,}"),
+    re.compile(r"dsk-[A-Za-z0-9_-]{20,}"),
+    re.compile(r"user_[A-Za-z0-9_-]{12,}"),
+    re.compile(r"(['\"]user_id['\"]\s*:\s*)['\"][^'\"]+['\"]"),
+]
+def contains_secret(text: str) -> bool:
+    return any(p.search(text or '') for p in SECRET_PATTERNS)
+def redact_secrets(text: str) -> str:
+    text = text or ''
+    for p in SECRET_PATTERNS:
+        if 'user_id' in p.pattern:
+            text=p.sub(r"\1'[REDACTED_USER_ID]'", text)
+        elif p.pattern.startswith('user_'):
+            text=p.sub('[REDACTED_USER_ID]', text)
+        else:
+            text=p.sub('[REDACTED_SECRET]', text)
+    return text

package/src/agent_apprenticeship_trace/evaluator.py

@@ -0,0 +1,64 @@
+from __future__ import annotations
+import json
+from pathlib import Path
+from pydantic import BaseModel
+from .schemas import EvaluatorFeedback, GraderResult, VerifierResult, AgentTrace, ActualOutputs, RevisionPlan
+from .config import get_settings
+from .io import read_json
+from .openai_structured import get_model_provider_status, run_structured_role
+from .public_sanitizer import sha256_text
+from .artifact_previews import build_artifact_previews
+
+def _mentor_provider_can_attempt() -> bool:
+    return bool(get_model_provider_status().get('provider_available'))
+
+def _mentor_provider_id() -> str:
+    settings=get_settings()
+    return settings.model_provider or 'openai'
+
+def _model_evaluator_active(settings) -> bool:
+    return settings.mentor_mode in {'model_assisted', 'hybrid'} or settings.evaluation_mode in {'hybrid', 'llm_required'}
+
+class LLMEvaluatorOutput(BaseModel):
+    evaluator_feedback: EvaluatorFeedback
+    revision_plan: RevisionPlan
+
+def deterministic_feedback(grader: GraderResult) -> EvaluatorFeedback:
+    return EvaluatorFeedback(feedback_id=f'feedback_{grader.attempt_id}', task_id=grader.task_id, attempt_id=grader.attempt_id, target_actor='worker', feedback_type='criteria_failure' if grader.failed_criteria else 'other', failed_rubric_items=grader.failed_criteria, evidence_refs=grader.evidence_refs, artifact_refs=grader.evidence_refs, feedback_summary='Improve any failed rubric items and preserve valid artifacts.', actionable_feedback=['Check every required artifact exists.', 'Address failed criteria explicitly.'], suggested_revision='Create or repair missing/weak deliverables.', revision_priority='high' if grader.failed_criteria else 'low', confidence=0.75, hidden_reference_used=False, hidden_reference_leaked=False, failed_or_weak_rubric_items=grader.failed_criteria, artifact_specific_comments=[], trace_specific_comments=[], revision_plan='Create or repair missing/weak deliverables.', model=None, provider=None, metadata_json={})
+
+def _revision_plan_from_feedback(fb: EvaluatorFeedback, target_attempt_id: str) -> RevisionPlan:
+    return RevisionPlan(revision_plan_id=f'revision_plan_{fb.task_id}', task_id=fb.task_id, source_attempt_id=fb.attempt_id, target_attempt_id=target_attempt_id, revision_kind='local_fix', revision_reason=fb.feedback_summary, failed_rubric_items=fb.failed_rubric_items, planned_changes=fb.actionable_feedback, expected_score_improvement=0.1, risk_of_regression='low', uses_evaluator_feedback=True, metadata_json={})
+
+def _evaluator_prompt(grader: GraderResult, verifier: VerifierResult, outputs: ActualOutputs, trace: AgentTrace | None, target_attempt_id: str, artifact_preview_bundle: dict | None=None) -> str:
+    return """Return only valid JSON. Do not include markdown. Do not add extra top-level fields; place extras under metadata_json.extra_model_fields.
+Required skeleton: {"evaluator_feedback":{"feedback_id":"feedback_<task_id>_<attempt_kind>","task_id":"...","attempt_id":"...","target_actor":"worker","feedback_type":"other","failed_rubric_items":[],"evidence_refs":[],"artifact_refs":[],"feedback_summary":"...","actionable_feedback":[],"suggested_revision":"...","revision_priority":"medium","confidence":0.7,"hidden_reference_used":false,"hidden_reference_leaked":false,"failed_or_weak_rubric_items":[],"artifact_specific_comments":[],"trace_specific_comments":[],"metadata_json":{}},"revision_plan":{"revision_plan_id":"revision_plan_<task_id>","task_id":"...","source_attempt_id":"...","target_attempt_id":"...","revision_kind":"local_fix","revision_reason":"...","failed_rubric_items":[],"planned_changes":[],"risk_of_regression":"medium","uses_evaluator_feedback":true,"metadata_json":{}}}.
+Evaluate the attempt using grader/verifier outputs, actual outputs, artifact content previews, artifacts, and trace. Produce actionable feedback and a concrete revision plan.
+Target revised attempt id: """ + target_attempt_id + "\nGraderResult JSON:\n" + json.dumps(grader.model_dump(), sort_keys=True) + "\nVerifierResult JSON:\n" + json.dumps(verifier.model_dump(), sort_keys=True) + "\nActualOutputs JSON:\n" + json.dumps(outputs.model_dump(), sort_keys=True) + "\nArtifact content previews JSON:\n" + json.dumps(artifact_preview_bundle or {}, sort_keys=True)[:16000] + "\nTrace summary JSON:\n" + json.dumps((trace.model_dump() if trace else {}), sort_keys=True)[:12000]
+
+def evaluate_attempt(grader: GraderResult, verifier: VerifierResult, outputs: ActualOutputs, trace: AgentTrace | None, target_attempt_id: str, role_root: Path | None=None, package_root: Path | None=None) -> tuple[EvaluatorFeedback, RevisionPlan]:
+    settings=get_settings(); role_root=role_root or Path('outputs/roles')
+    artifact_preview_bundle=build_artifact_previews(package_root, grader.evidence_refs or outputs.deliverable_refs or outputs.artifact_refs or outputs.files_created)
+    if _model_evaluator_active(settings) and _mentor_provider_can_attempt() and settings.llm_evaluator_enabled:
+        prompt=_evaluator_prompt(grader, verifier, outputs, trace, target_attempt_id, artifact_preview_bundle)
+        try:
+            provider=_mentor_provider_id()
+            model_override=settings.llm_evaluator_model if provider == 'openai' else None
+            rr=run_structured_role('evaluator_agent', prompt, LLMEvaluatorOutput, role_root/'evaluator_agent'/outputs.attempt_kind, allow_fallback=settings.allow_deterministic_eval_fallback, model_override=model_override, normalizer_context={'task_id': outputs.task_id, 'attempt_id': outputs.attempt_id, 'attempt_kind': outputs.attempt_kind, 'target_attempt_id': target_attempt_id, 'grader_result_id': grader.grader_result_id, 'verifier_result_id': verifier.verifier_result_id, 'artifact_content_refs': artifact_preview_bundle.get('artifact_content_refs'), 'artifact_content_previews': artifact_preview_bundle.get('artifact_content_previews'), 'artifact_content_hashes': artifact_preview_bundle.get('artifact_content_hashes'), 'artifact_content_preview_truncated': artifact_preview_bundle.get('artifact_content_preview_truncated'), 'model_grading_basis': artifact_preview_bundle.get('model_grading_basis'), 'model': model_override or settings.model_provider_model, 'provider':provider})
+            if rr.live_call_ok and rr.structured_output_validation_ok:
+                parsed=read_json(role_root/'evaluator_agent'/outputs.attempt_kind/'parsed_output.json')
+                fb=EvaluatorFeedback.model_validate(parsed['evaluator_feedback'])
+                rp=RevisionPlan.model_validate(parsed['revision_plan'])
+                fb.provider=rr.provider; fb.model=rr.model; fb.revision_plan=fb.revision_plan or rp.revision_reason
+                fb.metadata_json.update({'llm_prompt_ref_internal':str(role_root/'evaluator_agent'/outputs.attempt_kind/'prompt.md'),'llm_response_ref_internal':str(role_root/'evaluator_agent'/outputs.attempt_kind/'raw_output.txt'),'prompt_hash':sha256_text(prompt),'public_response_summary':'Model evaluator generated actionable feedback and revision plan.', **artifact_preview_bundle})
+                rp.metadata_json.update({'provider':rr.provider,'model':rr.model,'llm_prompt_ref_internal':str(role_root/'evaluator_agent'/outputs.attempt_kind/'prompt.md'),'llm_response_ref_internal':str(role_root/'evaluator_agent'/outputs.attempt_kind/'raw_output.txt')})
+                return fb,rp
+            if settings.evaluation_mode == 'llm_required' or settings.llm_fail_closed:
+                raise RuntimeError(rr.error_message or 'Model evaluator failed')
+        except Exception:
+            if settings.evaluation_mode == 'llm_required' or settings.llm_fail_closed:
+                raise
+    fb=deterministic_feedback(grader)
+    rp=_revision_plan_from_feedback(fb, target_attempt_id)
+    if _mentor_provider_can_attempt() and settings.llm_evaluator_enabled and settings.evaluation_mode != 'deterministic_only':
+        fb.metadata_json.update({'model_evaluator_enabled': True, 'llm_evaluator_enabled': True, 'llm_unavailable': True, 'deterministic_fallback': True, **artifact_preview_bundle})
+    return fb,rp

package/src/agent_apprenticeship_trace/grader.py

@@ -0,0 +1,194 @@
+from __future__ import annotations
+from pathlib import Path
+import json, re
+from .schemas import GraderResult, RubricSpec, ActualOutputs, RubricItemScore
+from .config import get_settings
+from .schemas import AgentTrace
+from .io import read_json, write_json
+from .openai_structured import get_model_provider_status, run_structured_role
+from .public_sanitizer import sha256_text
+from .artifact_previews import build_artifact_previews
+
+INPUT_ARTIFACT_NAMES = {
+    'payments.csv', 'invoices.csv', 'fx_rates.csv', 'vendor_aliases.csv', 'reconciliation_policy.md'
+}
+
+def _mentor_provider_can_attempt() -> bool:
+    return bool(get_model_provider_status().get('provider_available'))
+
+def _mentor_provider_id() -> str:
+    settings=get_settings()
+    return settings.model_provider or 'openai'
+
+def _model_grading_active(settings) -> bool:
+    return settings.mentor_mode in {'model_assisted', 'hybrid'} or settings.evaluation_mode in {'hybrid', 'llm_required'}
+
+def _is_output_artifact_name(name: str, output_names: set[str]) -> bool:
+    base=Path(str(name)).name
+    if base in INPUT_ARTIFACT_NAMES:
+        return False
+    return (not output_names) or base in output_names
+
+
+def _expected_items_from_outputs(outputs: ActualOutputs) -> list[str]:
+    md=outputs.metadata_json or {}
+    items=md.get('expected_deliverable_items') or []
+    return [Path(str(x)).name for x in items if x]
+
+def _produced_items(outputs: ActualOutputs) -> list[str]:
+    refs=list(outputs.deliverable_refs or []) + list(outputs.artifact_refs or []) + list(outputs.files_created or [])
+    md=outputs.metadata_json or {}
+    refs += [str(x) for x in (md.get('produced_deliverable_items') or [])]
+    return list(dict.fromkeys(Path(str(x)).name for x in refs if x))
+
+def _deliverable_match(outputs: ActualOutputs, evidence: list[str], missing: list[str]) -> dict[str, object]:
+    expected=_expected_items_from_outputs(outputs)
+    produced=_produced_items(outputs)
+    expected_set=set(expected)
+    produced_set=set(produced)
+    if not expected:
+        status='exact' if not missing else 'partial'
+        mismatches=[]
+    else:
+        missing_expected=sorted(expected_set - produced_set)
+        extra_produced=sorted(produced_set - expected_set)
+        mismatches=[{'expected': m, 'status':'missing_or_substituted'} for m in missing_expected]
+        status='exact' if not missing_expected else ('partial' if evidence else 'mismatch')
+        if missing_expected and extra_produced:
+            status='partial'
+    return {'expected_deliverable_items': expected, 'produced_deliverable_items': produced, 'deliverable_match_status': status, 'deliverable_mismatches': mismatches}
+
+def _candidate_refs(outputs: ActualOutputs, artifact_name: str, attempt_kind: str) -> list[str]:
+    refs=[]
+    refs.extend(outputs.deliverable_refs or [])
+    refs.extend(outputs.artifact_refs or [])
+    refs.extend(outputs.files_created or [])
+    refs.append(f'attempts/{attempt_kind}/artifacts/{artifact_name}')
+    refs.append(f'artifacts/{artifact_name}')
+    out=[]
+    for r in refs:
+        if not r: continue
+        r=str(r)
+        if r.endswith('/'+artifact_name) or Path(r).name == artifact_name:
+            if r.startswith('artifacts/'):
+                r=f'attempts/{attempt_kind}/{r}'
+            out.append(r)
+    return list(dict.fromkeys(out))
+
+def _resolve_required_artifacts(rubric: RubricSpec, outputs: ActualOutputs, attempt_kind: str, package_root: Path | None=None) -> tuple[list[str], list[str]]:
+    output_refs=list(outputs.deliverable_refs or []) + list(outputs.artifact_refs or []) + list(outputs.files_created or [])
+    output_names={Path(str(x)).name for x in output_refs if x and ('artifacts/' in str(x) or str(x).startswith(f'attempts/{attempt_kind}/'))}
+    required=_expected_items_from_outputs(outputs)
+    if not required:
+        required=[]
+    for item in ([] if required else rubric.rubric_items):
+        required.extend(item.required_artifacts)
+
+    if not required:
+        required.extend(rubric.required_artifacts)
+    required=[Path(x).name for x in required if x and _is_output_artifact_name(str(x), output_names)]
+    if not required:
+        required=[Path(x).name for x in output_refs if x and 'artifacts/' in str(x)]
+    required=list(dict.fromkeys(required))
+    evidence=[]; missing=[]
+    refs=set(outputs.deliverable_refs + outputs.artifact_refs + outputs.files_created)
+    for name in required:
+        cands=_candidate_refs(outputs, name, attempt_kind)
+        found=None
+        for c in cands:
+            path_ok = bool(package_root and (package_root/c).exists())
+            if c in refs or path_ok:
+                found=c; break
+        if not found and package_root:
+            c=f'attempts/{attempt_kind}/artifacts/{name}'
+            if (package_root/c).exists():
+                found=c
+        if not found and not package_root:
+            for c in cands:
+                if Path(c).name == name and (c in refs or outputs.status == 'success'):
+                    found=c; break
+        if found: evidence.append(found)
+        else: missing.append(name)
+    return evidence, missing
+
+def deterministic_grade(rubric: RubricSpec, outputs: ActualOutputs, attempt_kind: str, package_root: Path | None=None) -> GraderResult:
+    evidence, missing = _resolve_required_artifacts(rubric, outputs, attempt_kind, package_root)
+    match=_deliverable_match(outputs, evidence, missing)
+    required_count=max(1, len(match.get('expected_deliverable_items') or missing) or len(evidence)+len(missing))
+    artifact_contract_score = 1.0 if (outputs.status == 'success' and not missing and bool(evidence) and match['deliverable_match_status']=='exact') else max(0.0, min(1.0, len(evidence)/required_count))
+    artifact_contract_passed = outputs.status == 'success' and artifact_contract_score >= 1.0 and not missing and match['deliverable_match_status']=='exact'
+    item_scores=[]
+    for item in rubric.rubric_items:
+        output_names={Path(str(x)).name for x in (outputs.deliverable_refs or []) + (outputs.artifact_refs or []) + (outputs.files_created or []) if x and ('artifacts/' in str(x) or str(x).startswith(f'attempts/{attempt_kind}/'))}
+        req=[Path(x).name for x in item.required_artifacts if _is_output_artifact_name(str(x), output_names)]
+        item_evidence=[e for e in evidence if Path(e).name in req] or evidence
+        item_missing=[m for m in missing if m in req]
+        ok = outputs.status == 'success' and not item_missing and bool(item_evidence)
+        item_scores.append(RubricItemScore(rubric_item_id=item.rubric_item_id, criterion_name=item.criterion_name, score=1.0 if ok else 0.0, max_score=1.0, passed=ok, evidence_refs=item_evidence, failure_mode=None if ok else 'missing_artifact', notes='Deterministic artifact-contract precheck only; semantic correctness is not evaluated.', confidence=0.8, artifact_presence_ok=ok, semantic_correctness_score=None, reasoning_summary='Required artifact paths were resolved.' if ok else 'One or more required artifacts were not resolved.', improvement_suggestion=None if ok else 'Create the missing required artifact files.'))
+    score=artifact_contract_score
+    return GraderResult(grader_result_id=f'grader_{outputs.attempt_id}', task_id=outputs.task_id, attempt_id=outputs.attempt_id, attempt_kind=attempt_kind, rubric_id=rubric.rubric_id, grader_kind='deterministic', score_source='deterministic_artifact_contract', score=score, max_score=1.0, passed=score>=rubric.pass_threshold, rubric_item_scores=item_scores, failed_criteria=[s.rubric_item_id for s in item_scores if not s.passed], passed_criteria=[s.rubric_item_id for s in item_scores if s.passed], evidence_refs=evidence, confidence=0.8, reasoning_summary='Deterministic precheck verifies artifact presence/readiness only; semantic correctness is not evaluated.', limitations=['semantic_correctness_not_evaluated'], hidden_reference_used=False, hidden_reference_leaked=False, artifact_contract_score=artifact_contract_score, semantic_score=None, final_score=score, model=None, provider=None, deterministic_precheck_ref=None, public_response_summary='Artifact-contract precheck completed.', metadata_json={'artifact_contract_passed': artifact_contract_passed, 'semantic_correctness_not_evaluated': True, 'missing_artifacts': missing, **match})
+
+def _grader_prompt(rubric: RubricSpec, outputs: ActualOutputs, trace: AgentTrace | None, precheck: GraderResult, artifact_preview_bundle: dict | None=None) -> str:
+    return """Return only valid JSON matching GraderResult. Do not include markdown.
+You are the model-based grader. Score each rubric item using the task rubric, attempt trace, actual outputs, and artifact content previews when available. Return a JSON object with exactly these required top-level fields: grader_result_id, task_id, attempt_id, attempt_kind, rubric_id, grader_kind, score_source, artifact_contract_score, model_score, semantic_score, final_score, score, max_score, passed, confidence, evidence_refs, rubric_item_scores, limitations, hidden_reference_used, hidden_reference_leaked, metadata_json. Each rubric_item_scores entry must include artifact_presence_ok, semantic_correctness_score, evidence_refs, reasoning_summary, confidence, failure_mode, and improvement_suggestion. Cite evidence_refs, distinguish artifact presence from model-judged correctness, and set score_source=model_judged and grader_kind=model. Deterministic artifact-contract precheck is auxiliary only.
+Rubric JSON:
+""" + json.dumps(rubric.model_dump(), sort_keys=True) + "\nActualOutputs JSON:\n" + json.dumps(outputs.model_dump(), sort_keys=True) + "\nArtifact content previews JSON:\n" + json.dumps(artifact_preview_bundle or {}, sort_keys=True)[:16000] + "\nTrace summary JSON:\n" + json.dumps((trace.model_dump() if trace else {}), sort_keys=True)[:12000] + "\nDeterministic precheck JSON:\n" + json.dumps(precheck.model_dump(), sort_keys=True)
+
+def grade_attempt(rubric: RubricSpec, outputs: ActualOutputs, attempt_kind: str, trace: AgentTrace | None=None, role_root: Path | None=None, package_root: Path | None=None) -> GraderResult:
+    settings=get_settings()
+    pre=deterministic_grade(rubric, outputs, attempt_kind, package_root)
+    role_root=role_root or Path('outputs/roles')
+    artifact_preview_bundle=build_artifact_previews(package_root, pre.evidence_refs or outputs.deliverable_refs or outputs.artifact_refs or outputs.files_created)
+    pre.metadata_json.update(artifact_preview_bundle)
+    if _model_grading_active(settings) and _mentor_provider_can_attempt() and settings.llm_grader_enabled:
+        prompt=_grader_prompt(rubric, outputs, trace, pre, artifact_preview_bundle)
+        (role_root/'grader_agent'/attempt_kind).mkdir(parents=True, exist_ok=True)
+        write_json(role_root/'grader_agent'/attempt_kind/'deterministic_precheck.json', pre)
+        try:
+            provider=_mentor_provider_id()
+            model_override=settings.llm_grader_model if provider == 'openai' else None
+            rr=run_structured_role('grader_agent', prompt, GraderResult, role_root/'grader_agent'/attempt_kind, allow_fallback=settings.allow_deterministic_eval_fallback, model_override=model_override, normalizer_context={'task_id': outputs.task_id, 'attempt_id': outputs.attempt_id, 'attempt_kind': attempt_kind, 'rubric_id': rubric.rubric_id, 'artifact_contract_score': pre.artifact_contract_score, 'evidence_refs': pre.evidence_refs, 'artifact_content_refs': artifact_preview_bundle.get('artifact_content_refs'), 'artifact_content_previews': artifact_preview_bundle.get('artifact_content_previews'), 'artifact_content_hashes': artifact_preview_bundle.get('artifact_content_hashes'), 'artifact_content_preview_truncated': artifact_preview_bundle.get('artifact_content_preview_truncated'), 'model_grading_basis': artifact_preview_bundle.get('model_grading_basis'), 'model': model_override or settings.model_provider_model, 'provider': provider})
+            if rr.live_call_ok and rr.structured_output_validation_ok:
+                parsed=read_json(role_root/'grader_agent'/attempt_kind/'parsed_output.json')
+                g=GraderResult.model_validate(parsed)
+                g.grader_kind='model'; g.score_source='model_judged'; g.legacy_score_source='llm_semantic'; g.artifact_contract_score=pre.artifact_contract_score; g.semantic_score=g.semantic_score if g.semantic_score is not None else g.score; g.model_score=g.model_score if g.model_score is not None else g.semantic_score; g.legacy_semantic_score=g.legacy_semantic_score if g.legacy_semantic_score is not None else g.semantic_score; g.final_score=g.final_score if g.final_score is not None else g.semantic_score; g.score=g.final_score; g.model=rr.model; g.provider=rr.provider; g.deterministic_precheck_ref=str(role_root/'grader_agent'/attempt_kind/'deterministic_precheck.json'); g.llm_prompt_ref_internal=str(role_root/'grader_agent'/attempt_kind/'prompt.md'); g.llm_response_ref_internal=str(role_root/'grader_agent'/attempt_kind/'raw_output.txt'); g.public_prompt_hash=sha256_text(prompt); g.public_response_summary=g.public_response_summary or 'Model grader scored rubric items with evidence refs and artifact previews when available.'
+                g.metadata_json.update({'deterministic_precheck': pre.model_dump(), **artifact_preview_bundle, 'llm_role_result_ref_internal':str(role_root/'grader_agent'/attempt_kind/'role_result.json')})
+                return g
+            if settings.evaluation_mode == 'llm_required' or settings.llm_fail_closed:
+                raise RuntimeError(rr.error_message or 'Model grader failed')
+        except Exception:
+            if settings.evaluation_mode == 'llm_required' or settings.llm_fail_closed:
+                raise
+    if settings.evaluation_mode == 'deterministic_only':
+        pre.semantic_score=None; pre.final_score=pre.score
+    elif _mentor_provider_can_attempt() and settings.llm_grader_enabled:
+        pre.grader_kind='hybrid'; pre.score_source='deterministic_fallback'; pre.semantic_score=None; pre.final_score=pre.score
+        provider=_mentor_provider_id()
+        pre.model=settings.model_provider_model or settings.llm_grader_model; pre.provider=provider
+        pre.limitations.append('Model grader unavailable; deterministic fallback explicitly marked.')
+        pre.metadata_json.update({'llm_grader_enabled': True, 'llm_unavailable': True, 'deterministic_fallback': True})
+    return pre
+
+
+def apply_score_reliability(grader: GraderResult, verifier) -> GraderResult:
+    status=getattr(verifier, 'verification_status', None) or 'not_run'
+    issues=list(getattr(verifier, 'issues', None) or [])
+    if status == 'verified':
+        reliability='verified'
+    elif status == 'failed':
+        reliability='failed_verification'
+    else:
+        reliability='needs_review'
+    grader.score_reliability=reliability
+    grader.verifier_status=status
+    grader.verifier_confidence=getattr(verifier, 'confidence', None)
+    grader.verifier_issue_count=len(issues)
+    grader.verifier_issues_summary='; '.join(str(x) for x in issues[:5]) if issues else None
+    grader.metadata_json.update({
+        'score_reliability': reliability,
+        'verifier_status': status,
+        'verifier_confidence': getattr(verifier, 'confidence', None),
+        'verifier_issue_count': len(issues),
+        'verifier_issues_summary': grader.verifier_issues_summary,
+    })
+    return grader

package/src/agent_apprenticeship_trace/integration_status.py

@@ -0,0 +1,193 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from .config import get_settings, mentor_model_provider_readiness, apprentice_agent_readiness_status
+from .io import append_jsonl, read_jsonl
+from .recipes import MODEL_PROVIDER_RECIPES, WORKER_AGENT_RECIPES
+
+
+def smoke_cache_path(app_home: Path | None = None):
+    return (app_home or get_settings().app_home) / "integration_smoke_results.jsonl"
+
+
+def record_certification_result(
+    *,
+    provider_type: str,
+    provider_id: str,
+    result: str,
+    certification_kind: str,
+    agent_id: str | None = None,
+    model_provider_id: str | None = None,
+    error_type: str | None = None,
+    error_summary: str | None = None,
+    command_or_model: str | None = None,
+    app_home: Path | None = None,
+    metadata_json: dict[str, Any] | None = None,
+) -> None:
+    row = {
+        "provider_type": provider_type,
+        "provider_id": provider_id,
+        "result": result,
+        "certification_kind": certification_kind,
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+        "agent_id": agent_id,
+        "model_provider_id": model_provider_id,
+        "error_type": error_type,
+        "error_summary": error_summary,
+        "command_or_model": command_or_model,
+        "metadata_json": metadata_json or {},
+    }
+    try:
+        append_jsonl(smoke_cache_path(app_home), row)
+    except OSError:
+        # Live smokes are often run from restricted CI/sandbox contexts. A
+        # non-writable local app home should not hide the actual provider result.
+        return
+
+
+def record_live_smoke_result(
+    *,
+    provider_type: str,
+    provider_id: str,
+    result: str,
+    error_type: str | None = None,
+    error_summary: str | None = None,
+    command_or_model: str | None = None,
+) -> None:
+    kind = {
+        "apprentice_agent": "agent_live_smoke",
+        "mentor_model_provider": "model_live_smoke",
+    }.get(provider_type, "live_smoke")
+    record_certification_result(
+        provider_type=provider_type,
+        provider_id=provider_id,
+        result=result,
+        certification_kind=kind,
+        error_type=error_type,
+        error_summary=error_summary,
+        command_or_model=command_or_model,
+    )
+
+
+def latest_smoke_results(app_home: Path | None = None) -> dict[tuple[str, str], dict[str, Any]]:
+    path = smoke_cache_path(app_home)
+    if not path.exists():
+        return {}
+    latest: dict[tuple[str, str], dict[str, Any]] = {}
+    for row in read_jsonl(path):
+        key = (str(row.get("provider_type")), str(row.get("provider_id")))
+        latest[key] = row
+    return latest
+
+
+def latest_certification_results(app_home: Path | None = None) -> list[dict[str, Any]]:
+    path = smoke_cache_path(app_home)
+    if not path.exists():
+        return []
+    return list(read_jsonl(path))
+
+
+def _latest_for(rows: list[dict[str, Any]], **filters: str) -> dict[str, Any]:
+    match = None
+    for row in rows:
+        if all(str(row.get(key)) == value for key, value in filters.items()):
+            match = row
+    return match or {}
+
+
+def _latest_full_e2e_for(rows: list[dict[str, Any]], *, agent_id: str | None = None, provider_id: str | None = None) -> dict[str, Any]:
+    match = None
+    for row in rows:
+        if row.get("certification_kind") != "full_e2e":
+            continue
+        if agent_id is not None and row.get("agent_id") != agent_id:
+            continue
+        if provider_id is not None and row.get("model_provider_id") != provider_id:
+            continue
+        match = row
+    return match or {}
+
+
+def _last_tested(*rows: dict[str, Any]) -> str | None:
+    timestamps = [str(row.get("timestamp")) for row in rows if row.get("timestamp")]
+    return max(timestamps) if timestamps else None
+
+
+def integrations_report() -> dict[str, list[dict[str, Any]]]:
+    latest = latest_smoke_results()
+    rows = latest_certification_results()
+    settings = get_settings()
+    agents = []
+    for agent_id, recipe in WORKER_AGENT_RECIPES.items():
+        s = settings.model_copy(
+            update={
+                "worker_agent": agent_id,
+                "worker_agent_command": settings.worker_agent_command if settings.worker_agent == agent_id else None,
+                "custom_worker_command_template": settings.custom_worker_command_template if agent_id == "custom" else settings.custom_worker_command_template,
+                "custom_worker_display_name": settings.custom_worker_display_name if agent_id == "custom" else settings.custom_worker_display_name,
+            }
+        )
+        readiness = apprentice_agent_readiness_status(s)
+        smoke = latest.get(("apprentice_agent", agent_id), {})
+        if agent_id == "custom":
+            fixture_smoke = _latest_for(rows, provider_type="apprentice_agent", provider_id="custom", certification_kind="agent_live_smoke_fixture")
+            user_smoke = _latest_for(rows, provider_type="apprentice_agent", provider_id="custom", certification_kind="agent_live_smoke_user")
+            display_smoke = fixture_smoke or smoke
+        else:
+            fixture_smoke = {}
+            user_smoke = {}
+            display_smoke = smoke
+        full = _latest_full_e2e_for(rows, agent_id=agent_id)
+        agents.append(
+            {
+                "id": agent_id,
+                "display_name": recipe.display_name,
+                "adapter_implemented": True,
+                "command_expected": readiness.get("command") or recipe.command_name,
+                "command_found": bool(readiness.get("command_found")),
+                "fake_adapter_test_covered": True,
+                "live_smoke_script_available": True,
+                "latest_local_live_smoke_result": display_smoke.get("result") or "not_run",
+                "latest_full_e2e_result": full.get("result") or "not_run",
+                "last_tested_at": _last_tested(smoke, full, fixture_smoke, user_smoke),
+                "latest_error_summary": display_smoke.get("error_summary"),
+                "latest_full_e2e_error_summary": full.get("error_summary"),
+                "custom_fixture_live_smoke_result": fixture_smoke.get("result") if agent_id == "custom" else None,
+                "custom_user_live_smoke_result": user_smoke.get("result") if agent_id == "custom" else None,
+            }
+        )
+    providers = []
+    for provider_id, recipe in MODEL_PROVIDER_RECIPES.items():
+        s = settings.model_copy(
+            update={
+                "model_provider": provider_id,
+                "model_provider_api_key_env": settings.model_provider_api_key_env if settings.model_provider == provider_id else recipe.api_key_env_var,
+                "model_provider_model": settings.model_provider_model if settings.model_provider == provider_id else recipe.default_model,
+            }
+        )
+        readiness = mentor_model_provider_readiness(s)
+        smoke = latest.get(("mentor_model_provider", provider_id), {})
+        full = _latest_full_e2e_for(rows, provider_id=provider_id)
+        full_result = full.get("result") or "not_run"
+        if full_result == "passed" and str(smoke.get("result") or "").startswith("failed"):
+            full_result = "not_certified_due_to_provider_failure"
+        providers.append(
+            {
+                "id": provider_id,
+                "display_name": recipe.display_name,
+                "adapter_implemented": True,
+                "key_env_var": readiness.get("api_key_env_var") or recipe.api_key_env_var,
+                "key_visible": bool(readiness.get("api_key_visible")),
+                "fake_provider_test_covered": True,
+                "live_smoke_script_available": True,
+                "latest_local_live_smoke_result": smoke.get("result") or "not_run",
+                "latest_full_e2e_result": full_result,
+                "last_tested_at": _last_tested(smoke, full),
+                "latest_error_summary": smoke.get("error_summary"),
+                "latest_full_e2e_error_summary": full.get("error_summary"),
+            }
+        )
+    return {"apprentice_agents": agents, "mentor_model_providers": providers}

package/src/agent_apprenticeship_trace/io.py

@@ -0,0 +1,20 @@
+from __future__ import annotations
+import json, hashlib
+from pathlib import Path
+from pydantic import BaseModel
+
+def write_json(path: Path, obj):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    data = obj.model_dump(mode='json') if isinstance(obj, BaseModel) else obj
+    path.write_text(json.dumps(data, indent=2, sort_keys=True)+"\n")
+
+def read_json(path: Path): return json.loads(path.read_text())
+def append_jsonl(path: Path, obj):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    data = obj.model_dump(mode='json') if isinstance(obj, BaseModel) else obj
+    with path.open('a') as f: f.write(json.dumps(data, sort_keys=True)+"\n")
+def read_jsonl(path: Path):
+    if not path.exists(): return []
+    return [json.loads(l) for l in path.read_text().splitlines() if l.strip()]
+def sha256_file(path: Path):
+    h=hashlib.sha256(); h.update(path.read_bytes()); return h.hexdigest()