agent-apprenticeship 0.1.5 → 0.1.7

package/src/agent_apprenticeship_trace/evaluator.py

@@ -24,33 +24,147 @@ class LLMEvaluatorOutput(BaseModel):
     revision_plan: RevisionPlan
 
 def deterministic_feedback(grader: GraderResult) -> EvaluatorFeedback:
-    return EvaluatorFeedback(feedback_id=f'feedback_{grader.attempt_id}', task_id=grader.task_id, attempt_id=grader.attempt_id, target_actor='worker', feedback_type='criteria_failure' if grader.failed_criteria else 'other', failed_rubric_items=grader.failed_criteria, evidence_refs=grader.evidence_refs, artifact_refs=grader.evidence_refs, feedback_summary='Improve any failed rubric items and preserve valid artifacts.', actionable_feedback=['Check every required artifact exists.', 'Address failed criteria explicitly.'], suggested_revision='Create or repair missing/weak deliverables.', revision_priority='high' if grader.failed_criteria else 'low', confidence=0.75, hidden_reference_used=False, hidden_reference_leaked=False, failed_or_weak_rubric_items=grader.failed_criteria, artifact_specific_comments=[], trace_specific_comments=[], revision_plan='Create or repair missing/weak deliverables.', model=None, provider=None, metadata_json={})
+    failed = grader.failed_criteria if grader.score_source == 'structural_guardrail' else grader.failed_criteria
+    summary = 'Mentor evaluator unavailable; no outcome judgment was produced. Structural guardrails suggest preserving valid artifacts and repairing missing deliverables.'
+    return EvaluatorFeedback(feedback_id=f'feedback_{grader.attempt_id}', task_id=grader.task_id, attempt_id=grader.attempt_id, target_actor='apprentice', feedback_type='artifact_missing' if failed else 'other', failed_rubric_items=failed, evidence_refs=grader.evidence_refs, artifact_refs=grader.evidence_refs, feedback_summary=summary, actionable_feedback=['Preserve any valid generated artifacts.', 'Repair missing deliverables reported by structural guardrails.'] if failed else ['Preserve valid artifacts for any future revision.'], suggested_revision='Repair missing deliverables if the user or Mentor asks for a revision.', revision_priority='medium' if failed else 'low', confidence=0.35, hidden_reference_used=False, hidden_reference_leaked=False, failed_or_weak_rubric_items=failed, artifact_specific_comments=[], trace_specific_comments=[], revision_plan='Repair missing deliverables only if a revision is requested.', model=None, provider=None, metadata_json={'mentor_evaluator_status':'unavailable','structural_guidance':True})
 
 def _revision_plan_from_feedback(fb: EvaluatorFeedback, target_attempt_id: str) -> RevisionPlan:
     return RevisionPlan(revision_plan_id=f'revision_plan_{fb.task_id}', task_id=fb.task_id, source_attempt_id=fb.attempt_id, target_attempt_id=target_attempt_id, revision_kind='local_fix', revision_reason=fb.feedback_summary, failed_rubric_items=fb.failed_rubric_items, planned_changes=fb.actionable_feedback, expected_score_improvement=0.1, risk_of_regression='low', uses_evaluator_feedback=True, metadata_json={})
 
-def _evaluator_prompt(grader: GraderResult, verifier: VerifierResult, outputs: ActualOutputs, trace: AgentTrace | None, target_attempt_id: str, artifact_preview_bundle: dict | None=None) -> str:
+def _latest_review_packet(package_root: Path | None, attempt_kind: str) -> dict:
+    if not package_root:
+        return {}
+    packets = sorted((package_root / "loops" / "iterations").glob("*/review_packet.json"))
+    if not packets:
+        return {}
+    preferred = []
+    for path in packets:
+        try:
+            packet = read_json(path)
+        except Exception:
+            continue
+        actual_ref = str(packet.get("actual_outputs_ref") or "")
+        if f"attempts/{attempt_kind}/" in actual_ref:
+            preferred.append((path, packet))
+    if preferred:
+        path, packet = preferred[-1]
+        packet = dict(packet)
+        packet["review_packet_ref"] = str(path.relative_to(package_root))
+        return packet
+    try:
+        packet = read_json(packets[-1])
+    except Exception:
+        return {}
+    packet = dict(packet)
+    packet["review_packet_ref"] = str(packets[-1].relative_to(package_root))
+    return packet
+
+
+def _compact_trace_summary(trace: AgentTrace | None) -> dict:
+    if trace is None:
+        return {}
+    data = trace.model_dump()
+    steps = data.get("steps") or []
+    return {
+        "trace_id": data.get("trace_id"),
+        "attempt_id": data.get("attempt_id"),
+        "attempt_kind": data.get("attempt_kind"),
+        "step_count": len(steps),
+        "key_steps": [
+            {
+                "step": step.get("step"),
+                "operation": step.get("operation"),
+                "interaction_surface": step.get("interaction_surface"),
+                "environment_domain": step.get("environment_domain"),
+                "step_outcome": step.get("step_outcome"),
+                "artifact_refs": (step.get("artifact_refs") or [])[:5],
+                "state_change": str(step.get("state_change") or "")[:240],
+            }
+            for step in steps[:10]
+            if isinstance(step, dict) and step.get("action") != "user_message"
+        ],
+    }
+
+
+def _compact_grader(grader: GraderResult) -> dict:
+    data = grader.model_dump()
+    return {
+        "grader_result_id": data.get("grader_result_id"),
+        "attempt_id": data.get("attempt_id"),
+        "passed": data.get("passed"),
+        "score": data.get("score"),
+        "final_score": data.get("final_score"),
+        "score_source": data.get("score_source"),
+        "score_reliability": data.get("score_reliability"),
+        "failed_criteria": data.get("failed_criteria") or [],
+        "evidence_refs": (data.get("evidence_refs") or [])[:20],
+    }
+
+
+def _compact_verifier(verifier: VerifierResult) -> dict:
+    data = verifier.model_dump()
+    return {
+        "verifier_result_id": data.get("verifier_result_id"),
+        "attempt_id": data.get("attempt_id"),
+        "passed": data.get("passed"),
+        "verdict": data.get("verdict"),
+        "failed_checks": data.get("failed_checks") or data.get("failed_criteria") or [],
+        "evidence_refs": (data.get("evidence_refs") or [])[:20],
+        "notes": str(data.get("notes") or data.get("verifier_notes") or "")[:1000],
+    }
+
+
+def _compact_actual(outputs: ActualOutputs) -> dict:
+    data = outputs.model_dump()
+    return {
+        "task_id": data.get("task_id"),
+        "attempt_id": data.get("attempt_id"),
+        "attempt_kind": data.get("attempt_kind"),
+        "status": data.get("status"),
+        "output_summary": data.get("output_summary"),
+        "primary_output_ref": data.get("primary_output_ref"),
+        "deliverable_refs": data.get("deliverable_refs") or [],
+        "artifact_refs": data.get("artifact_refs") or [],
+        "files_created": data.get("files_created") or [],
+        "error_type": data.get("error_type"),
+        "error_message": data.get("error_message"),
+    }
+
+
+def _compact_artifact_previews(bundle: dict | None) -> dict:
+    bundle = bundle or {}
+    return {
+        "artifact_content_refs": (bundle.get("artifact_content_refs") or [])[:12],
+        "artifact_content_previews": (bundle.get("artifact_content_previews") or [])[:8],
+        "artifact_content_hashes": (bundle.get("artifact_content_hashes") or {}) if len(str(bundle.get("artifact_content_hashes") or {})) < 2000 else {},
+        "artifact_content_preview_truncated": bundle.get("artifact_content_preview_truncated"),
+        "model_grading_basis": bundle.get("model_grading_basis"),
+    }
+
+
+def _evaluator_prompt(grader: GraderResult, verifier: VerifierResult, outputs: ActualOutputs, trace: AgentTrace | None, target_attempt_id: str, artifact_preview_bundle: dict | None=None, review_packet: dict | None=None) -> str:
     return """Return only valid JSON. Do not include markdown. Do not add extra top-level fields; place extras under metadata_json.extra_model_fields.
-Required skeleton: {"evaluator_feedback":{"feedback_id":"feedback_<task_id>_<attempt_kind>","task_id":"...","attempt_id":"...","target_actor":"worker","feedback_type":"other","failed_rubric_items":[],"evidence_refs":[],"artifact_refs":[],"feedback_summary":"...","actionable_feedback":[],"suggested_revision":"...","revision_priority":"medium","confidence":0.7,"hidden_reference_used":false,"hidden_reference_leaked":false,"failed_or_weak_rubric_items":[],"artifact_specific_comments":[],"trace_specific_comments":[],"metadata_json":{}},"revision_plan":{"revision_plan_id":"revision_plan_<task_id>","task_id":"...","source_attempt_id":"...","target_attempt_id":"...","revision_kind":"local_fix","revision_reason":"...","failed_rubric_items":[],"planned_changes":[],"risk_of_regression":"medium","uses_evaluator_feedback":true,"metadata_json":{}}}.
-Evaluate the attempt using grader/verifier outputs, actual outputs, artifact content previews, artifacts, and trace. Produce actionable feedback and a concrete revision plan.
-Target revised attempt id: """ + target_attempt_id + "\nGraderResult JSON:\n" + json.dumps(grader.model_dump(), sort_keys=True) + "\nVerifierResult JSON:\n" + json.dumps(verifier.model_dump(), sort_keys=True) + "\nActualOutputs JSON:\n" + json.dumps(outputs.model_dump(), sort_keys=True) + "\nArtifact content previews JSON:\n" + json.dumps(artifact_preview_bundle or {}, sort_keys=True)[:16000] + "\nTrace summary JSON:\n" + json.dumps((trace.model_dump() if trace else {}), sort_keys=True)[:12000]
+Required skeleton: {"evaluator_feedback":{"feedback_id":"feedback_<task_id>_<attempt_kind>","task_id":"...","attempt_id":"...","target_actor":"apprentice","feedback_type":"other","failed_rubric_items":[],"evidence_refs":[],"artifact_refs":[],"feedback_summary":"...","actionable_feedback":[],"suggested_revision":"...","revision_priority":"medium","confidence":0.7,"hidden_reference_used":false,"hidden_reference_leaked":false,"failed_or_weak_rubric_items":[],"artifact_specific_comments":[],"trace_specific_comments":[],"metadata_json":{}},"revision_plan":{"revision_plan_id":"revision_plan_<task_id>","task_id":"...","source_attempt_id":"...","target_attempt_id":"...","revision_kind":"local_fix","revision_reason":"...","failed_rubric_items":[],"planned_changes":[],"risk_of_regression":"medium","uses_evaluator_feedback":true,"metadata_json":{}}}.
+Evaluate the attempt by first reading the loop review packet when provided, then checking compact Mentor grader/verifier outputs, actual_outputs.json, artifact refs, artifact content previews, and trace evidence. Treat the review packet as the current iteration dossier: it summarizes outputs, artifacts, trace refs, rubric/verifier status, known failures, diff evidence, and recommended review focus. Produce grounded actionable feedback and a concrete revision plan whose revision guidance can be mapped to feedback_type/feedback_content in future traces. Keep the response concise. Do not rely on Apprentice Agent self-assessment or self evaluation as outcome evidence. Do not use deterministic fallback judgement; structural guardrails are integrity checks only and are not outcome judgments.
+Target revised attempt id: """ + target_attempt_id + "\nLoop Review Packet JSON:\n" + json.dumps(review_packet or {}, sort_keys=True)[:5000] + "\nCompact GraderResult JSON:\n" + json.dumps(_compact_grader(grader), sort_keys=True) + "\nCompact VerifierResult JSON:\n" + json.dumps(_compact_verifier(verifier), sort_keys=True) + "\nCompact ActualOutputs JSON:\n" + json.dumps(_compact_actual(outputs), sort_keys=True) + "\nArtifact content previews JSON:\n" + json.dumps(_compact_artifact_previews(artifact_preview_bundle), sort_keys=True)[:5000] + "\nCompact Trace summary JSON:\n" + json.dumps(_compact_trace_summary(trace), sort_keys=True)[:3000]
 
 def evaluate_attempt(grader: GraderResult, verifier: VerifierResult, outputs: ActualOutputs, trace: AgentTrace | None, target_attempt_id: str, role_root: Path | None=None, package_root: Path | None=None) -> tuple[EvaluatorFeedback, RevisionPlan]:
     settings=get_settings(); role_root=role_root or Path('outputs/roles')
     artifact_preview_bundle=build_artifact_previews(package_root, grader.evidence_refs or outputs.deliverable_refs or outputs.artifact_refs or outputs.files_created)
-    if _model_evaluator_active(settings) and _mentor_provider_can_attempt() and settings.llm_evaluator_enabled:
-        prompt=_evaluator_prompt(grader, verifier, outputs, trace, target_attempt_id, artifact_preview_bundle)
+    review_packet=_latest_review_packet(package_root, outputs.attempt_kind)
+    if _model_evaluator_active(settings) and settings.llm_evaluator_enabled:
+        prompt=_evaluator_prompt(grader, verifier, outputs, trace, target_attempt_id, artifact_preview_bundle, review_packet)
         try:
             provider=_mentor_provider_id()
             model_override=settings.llm_evaluator_model if provider == 'openai' else None
-            rr=run_structured_role('evaluator_agent', prompt, LLMEvaluatorOutput, role_root/'evaluator_agent'/outputs.attempt_kind, allow_fallback=settings.allow_deterministic_eval_fallback, model_override=model_override, normalizer_context={'task_id': outputs.task_id, 'attempt_id': outputs.attempt_id, 'attempt_kind': outputs.attempt_kind, 'target_attempt_id': target_attempt_id, 'grader_result_id': grader.grader_result_id, 'verifier_result_id': verifier.verifier_result_id, 'artifact_content_refs': artifact_preview_bundle.get('artifact_content_refs'), 'artifact_content_previews': artifact_preview_bundle.get('artifact_content_previews'), 'artifact_content_hashes': artifact_preview_bundle.get('artifact_content_hashes'), 'artifact_content_preview_truncated': artifact_preview_bundle.get('artifact_content_preview_truncated'), 'model_grading_basis': artifact_preview_bundle.get('model_grading_basis'), 'model': model_override or settings.model_provider_model, 'provider':provider})
+            rr=run_structured_role('evaluator_agent', prompt, LLMEvaluatorOutput, role_root/'evaluator_agent'/outputs.attempt_kind, allow_fallback=settings.allow_deterministic_eval_fallback, model_override=model_override, normalizer_context={'task_id': outputs.task_id, 'attempt_id': outputs.attempt_id, 'attempt_kind': outputs.attempt_kind, 'target_attempt_id': target_attempt_id, 'grader_result_id': grader.grader_result_id, 'verifier_result_id': verifier.verifier_result_id, 'artifact_content_refs': artifact_preview_bundle.get('artifact_content_refs'), 'artifact_content_previews': artifact_preview_bundle.get('artifact_content_previews'), 'artifact_content_hashes': artifact_preview_bundle.get('artifact_content_hashes'), 'artifact_content_preview_truncated': artifact_preview_bundle.get('artifact_content_preview_truncated'), 'model_grading_basis': artifact_preview_bundle.get('model_grading_basis'), 'review_packet_ref': review_packet.get('review_packet_ref'), 'model': model_override or settings.model_provider_model, 'provider':provider})
             if rr.live_call_ok and rr.structured_output_validation_ok:
                 parsed=read_json(role_root/'evaluator_agent'/outputs.attempt_kind/'parsed_output.json')
                 fb=EvaluatorFeedback.model_validate(parsed['evaluator_feedback'])
                 rp=RevisionPlan.model_validate(parsed['revision_plan'])
                 fb.provider=rr.provider; fb.model=rr.model; fb.revision_plan=fb.revision_plan or rp.revision_reason
-                fb.metadata_json.update({'llm_prompt_ref_internal':str(role_root/'evaluator_agent'/outputs.attempt_kind/'prompt.md'),'llm_response_ref_internal':str(role_root/'evaluator_agent'/outputs.attempt_kind/'raw_output.txt'),'prompt_hash':sha256_text(prompt),'public_response_summary':'Model evaluator generated actionable feedback and revision plan.', **artifact_preview_bundle})
-                rp.metadata_json.update({'provider':rr.provider,'model':rr.model,'llm_prompt_ref_internal':str(role_root/'evaluator_agent'/outputs.attempt_kind/'prompt.md'),'llm_response_ref_internal':str(role_root/'evaluator_agent'/outputs.attempt_kind/'raw_output.txt')})
+                fb.metadata_json.update({'llm_prompt_ref_internal':str(role_root/'evaluator_agent'/outputs.attempt_kind/'prompt.md'),'llm_response_ref_internal':str(role_root/'evaluator_agent'/outputs.attempt_kind/'raw_output.txt'),'prompt_hash':sha256_text(prompt),'public_response_summary':'Mentor Model evaluator generated grounded feedback and a revision plan from the loop review packet and package evidence.','review_source':'review_packet','review_packet_ref':review_packet.get('review_packet_ref'), **artifact_preview_bundle})
+                rp.metadata_json.update({'provider':rr.provider,'model':rr.model,'review_source':'review_packet','review_packet_ref':review_packet.get('review_packet_ref'),'llm_prompt_ref_internal':str(role_root/'evaluator_agent'/outputs.attempt_kind/'prompt.md'),'llm_response_ref_internal':str(role_root/'evaluator_agent'/outputs.attempt_kind/'raw_output.txt')})
                 return fb,rp
             if settings.evaluation_mode == 'llm_required' or settings.llm_fail_closed:
                 raise RuntimeError(rr.error_message or 'Model evaluator failed')
@@ -59,6 +173,8 @@ def evaluate_attempt(grader: GraderResult, verifier: VerifierResult, outputs: Ac
                 raise
     fb=deterministic_feedback(grader)
     rp=_revision_plan_from_feedback(fb, target_attempt_id)
+    fb.metadata_json.update({'review_source':'review_packet', 'review_packet_ref':review_packet.get('review_packet_ref')})
+    rp.metadata_json.update({'review_source':'review_packet', 'review_packet_ref':review_packet.get('review_packet_ref')})
     if _mentor_provider_can_attempt() and settings.llm_evaluator_enabled and settings.evaluation_mode != 'deterministic_only':
-        fb.metadata_json.update({'model_evaluator_enabled': True, 'llm_evaluator_enabled': True, 'llm_unavailable': True, 'deterministic_fallback': True, **artifact_preview_bundle})
+        fb.metadata_json.update({'model_evaluator_enabled': True, 'llm_evaluator_enabled': True, 'llm_unavailable': True, 'mentor_evaluator_status': 'unavailable', 'review_source':'review_packet', 'review_packet_ref':review_packet.get('review_packet_ref'), **artifact_preview_bundle})
     return fb,rp

package/src/agent_apprenticeship_trace/grader.py

@@ -124,15 +124,15 @@ def deterministic_grade(rubric: RubricSpec, outputs: ActualOutputs, attempt_kind
         item_evidence=[e for e in evidence if Path(e).name in req] or evidence
         item_missing=[m for m in missing if m in req]
         ok = outputs.status == 'success' and not item_missing and bool(item_evidence)
-        item_scores.append(RubricItemScore(rubric_item_id=item.rubric_item_id, criterion_name=item.criterion_name, score=1.0 if ok else 0.0, max_score=1.0, passed=ok, evidence_refs=item_evidence, failure_mode=None if ok else 'missing_artifact', notes='Deterministic artifact-contract precheck only; semantic correctness is not evaluated.', confidence=0.8, artifact_presence_ok=ok, semantic_correctness_score=None, reasoning_summary='Required artifact paths were resolved.' if ok else 'One or more required artifacts were not resolved.', improvement_suggestion=None if ok else 'Create the missing required artifact files.'))
+        item_scores.append(RubricItemScore(rubric_item_id=item.rubric_item_id, criterion_name=item.criterion_name, score=1.0 if ok else 0.0, max_score=1.0, passed=ok, evidence_refs=item_evidence, failure_mode=None if ok else 'missing_artifact', notes='Structural artifact-contract guardrail only; semantic correctness is not judged.', confidence=0.8, artifact_presence_ok=ok, semantic_correctness_score=None, reasoning_summary='Required artifact paths were resolved.' if ok else 'One or more required artifacts were not resolved.', improvement_suggestion=None if ok else 'Create the missing required artifact files.'))
     score=artifact_contract_score
-    return GraderResult(grader_result_id=f'grader_{outputs.attempt_id}', task_id=outputs.task_id, attempt_id=outputs.attempt_id, attempt_kind=attempt_kind, rubric_id=rubric.rubric_id, grader_kind='deterministic', score_source='deterministic_artifact_contract', score=score, max_score=1.0, passed=score>=rubric.pass_threshold, rubric_item_scores=item_scores, failed_criteria=[s.rubric_item_id for s in item_scores if not s.passed], passed_criteria=[s.rubric_item_id for s in item_scores if s.passed], evidence_refs=evidence, confidence=0.8, reasoning_summary='Deterministic precheck verifies artifact presence/readiness only; semantic correctness is not evaluated.', limitations=['semantic_correctness_not_evaluated'], hidden_reference_used=False, hidden_reference_leaked=False, artifact_contract_score=artifact_contract_score, semantic_score=None, final_score=score, model=None, provider=None, deterministic_precheck_ref=None, public_response_summary='Artifact-contract precheck completed.', metadata_json={'artifact_contract_passed': artifact_contract_passed, 'semantic_correctness_not_evaluated': True, 'missing_artifacts': missing, **match})
+    return GraderResult(grader_result_id=f'grader_{outputs.attempt_id}', task_id=outputs.task_id, attempt_id=outputs.attempt_id, attempt_kind=attempt_kind, rubric_id=rubric.rubric_id, grader_kind='structural_guardrail', score_source='structural_guardrail', score=score, max_score=1.0, passed=artifact_contract_passed, rubric_item_scores=item_scores, failed_criteria=[s.rubric_item_id for s in item_scores if not s.passed], passed_criteria=[s.rubric_item_id for s in item_scores if s.passed], evidence_refs=evidence, confidence=0.8, reasoning_summary='Structural guardrail checked artifact presence/readiness only; Mentor grading is unavailable unless a Mentor Model or human grader runs.', limitations=['semantic_quality_not_judged','mentor_grade_unavailable'], hidden_reference_used=False, hidden_reference_leaked=False, artifact_contract_score=artifact_contract_score, semantic_score=None, final_score=score, model=None, provider=None, deterministic_precheck_ref=None, public_response_summary='Structural artifact-contract guardrail completed.', score_reliability='needs_review', metadata_json={'structural_guardrail': True, 'role_status': 'mentor_grader_unavailable', 'artifact_contract_passed': artifact_contract_passed, 'semantic_quality_not_judged': True, 'missing_artifacts': missing, **match})
 
 def _grader_prompt(rubric: RubricSpec, outputs: ActualOutputs, trace: AgentTrace | None, precheck: GraderResult, artifact_preview_bundle: dict | None=None) -> str:
     return """Return only valid JSON matching GraderResult. Do not include markdown.
-You are the model-based grader. Score each rubric item using the task rubric, attempt trace, actual outputs, and artifact content previews when available. Return a JSON object with exactly these required top-level fields: grader_result_id, task_id, attempt_id, attempt_kind, rubric_id, grader_kind, score_source, artifact_contract_score, model_score, semantic_score, final_score, score, max_score, passed, confidence, evidence_refs, rubric_item_scores, limitations, hidden_reference_used, hidden_reference_leaked, metadata_json. Each rubric_item_scores entry must include artifact_presence_ok, semantic_correctness_score, evidence_refs, reasoning_summary, confidence, failure_mode, and improvement_suggestion. Cite evidence_refs, distinguish artifact presence from model-judged correctness, and set score_source=model_judged and grader_kind=model. Deterministic artifact-contract precheck is auxiliary only.
+You are the Mentor Model grader. Score each rubric item using the Apprentice-generated plus Mentor-audited task rubric, attempt trace, actual_outputs.json, artifact refs, and artifact content previews when available. Return a JSON object with exactly these required top-level fields: grader_result_id, task_id, attempt_id, attempt_kind, rubric_id, grader_kind, score_source, artifact_contract_score, model_score, semantic_score, final_score, score, max_score, passed, confidence, evidence_refs, rubric_item_scores, limitations, hidden_reference_used, hidden_reference_leaked, metadata_json. Each rubric_item_scores entry must include artifact_presence_ok, semantic_correctness_score, evidence_refs, reasoning_summary, confidence, failure_mode, and improvement_suggestion. Cite evidence_refs from actual_outputs/artifacts/trace, distinguish artifact presence from Mentor-judged correctness, and set score_source=model_judged and grader_kind=model. Do not rely on Apprentice Agent self-assessment or self evaluation. Do not use deterministic fallback judgement; structural artifact-contract guardrails are auxiliary only and are not grader judgments.
 Rubric JSON:
-""" + json.dumps(rubric.model_dump(), sort_keys=True) + "\nActualOutputs JSON:\n" + json.dumps(outputs.model_dump(), sort_keys=True) + "\nArtifact content previews JSON:\n" + json.dumps(artifact_preview_bundle or {}, sort_keys=True)[:16000] + "\nTrace summary JSON:\n" + json.dumps((trace.model_dump() if trace else {}), sort_keys=True)[:12000] + "\nDeterministic precheck JSON:\n" + json.dumps(precheck.model_dump(), sort_keys=True)
+""" + json.dumps(rubric.model_dump(), sort_keys=True) + "\nActualOutputs JSON:\n" + json.dumps(outputs.model_dump(), sort_keys=True) + "\nArtifact content previews JSON:\n" + json.dumps(artifact_preview_bundle or {}, sort_keys=True)[:16000] + "\nTrace summary JSON:\n" + json.dumps((trace.model_dump() if trace else {}), sort_keys=True)[:12000] + "\nStructural guardrail JSON:\n" + json.dumps(precheck.model_dump(), sort_keys=True)
 
 def grade_attempt(rubric: RubricSpec, outputs: ActualOutputs, attempt_kind: str, trace: AgentTrace | None=None, role_root: Path | None=None, package_root: Path | None=None) -> GraderResult:
     settings=get_settings()
@@ -140,10 +140,11 @@ def grade_attempt(rubric: RubricSpec, outputs: ActualOutputs, attempt_kind: str,
     role_root=role_root or Path('outputs/roles')
     artifact_preview_bundle=build_artifact_previews(package_root, pre.evidence_refs or outputs.deliverable_refs or outputs.artifact_refs or outputs.files_created)
     pre.metadata_json.update(artifact_preview_bundle)
-    if _model_grading_active(settings) and _mentor_provider_can_attempt() and settings.llm_grader_enabled:
+    if _model_grading_active(settings) and settings.llm_grader_enabled:
         prompt=_grader_prompt(rubric, outputs, trace, pre, artifact_preview_bundle)
         (role_root/'grader_agent'/attempt_kind).mkdir(parents=True, exist_ok=True)
-        write_json(role_root/'grader_agent'/attempt_kind/'deterministic_precheck.json', pre)
+        structural_ref = role_root/'grader_agent'/attempt_kind/'structural_guardrail.json'
+        write_json(structural_ref, pre)
         try:
             provider=_mentor_provider_id()
             model_override=settings.llm_grader_model if provider == 'openai' else None
@@ -151,8 +152,8 @@ def grade_attempt(rubric: RubricSpec, outputs: ActualOutputs, attempt_kind: str,
             if rr.live_call_ok and rr.structured_output_validation_ok:
                 parsed=read_json(role_root/'grader_agent'/attempt_kind/'parsed_output.json')
                 g=GraderResult.model_validate(parsed)
-                g.grader_kind='model'; g.score_source='model_judged'; g.legacy_score_source='llm_semantic'; g.artifact_contract_score=pre.artifact_contract_score; g.semantic_score=g.semantic_score if g.semantic_score is not None else g.score; g.model_score=g.model_score if g.model_score is not None else g.semantic_score; g.legacy_semantic_score=g.legacy_semantic_score if g.legacy_semantic_score is not None else g.semantic_score; g.final_score=g.final_score if g.final_score is not None else g.semantic_score; g.score=g.final_score; g.model=rr.model; g.provider=rr.provider; g.deterministic_precheck_ref=str(role_root/'grader_agent'/attempt_kind/'deterministic_precheck.json'); g.llm_prompt_ref_internal=str(role_root/'grader_agent'/attempt_kind/'prompt.md'); g.llm_response_ref_internal=str(role_root/'grader_agent'/attempt_kind/'raw_output.txt'); g.public_prompt_hash=sha256_text(prompt); g.public_response_summary=g.public_response_summary or 'Model grader scored rubric items with evidence refs and artifact previews when available.'
-                g.metadata_json.update({'deterministic_precheck': pre.model_dump(), **artifact_preview_bundle, 'llm_role_result_ref_internal':str(role_root/'grader_agent'/attempt_kind/'role_result.json')})
+                g.grader_kind='model'; g.score_source='model_judged'; g.legacy_score_source='llm_semantic'; g.artifact_contract_score=pre.artifact_contract_score; g.semantic_score=g.semantic_score if g.semantic_score is not None else g.score; g.model_score=g.model_score if g.model_score is not None else g.semantic_score; g.legacy_semantic_score=g.legacy_semantic_score if g.legacy_semantic_score is not None else g.semantic_score; g.final_score=g.final_score if g.final_score is not None else g.semantic_score; g.score=g.final_score; g.model=rr.model; g.provider=rr.provider; g.deterministic_precheck_ref=None; g.llm_prompt_ref_internal=str(role_root/'grader_agent'/attempt_kind/'prompt.md'); g.llm_response_ref_internal=str(role_root/'grader_agent'/attempt_kind/'raw_output.txt'); g.public_prompt_hash=sha256_text(prompt); g.public_response_summary=g.public_response_summary or 'Mentor Model grader scored rubric items with evidence refs and artifact previews when available.'
+                g.metadata_json.update({'structural_guardrail_ref': str(structural_ref), 'structural_guardrail': pre.model_dump(), **artifact_preview_bundle, 'llm_role_result_ref_internal':str(role_root/'grader_agent'/attempt_kind/'role_result.json')})
                 return g
             if settings.evaluation_mode == 'llm_required' or settings.llm_fail_closed:
                 raise RuntimeError(rr.error_message or 'Model grader failed')
@@ -162,11 +163,11 @@ def grade_attempt(rubric: RubricSpec, outputs: ActualOutputs, attempt_kind: str,
     if settings.evaluation_mode == 'deterministic_only':
         pre.semantic_score=None; pre.final_score=pre.score
     elif _mentor_provider_can_attempt() and settings.llm_grader_enabled:
-        pre.grader_kind='hybrid'; pre.score_source='deterministic_fallback'; pre.semantic_score=None; pre.final_score=pre.score
+        pre.grader_kind='structural_guardrail'; pre.score_source='structural_guardrail'; pre.semantic_score=None; pre.final_score=pre.score
         provider=_mentor_provider_id()
         pre.model=settings.model_provider_model or settings.llm_grader_model; pre.provider=provider
-        pre.limitations.append('Model grader unavailable; deterministic fallback explicitly marked.')
-        pre.metadata_json.update({'llm_grader_enabled': True, 'llm_unavailable': True, 'deterministic_fallback': True})
+        pre.limitations.append('Mentor Model grader unavailable; only structural guardrails ran.')
+        pre.metadata_json.update({'llm_grader_enabled': True, 'llm_unavailable': True, 'role_status': 'mentor_grader_unavailable'})
     return pre
 
 

package/src/agent_apprenticeship_trace/learning.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import hashlib
 import json
 import os
+import re
 import shutil
 from datetime import datetime, timezone
 from pathlib import Path
@@ -10,7 +11,7 @@ from typing import Any
 
 from .config import Settings, get_settings
 from .env import contains_secret
-from .io import read_json, read_jsonl, write_json
+from .io import append_jsonl, read_json, read_jsonl, write_json
 
 
 PACK_STATUSES = {"draft", "active", "reverted", "removed"}
@@ -69,6 +70,18 @@ def experience_root(settings: Settings | None = None) -> Path:
     return root
 
 
+def installed_skills_root(settings: Settings | None = None) -> Path:
+    s = settings or get_settings()
+    root = s.app_home / "installed_skills"
+    root.mkdir(parents=True, exist_ok=True)
+    return root
+
+
+def installed_skills_index_path(settings: Settings | None = None) -> Path:
+    s = settings or get_settings()
+    return s.app_home / "installed_skills_index.jsonl"
+
+
 def _load_json_or_jsonl(path: Path) -> list[dict[str, Any]]:
     if not path.exists():
         return []
@@ -620,6 +633,262 @@ def compare_replay(before: dict[str, Any], after: dict[str, Any]) -> dict[str, A
     }
 
 
+def _find_bundle_root(path: Path) -> Path:
+    path = path.expanduser()
+    if path.is_file():
+        path = path.parent
+    candidates = [
+        path,
+        path / "contribution_bundle",
+        path / "package",
+    ]
+    for candidate in candidates:
+        if (candidate / "contribution_manifest.json").exists() or (candidate / "experience_compiler").exists():
+            return candidate
+    for candidate in path.rglob("contribution_manifest.json"):
+        return candidate.parent
+    raise FileNotFoundError(f"Experience Compilation not found: {path}")
+
+
+def _safe_copy_file(src: Path, dst: Path, files: list[str], root: Path) -> None:
+    if not src.exists() or not src.is_file():
+        return
+    dst.parent.mkdir(parents=True, exist_ok=True)
+    shutil.copy2(src, dst)
+    files.append(dst.relative_to(root).as_posix())
+
+
+def _safe_copytree(src: Path, dst: Path, files: list[str], root: Path) -> None:
+    if not src.exists() or not src.is_dir():
+        return
+    shutil.copytree(src, dst, dirs_exist_ok=True)
+    for file_path in sorted(dst.rglob("*")):
+        if file_path.is_file():
+            files.append(file_path.relative_to(root).as_posix())
+
+
+def _read_manifest_field(bundle: Path, key: str, default: Any = None) -> Any:
+    for rel in ["contribution_manifest.json", "experience_compiler/compiler_manifest.json"]:
+        path = bundle / rel
+        if path.exists():
+            try:
+                data = read_json(path)
+            except Exception:
+                continue
+            if isinstance(data, dict) and data.get(key) is not None:
+                return data.get(key)
+    return default
+
+
+def install_runtime_training_from_bundle(
+    source_item_id: str,
+    bundle_path: Path,
+    *,
+    settings: Settings | None = None,
+    agent_target: str = "current",
+    dry_run: bool = False,
+) -> dict[str, Any]:
+    """Install runtime-useful Experience Compiler outputs into the local registry."""
+    s = settings or get_settings()
+    bundle = _find_bundle_root(bundle_path)
+    compiler = bundle / "experience_compiler"
+    skill_dir = compiler / "skill_pack"
+    runtime_dir = compiler / "runtime_learning_package"
+    skill_md = skill_dir / "skill.md"
+    if not skill_md.exists():
+        raise FileNotFoundError(f"Runtime training skill.md not found in {bundle}")
+    package_id = str(_read_manifest_field(bundle, "bundle_id") or _read_manifest_field(bundle, "source_package_id") or bundle.name)
+    title = str(_read_manifest_field(bundle, "title") or _read_manifest_field(bundle, "source_task_id") or source_item_id)
+    digest = hashlib.sha256(f"{source_item_id}|{package_id}|{skill_md.read_text(errors='replace')[:5000]}".encode()).hexdigest()[:10]
+    installed_skill_id = f"aa-skill-{slugify(title, 'runtime-training')}-{digest}"
+    target = installed_skills_root(s) / installed_skill_id
+    files: list[str] = []
+    if dry_run:
+        return {
+            "installed_skill_id": installed_skill_id,
+            "source_item_id": source_item_id,
+            "source_package_id": package_id,
+            "dry_run": True,
+            "would_install": [
+                "skill.md",
+                "canonical_skill.json",
+                "retrieval_card.json",
+                "runtime_learning_package/",
+                "install_manifest.json",
+            ],
+            "target": str(target),
+        }
+    if target.exists():
+        shutil.rmtree(target)
+    target.mkdir(parents=True, exist_ok=True)
+    _safe_copy_file(skill_md, target / "skill.md", files, target)
+    _safe_copytree(runtime_dir, target / "runtime_learning_package", files, target)
+    for src_name, dst_name in [
+        ("canonical_skill.json", "canonical_skill.json"),
+        ("retrieval_card.json", "retrieval_card.json"),
+        ("skill_card.json", "skill_card.json"),
+        ("verifier_spec.json", "verifier_spec.json"),
+        ("verifier_checklist.json", "verifier_checklist.json"),
+        ("tool_recipe.json", "tool_recipe.json"),
+    ]:
+        _safe_copy_file(skill_dir / src_name, target / dst_name, files, target)
+    if not (target / "canonical_skill.json").exists():
+        write_json(
+            target / "canonical_skill.json",
+            {
+                "skill_id": installed_skill_id,
+                "title": title,
+                "source_item_id": source_item_id,
+                "source_package_id": package_id,
+                "skill_ref": "skill.md",
+            },
+        )
+        files.append("canonical_skill.json")
+    if not (target / "retrieval_card.json").exists():
+        write_json(
+            target / "retrieval_card.json",
+            {
+                "skill_id": installed_skill_id,
+                "title": title,
+                "domains": _read_manifest_field(bundle, "domains", []) or [],
+                "source_item_id": source_item_id,
+                "summary": f"Runtime training compiled from {title}.",
+            },
+        )
+        files.append("retrieval_card.json")
+    manifest = {
+        "installed_skill_id": installed_skill_id,
+        "source_item_id": source_item_id,
+        "source_package_id": package_id,
+        "source_url": _read_manifest_field(bundle, "source_url"),
+        "agent_target": agent_target,
+        "install_mode": "agent_apprenticeship_runtime",
+        "installed_at": utc_now(),
+        "enabled": True,
+        "title": title,
+        "files_installed": sorted(set(files + ["install_manifest.json"])),
+        "backup_refs": [],
+        "reversible": True,
+        "generation_notes": ["native_agent_files_unchanged"],
+    }
+    write_json(target / "install_manifest.json", manifest)
+    append_jsonl(installed_skills_index_path(s), manifest)
+    return manifest
+
+
+def list_installed_skills(settings: Settings | None = None, include_disabled: bool = True) -> list[dict[str, Any]]:
+    root = installed_skills_root(settings)
+    rows: list[dict[str, Any]] = []
+    for path in sorted(root.iterdir()) if root.exists() else []:
+        manifest = path / "install_manifest.json"
+        if not manifest.exists():
+            continue
+        try:
+            data = read_json(manifest)
+        except Exception:
+            continue
+        if not include_disabled and not data.get("enabled", True):
+            continue
+        data = dict(data)
+        data["path"] = str(path)
+        rows.append(data)
+    return rows
+
+
+def _installed_skill_path(installed_skill_id: str, settings: Settings | None = None) -> Path:
+    path = installed_skills_root(settings) / installed_skill_id
+    if not (path / "install_manifest.json").exists():
+        raise FileNotFoundError(f"Installed runtime training not found: {installed_skill_id}")
+    return path
+
+
+def set_installed_skill_enabled(installed_skill_id: str, enabled: bool, settings: Settings | None = None) -> dict[str, Any]:
+    path = _installed_skill_path(installed_skill_id, settings)
+    manifest = read_json(path / "install_manifest.json")
+    manifest["enabled"] = bool(enabled)
+    manifest["updated_at"] = utc_now()
+    write_json(path / "install_manifest.json", manifest)
+    append_jsonl(installed_skills_index_path(settings), manifest)
+    return manifest
+
+
+def uninstall_installed_skill(installed_skill_id: str, settings: Settings | None = None) -> Path:
+    root = installed_skills_root(settings)
+    path = _installed_skill_path(installed_skill_id, settings)
+    manifest = read_json(path / "install_manifest.json")
+    manifest["enabled"] = False
+    manifest["uninstalled_at"] = utc_now()
+    removed = root / "_removed" / installed_skill_id
+    removed.parent.mkdir(parents=True, exist_ok=True)
+    if removed.exists():
+        shutil.rmtree(removed)
+    write_json(path / "install_manifest.json", manifest)
+    shutil.move(str(path), str(removed))
+    append_jsonl(installed_skills_index_path(settings), manifest)
+    return removed
+
+
+def _skill_keywords(skill: dict[str, Any]) -> set[str]:
+    text = " ".join(
+        str(skill.get(key) or "")
+        for key in ["installed_skill_id", "title", "source_item_id", "source_package_id"]
+    )
+    path = Path(str(skill.get("path") or ""))
+    for rel in ["retrieval_card.json", "canonical_skill.json", "skill.md"]:
+        file_path = path / rel
+        if file_path.exists():
+            text += " " + file_path.read_text(errors="ignore")[:3000]
+    return {token.lower() for token in re.findall(r"[A-Za-z][A-Za-z0-9_-]{2,}", text)}
+
+
+def select_runtime_training_for_run(
+    instruction: str,
+    *,
+    settings: Settings | None = None,
+    force: bool = False,
+    limit: int = 2,
+) -> tuple[list[dict[str, Any]], str]:
+    enabled = list_installed_skills(settings, include_disabled=False)
+    if not enabled:
+        return [], ""
+    query_terms = {token.lower() for token in re.findall(r"[A-Za-z][A-Za-z0-9_-]{2,}", instruction)}
+    scored: list[tuple[int, dict[str, Any]]] = []
+    for skill in enabled:
+        overlap = len(query_terms & _skill_keywords(skill))
+        if force or overlap:
+            scored.append((overlap, skill))
+    scored.sort(key=lambda item: item[0], reverse=True)
+    selected = [skill for _, skill in scored[:limit]]
+    if not selected:
+        return [], ""
+    sections = ["\n\n---\nInstalled Runtime Training\nUse these enabled skills only when they are relevant. The task instruction remains authoritative.\n"]
+    for skill in selected:
+        path = Path(str(skill.get("path")))
+        skill_text = (path / "skill.md").read_text(errors="replace") if (path / "skill.md").exists() else ""
+        sections.append(
+            f"\n## {skill.get('title') or skill.get('installed_skill_id')}\n"
+            f"installed_skill_id: {skill.get('installed_skill_id')}\n"
+            f"selection_reason: matched runtime training against the task request.\n\n"
+            f"{skill_text[:5000]}\n"
+        )
+    sections.append("---\n")
+    return selected, "\n".join(sections)
+
+
+def installed_skill_run_refs(skills: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    refs: list[dict[str, Any]] = []
+    for skill in skills:
+        refs.append(
+            {
+                "installed_skill_id": skill.get("installed_skill_id"),
+                "title": skill.get("title"),
+                "source_item_id": skill.get("source_item_id"),
+                "source_package_id": skill.get("source_package_id"),
+            }
+        )
+    return refs
+
+
 def write_before_after_result(pack_path: Path, result: dict[str, Any]) -> Path:
     result = dict(result)
     result["created_at"] = utc_now()

package/src/agent_apprenticeship_trace/llm_output_normalizer.py

@@ -34,6 +34,7 @@ class RoleContext:
     task_instruction: str | None = None
     target_attempt_id: str | None = None
     verifier_result_id: str | None = None
+    review_packet_ref: str | None = None
 
 
 def _as_list(v: Any) -> list[Any]:
@@ -188,7 +189,7 @@ def _rubric_item(raw_item: Any, idx: int, weight: float, required_artifacts: lis
     evidence=raw_item.get('observable_evidence') or raw_item.get('evidence') or raw_item.get('evidence_requirements') or raw_item.get('success_criteria') or required_artifacts or ['final answer and artifacts']
     artifacts=raw_item.get('required_artifacts') or raw_item.get('artifacts') or required_artifacts or ['final_answer']
     scoring_method=raw_item.get('scoring_method')
-    if not isinstance(scoring_method, str) or scoring_method not in {'llm_rubric_judge','deterministic','schema_match','regex','unit_test','hybrid','human_future'}:
+    if not isinstance(scoring_method, str) or scoring_method not in {'llm_rubric_judge','deterministic','schema_match','regex','unit_test','hybrid','human_future','structural_guardrail'}:
         scoring_method='llm_rubric_judge'
     return {
         'rubric_item_id': str(raw_item.get('rubric_item_id') or raw_item.get('id') or f'ri_{idx}'),
@@ -253,7 +254,7 @@ def normalize_rubric_result(raw: dict[str, Any], context: RoleContext | None=Non
         'required_artifacts': required,
         'disqualifying_errors': [str(x) for x in _as_list(rub_raw.get('disqualifying_errors'))],
         'partial_credit_allowed': _bool(rub_raw.get('partial_credit_allowed'), True),
-        'grader_kind': rub_raw.get('grader_kind') if rub_raw.get('grader_kind') in {'llm_rubric_judge','deterministic','hybrid','human_future'} else 'llm_rubric_judge',
+        'grader_kind': rub_raw.get('grader_kind') if rub_raw.get('grader_kind') in {'llm_rubric_judge','deterministic','hybrid','human_future','structural_guardrail','unavailable'} else 'llm_rubric_judge',
         'rubric_generation_source': rub_raw.get('rubric_generation_source') if rub_raw.get('rubric_generation_source') in {'agent_assisted','family_template','task_specific_agent_draft','expert_override','deterministic_seed'} else 'task_specific_agent_draft',
         'rubric_generation_agent_provider': rub_raw.get('rubric_generation_agent_provider') or context.provider,
         'rubric_generation_agent_model': rub_raw.get('rubric_generation_agent_model') or context.model,
@@ -282,12 +283,12 @@ def normalize_evaluator_result(raw: dict[str, Any], context: RoleContext | None=
     actionable=[str(x) for x in _as_list(actionable)]
     weak=[str(x) for x in _as_list(fb_raw.get('failed_or_weak_rubric_items') or fb_raw.get('failed_rubric_items'))]
     md=_metadata(fb_raw, EVALUATOR_ALLOWED)
-    md.update({'attempt_kind': fb_raw.get('attempt_kind') or context.attempt_kind, 'grader_result_id': context.grader_result_id, 'verifier_result_id': context.verifier_result_id})
+    md.update({'attempt_kind': fb_raw.get('attempt_kind') or context.attempt_kind, 'grader_result_id': context.grader_result_id, 'verifier_result_id': context.verifier_result_id, 'review_source': 'review_packet' if context.review_packet_ref else md.get('review_source'), 'review_packet_ref': context.review_packet_ref})
     feedback={
         'feedback_id': str(fb_raw.get('feedback_id') or f'feedback_{task_id}_{context.attempt_kind}'),
         'task_id': task_id,
         'attempt_id': attempt_id,
-        'target_actor': fb_raw.get('target_actor') if fb_raw.get('target_actor') in {'worker','reviser','apprentice'} else 'worker',
+        'target_actor': fb_raw.get('target_actor') if fb_raw.get('target_actor') in {'worker','reviser','apprentice'} else 'apprentice',
         'feedback_type': fb_raw.get('feedback_type') if fb_raw.get('feedback_type') in {'criteria_failure','artifact_missing','format_error','logic_error','tool_error','quality_gap','strategy_gap','safety_or_privacy','other'} else 'other',
         'failed_rubric_items': weak,
         'evidence_refs': [str(x) for x in _as_list(fb_raw.get('evidence_refs'))],
@@ -322,7 +323,7 @@ def normalize_evaluator_result(raw: dict[str, Any], context: RoleContext | None=
         'uses_evaluator_feedback': _bool(rp_raw.get('uses_evaluator_feedback'), True),
         'metadata_json': rp_md,
     }
-    revision['metadata_json'].update({'source_attempt_kind': context.attempt_kind, 'revision_goal': summary, 'priority_items': weak, 'expected_improvements': []})
+    revision['metadata_json'].update({'source_attempt_kind': context.attempt_kind, 'revision_goal': summary, 'priority_items': weak, 'expected_improvements': [], 'review_source': 'review_packet' if context.review_packet_ref else revision['metadata_json'].get('review_source'), 'review_packet_ref': context.review_packet_ref})
     return {'evaluator_feedback': feedback, 'revision_plan': revision}
 
 def normalize_grader_result(raw: dict[str, Any], context: RoleContext | None=None) -> dict[str, Any]:
@@ -339,7 +340,7 @@ def normalize_grader_result(raw: dict[str, Any], context: RoleContext | None=Non
         semantic_score=0.0
         final_score=0.0
         limitations.append('Semantic grading impossible because artifact content was unavailable; score reflects missing outputs/log evidence.')
-    md['deterministic_precheck']={'artifact_contract_score': artifact_contract_score}
+    md['structural_guardrail']={'artifact_contract_score': artifact_contract_score}
     basis=context.model_grading_basis or ('artifact_content' if artifact_contract_score else ('logs_only' if refs else 'missing_outputs'))
     md['evidence_materialization_status']='materialized' if artifact_contract_score else 'missing_or_unverified'
     md['referenced_but_missing_artifacts']=[str(x) for x in _as_list(raw.get('referenced_but_missing_artifacts'))]
@@ -386,7 +387,7 @@ def normalize_grader_result(raw: dict[str, Any], context: RoleContext | None=Non
         'final_score': final_score,
         'model': raw.get('model') or context.model,
         'provider': raw.get('provider') or context.provider,
-        'deterministic_precheck_ref': raw.get('deterministic_precheck_ref'),
+        'deterministic_precheck_ref': None,
         'llm_prompt_ref_internal': raw.get('llm_prompt_ref_internal'),
         'llm_response_ref_internal': raw.get('llm_response_ref_internal'),
         'public_prompt_hash': raw.get('public_prompt_hash'),