agent-apprenticeship 0.1.0

package/src/agent_apprenticeship_trace/rubric_generation.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+import json
+from pathlib import Path
+from pydantic import BaseModel
+from .schemas import TaskIntakeSpec, RubricItem, RubricSpec, RubricQualityReport
+from .config import get_settings
+from .io import read_json
+from .openai_structured import get_model_provider_status, run_structured_role
+from .public_sanitizer import sanitize_public_obj, sha256_text
+
+class LLMRubricOutput(BaseModel):
+    rubric_spec: RubricSpec
+    rubric_quality_report: RubricQualityReport
+
+def _mentor_provider_can_attempt() -> bool:
+    return bool(get_model_provider_status().get('provider_available'))
+
+def _mentor_provider_id() -> str:
+    settings=get_settings()
+    return settings.model_provider or 'openai'
+
+def deterministic_rubric(spec: TaskIntakeSpec) -> tuple[RubricSpec, RubricQualityReport]:
+    reqs = spec.output_requirements or ['final deliverable']
+    weight = 1.0/len(reqs)
+    items=[RubricItem(rubric_item_id=f'ri_{i+1}', criterion_name=r, criterion_description=f'Output satisfies {r}', weight=weight, score_min=0, score_max=1, pass_threshold=0.7, observable_evidence=[r], required_artifacts=[r], scoring_method='deterministic', worker_visible=True, verifier_only=False, hidden_reference_required=False, failure_modes=['missing','incorrect'], partial_credit_rules=['partial if substantially present'], edge_cases=[], anti_cheat_notes=[], metadata_json={}) for i,r in enumerate(reqs)]
+    rub=RubricSpec(rubric_id=f'rubric_{spec.task_id}', task_id=spec.task_id, task_family_id=None, rubric_version='v0.1', rubric_items=items, total_weight=1.0, pass_threshold=0.7, worker_visible_rubric_ref='rubric/worker_visible_rubric.md', verifier_private_rubric_ref='rubric/verifier_private_rubric.json', hidden_reference_policy='No hidden references for deterministic seed.', scoring_aggregation='weighted_sum', required_artifacts=reqs, disqualifying_errors=['secret leak'], partial_credit_allowed=True, grader_kind='deterministic', rubric_generation_source='deterministic_seed', rubric_generation_agent_provider=None, rubric_generation_agent_model=None, rubric_generation_confidence=0.4, metadata_json={'rubric_source':'deterministic_fallback','rubric_limitations':['LLM rubric generation unavailable'],'rubric_confidence':'low'})
+    qr=RubricQualityReport(rubric_id=rub.rubric_id, task_id=spec.task_id, criteria_count=len(items), total_weight=1.0, weights_sum_valid=True, has_observable_evidence=True, has_required_artifacts=True, has_partial_credit_rules=True, has_disqualifying_errors=True, has_hidden_reference_policy=True, has_worker_visible_view=True, has_verifier_private_view=True, ambiguous_criteria_count=0, unverifiable_criteria_count=0, rubric_quality_score=0.85, quality_flags=[], blockers=[], metadata_json={})
+    settings=get_settings()
+    if settings.rubric_mode in {'hybrid','llm_default'} and _mentor_provider_can_attempt() and settings.llm_rubric_generation_enabled:
+        provider=_mentor_provider_id()
+        rub.grader_kind='hybrid'; rub.rubric_generation_agent_provider=provider; rub.rubric_generation_agent_model=settings.model_provider_model or settings.openai_model; rub.metadata_json.update({'llm_rubric_generation_enabled': True, 'llm_unavailable': True, 'rubric_source':'deterministic_fallback', 'provider': provider})
+        qr.quality_flags.append('deterministic_fallback_used')
+    return rub, qr
+
+def deterministic_rubric_quality_check(rubric: RubricSpec) -> RubricQualityReport:
+    vague=sum(1 for i in rubric.rubric_items if 'good quality' in (i.criterion_description or '').lower() and not i.observable_evidence)
+    return RubricQualityReport(rubric_id=rubric.rubric_id, task_id=rubric.task_id, criteria_count=len(rubric.rubric_items), total_weight=rubric.total_weight, weights_sum_valid=abs(sum(i.weight for i in rubric.rubric_items)-1.0)<1e-6, has_observable_evidence=all(bool(i.observable_evidence) for i in rubric.rubric_items), has_required_artifacts=all(bool(i.required_artifacts) for i in rubric.rubric_items), has_partial_credit_rules=all(bool(i.partial_credit_rules) for i in rubric.rubric_items), has_disqualifying_errors=bool(rubric.disqualifying_errors), has_hidden_reference_policy=bool(rubric.hidden_reference_policy), has_worker_visible_view=any(i.worker_visible for i in rubric.rubric_items), has_verifier_private_view=bool(rubric.verifier_private_rubric_ref), ambiguous_criteria_count=vague, unverifiable_criteria_count=sum(1 for i in rubric.rubric_items if not i.observable_evidence), rubric_quality_score=0.0 if vague else 0.85, quality_flags=['vague_rubric_item'] if vague else [], blockers=['vague criteria without evidence'] if vague else [], metadata_json={})
+
+def _rubric_prompt(spec: TaskIntakeSpec) -> str:
+    public_spec=sanitize_public_obj(spec.model_dump(mode='json'))
+    return """Return only valid JSON. Do not include markdown. Do not add extra top-level fields; place extras under metadata_json.extra_model_fields.
+Required skeleton: {"rubric_spec":{"rubric_id":"rubric_<task_id>","task_id":"...","rubric_version":"0.1","rubric_items":[{"rubric_item_id":"ri_1","criterion_name":"...","criterion_description":"...","weight":1.0,"score_min":0,"score_max":1,"pass_threshold":0.7,"observable_evidence":[],"required_artifacts":[],"scoring_method":"llm_rubric_judge","worker_visible":true,"verifier_only":false,"hidden_reference_required":false,"failure_modes":[],"partial_credit_rules":[],"edge_cases":[],"anti_cheat_notes":[],"metadata_json":{}}],"total_weight":1.0,"pass_threshold":0.7,"worker_visible_rubric_ref":"rubric/worker_visible_rubric.md","verifier_private_rubric_ref":"rubric/verifier_private_rubric.json","hidden_reference_policy":"no_hidden_reference_available","scoring_aggregation":"weighted_sum","required_artifacts":[],"disqualifying_errors":[],"partial_credit_allowed":true,"grader_kind":"llm_rubric_judge","rubric_generation_source":"task_specific_agent_draft","metadata_json":{}},"rubric_quality_report":{"rubric_id":"...","task_id":"...","criteria_count":1,"total_weight":1.0,"weights_sum_valid":true,"has_observable_evidence":true,"has_required_artifacts":true,"has_partial_credit_rules":true,"has_disqualifying_errors":false,"has_hidden_reference_policy":true,"has_worker_visible_view":true,"has_verifier_private_view":true,"ambiguous_criteria_count":0,"unverifiable_criteria_count":0,"rubric_quality_score":0.75,"quality_flags":[],"blockers":[],"metadata_json":{}}}.
+Create a task-specific rubric with weighted rubric_items that sum to 1.0. Include explicit required_artifacts, observable_evidence, success criteria, failure modes, grader/verifier/evaluator guidance in metadata, limitations, and public-safe metadata. Do not leak hidden/reference answers to worker-visible fields.
+TaskIntakeSpec JSON:
+""" + json.dumps(public_spec, sort_keys=True)
+
+def generate_rubric(spec: TaskIntakeSpec, role_root: Path | None=None) -> tuple[RubricSpec, RubricQualityReport]:
+    settings=get_settings(); role_root=role_root or Path('outputs/roles')
+    if settings.llm_rubric_generation_enabled and settings.rubric_mode in {'hybrid','llm_default','llm_required'} and _mentor_provider_can_attempt():
+        prompt=_rubric_prompt(spec)
+        try:
+            provider=_mentor_provider_id()
+            model_override=settings.llm_rubric_model if provider == 'openai' else None
+            rr=run_structured_role('rubric_agent', prompt, LLMRubricOutput, role_root/'rubric_agent', allow_fallback=settings.allow_deterministic_eval_fallback, model_override=model_override, normalizer_context={'task_id': spec.task_id, 'task_title': spec.normalized_title, 'task_instruction': spec.normalized_instruction, 'model': model_override or settings.model_provider_model, 'provider':provider})
+            if rr.live_call_ok and rr.structured_output_validation_ok:
+                parsed=read_json(role_root/'rubric_agent/parsed_output.json')
+                rub=RubricSpec.model_validate(parsed['rubric_spec'])
+                qr=deterministic_rubric_quality_check(rub)
+                rub.grader_kind='hybrid'; rub.rubric_generation_agent_provider=rr.provider; rub.rubric_generation_agent_model=rr.model; rub.rubric_generation_confidence=rub.rubric_generation_confidence or 0.75
+                rub.metadata_json.update({'rubric_source':'llm','provider':rr.provider,'model':rr.model,'llm_prompt_ref_internal':str(role_root/'rubric_agent/prompt.md'),'llm_response_ref_internal':str(role_root/'rubric_agent/raw_output.txt'),'prompt_template_id':'rubric_generation_agent_v0','prompt_template_version':'0.1','prompt_hash':sha256_text(prompt),'public_response_summary':'Model-generated rubric passed deterministic quality checking.'})
+                qr.metadata_json.update({'rubric_source':'llm','role_result_ref_internal':str(role_root/'rubric_agent/role_result.json')})
+                return rub, qr
+            if settings.rubric_mode == 'llm_required' or settings.llm_fail_closed:
+                raise RuntimeError(rr.error_message or 'Model rubric generation failed')
+        except Exception:
+            if settings.rubric_mode == 'llm_required' or settings.llm_fail_closed:
+                raise
+    return deterministic_rubric(spec)
+
+def worker_visible_markdown(rubric: RubricSpec) -> str:
+    lines=[f'# Worker-visible rubric for {rubric.task_id}', '']
+    for item in rubric.rubric_items:
+        if item.worker_visible and not item.verifier_only:
+            lines.append(f'- **{item.criterion_name}** ({item.weight}): {item.criterion_description}')
+    return '\n'.join(lines)+'\n'

package/src/agent_apprenticeship_trace/schemas.py

@@ -0,0 +1,273 @@
+from __future__ import annotations
+from typing import Literal, Any
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+DictAny = dict[str, Any]
+
+class StrictModel(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+class ArtifactRef(StrictModel):
+    artifact_id: str
+    task_id: str
+    attempt_id: str | None = None
+    artifact_kind: Literal["input","output","intermediate","log","trace","score","rubric","feedback","lesson","package_file","media","document","unknown"]
+    artifact_role: Literal["task_input","worker_output","reviser_output","apprentice_output","reference","hidden_reference","grader_output","verifier_output","evaluator_output","system_log","trace_file","other"]
+    workspace_path: str | None = None
+    package_relative_path: str
+    release_relative_path: str | None = None
+    mime_type: str | None = None
+    media_type: Literal["text","code","data","document","image","audio","video","archive","binary","unknown"]
+    size_bytes: int | None = None
+    content_hash: str | None = None
+    secret_scan_ok: bool
+    metadata_json: DictAny = Field(default_factory=dict)
+
+class ActualOutputs(StrictModel):
+    task_id: str
+    attempt_id: str
+    attempt_kind: str
+    status: Literal["success","partial","failed","timeout","error"]
+    output_summary: str
+    primary_output_ref: str | None = None
+    deliverable_refs: list[str] = Field(default_factory=list)
+    final_message_ref: str | None = None
+    artifact_refs: list[str] = Field(default_factory=list)
+    files_created: list[str] = Field(default_factory=list)
+    files_modified: list[str] = Field(default_factory=list)
+    files_deleted: list[str] = Field(default_factory=list)
+    stdout_ref: str | None = None
+    stderr_ref: str | None = None
+    raw_log_refs: list[str] = Field(default_factory=list)
+    error_type: str | None = None
+    error_message: str | None = None
+    metadata_json: DictAny = Field(default_factory=dict)
+
+class AgentTraceStep(StrictModel):
+    step: int
+    turn: int
+    actor: str
+    action: Literal["user_message","agent_step","output","error"]
+    operation: Literal["plan","analyze","search","read","write","edit","execute","verify","download","install","ask_user","answer","select","grade","evaluate","revise","other"] | None = None
+    tool: str | None = None
+    execution_mode: Literal["serial","parallel"] | None = None
+    parallel_group: str | None = None
+    observation: str | None = None
+    input: str | None = None
+    input_source: DictAny | None = None
+    output: str | None = None
+    state_change: str | None = None
+    reasoning: str | None = None
+    caused_by: list[int] | None = None
+    causal_type: Literal["user_request","follow_up_user_request","answer_to_agent_question","execution_of_plan","dependency_on_tool_result","retry_after_failure","correction_response","approval_response","verification_of_prior_step","dependency_on_multiple_prior_steps","delegation_to_subagent","delegated_work","used_subagent_result","handoff_from_subagent","parallel_work","other"] | None = None
+    causal_note: str | None = None
+    alternatives_considered: str | None = None
+    success: bool | None = None
+    step_outcome: Literal["progress","neutral","blocked","failed","corrected","completed"] | None = None
+    error_type: str | None = None
+    error_message: str | None = None
+    message_role: Literal["direct_request","answer_to_agent_question","correction","approval","clarification","selection","status_update","new_constraint","other"] | None = None
+    feedback_type: Literal["correction","approval","clarification","new_instruction","other"] | None = None
+    feedback_content: str | None = None
+    started_at: str | None = None
+    ended_at: str | None = None
+    retry_of: int | None = None
+    artifact_refs: list[str] = Field(default_factory=list)
+    metadata_json: DictAny = Field(default_factory=dict)
+
+    @model_validator(mode="after")
+    def validate_step(self):
+        if self.step < 1: raise ValueError("step must start at 1")
+        if self.action == "user_message":
+            required_null = ["operation","tool","execution_mode","observation","reasoning","success","step_outcome","output"]
+            bad = [name for name in required_null if getattr(self, name) is not None]
+            if self.actor != "user" or bad:
+                raise ValueError(f"user_message invariants failed: {bad}")
+        elif self.operation is None:
+            raise ValueError("operation is required for non-user steps")
+        return self
+
+class AgentTrace(StrictModel):
+    schema_version: str = "aa-trace-v0.1"
+    trace_id: str
+    collection_id: str | None = None
+    prior_trace_id: str | None = None
+    trace_mode: Literal["live","retraced","hybrid"]
+    task: str
+    task_id: str
+    task_family_id: str | None = None
+    attempt_id: str
+    attempt_kind: Literal["baseline","revised","apprentice_without_lessons","apprentice_with_lessons","other"]
+    attempt_status: Literal["completed","failed","blocked","fallback","partial"] | None = None
+    agent_tools: list[str]
+    started_at: str | None = None
+    ended_at: str | None = None
+    system_prompt: str | None = None
+    system_prompt_hash: str | None = None
+    skills: list[str] | None = None
+    memory: str | None = None
+    agent_config: DictAny | None = None
+    learning: str | None = None
+    termination_reason: Literal["task_complete","verifier_passed","verifier_failed","max_iterations_reached","agent_blocked","timeout","error_unrecoverable","partial_then_stopped","provider_usage_limit","other"]
+    steps: list[AgentTraceStep]
+    actual_outputs: ActualOutputs | None = None
+    artifacts: list[ArtifactRef] = Field(default_factory=list)
+    iteration_index: int | None = None
+    previous_attempt_id: str | None = None
+    revision_group_id: str | None = None
+    completion_reason: str | None = None
+    final_attempt_id: str | None = None
+    preferred_attempt_id: str | None = None
+    initial_attempt_id: str | None = None
+    revision_attempt_ids: list[str] = Field(default_factory=list)
+    metadata_json: DictAny = Field(default_factory=dict)
+
+    @model_validator(mode="after")
+    def validate_trace(self):
+        if not self.steps: raise ValueError("trace must have steps")
+        nums = [s.step for s in self.steps]
+        if nums != list(range(1, len(nums)+1)): raise ValueError("step numbers must be monotonic starting at 1")
+        seen=set()
+        for s in self.steps:
+            if s.caused_by and any(c >= s.step or c < 1 for c in s.caused_by):
+                raise ValueError("causality refs must point to earlier steps")
+            seen.add(s.step)
+        return self
+
+class RawTaskRecord(StrictModel):
+    raw_task_id: str
+    source_kind: str
+    source_url: str | None = None
+    source_license: str | None = None
+    raw_title: str
+    raw_description: str
+    raw_payload: DictAny = Field(default_factory=dict)
+    task_id: str | None = None
+    normalized_title: str | None = None
+    normalized_instruction: str | None = None
+    input_artifact_refs: list[str] = Field(default_factory=list)
+    created_at: str | None = None
+    metadata_json: DictAny = Field(default_factory=dict)
+    normalized_domain: str | None = None
+    normalized_subdomain: str | None = None
+    apprenticeship_role: str | None = None
+    task_family: str | None = None
+    expected_deliverable: str | None = None
+    expected_economic_value: str | None = None
+    expected_economic_value_for_agent_apprentice: str | None = None
+    expected_pay: str | None = None
+    expected_apprentice_pay: str | None = None
+    source_url_or_ref: str | None = None
+    difficulty_tier: Literal["easy","medium","hard","expert"] | None = None
+    needs_expert_review: bool | None = None
+
+    @model_validator(mode="before")
+    @classmethod
+    def accept_task_sheet(cls, data):
+        if not isinstance(data, dict):
+            return data
+        d=dict(data)
+        payload=dict(d.get('raw_payload') or {})
+        if 'task_id' in d and 'raw_task_id' not in d:
+            d['raw_task_id']=d['task_id']
+        if 'normalized_title' in d and 'raw_title' not in d:
+            d['raw_title']=d['normalized_title']
+        if 'normalized_instruction' in d and 'raw_description' not in d:
+            d['raw_description']=d['normalized_instruction']
+        if 'source_url_or_ref' in d and 'source_url' not in d:
+            d['source_url']=d['source_url_or_ref']
+        if 'expected_pay' in d and 'expected_economic_value' not in d:
+            d['expected_economic_value'] = d['expected_pay']
+        if 'expected_apprentice_pay' in d and 'expected_economic_value_for_agent_apprentice' not in d:
+            d['expected_economic_value_for_agent_apprentice'] = d['expected_apprentice_pay']
+        for key in ['normalized_domain','normalized_subdomain','apprenticeship_role','task_family','expected_deliverable','expected_economic_value','expected_economic_value_for_agent_apprentice','expected_pay','expected_apprentice_pay','difficulty_tier','needs_expert_review']:
+            if key in d and key not in payload:
+                payload[key]=d[key]
+        d.setdefault('source_kind', d.get('source_kind') or 'manual_seed')
+        d.setdefault('raw_title', d.get('raw_task_id','untitled_task'))
+        d.setdefault('raw_description', d.get('expected_deliverable') or '')
+        d['raw_payload']=payload
+        return d
+
+class TaskIntakeSpec(StrictModel):
+    task_id: str; normalized_title: str; normalized_instruction: str; domain: str
+    subdomain: str | None = None; professional_role: str | None = None; apprenticeship_role: str | None = None; task_family: str | None = None; expected_economic_value: str | None = None; expected_economic_value_for_agent_apprentice: str | None = None; expected_pay: str | None = None; expected_apprentice_pay: str | None = None; workflow_type: str
+    skill_targets: list[str] = Field(default_factory=list)
+    difficulty_tier: Literal["easy","medium","hard","expert"]
+    expected_human_deliverable: str; expected_agent_deliverable: str
+    input_requirements: list[str] = Field(default_factory=list); output_requirements: list[str] = Field(default_factory=list)
+    required_context: list[str] = Field(default_factory=list); assumptions: list[str] = Field(default_factory=list)
+    constraints: list[str] = Field(default_factory=list); allowed_tools: list[str] = Field(default_factory=list); disallowed_tools: list[str] = Field(default_factory=list)
+    privacy_classification: Literal["public","synthetic","sensitive_possible","contains_pii","unknown"]
+    license: str | None = None; allowed_use: str | None = None
+    rubricability_score: float; verifiability_score: float; artifactability_score: float
+    needs_expert_review: bool; metadata_json: DictAny = Field(default_factory=dict)
+
+    @model_validator(mode="before")
+    @classmethod
+    def accept_legacy_economic_fields(cls, data):
+        if not isinstance(data, dict):
+            return data
+        d = dict(data)
+        if d.get('expected_pay') is not None and d.get('expected_economic_value') is None:
+            d['expected_economic_value'] = d.get('expected_pay')
+        if d.get('expected_apprentice_pay') is not None and d.get('expected_economic_value_for_agent_apprentice') is None:
+            d['expected_economic_value_for_agent_apprentice'] = d.get('expected_apprentice_pay')
+        if d.get('expected_economic_value') is not None and d.get('expected_pay') is None:
+            d['expected_pay'] = d.get('expected_economic_value')
+        if d.get('expected_economic_value_for_agent_apprentice') is not None and d.get('expected_apprentice_pay') is None:
+            d['expected_apprentice_pay'] = d.get('expected_economic_value_for_agent_apprentice')
+        return d
+
+class TaskIntakeQualityReport(StrictModel):
+    task_id: str; instruction_clarity_score: float; input_completeness_score: float; output_contract_score: float
+    rubricability_score: float; verifiability_score: float; artifactability_score: float; privacy_risk_score: float; license_risk_score: float; ambiguity_score: float; overall_intake_quality_score: float
+    quality_flags: list[str] = Field(default_factory=list); blockers: list[str] = Field(default_factory=list); recommended_fix: str | None = None; metadata_json: DictAny = Field(default_factory=dict)
+
+class RubricItem(StrictModel):
+    rubric_item_id: str; criterion_name: str; criterion_description: str; weight: float; score_min: float; score_max: float; pass_threshold: float
+    observable_evidence: list[str]; required_artifacts: list[str]
+    scoring_method: Literal["llm_rubric_judge","deterministic","schema_match","regex","unit_test","hybrid","human_future"]
+    worker_visible: bool; verifier_only: bool; hidden_reference_required: bool
+    failure_modes: list[str] = Field(default_factory=list); partial_credit_rules: list[str] = Field(default_factory=list); edge_cases: list[str] = Field(default_factory=list); anti_cheat_notes: list[str] = Field(default_factory=list); metadata_json: DictAny = Field(default_factory=dict)
+
+class RubricSpec(StrictModel):
+    rubric_id: str; task_id: str; task_family_id: str | None = None; rubric_version: str; rubric_items: list[RubricItem]; total_weight: float; pass_threshold: float
+    worker_visible_rubric_ref: str; verifier_private_rubric_ref: str; hidden_reference_policy: str
+    scoring_aggregation: Literal["weighted_sum","sum","all_required","custom"]
+    required_artifacts: list[str]; disqualifying_errors: list[str] = Field(default_factory=list); partial_credit_allowed: bool
+    grader_kind: Literal["llm_rubric_judge","deterministic","hybrid","human_future"]
+    rubric_generation_source: Literal["agent_assisted","family_template","task_specific_agent_draft","expert_override","deterministic_seed"]
+    rubric_generation_agent_provider: str | None = None; rubric_generation_agent_model: str | None = None; rubric_generation_confidence: float | None = None; metadata_json: DictAny = Field(default_factory=dict)
+    @model_validator(mode="after")
+    def valid_rubric(self):
+        if not self.rubric_items: raise ValueError("rubric_items required")
+        if not (abs(sum(i.weight for i in self.rubric_items)-1.0)<1e-6 or abs(sum(i.weight for i in self.rubric_items)-100)<1e-6): raise ValueError("weights must sum to 1.0 or 100")
+        if any(not i.observable_evidence or not i.required_artifacts for i in self.rubric_items): raise ValueError("observable evidence and artifacts required")
+        return self
+
+class RubricQualityReport(StrictModel):
+    rubric_id: str; task_id: str; criteria_count: int; total_weight: float; weights_sum_valid: bool; has_observable_evidence: bool; has_required_artifacts: bool; has_partial_credit_rules: bool; has_disqualifying_errors: bool; has_hidden_reference_policy: bool; has_worker_visible_view: bool; has_verifier_private_view: bool; ambiguous_criteria_count: int; unverifiable_criteria_count: int; rubric_quality_score: float; quality_flags: list[str] = Field(default_factory=list); blockers: list[str] = Field(default_factory=list); metadata_json: DictAny = Field(default_factory=dict)
+
+class RubricItemScore(StrictModel):
+    rubric_item_id: str; criterion_name: str; score: float; max_score: float; passed: bool; evidence_refs: list[str] = Field(default_factory=list); failure_mode: str | None = None; notes: str | None = None; confidence: float | None = None; artifact_presence_ok: bool | None = None; semantic_correctness_score: float | None = None; reasoning_summary: str | None = None; improvement_suggestion: str | None = None
+class GraderResult(StrictModel):
+    grader_result_id: str; task_id: str; attempt_id: str; attempt_kind: str; rubric_id: str; grader_kind: Literal["llm_rubric_judge","llm","model","deterministic","hybrid","human_future"]; score_source: Literal["llm","llm_semantic","model_judged","deterministic","deterministic_artifact_contract","deterministic_fallback","hybrid","human_future"]; score: float; max_score: float; passed: bool; rubric_item_scores: list[RubricItemScore]; failed_criteria: list[str]; passed_criteria: list[str]; evidence_refs: list[str]; confidence: float; reasoning_summary: str | None = None; limitations: list[str] = Field(default_factory=list); hidden_reference_used: bool; hidden_reference_leaked: bool; artifact_contract_score: float | None = None; semantic_score: float | None = None; model_score: float | None = None; legacy_semantic_score: float | None = None; legacy_score_source: str | None = None; final_score: float | None = None; model: str | None = None; provider: str | None = None; deterministic_precheck_ref: str | None = None; llm_prompt_ref_internal: str | None = None; llm_response_ref_internal: str | None = None; public_prompt_hash: str | None = None; public_response_summary: str | None = None; score_reliability: Literal["verified","unverified","needs_review","failed_verification"] | None = None; verifier_status: str | None = None; verifier_confidence: float | None = None; verifier_issue_count: int | None = None; verifier_issues_summary: str | None = None; metadata_json: DictAny = Field(default_factory=dict)
+class VerifierResult(StrictModel):
+    verifier_result_id: str; task_id: str; attempt_id: str; attempt_kind: str; grader_result_id: str | None = None; verification_status: Literal["verified","partially_verified","failed","not_run"]; artifact_contract_ok: bool; evidence_grounding_ok: bool; score_consistency_ok: bool; hidden_reference_leaked: bool; issues: list[str] = Field(default_factory=list); confidence: float; verifier_notes: str | None = None; semantic_evidence_grounding_ok: bool | None = None; unsupported_claims: list[str] = Field(default_factory=list); leakage_check_ok: bool | None = None; model: str | None = None; provider: str | None = None; metadata_json: DictAny = Field(default_factory=dict)
+class EvaluatorFeedback(StrictModel):
+    feedback_id: str; task_id: str; attempt_id: str; target_actor: Literal["worker","reviser","apprentice"]; feedback_type: Literal["criteria_failure","artifact_missing","format_error","logic_error","tool_error","quality_gap","strategy_gap","safety_or_privacy","other"]; failed_rubric_items: list[str]; evidence_refs: list[str]; artifact_refs: list[str]; feedback_summary: str; actionable_feedback: list[str]; suggested_revision: str; revision_priority: Literal["low","medium","high"]; confidence: float; hidden_reference_used: bool; hidden_reference_leaked: bool; failed_or_weak_rubric_items: list[str] = Field(default_factory=list); artifact_specific_comments: list[str] = Field(default_factory=list); trace_specific_comments: list[str] = Field(default_factory=list); revision_plan: str | None = None; model: str | None = None; provider: str | None = None; metadata_json: DictAny = Field(default_factory=dict)
+class RevisionPlan(StrictModel):
+    revision_plan_id: str; task_id: str; source_attempt_id: str; target_attempt_id: str; revision_kind: Literal["local_fix","strategy_shift","tool_change","decomposition_change","artifact_rebuild","format_repair","other"]; revision_reason: str; failed_rubric_items: list[str]; planned_changes: list[str]; expected_score_improvement: float | None = None; risk_of_regression: Literal["low","medium","high"]; uses_evaluator_feedback: bool; metadata_json: DictAny = Field(default_factory=dict)
+class HillclimbResult(StrictModel):
+    hillclimb_id: str; task_id: str; baseline_attempt_id: str; revised_attempt_id: str; baseline_score: float; revised_score: float; revision_score_delta: float; baseline_passed: bool; revised_passed: bool; failed_criteria_before: list[str]; failed_criteria_after: list[str]; criteria_improved: list[str]; criteria_regressed: list[str]; artifact_completeness_before: float; artifact_completeness_after: float; artifact_completeness_delta: float; regression_count: int; improvement_kind: Literal["score_delta","pass_delta","criteria_delta","artifact_delta","none","regression"]; hillclimb_evidence_strength: Literal["observed_improvement","no_observed_improvement","regression_observed"]; revision_success: bool; metadata_json: DictAny = Field(default_factory=dict)
+class LessonPack(StrictModel):
+    lesson_id: str; task_id: str; source_attempt_ids: list[str]; lesson_summary: str; strategy_lessons: list[str]; common_failure_modes: list[str]; rubric_reminders: list[str]; artifact_requirements: list[str]; verifier_feedback_summary: str; hidden_reference_leaked: bool; metadata_json: DictAny = Field(default_factory=dict)
+class TrainingSignal(StrictModel):
+    signal_id: str; task_id: str; signal_type: Literal["rollout","process_supervision","verifier_training","reward_modeling","revision_preference","lesson_transfer"]; source_attempt_ids: list[str]; baseline_score: float | None = None; revised_score: float | None = None; score_delta: float | None = None; criteria_improved: list[str]=Field(default_factory=list); criteria_regressed: list[str]=Field(default_factory=list); failed_criteria_before: list[str]=Field(default_factory=list); failed_criteria_after: list[str]=Field(default_factory=list); feedback_ref: str | None = None; revision_plan_ref: str | None = None; grader_result_refs: list[str]=Field(default_factory=list); verifier_result_refs: list[str]=Field(default_factory=list); trace_refs: list[str]=Field(default_factory=list); artifact_refs: list[str]=Field(default_factory=list); score_source: str; grader_kind: str; confidence: float | None = None; training_use_cases: list[str]=Field(default_factory=list); limitations: list[str]=Field(default_factory=list); metadata_json: DictAny=Field(default_factory=dict)
+class ProcessSupervisionExample(StrictModel):
+    example_id: str; task_id: str; attempt_id: str; attempt_kind: str | None = None; trace_id: str; step: int; actor: str; action: str; operation: str | None = None; tool: str | None = None; observation: str | None = None; input: str | None = None; output: str | None = None; state_change: str | None = None; reasoning: str | None = None; caused_by: list[int] | None = None; causal_type: str | None = None; success: bool | None = None; step_outcome: str | None = None; step_quality_label: Literal["positive","neutral","negative","unknown"]; local_reward: float | None = None; failure_mode: str | None = None; grader_feedback: str | None = None; verifier_feedback: str | None = None; evaluator_feedback: str | None = None; revision_reason: str | None = None; final_outcome_score: float | None = None; label_source: Literal["grader","verifier","evaluator","heuristic","human_future","none"]; label: str = ""; metadata_json: DictAny = Field(default_factory=dict)
+class RewardModelingExample(StrictModel):
+    example_id: str; task_id: str; attempt_id: str; rubric_ref: str; output_refs: list[str]; attempt_summary: str; rubric_item_scores: list[RubricItemScore]; final_score: float; passed: bool; failure_modes: list[str]; grader_notes: str | None = None; evidence_refs: list[str]; score_source: str; grader_kind: str; confidence: float; score_reliability: str | None = None; verifier_status: str | None = None; verifier_confidence: float | None = None; verifier_issue_count: int | None = None; verifier_issues_summary: str | None = None; metadata_json: DictAny=Field(default_factory=dict)
+class RevisionPreferencePair(StrictModel):
+    pair_id: str; task_id: str; rubric_ref: str; baseline_attempt_ref: str; revised_attempt_ref: str; chosen_attempt_id: str; rejected_attempt_id: str; baseline_score: float; revised_score: float; score_delta: float; criteria_improved: list[str]; criteria_regressed: list[str]; preference_reason: str; score_source: str; grader_kind: str; confidence: float; metadata_json: DictAny=Field(default_factory=dict)

package/src/agent_apprenticeship_trace/session_events.py

@@ -0,0 +1,99 @@
+from __future__ import annotations
+
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from .io import append_jsonl, read_jsonl
+
+
+SessionEventType = Literal[
+    "task_instruction",
+    "task_assets_added",
+    "agent_attempt",
+    "evaluation",
+    "revision",
+    "user_followup",
+    "followup_assets_added",
+    "session_finished",
+]
+
+
+class SessionEvent(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    event_type: SessionEventType
+    event_index: int
+    created_at: str
+    run_id: str
+    task_id: str | None = None
+    session_id: str | None = None
+    attempt_id: str | None = None
+    feedback_source: str | None = None
+    feedback_type: str | None = None
+    applies_to_attempt: str | None = None
+    followup_index: int | None = None
+    instruction: str | None = None
+    followup_instruction: str | None = None
+    assets: list[str] = Field(default_factory=list)
+    followup_assets: list[str] = Field(default_factory=list)
+    metadata_json: dict[str, Any] = Field(default_factory=dict)
+
+
+def utc_now() -> str:
+    return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
+
+
+def events_path(run_root: Path) -> Path:
+    return run_root / "session_events.jsonl"
+
+
+def read_session_events(run_root: Path) -> list[dict[str, Any]]:
+    return read_jsonl(events_path(run_root))
+
+
+def next_event_index(run_root: Path) -> int:
+    return len(read_session_events(run_root)) + 1
+
+
+def append_session_event(run_root: Path, **kwargs: Any) -> SessionEvent:
+    run_root.mkdir(parents=True, exist_ok=True)
+    event = SessionEvent(
+        event_index=next_event_index(run_root),
+        created_at=utc_now(),
+        **kwargs,
+    )
+    append_jsonl(events_path(run_root), event)
+    return event
+
+
+def backfill_session_event_task_ids(run_root: Path, task_id: str | None) -> None:
+    if not task_id:
+        return
+    path = events_path(run_root)
+    rows = read_jsonl(path)
+    if not rows:
+        return
+    changed = False
+    for row in rows:
+        if isinstance(row, dict) and not row.get("task_id"):
+            row["task_id"] = task_id
+            changed = True
+    if not changed:
+        return
+    with path.open("w") as f:
+        for row in rows:
+            f.write(json.dumps(row, sort_keys=True) + "\n")
+
+
+def next_followup_index(run_root: Path) -> int:
+    events = read_session_events(run_root)
+    values = [
+        int(e.get("followup_index") or 0)
+        for e in events
+        if e.get("event_type") == "user_followup"
+    ]
+    return (max(values) if values else 0) + 1

package/src/agent_apprenticeship_trace/task_intake.py

@@ -0,0 +1,112 @@
+from __future__ import annotations
+import json
+from pathlib import Path
+from pydantic import BaseModel
+from .schemas import RawTaskRecord, TaskIntakeSpec, TaskIntakeQualityReport
+from .config import get_settings
+from .io import read_json
+from .openai_structured import get_model_provider_status, run_structured_role
+from .public_sanitizer import sanitize_public_obj, sha256_text
+
+class LLMTaskIntakeOutput(BaseModel):
+    task_intake_spec: TaskIntakeSpec
+    task_intake_quality_report: TaskIntakeQualityReport
+
+SOURCE_FIELD_KEYS={'source_url_or_ref','source_kind','source_url','source_ref','source_license'}
+
+def _mentor_provider_can_attempt() -> bool:
+    return bool(get_model_provider_status().get('provider_available'))
+
+def _mentor_provider_id() -> str:
+    settings=get_settings()
+    return settings.model_provider or 'openai'
+
+def _drop_source_fields(obj):
+    if isinstance(obj, list):
+        return [_drop_source_fields(v) for v in obj]
+    if isinstance(obj, dict):
+        return {k:_drop_source_fields(v) for k,v in obj.items() if k not in SOURCE_FIELD_KEYS and v is not None}
+    return obj
+
+def _task_record_for_intake(raw: RawTaskRecord) -> dict:
+    data=raw.model_dump(mode='json')
+    return sanitize_public_obj(_drop_source_fields(data))
+
+def direct_task_sheet_metadata(raw: RawTaskRecord) -> dict[str, object]:
+    payload=raw.raw_payload or {}
+    expected_economic_value = raw.expected_economic_value or payload.get('expected_economic_value') or raw.expected_pay or payload.get('expected_pay')
+    expected_economic_value_for_agent_apprentice = (
+        raw.expected_economic_value_for_agent_apprentice
+        or payload.get('expected_economic_value_for_agent_apprentice')
+        or raw.expected_apprentice_pay
+        or payload.get('expected_apprentice_pay')
+    )
+    return {
+        'domain': raw.normalized_domain or payload.get('normalized_domain') or payload.get('domain'),
+        'subdomain': raw.normalized_subdomain or payload.get('normalized_subdomain') or payload.get('subdomain'),
+        'apprenticeship_role': raw.apprenticeship_role or payload.get('apprenticeship_role'),
+        'expected_economic_value': expected_economic_value,
+        'expected_economic_value_for_agent_apprentice': expected_economic_value_for_agent_apprentice,
+        'expected_pay': expected_economic_value,
+        'expected_apprentice_pay': expected_economic_value_for_agent_apprentice,
+        'task_family': raw.task_family or payload.get('task_family'),
+        'difficulty_tier': raw.difficulty_tier or payload.get('difficulty_tier'),
+        'needs_expert_review': raw.needs_expert_review if raw.needs_expert_review is not None else payload.get('needs_expert_review'),
+    }
+
+def apply_direct_task_sheet_metadata(spec: TaskIntakeSpec, raw: RawTaskRecord) -> TaskIntakeSpec:
+    direct={k:v for k,v in direct_task_sheet_metadata(raw).items() if v is not None}
+    updates={}
+    for src,dst in [('domain','domain'),('subdomain','subdomain'),('apprenticeship_role','apprenticeship_role'),('expected_economic_value','expected_economic_value'),('expected_economic_value_for_agent_apprentice','expected_economic_value_for_agent_apprentice'),('expected_pay','expected_pay'),('expected_apprentice_pay','expected_apprentice_pay'),('task_family','task_family'),('difficulty_tier','difficulty_tier'),('needs_expert_review','needs_expert_review')]:
+        if src in direct:
+            updates[dst]=direct[src]
+    md=dict(spec.metadata_json or {})
+    md['direct_task_sheet_fields']={k:v for k,v in direct.items() if k not in {'expected_pay','expected_apprentice_pay'}}
+    md['direct_task_sheet_fields_preserved']=bool(direct)
+    return spec.model_copy(update={**updates, 'metadata_json': md})
+
+def deterministic_intake(raw: RawTaskRecord) -> tuple[TaskIntakeSpec, TaskIntakeQualityReport]:
+    tid = raw.raw_task_id.replace('raw_','task_')
+    direct=direct_task_sheet_metadata(raw)
+    expected=raw.raw_payload.get('expected_deliverable') or raw.expected_deliverable or raw.raw_payload.get('expected_agent_deliverable','Completed deliverables and audit notes')
+    spec=TaskIntakeSpec(task_id=tid, normalized_title=raw.raw_title, normalized_instruction=raw.raw_description, domain=direct.get('domain') or raw.raw_payload.get('domain','general'), subdomain=direct.get('subdomain') or raw.raw_payload.get('subdomain'), professional_role=raw.raw_payload.get('professional_role'), apprenticeship_role=direct.get('apprenticeship_role'), task_family=direct.get('task_family'), expected_economic_value=direct.get('expected_economic_value'), expected_economic_value_for_agent_apprentice=direct.get('expected_economic_value_for_agent_apprentice'), expected_pay=direct.get('expected_pay'), expected_apprentice_pay=direct.get('expected_apprentice_pay'), workflow_type=raw.raw_payload.get('workflow_type','analysis'), skill_targets=raw.raw_payload.get('skill_targets',['analysis','artifact_generation']), difficulty_tier=direct.get('difficulty_tier') or raw.raw_payload.get('difficulty_tier','medium'), expected_human_deliverable=raw.raw_payload.get('expected_human_deliverable', expected), expected_agent_deliverable=expected, input_requirements=raw.raw_payload.get('input_requirements',[]), output_requirements=raw.raw_payload.get('output_requirements',[]), required_context=raw.raw_payload.get('required_context',[]), assumptions=[], constraints=raw.raw_payload.get('constraints',[]), allowed_tools=['python','file_read','file_write','bash'], disallowed_tools=['browser'], privacy_classification=raw.raw_payload.get('privacy_classification','unknown'), license=None, allowed_use='local apprenticeship data generation', rubricability_score=0.8, verifiability_score=0.8, artifactability_score=0.9, needs_expert_review=bool(direct.get('needs_expert_review')) if direct.get('needs_expert_review') is not None else False, metadata_json={'runner':'deterministic','intake_source':'deterministic_fallback'})
+    spec=apply_direct_task_sheet_metadata(spec, raw)
+    settings=get_settings()
+    if settings.rubric_mode in {'hybrid','llm_default'} and _mentor_provider_can_attempt() and settings.llm_task_intake_enabled:
+        spec.metadata_json.update({'llm_task_intake_enabled': True, 'llm_unavailable': True, 'intake_source':'deterministic_fallback', 'provider': _mentor_provider_id()})
+    q=TaskIntakeQualityReport(task_id=tid, instruction_clarity_score=0.8, input_completeness_score=0.8, output_contract_score=0.9, rubricability_score=spec.rubricability_score, verifiability_score=spec.verifiability_score, artifactability_score=spec.artifactability_score, privacy_risk_score=0.1, license_risk_score=0.1, ambiguity_score=0.2, overall_intake_quality_score=0.82, quality_flags=[], blockers=[], recommended_fix=None, metadata_json={})
+    return spec,q
+
+def _intake_prompt(raw: RawTaskRecord) -> str:
+    return """Return only valid JSON. Do not include markdown. Do not add extra top-level fields; place extras under metadata_json.extra_model_fields.
+Required skeleton: {"task_intake_spec":{"task_id":"...","normalized_title":"...","normalized_instruction":"...","domain":"general","workflow_type":"artifact_generation","difficulty_tier":"medium","expected_human_deliverable":"...","expected_agent_deliverable":"...","input_requirements":[],"output_requirements":[],"required_context":[],"assumptions":[],"constraints":[],"allowed_tools":[],"disallowed_tools":[],"privacy_classification":"unknown","rubricability_score":0.7,"verifiability_score":0.7,"artifactability_score":0.7,"needs_expert_review":false,"metadata_json":{}},"task_intake_quality_report":{"task_id":"...","instruction_clarity_score":0.7,"input_completeness_score":0.7,"output_contract_score":0.7,"rubricability_score":0.7,"verifiability_score":0.7,"artifactability_score":0.7,"privacy_risk_score":0.2,"license_risk_score":0.2,"ambiguity_score":0.3,"overall_intake_quality_score":0.7,"quality_flags":[],"blockers":[],"metadata_json":{}}}.
+Create a publishable task intake spec for reusable agent work experience. Populate title, domain, subdomain, agent apprenticeship role, expected_economic_value, and expected_economic_value_for_agent_apprentice when inferable from the task. Include task_family_guess, task_type, required_artifacts, hidden_reference_policy, risk_notes, subjectivity_level, evaluation_difficulty, and suggested_evaluator_type under metadata_json when not schema fields.
+Raw task JSON:
+""" + json.dumps(_task_record_for_intake(raw), sort_keys=True)
+
+def task_intake(raw: RawTaskRecord, role_root: Path | None=None) -> tuple[TaskIntakeSpec, TaskIntakeQualityReport]:
+    settings=get_settings()
+    role_root=role_root or Path('outputs/roles')
+    if settings.llm_task_intake_enabled and settings.rubric_mode in {'hybrid','llm_default','llm_required'} and _mentor_provider_can_attempt():
+        prompt=_intake_prompt(raw)
+        try:
+            provider=_mentor_provider_id()
+            model_override=settings.llm_task_intake_model if provider == 'openai' else None
+            rr=run_structured_role('intake_agent', prompt, LLMTaskIntakeOutput, role_root/'intake_agent', allow_fallback=settings.allow_deterministic_eval_fallback, model_override=model_override, normalizer_context={'task_id': raw.raw_task_id.replace('raw_','task_'), 'task_title': raw.raw_title, 'task_instruction': raw.raw_description, 'model': model_override or settings.model_provider_model, 'provider':provider})
+            if rr.live_call_ok and rr.structured_output_validation_ok:
+                parsed=read_json(role_root/'intake_agent/parsed_output.json')
+                spec=TaskIntakeSpec.model_validate(parsed['task_intake_spec'])
+                spec=apply_direct_task_sheet_metadata(spec, raw)
+                q=TaskIntakeQualityReport.model_validate(parsed['task_intake_quality_report'])
+                spec.metadata_json.update({'intake_source':'llm','provider':rr.provider,'model':rr.model,'llm_prompt_ref_internal':str(role_root/'intake_agent/prompt.md'),'llm_response_ref_internal':str(role_root/'intake_agent/raw_output.txt'),'prompt_template_id':'task_intake_agent_v0','prompt_template_version':'0.1','prompt_hash':sha256_text(prompt),'public_response_summary':'Model-assisted task intake generated structured task spec.'})
+                q.metadata_json.update({'intake_source':'llm','role_result_ref_internal':str(role_root/'intake_agent/role_result.json')})
+                return spec,q
+            if settings.rubric_mode == 'llm_required' or settings.llm_fail_closed:
+                raise RuntimeError(rr.error_message or 'Model task intake failed')
+        except Exception:
+            if settings.rubric_mode == 'llm_required' or settings.llm_fail_closed:
+                raise
+    spec,q=deterministic_intake(raw)
+    if settings.llm_task_intake_enabled and _mentor_provider_can_attempt():
+        spec.metadata_json.update({'intake_source':'deterministic_fallback','llm_unavailable':True,'provider':_mentor_provider_id()})
+    return spec,q