agent-apprenticeship 0.1.8 → 0.2.0

package/src/agent_apprenticeship_trace/ecosystem_r2.py

@@ -24,6 +24,7 @@ from .config import (
 from .env import redact_secrets
 from .io import append_jsonl, read_json, read_jsonl, write_json
 from .public_sanitizer import sanitize_public_obj
+from .version import get_package_version
 
 
 def utc_now() -> str:
@@ -236,7 +237,7 @@ def ecosystem_item_from_bundle(
         str(metadata.get("economic_value_summary") or manifest.get("economic_value_summary") or "").strip()
         or f"{expected_value.title()} reuse value based on trace evidence, artifact coverage, and generated training rows."
     )
-    package_version = str(manifest.get("package_version") or manifest.get("agent_apprenticeship_version") or "0.1.7")
+    package_version = str(manifest.get("package_version") or manifest.get("agent_apprenticeship_version") or get_package_version())
     loop_count = int(loop_manifest.get("loop_iteration_count") or manifest.get("loop_iteration_count") or 0)
     has_skill = (compiler_dir / "skill_pack").exists()
     has_training = (compiler_dir / "training_time_package").exists()

package/src/agent_apprenticeship_trace/learning.py

@@ -12,6 +12,7 @@ from typing import Any
 from .config import Settings, get_settings
 from .env import contains_secret
 from .io import append_jsonl, read_json, read_jsonl, write_json
+from .version import get_package_version
 
 
 PACK_STATUSES = {"draft", "active", "reverted", "removed"}
@@ -697,6 +698,7 @@ def install_runtime_training_from_bundle(
     skill_md = skill_dir / "skill.md"
     if not skill_md.exists():
         raise FileNotFoundError(f"Runtime training skill.md not found in {bundle}")
+    quality = _runtime_training_quality(bundle)
     package_id = str(_read_manifest_field(bundle, "bundle_id") or _read_manifest_field(bundle, "source_package_id") or bundle.name)
     title = str(_read_manifest_field(bundle, "title") or _read_manifest_field(bundle, "source_task_id") or source_item_id)
     digest = hashlib.sha256(f"{source_item_id}|{package_id}|{skill_md.read_text(errors='replace')[:5000]}".encode()).hexdigest()[:10]
@@ -761,6 +763,7 @@ def install_runtime_training_from_bundle(
         "source_item_id": source_item_id,
         "source_package_id": package_id,
         "source_url": _read_manifest_field(bundle, "source_url"),
+        "package_version": get_package_version(),
         "agent_target": agent_target,
         "install_mode": "agent_apprenticeship_runtime",
         "installed_at": utc_now(),
@@ -770,12 +773,80 @@ def install_runtime_training_from_bundle(
         "backup_refs": [],
         "reversible": True,
         "generation_notes": ["native_agent_files_unchanged"],
+        "transfer_confidence": quality["transfer_confidence"],
+        "runtime_training_confidence": quality["transfer_confidence"],
+        "low_confidence_reasons": [quality["low_confidence_reason"]] if quality.get("low_confidence_reason") else [],
+        "source_quality_gate": quality,
     }
     write_json(target / "install_manifest.json", manifest)
     append_jsonl(installed_skills_index_path(s), manifest)
     return manifest
 
 
+def _runtime_training_quality(bundle: Path) -> dict[str, Any]:
+    runtime = bundle / "experience_compiler" / "runtime_learning_package"
+    compiler = bundle / "experience_compiler"
+    has_actual = any((bundle / "attempts").glob("*/actual_outputs.json")) if (bundle / "attempts").exists() else False
+    has_artifacts = any((bundle / "attempts").glob("*/artifacts/*")) if (bundle / "attempts").exists() else False
+    required = {
+        "transfer_runbook": runtime / "transfer_runbook.md",
+        "minimal_prompt_context": runtime / "minimal_prompt_context.md",
+        "output_contract": runtime / "output_contract.json",
+        "artifact_contract": runtime / "artifact_contract.json",
+        "prefinal_verification_checklist": runtime / "prefinal_verification_checklist.json",
+    }
+    missing = [name for name, path in required.items() if not path.exists()]
+    evidence_map = compiler / "source_evidence_map.json"
+    source_score = _runtime_training_source_score(bundle)
+    checks = {
+        "actual_outputs_exist": has_actual,
+        "artifacts_exist": has_artifacts,
+        "source_evidence_map_exists": evidence_map.exists(),
+        "runtime_package_non_generic": bool((runtime / "transfer_runbook.md").exists() and len((runtime / "transfer_runbook.md").read_text(errors="replace")) > 500),
+        "required_runtime_contract_files_present": not missing,
+        "missing_runtime_contract_files": missing,
+        "source_score": source_score,
+        "source_score_passes_transfer_gate": source_score is None or source_score >= 0.7,
+    }
+    confidence = "high" if all(v for k, v in checks.items() if k != "missing_runtime_contract_files") else "low"
+    checks["transfer_confidence"] = confidence
+    reasons = list(missing)
+    if source_score is not None and source_score < 0.7:
+        reasons.append(f"source_score_below_transfer_gate:{source_score:.3f}")
+    checks["low_confidence_reason"] = "; ".join(reasons) if reasons else None
+    return checks
+
+
+def _runtime_training_source_score(bundle: Path) -> float | None:
+    for rel in [
+        "grading/baseline_grader_result.json",
+        "evaluation/grader_result.json",
+        "evaluation/grader_results.json",
+    ]:
+        path = bundle / rel
+        if not path.exists():
+            continue
+        try:
+            payload = read_json(path)
+        except Exception:
+            continue
+        if isinstance(payload, dict):
+            value = payload.get("score")
+            try:
+                return float(value)
+            except Exception:
+                pass
+    jsonl = bundle / "evaluation/grader_results.jsonl"
+    if jsonl.exists():
+        for row in read_jsonl(jsonl):
+            if isinstance(row, dict) and row.get("score") is not None:
+                try:
+                    return float(row.get("score"))
+                except Exception:
+                    continue
+    return None
+
+
 def list_installed_skills(settings: Settings | None = None, include_disabled: bool = True) -> list[dict[str, Any]]:
     root = installed_skills_root(settings)
     rows: list[dict[str, Any]] = []
@@ -852,25 +923,45 @@ def select_runtime_training_for_run(
     if not enabled:
         return [], ""
     query_terms = {token.lower() for token in re.findall(r"[A-Za-z][A-Za-z0-9_-]{2,}", instruction)}
-    scored: list[tuple[int, dict[str, Any]]] = []
+    scored: list[tuple[int, dict[str, Any], str]] = []
     for skill in enabled:
         overlap = len(query_terms & _skill_keywords(skill))
         if force or overlap:
-            scored.append((overlap, skill))
+            confidence = skill.get("transfer_confidence") or (skill.get("source_quality_gate") or {}).get("transfer_confidence") or "unknown"
+            source_gate = skill.get("source_quality_gate") if isinstance(skill.get("source_quality_gate"), dict) else {}
+            source_score = source_gate.get("source_score")
+            quality_penalty = 0
+            reason_bits = ["matched runtime training against the task request"]
+            if confidence == "low":
+                quality_penalty -= 50
+                reason_bits.append("low-confidence source; use cautiously")
+            if isinstance(source_score, (int, float)) and float(source_score) < 0.7:
+                quality_penalty -= 50
+                reason_bits.append(f"source score {float(source_score):.2f} below transfer gate")
+            if force:
+                reason_bits.append("explicit --use-installed-skills requested")
+            scored.append((overlap + quality_penalty, skill, "; ".join(reason_bits)))
     scored.sort(key=lambda item: item[0], reverse=True)
-    selected = [skill for _, skill in scored[:limit]]
+    selected = [dict(skill, runtime_training_selection_reason=reason) for _, skill, reason in scored[:limit]]
     if not selected:
         return [], ""
     sections = ["\n\n---\nInstalled Runtime Training\nUse these enabled skills only when they are relevant. The task instruction remains authoritative.\n"]
-    for skill in selected:
+    for idx, skill in enumerate(selected):
         path = Path(str(skill.get("path")))
-        skill_text = (path / "skill.md").read_text(errors="replace") if (path / "skill.md").exists() else ""
+        runtime_text, runtime_meta = _runtime_prompt_context(path)
+        skill = dict(skill)
+        skill.update(runtime_meta)
+        if runtime_text:
+            skill_text = runtime_text
+        else:
+            skill_text = (path / "skill.md").read_text(errors="replace") if (path / "skill.md").exists() else ""
         sections.append(
             f"\n## {skill.get('title') or skill.get('installed_skill_id')}\n"
             f"installed_skill_id: {skill.get('installed_skill_id')}\n"
-            f"selection_reason: matched runtime training against the task request.\n\n"
+            f"selection_reason: {skill.get('runtime_training_selection_reason') or 'matched runtime training against the task request'}.\n\n"
             f"{skill_text[:5000]}\n"
         )
+        selected[idx] = skill
     sections.append("---\n")
     return selected, "\n".join(sections)
 
@@ -884,11 +975,117 @@ def installed_skill_run_refs(skills: list[dict[str, Any]]) -> list[dict[str, Any
                 "title": skill.get("title"),
                 "source_item_id": skill.get("source_item_id"),
                 "source_package_id": skill.get("source_package_id"),
+                "runtime_training_injected_sections": skill.get("runtime_training_injected_sections") or [],
+                "runtime_training_context_char_count": skill.get("runtime_training_context_char_count") or 0,
+                "runtime_training_content_hash": skill.get("runtime_training_content_hash"),
+                "runtime_training_selection_reason": skill.get("runtime_training_selection_reason"),
+                "transfer_confidence": skill.get("transfer_confidence"),
+                "runtime_training_confidence": skill.get("runtime_training_confidence") or skill.get("transfer_confidence"),
+                "low_confidence_reasons": skill.get("low_confidence_reasons") or [],
+                "source_quality_gate": skill.get("source_quality_gate"),
+                "selected_transfer_runbook_path": skill.get("selected_transfer_runbook_path"),
+                "selected_transfer_checklist_path": skill.get("selected_transfer_checklist_path"),
+                "injected_minimal_prompt_context": bool(skill.get("injected_minimal_prompt_context")),
+                "injected_output_contract": bool(skill.get("injected_output_contract")),
+                "injected_artifact_contract": bool(skill.get("injected_artifact_contract")),
+                "injected_prefinal_checklist": bool(skill.get("injected_prefinal_checklist")),
+                "prefinal_verification_passed": skill.get("prefinal_verification_passed"),
+                "repair_loop_count": skill.get("repair_loop_count", 0),
+                "contract_failures_json": skill.get("contract_failures_json") or [],
+                "runtime_training_injection_mode": skill.get("runtime_training_injection_mode"),
             }
         )
     return refs
 
 
+def _jsonl_preview(path: Path, limit: int = 8) -> str:
+    if not path.exists() or not path.is_file():
+        return ""
+    lines: list[str] = []
+    for line in path.read_text(errors="replace").splitlines():
+        line = line.strip()
+        if line:
+            lines.append(line)
+        if len(lines) >= limit:
+            break
+    return "\n".join(lines)
+
+
+def _runtime_prompt_context(skill_root: Path) -> tuple[str, dict[str, Any]]:
+    """Return compact runtime-training context that is actually injected."""
+    runtime = skill_root / "runtime_learning_package"
+    injection_mode = (os.getenv("AA_RUNTIME_TRAINING_INJECTION_MODE") or "contract").strip().lower()
+    if injection_mode == "full":
+        ordered_files = [
+            ("minimal_prompt_context.md", "minimal_prompt_context"),
+            ("transfer_runbook.md", "transfer_runbook"),
+            ("transfer_checklist.json", "transfer_checklist"),
+            ("tool_execution_recipe.json", "tool_execution_recipe"),
+            ("output_verification_steps.jsonl", "output_verification_steps"),
+            ("common_errors_to_avoid.jsonl", "common_errors_to_avoid"),
+            ("runtime_instructions.md", "runtime_instructions"),
+        ]
+    else:
+        ordered_files = [
+            ("minimal_prompt_context.md", "minimal_prompt_context"),
+            ("output_contract.json", "output_contract"),
+            ("artifact_contract.json", "artifact_contract"),
+            ("prefinal_verification_checklist.json", "prefinal_verification_checklist"),
+        ]
+    parts: list[str] = []
+    injected_sections: list[str] = []
+    selected_runbook = None
+    selected_checklist = None
+    for rel, section in ordered_files:
+        path = runtime / rel
+        if not path.exists() or not path.is_file():
+            continue
+        if rel.endswith(".jsonl"):
+            text = _jsonl_preview(path)
+        elif rel.endswith(".json"):
+            try:
+                payload = read_json(path)
+                text = json.dumps(payload, indent=2, sort_keys=True)
+            except Exception:
+                text = path.read_text(errors="replace")
+        else:
+            text = path.read_text(errors="replace")
+        text = text.strip()
+        if not text:
+            continue
+        if rel == "transfer_runbook.md":
+            selected_runbook = f"runtime_learning_package/{rel}"
+        if rel == "transfer_checklist.json":
+            selected_checklist = f"runtime_learning_package/{rel}"
+        injected_sections.append(section)
+        section_limit = 2200 if injection_mode == "full" else 950
+        parts.append(f"### {section.replace('_', ' ').title()}\n{text[:section_limit]}")
+    if selected_runbook is None and (runtime / "transfer_runbook.md").exists():
+        selected_runbook = "runtime_learning_package/transfer_runbook.md"
+    if selected_checklist is None:
+        if (runtime / "prefinal_verification_checklist.json").exists():
+            selected_checklist = "runtime_learning_package/prefinal_verification_checklist.json"
+        elif (runtime / "transfer_checklist.json").exists():
+            selected_checklist = "runtime_learning_package/transfer_checklist.json"
+    combined = "\n\n".join(parts)
+    digest = hashlib.sha256(combined.encode()).hexdigest()[:16] if combined else None
+    return combined, {
+        "runtime_training_injected_sections": injected_sections,
+        "runtime_training_context_char_count": len(combined),
+        "runtime_training_content_hash": digest,
+        "selected_transfer_runbook_path": selected_runbook,
+        "selected_transfer_checklist_path": selected_checklist,
+        "injected_minimal_prompt_context": "minimal_prompt_context" in injected_sections,
+        "injected_output_contract": "output_contract" in injected_sections,
+        "injected_artifact_contract": "artifact_contract" in injected_sections,
+        "injected_prefinal_checklist": "prefinal_verification_checklist" in injected_sections,
+        "prefinal_verification_passed": None,
+        "repair_loop_count": 0,
+        "contract_failures_json": [],
+        "runtime_training_injection_mode": injection_mode,
+    }
+
+
 def write_before_after_result(pack_path: Path, result: dict[str, Any]) -> Path:
     result = dict(result)
     result["created_at"] = utc_now()

package/src/agent_apprenticeship_trace/public_run.py

@@ -52,7 +52,7 @@ def _write_failed_tdo_reports(run_root: Path, settings: Settings, error: str) ->
         }
         write_json(tdo_dir / "tdo_report.json", report)
         write_json(tdo_dir / "tdo_manifest.json", {
-            "schema_version": "aa-tdo-v0.1",
+            "schema_version": "aa-experience-compiler-v0.2",
             "tdo_id": report["tdo_id"],
             "tdo_status": "failed",
             "source_package_id": pkg.name,
@@ -73,7 +73,7 @@ def _write_failed_tdo_reports(run_root: Path, settings: Settings, error: str) ->
             "created_at": report["created_at"],
         })
         write_json(compiler_dir / "compiler_manifest.json", {
-            "schema_version": "aa-experience-compiler-v0.1",
+            "schema_version": "aa-experience-compiler-v0.2",
             "compiler_status": "failed",
             "source_package_id": pkg.name,
             "compiler_output_refs": ["experience_compiler/compiler_report.json", "experience_compiler/compiler_manifest.json"],
@@ -994,11 +994,39 @@ def _runtime_training_session_fields(runtime_training_refs: list[dict] | None) -
             "installed_skill_refs": [],
             "installed_skill_ids": [],
         }
+    injected_sections: list[str] = []
+    confidence_values: list[str] = []
+    selection_reasons: list[str] = []
+    total_chars = 0
+    content_hashes: list[str] = []
+    for ref in refs:
+        for section in ref.get("runtime_training_injected_sections") or []:
+            if section not in injected_sections:
+                injected_sections.append(section)
+        source_gate = ref.get("source_quality_gate") if isinstance(ref.get("source_quality_gate"), dict) else {}
+        confidence = ref.get("runtime_training_confidence") or ref.get("transfer_confidence") or source_gate.get("transfer_confidence")
+        if confidence:
+            confidence_values.append(str(confidence))
+        reason = ref.get("runtime_training_selection_reason") or ref.get("selection_reason")
+        if reason:
+            selection_reasons.append(str(reason))
+        total_chars += int(ref.get("runtime_training_context_char_count") or 0)
+        if ref.get("runtime_training_content_hash"):
+            content_hashes.append(str(ref.get("runtime_training_content_hash")))
     return {
         "runtime_training_used": True,
         "installed_skill_refs": refs,
         "installed_skill_ids": [ref.get("installed_skill_id") for ref in refs if ref.get("installed_skill_id")],
-        "runtime_training_selection_reason": "enabled installed runtime training matched this task or was explicitly requested",
+        "runtime_training_selection_reason": "; ".join(selection_reasons) or "enabled installed runtime training matched this task or was explicitly requested",
+        "runtime_training_confidence": "low" if "low" in confidence_values else "high" if confidence_values else "unknown",
+        "runtime_training_injected_sections": injected_sections,
+        "runtime_training_injected_char_count": total_chars,
+        "runtime_training_content_hashes": content_hashes,
+        "output_contract_used": "output_contract" in injected_sections,
+        "artifact_contract_used": "artifact_contract" in injected_sections,
+        "prefinal_verification_status": "not_run",
+        "repair_loop_count": max((int(ref.get("repair_loop_count") or 0) for ref in refs), default=0),
+        "contract_failures_json": [failure for ref in refs for failure in (ref.get("contract_failures_json") or [])],
     }
 
 

package/src/agent_apprenticeship_trace/public_sanitizer.py

@@ -56,9 +56,9 @@ PUBLIC_TEXT_REPLACEMENTS=[
     ("T" + "DO", "Experience Compiler"),
     ("Honest " + "limitations", "Generation notes"),
     ("honest " + "limitations", "generation notes"),
-    ("Limitations", "Generation Notes"),
-    ("limitations", "generation notes"),
-    ("limitation", "generation note"),
+    ("Limit" + "ations", "Generation Notes"),
+    ("limit" + "ations", "generation notes"),
+    ("limit" + "ation", "generation note"),
     ("Applic" + "ability", "Best Use"),
     ("applic" + "ability", "fit"),
     ("Required " + "Conditions", "Required Inputs"),