okstra 0.1.0

package/runtime/validators/validate-run.py

@@ -0,0 +1,568 @@
+#!/usr/bin/env python3
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+TERMINAL_STATUSES = {"completed", "timeout", "error", "not-run"}
+ATTEMPTED_STATUSES = {"completed", "timeout", "error"}
+
+
+def utc_now() -> str:
+    return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def load_json(path: Path) -> dict:
+    try:
+        return json.loads(path.read_text())
+    except FileNotFoundError:
+        raise
+    except Exception as exc:
+        raise ValueError(f"failed to parse JSON: {path}") from exc
+
+
+def write_json(path: Path, payload: dict) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n")
+
+
+def default_next_phase(task_type: str) -> str:
+    mapping = {
+        "requirements-discovery": "pending-routing-decision",
+        "error-analysis": "implementation-planning",
+        "implementation-planning": "implementation",
+        "final-verification": "done-or-follow-up",
+    }
+    return mapping.get(task_type, "unknown")
+
+
+def update_workflow_metadata(
+    run_manifest: dict,
+    task_manifest: dict,
+    validation_status: str,
+) -> None:
+    workflow = task_manifest.get("workflow", {})
+    if not isinstance(workflow, dict):
+        workflow = {}
+
+    current_phase = (
+        workflow.get("currentPhase")
+        or task_manifest.get("taskType")
+        or run_manifest.get("taskType")
+        or ""
+    )
+    phase_sequence = workflow.get("phaseSequence", [])
+    if not isinstance(phase_sequence, list) or not phase_sequence:
+        phase_sequence = [
+            "requirements-discovery",
+            "error-analysis",
+            "implementation-planning",
+            "implementation",
+            "final-verification",
+        ]
+
+    phase_states = workflow.get("phaseStates", {})
+    if not isinstance(phase_states, dict):
+        phase_states = {}
+    for phase in phase_sequence:
+        phase_states.setdefault(phase, "not-started")
+
+    if validation_status == "passed":
+        current_phase_state = "completed"
+        if current_phase:
+            phase_states[current_phase] = current_phase_state
+        last_completed_phase = current_phase or workflow.get("lastCompletedPhase", "")
+        next_recommended_phase = (
+            workflow.get("nextRecommendedPhase") or default_next_phase(current_phase)
+        )
+    else:
+        current_phase_state = "blocked"
+        if current_phase:
+            phase_states[current_phase] = current_phase_state
+        last_completed_phase = workflow.get("lastCompletedPhase", "")
+        next_recommended_phase = current_phase or workflow.get(
+            "nextRecommendedPhase", ""
+        )
+
+    routing_status = workflow.get("routingStatus")
+    if not isinstance(routing_status, str) or not routing_status.strip():
+        routing_status = (
+            "pending" if current_phase == "requirements-discovery" else "not-applicable"
+        )
+
+    awaiting_approval = workflow.get("awaitingApproval")
+    if not isinstance(awaiting_approval, bool):
+        awaiting_approval = False
+
+    last_safe_checkpoint = workflow.get("lastSafeCheckpoint", {})
+    if not isinstance(last_safe_checkpoint, dict):
+        last_safe_checkpoint = {}
+    last_safe_checkpoint.update(
+        {
+            "label": (
+                "validation-passed"
+                if validation_status == "passed"
+                else "validation-failed"
+            ),
+            "taskManifestPath": task_manifest.get(
+                "taskManifestPath", ""
+            ),
+            "taskIndexPath": task_manifest.get(
+                "taskIndexPath", ""
+            ),
+            "latestRunPath": task_manifest.get(
+                "latestRunPath", ""
+            ),
+            "latestRunManifestPath": run_manifest.get(
+                "runManifestPath", ""
+            ),
+            "latestTeamStatePath": run_manifest.get(
+                "teamStatePath", ""
+            ),
+            "latestReportPath": task_manifest.get(
+                "latestReportPath", ""
+            ),
+            "latestResumeCommandPath": task_manifest.get(
+                "latestResumeCommandPath", ""
+            ),
+        }
+    )
+
+    workflow.update(
+        {
+            "phaseSequence": phase_sequence,
+            "currentPhase": current_phase,
+            "currentPhaseState": current_phase_state,
+            "phaseStates": phase_states,
+            "lastCompletedPhase": last_completed_phase,
+            "nextRecommendedPhase": next_recommended_phase,
+            "awaitingApproval": awaiting_approval,
+            "routingStatus": routing_status,
+            "lastSafeCheckpoint": last_safe_checkpoint,
+        }
+    )
+    task_manifest["workflow"] = workflow
+
+    workflow_snapshot = run_manifest.get("workflowSnapshot", {})
+    if not isinstance(workflow_snapshot, dict):
+        workflow_snapshot = {}
+    workflow_snapshot.update(
+        {
+            "phaseSequence": workflow["phaseSequence"],
+            "currentPhase": workflow["currentPhase"],
+            "currentPhaseState": workflow["currentPhaseState"],
+            "phaseStates": workflow["phaseStates"],
+            "lastCompletedPhase": workflow["lastCompletedPhase"],
+            "nextRecommendedPhase": workflow["nextRecommendedPhase"],
+            "awaitingApproval": workflow["awaitingApproval"],
+            "routingStatus": workflow["routingStatus"],
+            "lastSafeCheckpoint": workflow["lastSafeCheckpoint"],
+        }
+    )
+    run_manifest["workflowSnapshot"] = workflow_snapshot
+
+
+def update_validation_metadata(
+    team_state: dict,
+    run_manifest: dict,
+    task_manifest: dict,
+    validation_status: str,
+    failures: list[str],
+) -> None:
+    checked_at = utc_now()
+
+    team_state.setdefault("validator", {})
+    team_state["validator"]["status"] = validation_status
+    team_state["validator"]["lastValidatedAt"] = checked_at
+    team_state["validator"]["failures"] = failures
+
+    run_manifest.setdefault("validation", {})
+    run_manifest["validation"]["required"] = True
+    run_manifest["validation"]["status"] = validation_status
+    run_manifest["validation"]["lastCheckedAt"] = checked_at
+    run_manifest["validation"]["passed"] = validation_status == "passed"
+    run_manifest["validation"]["failures"] = failures
+    run_manifest["status"] = (
+        "completed" if validation_status == "passed" else "contract-violated"
+    )
+
+    task_manifest.setdefault("contractValidation", {})
+    task_manifest["contractValidation"]["required"] = True
+    task_manifest["contractValidation"]["status"] = validation_status
+    task_manifest["contractValidation"]["lastCheckedAt"] = checked_at
+    task_manifest["contractValidation"]["passed"] = validation_status == "passed"
+    task_manifest["contractValidation"]["failures"] = failures
+    task_manifest["latestRunStatus"] = run_manifest["status"]
+    task_manifest["currentStatus"] = (
+        "completed" if validation_status == "passed" else "contract-violated"
+    )
+    update_workflow_metadata(run_manifest, task_manifest, validation_status)
+
+
+def extract_contract(
+    run_manifest: dict, task_manifest: dict, failures: list[str]
+) -> dict:
+    run_contract = run_manifest.get("teamContract")
+    task_contract = task_manifest.get("resultContract")
+
+    if not isinstance(run_contract, dict):
+        run_contract = {}
+    if not isinstance(task_contract, dict):
+        task_contract = {}
+
+    required_worker_roles = run_contract.get("requiredWorkerRoles")
+    if not isinstance(required_worker_roles, list):
+        required_worker_roles = task_contract.get("requiredWorkerRoles")
+    if not isinstance(required_worker_roles, list):
+        required_worker_roles = []
+        failures.append("requiredWorkerRoles is missing from run/task manifest")
+
+    required_agent_status_entries = run_contract.get("requiredAgentStatusEntries")
+    if not isinstance(required_agent_status_entries, list):
+        required_agent_status_entries = task_contract.get("requiredAgentStatusEntries")
+    if not isinstance(required_agent_status_entries, list):
+        required_agent_status_entries = ["Claude lead"] + [
+            item.get("role", "")
+            for item in required_worker_roles
+            if isinstance(item, dict) and item.get("role")
+        ]
+
+    return {
+        "lead_role": run_contract.get("leadRole")
+        or task_contract.get("leadRole")
+        or "Claude lead",
+        "lead_agent": run_contract.get("leadAgent")
+        or task_contract.get("leadAgent")
+        or "claude",
+        "lead_model": run_contract.get("leadModel")
+        or task_contract.get("leadModel")
+        or "",
+        "lead_model_execution_value": (
+            run_contract.get("leadModelExecutionValue")
+            or task_contract.get("leadModelExecutionValue")
+            or ""
+        ),
+        "required_worker_roles": required_worker_roles,
+        "required_agent_status_entries": [
+            item
+            for item in required_agent_status_entries
+            if isinstance(item, str) and item.strip()
+        ],
+    }
+
+
+def validate_team_state(
+    team_state: dict, project_root: Path, contract: dict, failures: list[str]
+) -> None:
+    artifacts = team_state.get("artifacts")
+    if not isinstance(artifacts, dict):
+        failures.append("team-state.artifacts must be an object")
+    elif not str(artifacts.get("workerPromptsDirectoryPath", "")).strip():
+        failures.append(
+            "team-state.artifacts.workerPromptsDirectoryPath is missing"
+        )
+    lead = team_state.get("lead")
+    if not isinstance(lead, dict):
+        failures.append("team-state.lead is missing")
+    else:
+        if lead.get("role") != contract["lead_role"]:
+            failures.append(f"team-state.lead.role must be `{contract['lead_role']}`")
+        if lead.get("agent") != contract["lead_agent"]:
+            failures.append(f"team-state.lead.agent must be `{contract['lead_agent']}`")
+        expected_lead_model = contract.get("lead_model")
+        if expected_lead_model and lead.get("model") != expected_lead_model:
+            failures.append(f"team-state.lead.model must be `{expected_lead_model}`")
+        expected_lead_model_execution_value = contract.get("lead_model_execution_value")
+        if (
+            expected_lead_model_execution_value
+            and lead.get("modelExecutionValue") != expected_lead_model_execution_value
+        ):
+            failures.append(
+                "team-state.lead.modelExecutionValue must be "
+                f"`{expected_lead_model_execution_value}`"
+            )
+
+    workers = team_state.get("workers")
+    if not isinstance(workers, list):
+        failures.append("team-state.workers must be a list")
+        return
+
+    by_role: dict[str, dict] = {}
+    for worker in workers:
+        if not isinstance(worker, dict):
+            failures.append("team-state.workers contains a non-object entry")
+            continue
+        role = str(worker.get("role", "")).strip()
+        if not role:
+            failures.append("team-state.workers contains an entry without role")
+            continue
+        if role in by_role:
+            failures.append(f"duplicate worker role detected: {role}")
+            continue
+        by_role[role] = worker
+
+    expected_workers: dict[str, dict] = {}
+    for worker in contract["required_worker_roles"]:
+        if not isinstance(worker, dict):
+            failures.append("requiredWorkerRoles contains a non-object entry")
+            continue
+        role = str(worker.get("role", "")).strip()
+        if not role:
+            failures.append("requiredWorkerRoles contains an entry without role")
+            continue
+        expected_workers[role] = worker
+
+    for role, expected in expected_workers.items():
+        worker = by_role.get(role)
+        if worker is None:
+            failures.append(f"missing required worker role: {role}")
+            continue
+
+        expected_worker_id = expected.get("workerId")
+        if expected_worker_id and worker.get("workerId") != expected_worker_id:
+            failures.append(f"{role} must use workerId `{expected_worker_id}`")
+
+        expected_agent = expected.get("agent")
+        if expected_agent and worker.get("agent") != expected_agent:
+            failures.append(f"{role} must use agent `{expected_agent}`")
+
+        expected_model = expected.get("model")
+        if expected_model and worker.get("model") != expected_model:
+            failures.append(f"{role} must use model `{expected_model}`")
+
+        expected_model_execution_value = expected.get("modelExecutionValue")
+        if (
+            expected_model_execution_value
+            and worker.get("modelExecutionValue") != expected_model_execution_value
+        ):
+            failures.append(
+                f"{role} must use modelExecutionValue `{expected_model_execution_value}`"
+            )
+
+        expected_result_relative = expected.get("resultPath")
+        result_relative = worker.get("resultPath", "")
+        if expected_result_relative and result_relative != expected_result_relative:
+            failures.append(
+                f"{role} must use resultPath `{expected_result_relative}`"
+            )
+        expected_prompt_relative = expected.get("promptPath")
+        prompt_relative = worker.get("promptPath", "")
+        if expected_prompt_relative and prompt_relative != expected_prompt_relative:
+            failures.append(
+                f"{role} must use promptPath `{expected_prompt_relative}`"
+            )
+
+        status = worker.get("status")
+        if status not in TERMINAL_STATUSES:
+            failures.append(f"{role} has invalid terminal status: {status}")
+            continue
+
+        reason = str(worker.get("reason", "")).strip()
+        prompt_exists = bool(prompt_relative) and (project_root / prompt_relative).exists()
+        result_exists = (
+            bool(result_relative) and (project_root / result_relative).exists()
+        )
+        if status in ATTEMPTED_STATUSES and not prompt_relative:
+            failures.append(
+                f"{role} with status `{status}` must include promptPath"
+            )
+        if status in ATTEMPTED_STATUSES and prompt_relative and not prompt_exists:
+            failures.append(
+                f"{role} with status `{status}` is missing worker prompt history file: {prompt_relative}"
+            )
+
+        if status == "completed" and not result_exists:
+            failures.append(
+                f"{role} is completed but worker result file is missing: {result_relative}"
+            )
+        if status != "completed" and not reason:
+            failures.append(f"{role} with status `{status}` must include a reason")
+
+    unexpected_roles = set(by_role) - set(expected_workers)
+    for role in sorted(unexpected_roles):
+        failures.append(f"unexpected worker role detected: {role}")
+
+    generic_roles = [
+        role
+        for role in by_role
+        if "generic" in role.lower() or "parallel worker" in role.lower()
+    ]
+    for role in generic_roles:
+        failures.append(f"generic worker role is not allowed: {role}")
+
+
+TOKEN_PLACEHOLDERS = (
+    "{{LEAD_TOTAL_TOKENS}}",
+    "{{LEAD_BILLABLE_TOKENS}}",
+    "{{LEAD_COST_USD}}",
+    "{{WORKER_TOTAL_TOKENS}}",
+    "{{WORKER_BILLABLE_TOKENS}}",
+    "{{WORKER_COST_USD}}",
+    "{{GRAND_TOTAL_TOKENS}}",
+    "{{GRAND_BILLABLE_TOKENS}}",
+    "{{GRAND_COST_USD}}",
+    "{{CLI_COST_USD}}",
+)
+
+
+def validate_report(
+    report_path: Path, required_agent_status_entries: list[str], failures: list[str]
+) -> None:
+    if not report_path.exists():
+        failures.append(f"final report is missing: {report_path}")
+        return
+
+    content = report_path.read_text()
+    for label in required_agent_status_entries:
+        if label not in content:
+            failures.append(
+                f"final report does not include required agent status entry: {label}"
+            )
+
+    for placeholder in TOKEN_PLACEHOLDERS:
+        if placeholder in content:
+            failures.append(
+                f"final report contains unsubstituted token placeholder `{placeholder}` — "
+                "run `okstra-token-usage.py ... --substitute-final-report <report-path>` during Phase 7"
+            )
+
+
+def validate_team_state_usage(team_state: dict, failures: list[str]) -> None:
+    summary = team_state.get("usageSummary") or {}
+    if not summary or not summary.get("collectedAt"):
+        failures.append(
+            "team-state.usageSummary is empty — Phase 7 token-usage collection was skipped. "
+            "Run `python3 scripts/okstra-token-usage.py <team-state> --write --summary "
+            "--substitute-final-report <final-report>`."
+        )
+
+
+PLANNING_REQUIRED_SECTIONS = (
+    "Option Candidates",
+    "Trade-off",
+    "Recommended Option",
+    "Stepwise Execution Order",
+    "Dependency",
+    "Validation Checklist",
+    "Rollback",
+    "User Approval Request",
+)
+
+
+def validate_phase_boundary(
+    task_type: str,
+    report_path: Path,
+    failures: list[str],
+) -> None:
+    """Phase-specific contract checks.
+
+    For `implementation-planning` runs, the final report must contain the
+    required deliverable sections; absence indicates a planning run that
+    skipped its core outputs (or an implementation run that ran under the
+    wrong task type).
+    """
+    if task_type != "implementation-planning":
+        return
+    if not report_path.exists():
+        return
+    content = report_path.read_text()
+    for needle in PLANNING_REQUIRED_SECTIONS:
+        if needle not in content:
+            failures.append(
+                "implementation-planning report is missing required section: "
+                f"`{needle}`"
+            )
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="Validate okstra run contract artifacts."
+    )
+    parser.add_argument(
+        "--team-state",
+        required=True,
+        help="Project-relative or absolute path to the team state JSON.",
+    )
+    parser.add_argument(
+        "--report",
+        required=True,
+        help="Project-relative or absolute path to the final report Markdown file.",
+    )
+    parser.add_argument(
+        "--run-manifest",
+        required=True,
+        help="Project-relative or absolute path to the run manifest JSON.",
+    )
+    parser.add_argument(
+        "--task-manifest",
+        required=True,
+        help="Project-relative or absolute path to the task manifest JSON.",
+    )
+    parser.add_argument(
+        "--final-status", required=False, help="Optional final status file to write."
+    )
+    args = parser.parse_args()
+
+    run_manifest_path = Path(args.run_manifest).resolve()
+    run_manifest = load_json(run_manifest_path)
+    task_manifest_path = Path(args.task_manifest).resolve()
+    task_manifest = load_json(task_manifest_path)
+
+    project_root_raw = str(task_manifest.get("projectRoot") or "").strip()
+    if not project_root_raw:
+        raise ValueError("projectRoot is missing from task manifest")
+    project_root = Path(project_root_raw)
+
+    def resolve_input(raw_path: str) -> Path:
+        path = Path(raw_path)
+        if path.is_absolute():
+            return path
+        return (project_root / raw_path).resolve()
+
+    team_state_path = resolve_input(args.team_state)
+    report_path = resolve_input(args.report)
+    team_state = load_json(team_state_path)
+
+    failures: list[str] = []
+    contract = extract_contract(run_manifest, task_manifest, failures)
+    validate_team_state(team_state, project_root, contract, failures)
+    validate_report(report_path, contract["required_agent_status_entries"], failures)
+    validate_team_state_usage(team_state, failures)
+
+    task_type = str(task_manifest.get("taskType") or run_manifest.get("taskType") or "").strip()
+    validate_phase_boundary(task_type, report_path, failures)
+
+    validation_status = "passed" if not failures else "failed"
+    update_validation_metadata(
+        team_state, run_manifest, task_manifest, validation_status, failures
+    )
+
+    write_json(team_state_path, team_state)
+    write_json(run_manifest_path, run_manifest)
+    write_json(task_manifest_path, task_manifest)
+
+    if args.final_status:
+        final_status_path = resolve_input(args.final_status)
+        final_status_path.parent.mkdir(parents=True, exist_ok=True)
+        final_status_path.write_text(
+            ("completed" if validation_status == "passed" else "contract-violated")
+            + "\n"
+        )
+
+    result = {
+        "validationStatus": validation_status,
+        "finalRunStatus": run_manifest.get("status"),
+        "failures": failures,
+        "teamStatePath": str(team_state_path),
+        "reportPath": str(report_path),
+    }
+    print(json.dumps(result, ensure_ascii=False, indent=2))
+    return 0 if validation_status == "passed" else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())