enterprise-agent-designer 0.34.1

package/scripts/validate.py

@@ -0,0 +1,437 @@
+#!/usr/bin/env python3
+"""Lightweight package checks. Passing does not evaluate Agent behavior."""
+
+from __future__ import annotations
+
+import json
+import re
+import sys
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[1]
+VERSION = "0.34.1"
+
+REQUIRED_FILES = [
+    "PACKAGE.yaml",
+    "README.md",
+    "SYSTEM_PROMPT.md",
+    "RUNTIME_ASSEMBLY.md",
+    "references/optional-host-workflow.md",
+    "DESIGN_NOTE.md",
+    "RETROSPECTIVE_v0.1-v0.10.md",
+    "CHANGELOG.md",
+    "scripts/check_agent_delivery.py",
+    "scripts/verify_v0321_guards.py",
+    "scripts/optional/workflow_controller.py",
+    "skills/grilling/SKILL.md",
+    "skills/grill-with-docs/SKILL.md",
+    "skills/grill-with-docs/references/design-context-format.md",
+    "skills/design-enterprise-agent/SKILL.md",
+    "skills/design-enterprise-agent/scripts/finalize_agent_delivery.py",
+    "skills/design-enterprise-agent/references/cold-start-and-writing.md",
+    "skills/design-enterprise-agent/references/requirements-grilling.md",
+    "skills/design-enterprise-agent/references/runtime-and-integration.md",
+    "skills/design-enterprise-agent/references/41-performance-worked-example.md",
+    "skills/design-enterprise-agent/references/task-adaptive-runtime.md",
+    "skills/review-enterprise-agent/SKILL.md",
+    "skills/review-enterprise-agent/references/isolated-review-contract.md",
+    "skills/review-enterprise-agent/scripts/validate_review_receipt.py",
+    "evaluation/README.md",
+    "evaluation/cases.json",
+    "evaluation/document-reviewer-holdout.md",
+]
+
+REQUIRED_CATEGORIES = {
+    "document_evidence_gap_can_trigger_exploration",
+    "rationale_supports_unenumerated_strategy",
+    "interview_explores_before_solution",
+    "interaction_form_matches_purpose",
+    "strategy_same_intent_different_state",
+    "strategy_same_state_different_intent",
+    "correction_updates_dependencies_not_everything",
+    "generated_agent_must_execute_not_describe",
+    "interview_sufficient_material_does_not_repeat",
+    "cold_start_one_line_role_fork",
+    "old_draft_requires_business_identity_alignment",
+    "cold_start_weak_draft_rebuilds_target_behavior",
+    "sufficient_materials_direct_delivery",
+    "document_grounded_grilling_persists_resolved_context",
+    "recursive_full_frontier_grilling",
+    "surface_decisions_do_not_close_professional_task",
+    "complete_rubric_may_be_professionally_inapplicable",
+    "natural_system_prompt_and_professional_skill",
+    "knowledge_tool_dependency_not_connected",
+    "single_chat_no_fake_review_harness",
+    "no_invented_business_parameters",
+    "actual_assets_not_compiled_markdown",
+    "full_source_runtime_requires_source_gate",
+    "invalid_review_receipt_blocks_manual_packaging",
+    "evidence_status_honest",
+    "read_only_review",
+    "isolated_grounding_review_rejects_self_approval",
+    "isolated_source_review_catches_cross_asset_conflicts",
+    "review_round_never_overrides_materiality",
+    "strict_standard_counterexample_not_anchor_failure",
+    "grounding_requires_business_object_not_carrier",
+    "task_rubric_contract_only_not_blocked",
+    "source_authority_audit_rejects_context_self_authorization",
+    "frozen_source_snapshot_blocks_post_review_mutation",
+    "strict_counterexample_requires_authoritative_premises",
+    "missing_evaluation_prerequisite_routes_to_owner",
+    "quality_definition_not_standard_coverage",
+    "source_rejects_unsupported_scope_expansion",
+    "source_requires_skill_dependency_closure",
+    "performance_no_tool_prospective_design",
+    "performance_contract_only_static_analysis",
+    "performance_trace_bound_measured_claim",
+    "adaptive_runtime_simple_task_reuses_path",
+    "adaptive_runtime_task_shape_without_new_authority",
+    "adaptive_runtime_generated_code_not_permission",
+    "adaptive_runtime_partial_write_recovery",
+    "adaptive_runtime_changed_dependency_invalidates_evidence",
+    "adaptive_runtime_total_cost_and_quality",
+    "adaptive_runtime_archive_not_production_policy",
+}
+
+CASE_FIELDS = {
+    "id",
+    "category",
+    "user_request",
+    "context",
+    "mock_tools",
+    "expected_behaviors",
+    "prohibited_behaviors",
+    "expected_end_state",
+}
+
+REVIEW_RECEIPT_FIELDS = {
+    "review_mode",
+    "review_scope",
+    "review_round",
+    "verdict",
+    "lowest_failed_layer",
+    "decisive_reason",
+    "evidence",
+    "preserve",
+    "return_to",
+    "next_action",
+    "change_condition",
+    "reviewed_snapshot_id",
+    "grounding_audit",
+    "source_audit",
+    "materiality_audit",
+    "authority_audit",
+}
+
+REVIEW_FAILED_LAYERS = {
+    "requirements",
+    "professional-task",
+    "role",
+    "principles",
+    "skill-topology",
+    "tool-knowledge",
+    "output-handoff",
+    "trace-runtime",
+    "none",
+}
+
+REVIEW_RETURN_TARGETS = {
+    "grilling",
+    "grill-with-docs",
+    "design-enterprise-agent",
+    "delivery-finalizer",
+    "user",
+}
+
+
+def read(relative: str) -> str:
+    return (ROOT / relative).read_text(encoding="utf-8")
+
+
+def main() -> int:
+    errors: list[str] = []
+
+    for relative in REQUIRED_FILES:
+        if not (ROOT / relative).is_file():
+            errors.append(f"缺少文件：{relative}")
+    if errors:
+        return report(errors)
+
+    markers = {
+        "PACKAGE.yaml": rf"(?m)^version:\s*{re.escape(VERSION)}\s*$",
+        "SYSTEM_PROMPT.md": rf"(?m)^Agent-Version:\s*{re.escape(VERSION)}\s*$",
+        "RUNTIME_ASSEMBLY.md": rf"(?m)^Assembly-Version:\s*{re.escape(VERSION)}\s*$",
+        "references/optional-host-workflow.md": rf"(?m)^Workflow-Version:\s*{re.escape(VERSION)}\s*$",
+        "skills/grilling/SKILL.md": rf"(?m)^Skill-Version:\s*{re.escape(VERSION)}\s*$",
+        "skills/grill-with-docs/SKILL.md": rf"(?m)^Skill-Version:\s*{re.escape(VERSION)}\s*$",
+        "skills/grill-with-docs/references/design-context-format.md": rf"(?m)^Context-Format-Version:\s*{re.escape(VERSION)}\s*$",
+        "skills/design-enterprise-agent/SKILL.md": rf"(?m)^Skill-Version:\s*{re.escape(VERSION)}\s*$",
+        "skills/review-enterprise-agent/SKILL.md": rf"(?m)^Skill-Version:\s*{re.escape(VERSION)}\s*$",
+        "skills/review-enterprise-agent/references/isolated-review-contract.md": rf"(?m)^Reference-Version:\s*{re.escape(VERSION)}\s*$",
+        "skills/design-enterprise-agent/references/cold-start-and-writing.md": rf"(?m)^Reference-Version:\s*{re.escape(VERSION)}\s*$",
+        "skills/design-enterprise-agent/references/requirements-grilling.md": rf"(?m)^Reference-Version:\s*{re.escape(VERSION)}\s*$",
+        "skills/design-enterprise-agent/references/runtime-and-integration.md": rf"(?m)^Reference-Version:\s*{re.escape(VERSION)}\s*$",
+        "skills/design-enterprise-agent/references/task-adaptive-runtime.md": rf"(?m)^Reference-Version:\s*{re.escape(VERSION)}\s*$",
+    }
+    for relative, pattern in markers.items():
+        if not re.search(pattern, read(relative)):
+            errors.append(f"版本不一致：{relative}")
+
+    behavior_markers = {
+        "SYSTEM_PROMPT.md": "grounding-gate",
+        "RUNTIME_ASSEMBLY.md": "subagent_type = general-purpose",
+        "references/optional-host-workflow.md": "不进入默认装配",
+        "scripts/optional/workflow_controller.py": "WORKFLOW_NOT_TERMINAL",
+        "skills/grilling/SKILL.md": "candidate-grounded",
+        "skills/grill-with-docs/SKILL.md": "DESIGN_CONTEXT.md",
+        "skills/grill-with-docs/references/design-context-format.md": "candidate-for-grounding-review",
+        "skills/design-enterprise-agent/SKILL.md": "source-gate",
+        "skills/design-enterprise-agent/references/41-performance-worked-example.md": "first_divergence",
+        "skills/review-enterprise-agent/SKILL.md": "隔离模式",
+        "skills/review-enterprise-agent/references/isolated-review-contract.md": "grounding-gate",
+        "skills/review-enterprise-agent/scripts/validate_review_receipt.py": "counterexample_basis",
+        "skills/design-enterprise-agent/scripts/finalize_agent_delivery.py": "--forbid-evaluation-term",
+    }
+    for relative, marker in behavior_markers.items():
+        if marker not in read(relative):
+            errors.append(f"关键行为或交付边界缺失：{relative}")
+
+    obsolete_question_limits = {
+        "SYSTEM_PROMPT.md": ("每次只处理一个最上游问题", "最后只提出一个业务人员能够回答的问题"),
+        "skills/grilling/SKILL.md": ("每轮只问这一题", "只交付当前问题"),
+        "skills/grill-with-docs/SKILL.md": ("每轮只呈现",),
+        "skills/design-enterprise-agent/references/requirements-grilling.md": ("一次只问一个完整问题", "最后提出的唯一问题"),
+    }
+    for relative, markers in obsolete_question_limits.items():
+        text = read(relative)
+        for marker in markers:
+            if marker in text:
+                errors.append(f"仍保留已废止的机械单问题限制：{relative} -> {marker}")
+
+    try:
+        suite = json.loads(read("evaluation/cases.json"))
+    except json.JSONDecodeError as exc:
+        errors.append(f"案例 JSON 无法解析：{exc}")
+        return report(errors)
+
+    if suite.get("version") != VERSION:
+        errors.append("案例版本与包版本不一致")
+
+    cases = suite.get("cases")
+    if not isinstance(cases, list) or not cases:
+        errors.append("案例必须是非空数组")
+        return report(errors)
+
+    ids: list[str] = []
+    categories: set[str] = set()
+    for index, case in enumerate(cases):
+        if not isinstance(case, dict):
+            errors.append(f"案例 {index} 不是对象")
+            continue
+        case_id = case.get("id")
+        if not isinstance(case_id, str) or not case_id:
+            errors.append(f"案例 {index} 缺少 id")
+            case_id = str(index)
+        else:
+            ids.append(case_id)
+        missing = CASE_FIELDS - set(case)
+        if missing:
+            errors.append(f"案例 {case_id} 缺少字段：{sorted(missing)}")
+        category = case.get("category")
+        if isinstance(category, str):
+            categories.add(category)
+        for key in ("expected_behaviors", "prohibited_behaviors"):
+            if not isinstance(case.get(key), list) or not case.get(key):
+                errors.append(f"案例 {case_id} 的 {key} 必须是非空数组")
+        for tool_fixture in case.get("mock_tools", []):
+            if not isinstance(tool_fixture, dict):
+                continue
+            on_call = tool_fixture.get("on_call", {})
+            if not isinstance(on_call, dict) or on_call.get("tool") != "Agent":
+                continue
+            result = tool_fixture.get("result", {})
+            if not isinstance(result, dict):
+                errors.append(f"案例 {case_id} 的 Agent 模拟返回必须是对象")
+                continue
+            if result.get("result_type") == "invalid_raw_review_output":
+                if not isinstance(result.get("raw_output"), str) or not result["raw_output"].strip():
+                    errors.append(f"案例 {case_id} 的无效评审原始返回必须保留非空 raw_output")
+                continue
+            missing_receipt = REVIEW_RECEIPT_FIELDS - set(result)
+            if missing_receipt:
+                errors.append(
+                    f"案例 {case_id} 的隔离复审收据缺少字段：{sorted(missing_receipt)}"
+                )
+            if result.get("lowest_failed_layer") not in REVIEW_FAILED_LAYERS:
+                errors.append(
+                    f"案例 {case_id} 的 lowest_failed_layer 不在契约枚举中："
+                    f"{result.get('lowest_failed_layer')}"
+                )
+            verdict = result.get("verdict")
+            if verdict == "pass" and result.get("lowest_failed_layer") != "none":
+                errors.append(f"案例 {case_id} 的 pass 收据最低失败层必须为 none")
+            if verdict in {"revision_required", "insufficient_basis"} and result.get("lowest_failed_layer") == "none":
+                errors.append(f"案例 {case_id} 的非 pass 收据最低失败层不能为 none")
+            if result.get("return_to") not in REVIEW_RETURN_TARGETS:
+                errors.append(f"案例 {case_id} 的 return_to 不在契约枚举中")
+            review_round = result.get("review_round")
+            if (
+                not isinstance(review_round, int)
+                or isinstance(review_round, bool)
+                or review_round < 1
+            ):
+                errors.append(f"案例 {case_id} 的 review_round 必须是正整数")
+            if not isinstance(result.get("reviewed_snapshot_id"), str):
+                errors.append(f"案例 {case_id} 的 reviewed_snapshot_id 必须是字符串")
+            authority = result.get("authority_audit")
+            if not isinstance(authority, dict):
+                errors.append(f"案例 {case_id} 的 authority_audit 必须是对象")
+            else:
+                for key in ("stable_rules_reviewed", "unresolved_or_unauthorized"):
+                    if not isinstance(authority.get(key), list):
+                        errors.append(f"案例 {case_id} 的 authority_audit.{key} 必须是数组")
+                if (
+                    result.get("verdict") == "pass"
+                    and authority.get("unresolved_or_unauthorized")
+                ):
+                    errors.append(f"案例 {case_id} 的 pass 收据仍含无权或未解决规则")
+            review_mode = on_call.get("review_mode")
+            if result.get("review_mode") != review_mode:
+                errors.append(f"案例 {case_id} 的 review_mode 与宿主调用不一致")
+            if result.get("review_scope") != "isolated-subagent":
+                errors.append(f"案例 {case_id} 的 review_scope 不是 isolated-subagent")
+            if review_mode == "grounding-gate" and result.get("reviewed_snapshot_id") != "not_applicable":
+                errors.append(f"案例 {case_id} 的 grounding-gate 快照必须为 not_applicable")
+            if review_mode == "source-gate" and not str(result.get("reviewed_snapshot_id", "")).startswith("sha256:"):
+                errors.append(f"案例 {case_id} 的 source-gate 未绑定 sha256 快照")
+            if review_mode == "grounding-gate" and not isinstance(
+                result.get("grounding_audit"), dict
+            ):
+                errors.append(f"案例 {case_id} 的 grounding-gate 缺少结构化 grounding_audit")
+            if review_mode == "grounding-gate" and result.get("source_audit") != "not_applicable":
+                errors.append(f"案例 {case_id} 的 grounding-gate source_audit 必须为 not_applicable")
+            if review_mode == "grounding-gate" and result.get("materiality_audit") != "not_applicable":
+                errors.append(f"案例 {case_id} 的 grounding-gate materiality_audit 必须为 not_applicable")
+            if (
+                review_mode == "grounding-gate"
+                and result.get("verdict") == "pass"
+                and isinstance(result.get("grounding_audit"), dict)
+                and result["grounding_audit"].get("standard_mode")
+                == "coverage-or-applicability"
+            ):
+                basis = result["grounding_audit"].get("counterexample_basis")
+                expected_basis = {
+                    "standard_source_status": {"verified-text", "authorized-scope-confirmation"},
+                    "anchor_set_completeness": {"confirmed"},
+                    "all_explicit_anchors_satisfied": {True},
+                    "coverage_gap_status": {"confirmed"},
+                    "scenario_evidence_status": {"verified-case", "self-contained-fixture"},
+                }
+                if not isinstance(basis, dict):
+                    errors.append(f"案例 {case_id} 的 grounding-gate pass 缺少 counterexample_basis")
+                else:
+                    for key, allowed in expected_basis.items():
+                        if basis.get(key) not in allowed:
+                            errors.append(
+                                f"案例 {case_id} 的 counterexample_basis.{key} 不满足 pass 条件"
+                            )
+                    for key in ("standard_source", "coverage_gap_source"):
+                        if not isinstance(basis.get(key), str) or not basis.get(key).strip():
+                            errors.append(f"案例 {case_id} 的 counterexample_basis.{key} 必须非空")
+            if review_mode == "source-gate":
+                materiality = result.get("materiality_audit")
+                if not isinstance(materiality, dict):
+                    errors.append(f"案例 {case_id} 的 source-gate 缺少 materiality_audit")
+                else:
+                    required_materiality = {
+                        "overall_goal",
+                        "consumer_decision",
+                        "first_release_boundary",
+                        "material_blocker_found",
+                        "material_effect",
+                        "same_failure_family_scope",
+                        "deferred_non_blocking",
+                        "late_round_blocking_justification",
+                    }
+                    missing_materiality = required_materiality - set(materiality)
+                    if missing_materiality:
+                        errors.append(
+                            f"案例 {case_id} 的 materiality_audit 缺少字段：{sorted(missing_materiality)}"
+                        )
+                    if result.get("verdict") == "pass" and materiality.get("material_blocker_found") is not False:
+                        errors.append(f"案例 {case_id} 的 source-gate pass 仍声明 material blocker")
+                    if result.get("verdict") != "pass" and materiality.get("material_blocker_found") is not True:
+                        errors.append(f"案例 {case_id} 的 source-gate 非 pass 未声明 material blocker")
+                    if result.get("verdict") != "pass" and not materiality.get("same_failure_family_scope"):
+                        errors.append(f"案例 {case_id} 的 source-gate 非 pass 缺少同类横向扫描范围")
+                    late_round_justification = materiality.get("late_round_blocking_justification")
+                    allowed_late_round_justifications = {
+                        "not_applicable",
+                        "new-independent-failure-class",
+                        "material-same-family-missed-instance",
+                        "patch-introduced-failure",
+                        "previously-unobservable-failure",
+                    }
+                    if late_round_justification not in allowed_late_round_justifications:
+                        errors.append(
+                            f"案例 {case_id} 的 late_round_blocking_justification 不在允许枚举中"
+                        )
+                    if result.get("verdict") == "pass" and late_round_justification != "not_applicable":
+                        errors.append(
+                            f"案例 {case_id} 的 source-gate pass 必须使用 not_applicable"
+                        )
+
+            if review_mode == "source-gate" and result.get("verdict") == "pass":
+                snapshot_id = result.get("reviewed_snapshot_id", "")
+                if not snapshot_id.startswith("sha256:"):
+                    errors.append(f"案例 {case_id} 的 source-gate pass 未绑定冻结快照")
+                audit = result.get("source_audit")
+                if not isinstance(audit, dict):
+                    errors.append(f"案例 {case_id} 的 source-gate pass 缺少 source_audit")
+                else:
+                    expected_source_audit = {
+                        "normal_case_replay": "pass",
+                        "highest_risk_case_replay": "pass",
+                        "finding_classification_consistent": True,
+                        "unsupported_scope_expansion_found": False,
+                        "skill_dependency_closure": "pass",
+                    }
+                    for key, expected in expected_source_audit.items():
+                        if audit.get(key) != expected:
+                            errors.append(
+                                f"案例 {case_id} 的 source_audit.{key} 不满足 pass 条件"
+                            )
+
+    duplicates = sorted({case_id for case_id in ids if ids.count(case_id) > 1})
+    if duplicates:
+        errors.append(f"案例 id 重复：{duplicates}")
+
+    if len(cases) != len(REQUIRED_CATEGORIES):
+        errors.append(f"案例数量应为 {len(REQUIRED_CATEGORIES)}，实际为 {len(cases)}")
+
+    missing_categories = REQUIRED_CATEGORIES - categories
+    extra_categories = categories - REQUIRED_CATEGORIES
+    if missing_categories:
+        errors.append(f"核心行为类别缺失：{sorted(missing_categories)}")
+    if extra_categories:
+        errors.append(f"存在未裁决类别：{sorted(extra_categories)}")
+
+    return report(errors)
+
+
+def report(errors: list[str]) -> int:
+    if errors:
+        print(f"FAIL：{len(errors)} 个源码结构问题")
+        for error in errors:
+            print(f"- {error}")
+        return 1
+    count = len(json.loads(read("evaluation/cases.json"))["cases"])
+    print(f"PASS：文件、版本、四个 Skill 入口和 {count} 个高信号案例结构有效")
+    print("说明：该结果不评价 Agent 行为，不构成 L1，也不是模型上限测试。")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
+