@guanlin4924/devflow-pi 0.1.0

package/plugins/devflow-quality/skills/workflow-generate-mainflow-cases/scripts/validate_case_payloads.py

@@ -0,0 +1,370 @@
+#!/usr/bin/env python3
+"""Validate normalized mainflow case import payloads."""
+
+from __future__ import annotations
+
+import argparse
+import importlib
+import json
+import re
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+
+MAINFLOW_TAGS = {"api-mainflow", "ui-mainflow"}
+ALLOWED_DRIVERS = {"api", "web", "ui"}
+VALID_TARGET_KEYS = {"css", "xpath", "label", "placeholder", "role", "name", "near_text", "exact"}
+# 精确匹配器：严格对应DOM结构，必须来自 source code 证据，不能凭空编造
+EXACT_LOCATOR_KEYS = {"css", "xpath", "role"}
+# 语义匹配器：通过用户可见文本/属性定位，更抗DOM变化但需检查唯一性
+SEMANTIC_LOCATOR_KEYS = {"label", "placeholder", "near_text"}
+# 表格行常见操作文本 — 用 label 定位时几乎一定不唯一
+HIGH_RISK_LABELS = {
+    "编辑", "删除", "查看", "详情", "启用", "禁用", "停用", "启用",
+    "新增", "添加", "修改", "复制", "移动", "导出", "导入", "下载",
+    "上传", "设置", "配置", "操作", "更多", "取消", "确定", "保存",
+    "提交", "重置", "刷新", "搜索", "查询", "筛选", "排序",
+}
+# CSS 选择器最小长度 — 太短的选择器通常是编造的，缺乏 scope
+MIN_CSS_LENGTH = 10
+# 验证类 action — 应使用 ai_only
+VERIFICATION_ACTIONS = {"assert_visible", "assert_text", "assert_url_contains", "assert_any_visible"}
+VALID_ASSERTION_TYPES = {
+    "equals", "json_path_equals", "status_code", "not_equals", "contains",
+    "not_contains", "greater_than", "less_than", "json_path_exists",
+    "regex", "is_empty", "is_not_empty", "in", "not_in",
+}
+API_ACTIONS = {"get", "post", "put", "patch", "delete", "head", "options"}
+WEB_UI_ACTIONS = {
+    "goto",
+    "navigate",
+    "click",
+    "fill",
+    "input",
+    "select",
+    "hover",
+    "wait",
+    "assert_visible",
+    "assert_text",
+    "assert_url_contains",
+    "assert_any_visible",
+    "evaluate_js",
+    "screenshot",
+    "act",
+}
+SECRET_KEY_RE = re.compile(r"(token|password|passwd|secret|api[_-]?key|authorization|cookie)", re.I)
+SECRET_VALUE_RE = re.compile(
+    r"(sk-[A-Za-z0-9_\-]{16,}|ghp_[A-Za-z0-9_]{16,}|xox[baprs]-[A-Za-z0-9\-]{16,}|"
+    r"Bearer\s+[A-Za-z0-9_\-.]{16,}|eyJ[A-Za-z0-9_\-]{16,}\.[A-Za-z0-9_\-]{8,})"
+)
+PLACEHOLDER_RE = re.compile(r"^(\$\{[A-Za-z_][A-Za-z0-9_]*\}|<[^>]+>|REPLACE_ME|dummy|example|your-.+)$", re.I)
+
+
+def load_payload(path: Path) -> dict[str, Any]:
+    raw = json.loads(path.read_text(encoding="utf-8"))
+    if isinstance(raw, list):
+        return {"metadata": {}, "cases": raw}
+    if isinstance(raw, dict) and isinstance(raw.get("cases"), list):
+        return raw
+    raise ValueError("payload must be a JSON object with cases[] or a JSON array")
+
+
+def backend_root_from_discovery(path: Path) -> Path:
+    data = json.loads(path.read_text(encoding="utf-8"))
+    for codebase in data.get("codebases", []):
+        if codebase.get("domain") == "backend" or "后端" in str(codebase.get("type", "")):
+            root = Path(str(codebase.get("root", "")))
+            if root.exists():
+                return root
+    raise ValueError("no existing backend codebase root found in discovery")
+
+
+def load_project_validators(discovery_path: Path):
+    backend_root = backend_root_from_discovery(discovery_path)
+    sys.path.insert(0, str(backend_root.resolve()))
+    case_module = importlib.import_module("app.schemas.case")
+    dsl_module = importlib.import_module("app.services.dsl_parser")
+    return case_module.CaseCreate, dsl_module.validate_dsl
+
+
+def is_placeholder(value: str) -> bool:
+    return bool(PLACEHOLDER_RE.match(value.strip())) or "${" in value
+
+
+def walk_values(value: Any, path: str = "$") -> list[tuple[str, Any]]:
+    found = [(path, value)]
+    if isinstance(value, dict):
+        for key, child in value.items():
+            found.extend(walk_values(child, f"{path}.{key}"))
+    elif isinstance(value, list):
+        for index, child in enumerate(value):
+            found.extend(walk_values(child, f"{path}[{index}]"))
+    return found
+
+
+def detect_secrets(case: dict[str, Any]) -> list[str]:
+    errors: list[str] = []
+    for path, value in walk_values(case):
+        if isinstance(value, str) and SECRET_VALUE_RE.search(value) and not is_placeholder(value):
+            errors.append(f"{path} contains a token-like literal")
+        key = path.rsplit(".", 1)[-1].lower()
+        if SECRET_KEY_RE.search(key) and isinstance(value, str) and value.strip() and not is_placeholder(value):
+            errors.append(f"{path} contains a secret-looking literal")
+    return errors
+
+
+def validate_case_payload_shape(case: dict[str, Any], CaseCreate: Any) -> list[str]:
+    errors: list[str] = []
+    try:
+        if hasattr(CaseCreate, "model_validate"):
+            CaseCreate.model_validate(case)
+        else:
+            CaseCreate(**case)
+    except Exception as exc:  # noqa: BLE001
+        errors.append(f"CaseCreate validation failed: {exc}")
+    return errors
+
+
+def validate_dsl_static(dsl: dict[str, Any], tags: list[str]) -> list[str]:
+    errors: list[str] = []
+    if not dsl.get("name"):
+        errors.append("dsl_json.name is required")
+    steps = dsl.get("steps")
+    if not isinstance(steps, list) or not steps:
+        errors.append("dsl_json.steps must be a non-empty list")
+        return errors
+
+    step_ids: set[str] = set()
+    for index, step in enumerate(steps):
+        path = f"dsl_json.steps[{index}]"
+        if not isinstance(step, dict):
+            errors.append(f"{path} must be an object")
+            continue
+        step_id = step.get("id")
+        if not step_id:
+            errors.append(f"{path}.id is required")
+        elif step_id in step_ids:
+            errors.append(f"{path}.id duplicates '{step_id}'")
+        else:
+            step_ids.add(step_id)
+
+        driver = step.get("driver")
+        action = str(step.get("action") or "").lower()
+        if driver not in ALLOWED_DRIVERS:
+            errors.append(f"{path}.driver must be one of {sorted(ALLOWED_DRIVERS)}")
+        if not action:
+            errors.append(f"{path}.action is required")
+        elif driver == "api" and action not in API_ACTIONS:
+            errors.append(f"{path}.action is not a valid API method")
+        elif driver in {"web", "ui"} and action not in WEB_UI_ACTIONS:
+            errors.append(f"{path}.action is not a valid UI/Web action")
+
+        params = step.get("params") or {}
+        if driver == "api" and not params.get("url"):
+            errors.append(f"{path}.params.url is required for API steps")
+        if driver == "ui":
+            strategy = step.get("strategy") or {}
+            target = step.get("target") or {}
+            strategy_mode = strategy.get("mode", "")
+            if strategy_mode != "ai_only" and not any(target.values()):
+                errors.append(f"{path}.target is required for non-ai_only UI steps")
+            unknown_keys = set(target.keys()) - VALID_TARGET_KEYS
+            if unknown_keys:
+                errors.append(
+                    f"{path}.target contains invalid keys {sorted(unknown_keys)}. "
+                    f"Valid keys: {sorted(VALID_TARGET_KEYS)}"
+                )
+
+            # --- Locator quality rules ---
+
+            # Rule 1: 精确匹配器 (css/xpath/role) 必须来自 source evidence，检查选择器质量
+            for exact_key in EXACT_LOCATOR_KEYS & set(target.keys()):
+                value = str(target.get(exact_key, "")).strip()
+                if not value:
+                    continue
+                if exact_key == "css":
+                    if len(value) < MIN_CSS_LENGTH:
+                        errors.append(
+                            f"{path}.target.css='{value}' is too generic ({len(value)} chars, min {MIN_CSS_LENGTH}). "
+                            f"CSS selectors are exact matchers and must be scoped. "
+                            f"Add context: dialog footer, table row, form item, etc."
+                        )
+                    # CSS 选择器不应是裸的 tag/class 名（无组合器、无伪类）
+                    if " " not in value and "." not in value and "#" not in value:
+                        errors.append(
+                            f"{path}.target.css='{value}' looks like a bare identifier. "
+                            f"Use a scoped selector like '.el-dialog__footer .el-button--primary'."
+                        )
+                if exact_key == "role":
+                    if "name" not in target or not str(target.get("name", "")).strip():
+                        errors.append(
+                            f"{path}.target has 'role' but missing 'name'. "
+                            f"role+name is a paired exact matcher; role alone is not scoped enough."
+                        )
+
+            # Rule 2: 语义匹配器 — label 用于高风险文本时检查唯一性
+            label_value = str(target.get("label", "")).strip()
+            if label_value and label_value in HIGH_RISK_LABELS:
+                has_scope = bool(target.get("css") or target.get("near_text") or target.get("xpath"))
+                if not has_scope:
+                    errors.append(
+                        f"{path}.target.label='{label_value}' is a high-risk label "
+                        f"(common table-row action, likely appears on multiple rows). "
+                        f"Add scope with 'css' for row-specific selector, or use 'near_text' to bind to a row identifier."
+                    )
+
+            # Rule 3: 验证步骤必须用 ai_only
+            if action in VERIFICATION_ACTIONS and strategy_mode != "ai_only":
+                errors.append(
+                    f"{path}.action='{action}' is a verification step but strategy.mode='{strategy_mode}'. "
+                    f"Verification steps must use strategy.mode='ai_only'. "
+                    f"Visual verification of transient UI (toast, dialogs) is inherently an AI task."
+                )
+            # act + prompt 也是验证，同样要求 ai_only
+            if action == "act" and step.get("prompt") and strategy_mode != "ai_only":
+                errors.append(
+                    f"{path}.action='act' with prompt indicates an AI-driven step but strategy.mode='{strategy_mode}'. "
+                    f"Use strategy.mode='ai_only' for AI-driven verification/action steps."
+                )
+
+        if driver == "api":
+            for ai, assertion in enumerate(step.get("assertions") or []):
+                atype = str(assertion.get("type") or "")
+                if atype and atype not in VALID_ASSERTION_TYPES:
+                    errors.append(
+                        f"{path}.assertions[{ai}].type '{atype}' is not a valid assertion type. "
+                        f"Valid types: {sorted(VALID_ASSERTION_TYPES)}"
+                    )
+
+        # Web/UI steps: catch assertions that look like confused actions
+        if driver in {"web", "ui"}:
+            for ai, assertion in enumerate(step.get("assertions") or []):
+                atype = str(assertion.get("type") or "").lower()
+                if atype in {"text", "visible", "url_contains", "any_visible"}:
+                    errors.append(
+                        f"{path}.assertions[{ai}].type '{atype}' is not a valid assertion type. "
+                        f"For web/ui steps, use action='assert_{atype}' instead of assertions[]."
+                    )
+
+    for index, step in enumerate(steps):
+        for dep in step.get("depends_on") or []:
+            if dep not in step_ids:
+                errors.append(f"dsl_json.steps[{index}].depends_on references missing step '{dep}'")
+
+    case_tags = set(tags)
+    dsl_tags = set(dsl.get("tags") or [])
+    if not case_tags:
+        errors.append("tags must include api-mainflow or ui-mainflow")
+    elif not case_tags.issubset(MAINFLOW_TAGS):
+        errors.append(f"tags contain non-mainflow values: {tags}")
+    if not dsl_tags:
+        errors.append("dsl_json.tags must include api-mainflow or ui-mainflow")
+    elif not dsl_tags.issubset(MAINFLOW_TAGS):
+        errors.append(f"dsl_json.tags contain non-mainflow values: {dsl.get('tags')}")
+
+    return errors
+
+
+def validate_project_dsl(dsl: dict[str, Any], validate_dsl: Any) -> list[str]:
+    result = validate_dsl(dsl)
+    if getattr(result, "valid", False):
+        return []
+    return [f"project dsl_parser: {error}" for error in getattr(result, "errors", [])]
+
+
+def render_report(results: list[dict[str, Any]], warnings: list[str], payload_path: Path) -> str:
+    total_errors = sum(len(item["errors"]) for item in results)
+    status = "passed" if total_errors == 0 else "failed"
+    lines = [
+        "# Mainflow Case Validation Report",
+        "",
+        f"- Payload: `{payload_path}`",
+        f"- Generated at: {datetime.now(timezone.utc).isoformat()}",
+        f"- Status: **{status}**",
+        f"- Cases: {len(results)}",
+        f"- Errors: {total_errors}",
+        f"- Warnings: {len(warnings)}",
+        "",
+        "## Case Results",
+        "",
+        "| Case | Tags | Status | Errors |",
+        "|---|---|---:|---:|",
+    ]
+    for item in results:
+        case_status = "passed" if not item["errors"] else "failed"
+        lines.append(f"| {item['name']} | {', '.join(item['tags'])} | {case_status} | {len(item['errors'])} |")
+
+    if warnings:
+        lines.extend(["", "## Warnings", ""])
+        lines.extend(f"- {warning}" for warning in warnings)
+
+    failed = [item for item in results if item["errors"]]
+    if failed:
+        lines.extend(["", "## Errors", ""])
+        for item in failed:
+            lines.append(f"### {item['name']}")
+            for error in item["errors"]:
+                lines.append(f"- {error}")
+            lines.append("")
+
+    return "\n".join(lines).rstrip() + "\n"
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("input", help="case-import-payloads.json")
+    parser.add_argument("--discovery", required=True, help="codebase-discovery.json")
+    parser.add_argument("--output", required=True, help="validation-report.md")
+    args = parser.parse_args()
+
+    payload_path = Path(args.input)
+    discovery_path = Path(args.discovery)
+    output_path = Path(args.output)
+    warnings: list[str] = []
+    results: list[dict[str, Any]] = []
+
+    try:
+        payload = load_payload(payload_path)
+        if payload.get("cases"):
+            CaseCreate, project_validate_dsl = load_project_validators(discovery_path)
+        else:
+            CaseCreate = project_validate_dsl = None
+    except Exception as exc:  # noqa: BLE001
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        output_path.write_text(
+            "# Mainflow Case Validation Report\n\n"
+            f"- Payload: `{payload_path}`\n"
+            "- Status: **failed**\n\n"
+            "## Errors\n\n"
+            f"- Validation bootstrap failed: {exc}\n",
+            encoding="utf-8",
+        )
+        print(f"ERROR: {exc}", file=sys.stderr)
+        return 2
+
+    if not payload.get("cases"):
+        warnings.append("No cases were generated.")
+
+    for index, case in enumerate(payload.get("cases") or [], start=1):
+        name = str(case.get("name") or f"case-{index}")
+        tags = list(case.get("tags") or [])
+        dsl = case.get("dsl_json") or {}
+        errors: list[str] = []
+        errors.extend(validate_case_payload_shape(case, CaseCreate))
+        errors.extend(validate_dsl_static(dsl, tags))
+        errors.extend(detect_secrets(case))
+        errors.extend(validate_project_dsl(dsl, project_validate_dsl))
+        results.append({"name": name, "tags": tags, "errors": errors})
+
+    report = render_report(results, warnings, payload_path)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    output_path.write_text(report, encoding="utf-8")
+    total_errors = sum(len(item["errors"]) for item in results)
+    print(json.dumps({"status": "passed" if total_errors == 0 else "failed", "errors": total_errors}, ensure_ascii=False))
+    return 0 if total_errors == 0 else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/plugins/devflow-quality/skills/workflow-verify-work/SKILL.md

@@ -0,0 +1,106 @@
+---
+name: workflow-verify-work
+description: "Use when implemented features need UAT, verification evidence, user-facing behavior checks, or failure routing before milestone audit."
+---
+
+<objective>
+Validate built features through conversational testing with persistent state.
+
+Purpose: Confirm what the agent built actually works from the user's perspective. One test at a time, plain text responses, no interrogation. When issues are found, automatically diagnose, plan fixes, and prepare for execution.
+
+Output:
+- `{phase_num}-UAT.md` — raw test log (items, results, user responses)
+- `{phase_num}-VERIFICATION.md` — formal verification report mapping REQ-IDs / FP-IDs to passed UAT items, with evidence summary and closure statement. Must be generated when all UAT items pass (no failures, no skips that block closure). If VERIFICATION.md already exists from execute-phase, merge UAT results into it rather than creating a duplicate.
+- `.planning/phases/{phase}/01-VERIFICATION.meta.json` — structured metadata (`uat_passed`, `uat_failed`, `uat_skipped`, `gate_passed`, `gate_total`, `known_issues`). Schema in `feature-development.yaml` under `steps[verify].meta_output.fields`.
+</objective>
+
+<execution_context>
+references/verify-work.md
+assets/UAT.md
+</execution_context>
+
+<context>
+Phase: $ARGUMENTS (optional)
+- If provided: Test specific phase (e.g., "4")
+- If not provided: Check for active sessions or prompt for phase
+
+**Project rule package:**
+During verification, discover and read applicable rule sources:
+- `.planning/rules/INDEX.md`
+- `.planning/execution-contract.md`
+- `.planning/toolchain.md`
+- `.planning/lessons/README.md`
+- `.planning/milestones/`
+- `.planning/requirements/`
+- `.planning/code-index/`
+- `.context/spec/`
+- `.context/workflow/verification.md`
+- `.context/workflow/checklist.md`
+- `.context/lessons/README.md`（如存在）
+- root `AGENTS.md`, `CLAUDE.md`, or equivalent project instruction files
+
+Context files are resolved inside the workflow (`init verify-work`) and delegated via `<files_to_read>` blocks.
+Use `.planning/milestones/`, `.planning/requirements/`, `.planning/lessons/`, and phase `CONTEXT.md` as primary history when verification needs prior decisions. Treat `features/` or `.context/lessons/` as optional legacy/project-rule sources only when present.
+</context>
+
+<verification_gates>
+Verification must include these gates when project artifacts exist:
+
+1. **Change-list consistency gate**
+   - Locate the relevant `.planning/features/**/*变更清单.md` or phase plan. If missing, check legacy `features/**/*变更清单.md`.
+   - Check expected changes against implementation.
+   - If `.planning/code-index/` exists, use it to verify changed files, symbols, callers/callees, imports, and resource reads/writes before source readback.
+   - Report missing implementation, extra implementation, and deviations before functional UAT conclusions.
+
+2. **Project rule review gate**
+   - Review changed files against applicable project rules, task-level rules, and lessons.
+   - Existing code style may inform local patterns but must not override explicit project rules.
+   - Review the whole changed file or component boundary, not only the diff, when the project rules apply at file or layer level.
+   - Check both omissions and extras: missing required pieces, extra unplanned code, unused imports/fields/methods, stale generated artifacts, and duplicated implementations.
+   - If issues exist, produce a fix list and route back to execution.
+
+3. **Mainflow automation gate**
+   - Read `.planning/phases/{phase}/MAINFLOW-CASES.md`, `.planning/generated-cases/{feature}/validation-report.md`, and `.planning/generated-cases/{feature}/case-import-payloads.json` when they exist.
+   - If generated DSL validation failed, UAT does not pass.
+   - If mainflow case execution failed after import or direct execution, UAT does not pass.
+   - If environment or platform dependencies are missing and execution cannot run, mark verification `blocked`, not passed.
+   - If cases were not imported into the platform, complete at least static validation and record that import/execution remains pending.
+   - If cases were imported into the platform, record `execution_id` and execution result for each imported mainflow case.
+   - Mainflow cases must be limited to `api-mainflow` and `ui-mainflow`; any non-mainflow tag is a gate failure.
+
+4. **Toolchain and test gate**
+   - Run or require the commands declared by `.planning/toolchain.md`, `.planning/execution-contract.md`, `.context/workflow/verification.md`, or the change list.
+   - Include target tests, full tests, build, codegen/mock/DI checks, migration checks, and integration tests when applicable.
+
+5. **Compatibility gate**
+   - Check interface, database, config, data migration, permission, and consumer compatibility risks.
+   - For frontend-visible API/VO/schema changes, record backward compatibility or migration handling.
+   - For database/model/config changes, record migration, rollback, and deployment ordering risk.
+   - Record accepted residual risks explicitly.
+
+6. **Baseline decision gate**
+   - Compare implementation against `.planning/requirements/baseline.md`, especially `反向审视与用户拍板`, `待确认事项`, and `下游输入契约`.
+   - Confirm implemented behavior maps to `FP-*`, `BR-*`, `BC-*`, and confirmed `CR-*` decisions.
+   - Any implemented scope based on a pending decision is a verification failure unless the user explicitly approved it.
+
+7. **Lessons extraction gate**
+   - After verification is complete, review the findings for lessons worth preserving.
+   - Candidate triggers for a lesson:
+     - A regression was caught early — what pattern prevented it?
+     - A non-obvious interaction between modules was discovered
+     - A test gap was identified that exists beyond this feature
+     - A design decision was made that future tasks must reference
+   - If a lesson candidate exists, write to `.context/lessons/YYYY-MM-DD-{slug}.md`.
+   - If no candidate, skip.
+   - This gate covers both positive patterns (successful approach worth repeating) and negative patterns (regression prevention).
+
+The feature is complete only when UAT, mainflow automation, baseline decision, consistency, project rule review, compatibility, and required toolchain/test gates pass or have explicit user-approved exceptions.
+
+When all gates pass: write `VERIFICATION.md` before exiting. This is non-negotiable — downstream `workflow-audit-milestone` requires it.
+</verification_gates>
+
+<process>
+Execute the verify-work workflow from references/verify-work.md end-to-end.
+Preserve all workflow gates (session management, test presentation, diagnosis, fix planning, routing).
+Additionally preserve the verification gates above before declaring the work complete.
+</process>