@guanlin4924/devflow-pi 0.1.19 → 0.1.21

package/plugins/devflow-core/skills/workflow-orchestrator/scripts/acceptance_test_runner.py

@@ -0,0 +1,394 @@
+#!/usr/bin/env python3
+"""Execute only adapters bound to a frozen TC suite; never derive scope from delivery prose."""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import re
+import subprocess
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+from urllib.error import HTTPError, URLError
+from urllib.parse import urlencode, urljoin
+from urllib.request import Request, urlopen
+
+from test_command_policy import validate_test_command
+
+
+FEATURE_RE = re.compile(r"^[^/\\\x00-\x1f\x7f]+$")
+AUTOMATION_MODES = {"automated", "manual", "blocked"}
+
+
+def utc_now() -> str:
+    return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
+
+
+def digest(value: str | bytes) -> str:
+    if isinstance(value, str):
+        value = value.encode("utf-8")
+    return hashlib.sha256(value).hexdigest()
+
+
+def read_json(path: Path, errors: list[str], label: str) -> dict[str, Any] | None:
+    try:
+        value = json.loads(path.read_text(encoding="utf-8"))
+    except FileNotFoundError:
+        errors.append(f"missing {label}: {path.as_posix()}")
+        return None
+    except json.JSONDecodeError as error:
+        errors.append(f"invalid {label}: {error.msg}")
+        return None
+    if not isinstance(value, dict):
+        errors.append(f"invalid {label}: root must be an object")
+        return None
+    return value
+
+
+def write_json(path: Path, value: dict[str, Any]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary = path.with_name(f".{path.name}.tmp")
+    temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    temporary.replace(path)
+
+
+def valid_argv(value: Any) -> bool:
+    return isinstance(value, list) and bool(value) and all(isinstance(token, str) and token and "\x00" not in token for token in value)
+
+
+ENV_NAME_RE = re.compile(r"^[A-Z_][A-Z0-9_]*$")
+
+
+def valid_http_adapter(value: Any) -> bool:
+    if not isinstance(value, dict) or value.get("kind") != "http" or not isinstance(value.get("base_url_env"), str) or not ENV_NAME_RE.fullmatch(value["base_url_env"]):
+        return False
+    headers_env = value.get("headers_env", {})
+    if not isinstance(headers_env, dict) or not all(isinstance(header, str) and header and isinstance(env, str) and ENV_NAME_RE.fullmatch(env) for header, env in headers_env.items()):
+        return False
+    setup = value.get("setup")
+    return setup is None or (isinstance(setup, dict) and valid_argv(setup.get("argv")))
+
+
+def pointer_value(document: Any, pointer: str) -> Any:
+    if pointer == "":
+        return document
+    if not isinstance(pointer, str) or not pointer.startswith("/"):
+        raise ValueError("error_code_pointer must be an RFC 6901 JSON pointer")
+    current = document
+    for token in pointer[1:].split("/"):
+        token = token.replace("~1", "/").replace("~0", "~")
+        if isinstance(current, dict) and token in current:
+            current = current[token]
+        elif isinstance(current, list) and token.isdigit() and int(token) < len(current):
+            current = current[int(token)]
+        else:
+            raise KeyError(pointer)
+    return current
+
+
+def http_execution_error(source: dict[str, Any]) -> str | None:
+    execution = source.get("execution")
+    if not isinstance(execution, dict) or execution.get("protocol") != "http":
+        return "frozen TC has no HTTP execution definition"
+    request = execution.get("request")
+    expect = execution.get("expect")
+    if not isinstance(request, dict) or not isinstance(expect, dict):
+        return "frozen TC HTTP execution requires request and expect"
+    if not isinstance(request.get("method"), str) or not re.fullmatch(r"[A-Z]+", request["method"]):
+        return "frozen TC HTTP request method is invalid"
+    if not isinstance(request.get("path"), str) or not request["path"].startswith("/"):
+        return "frozen TC HTTP request path is invalid"
+    if not isinstance(expect.get("http_status"), int) or not 100 <= expect["http_status"] <= 599:
+        return "frozen TC HTTP expectation http_status is invalid"
+    for key in ("path_params", "query", "headers", "body"):
+        if key in request and not isinstance(request[key], dict):
+            return f"frozen TC HTTP request {key} must be an object"
+    if "error_code" in expect and (not isinstance(expect["error_code"], str) or not expect["error_code"].strip()):
+        return "frozen TC HTTP expectation error_code is invalid"
+    if "error_code_pointer" in expect and (not isinstance(expect["error_code_pointer"], str) or not expect["error_code_pointer"].startswith("/")):
+        return "frozen TC HTTP expectation error_code_pointer is invalid"
+    return None
+
+
+def frozen_cases(suite: dict[str, Any], errors: list[str]) -> dict[str, dict[str, Any]]:
+    cases: dict[str, dict[str, Any]] = {}
+    for node in suite.get("nodes", []):
+        if not isinstance(node, dict) or node.get("kind") != "test_case":
+            continue
+        tc_id = node.get("id")
+        data = node.get("data")
+        if not isinstance(tc_id, str) or not isinstance(data, dict):
+            errors.append("invalid test_case node in TC suite")
+            continue
+        if node.get("status") != "confirmed" or node.get("stale") is True:
+            errors.append(f"TC is not frozen and fresh: {tc_id}")
+        cases[tc_id] = data
+    if not cases:
+        errors.append("TC suite contains no test_case nodes")
+    return cases
+
+
+def paths(root: Path, feature: str) -> tuple[Path, Path, Path]:
+    return (
+        root / ".planning" / "requirements" / feature / "tests.json",
+        root / ".planning" / "features" / feature / "acceptance-test-bindings.json",
+        root / ".planning" / "test-runs" / feature / "acceptance-results.json",
+    )
+
+
+def file_fingerprint(path: Path) -> str | None:
+    return digest(path.read_bytes()) if path.is_file() else None
+
+
+def run_http_case(source: dict[str, Any], adapter: dict[str, Any]) -> tuple[bool, int, str]:
+    execution = source["execution"]
+    request_spec = execution["request"]
+    expect = execution["expect"]
+    base_url = os.environ.get(adapter["base_url_env"])
+    if not base_url:
+        return False, 127, f"required test base URL environment variable is unset: {adapter['base_url_env']}"
+    setup_output = ""
+    setup = adapter.get("setup")
+    if isinstance(setup, dict):
+        try:
+            completed = subprocess.run(setup["argv"], text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=180, check=False)
+            setup_output = f"setup exit={completed.returncode}: {completed.stdout}\n{completed.stderr}"
+            if completed.returncode != 0:
+                return False, completed.returncode, setup_output
+        except subprocess.TimeoutExpired as error:
+            return False, 124, f"setup command timed out: {error}"
+        except OSError as error:
+            return False, 127, f"setup command could not start: {error}"
+    path = request_spec["path"]
+    for key, value in request_spec.get("path_params", {}).items():
+        path = path.replace("{" + str(key) + "}", str(value)).replace(":" + str(key), str(value))
+    query = request_spec.get("query", {})
+    url = urljoin(base_url.rstrip("/") + "/", path.lstrip("/"))
+    if query:
+        url += ("&" if "?" in url else "?") + urlencode(query, doseq=True)
+    headers = {str(key): str(value) for key, value in request_spec.get("headers", {}).items()}
+    for header, env_name in adapter.get("headers_env", {}).items():
+        value = os.environ.get(env_name)
+        if not value:
+            return False, 127, f"required test header environment variable is unset: {env_name}"
+        headers[header] = value
+    body = request_spec.get("body")
+    payload: bytes | None = None
+    if body is not None:
+        payload = json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
+        headers.setdefault("Content-Type", "application/json")
+    request = Request(url, data=payload, headers=headers, method=request_spec["method"])
+    try:
+        with urlopen(request, timeout=20) as response:
+            http_status = response.status
+            output = response.read().decode("utf-8", errors="replace")
+    except HTTPError as error:
+        http_status = error.code
+        output = error.read().decode("utf-8", errors="replace")
+    except (URLError, OSError) as error:
+        return False, 127, f"{setup_output}\nHTTP adapter could not complete request: {error}"
+    expected_status = expect["http_status"]
+    if http_status != expected_status:
+        return False, 1, f"{setup_output}\nexpected HTTP {expected_status}, got {http_status}; body={output[:1000]}"
+    if "error_code" in expect:
+        try:
+            body_json = json.loads(output)
+            actual_code = pointer_value(body_json, expect.get("error_code_pointer", "/error/code"))
+        except (json.JSONDecodeError, KeyError, ValueError) as error:
+            return False, 1, f"{setup_output}\ncould not read expected error code from response: {error}; body={output[:1000]}"
+        if actual_code != expect["error_code"]:
+            return False, 1, f"{setup_output}\nexpected error code {expect['error_code']}, got {actual_code!r}; body={output[:1000]}"
+    return True, 0, f"{setup_output}\nHTTP {http_status} matched expectation"
+
+
+def validate(root: Path, feature: str) -> tuple[dict[str, Any], dict[str, Any] | None, dict[str, Any] | None]:
+    errors: list[str] = []
+    suite_path, binding_path, _ = paths(root, feature)
+    suite = read_json(suite_path, errors, "TC suite")
+    bindings = read_json(binding_path, errors, "acceptance-test bindings")
+    if suite is None or bindings is None:
+        return {"ok": False, "feature": feature, "errors": errors}, suite, bindings
+    cases = frozen_cases(suite, errors)
+    if bindings.get("schema_version") != 1:
+        errors.append("acceptance-test bindings schema_version must be 1")
+    if bindings.get("origin") != "devflow_acceptance_test_execution":
+        errors.append("acceptance-test bindings origin must be devflow_acceptance_test_execution")
+    if bindings.get("feature") != feature:
+        errors.append("acceptance-test bindings feature does not match requested feature")
+    pin = bindings.get("test_suite")
+    meta = suite.get("view_meta") if isinstance(suite.get("view_meta"), dict) else {}
+    if not isinstance(pin, dict):
+        errors.append("test_suite pin is required")
+    else:
+        if pin.get("graph_revision") != suite.get("graph_revision"):
+            errors.append("test_suite.graph_revision does not match current TC suite")
+        if pin.get("fingerprint") != meta.get("fingerprint"):
+            errors.append("test_suite.fingerprint does not match current TC suite")
+
+    entries = bindings.get("cases")
+    if not isinstance(entries, list) or not entries:
+        errors.append("cases must be a non-empty array")
+        entries = []
+    seen: set[str] = set()
+    for index, entry in enumerate(entries):
+        label = f"cases[{index}]"
+        if not isinstance(entry, dict):
+            errors.append(f"{label} must be an object")
+            continue
+        tc_id = entry.get("tc_id")
+        if not isinstance(tc_id, str) or not tc_id:
+            errors.append(f"{label}.tc_id is required")
+            continue
+        if tc_id in seen:
+            errors.append(f"duplicate acceptance-test binding: {tc_id}")
+        seen.add(tc_id)
+        source = cases.get(tc_id)
+        if source is None:
+            errors.append(f"unknown TC reference: {tc_id}")
+            continue
+        mode = entry.get("mode")
+        if mode not in AUTOMATION_MODES:
+            errors.append(f"{label}.mode is invalid")
+            continue
+        if mode != source.get("automation_mode"):
+            errors.append(f"{label}.mode must match frozen TC automation_mode")
+        if mode == "automated":
+            adapter = entry.get("adapter")
+            if not isinstance(adapter, dict):
+                errors.append(f"{label}.adapter is required")
+            elif adapter.get("kind") == "command":
+                if not valid_argv(adapter.get("argv")):
+                    errors.append(f"{label}.adapter command requires non-empty argv")
+                    continue
+                command_error = validate_test_command(root, adapter["argv"])
+                if command_error:
+                    errors.append(f"{label}.adapter: {command_error}")
+            elif adapter.get("kind") == "http":
+                if not valid_http_adapter(adapter):
+                    errors.append(f"{label}.adapter HTTP requires base_url_env, valid optional headers_env, and optional setup.argv")
+                else:
+                    execution_error = http_execution_error(source)
+                    if execution_error:
+                        errors.append(f"{label}.adapter: {execution_error}")
+                    if isinstance(adapter.get("setup"), dict):
+                        setup_error = validate_test_command(root, adapter["setup"]["argv"])
+                        if setup_error:
+                            errors.append(f"{label}.adapter.setup: {setup_error}")
+            else:
+                errors.append(f"{label}.adapter kind must be command or http")
+        elif mode == "manual":
+            if not isinstance(entry.get("owner"), str) or not entry["owner"].strip() or not isinstance(entry.get("reason"), str) or not entry["reason"].strip():
+                errors.append(f"{label}.manual requires owner and reason")
+        elif mode == "blocked":
+            if not isinstance(entry.get("blocked_by"), str) or not entry["blocked_by"].strip() or not isinstance(entry.get("owner"), str) or not entry["owner"].strip():
+                errors.append(f"{label}.blocked requires blocked_by and owner")
+    for tc_id in sorted(set(cases) - seen):
+        errors.append(f"unbound frozen TC: {tc_id}")
+    return {"ok": not errors, "feature": feature, "tc_ids": sorted(cases), "errors": errors}, suite, bindings
+
+
+def run(root: Path, feature: str, write: bool) -> dict[str, Any]:
+    validation, suite, bindings = validate(root, feature)
+    if not validation["ok"]:
+        return validation
+    assert suite is not None and bindings is not None
+    cases = frozen_cases(suite, [])
+    _, _, result_path = paths(root, feature)
+    previous = read_json(result_path, [], "previous acceptance-test results")
+    task_bindings_path = root / ".planning" / "features" / feature / "test-bindings.json"
+    execution_bindings_path = root / ".planning" / "features" / feature / "acceptance-test-bindings.json"
+    results: list[dict[str, Any]] = []
+    for entry in bindings["cases"]:
+        tc_id = entry["tc_id"]
+        mode = entry["mode"]
+        if mode == "automated":
+            started_at = utc_now()
+            adapter = entry["adapter"]
+            if adapter["kind"] == "command":
+                argv = adapter["argv"]
+                try:
+                    completed = subprocess.run(argv, cwd=root, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=180, check=False)
+                    exit_code = completed.returncode
+                    output = f"{completed.stdout}\n{completed.stderr}"
+                except subprocess.TimeoutExpired as error:
+                    exit_code = 124
+                    output = f"command timed out: {error}"
+                except OSError as error:
+                    exit_code = 127
+                    output = f"command could not start: {error}"
+                result = {"tc_id": tc_id, "status": "passed" if exit_code == 0 else "failed", "mode": mode, "adapter_kind": "command", "argv": argv,
+                          "command_hash": digest(json.dumps(argv, ensure_ascii=False, separators=(",", ":"))), "started_at": started_at,
+                          "finished_at": utc_now(), "exit_code": exit_code, "output_sha256": digest(output)}
+            else:
+                passed, exit_code, output = run_http_case(cases[tc_id], adapter)
+                result = {"tc_id": tc_id, "status": "passed" if passed else "failed", "mode": mode, "adapter_kind": "http",
+                          "adapter_fingerprint": digest(json.dumps(adapter, ensure_ascii=False, separators=(",", ":"))), "started_at": started_at,
+                          "finished_at": utc_now(), "exit_code": exit_code, "output_sha256": digest(output)}
+            results.append(result)
+        elif mode == "manual":
+            results.append({"tc_id": tc_id, "status": "blocked", "mode": mode, "owner": entry["owner"], "reason": entry["reason"],
+                            "blocked_by": "manual execution has no Core-observed result"})
+        else:
+            results.append({"tc_id": tc_id, "status": "blocked", "mode": mode, "owner": entry["owner"], "blocked_by": entry["blocked_by"]})
+    all_passed = all(item["status"] == "passed" for item in results)
+    history = previous.get("history", []) if isinstance(previous, dict) and isinstance(previous.get("history"), list) else []
+    if isinstance(previous, dict) and isinstance(previous.get("results"), list):
+        history = [*history, {"executed_at": previous.get("executed_at"), "passed": previous.get("passed"), "test_suite": previous.get("test_suite"),
+                              "implementation_task_bindings": previous.get("implementation_task_bindings"), "execution_binding_fingerprint": previous.get("execution_binding_fingerprint"),
+                              "results": previous["results"]}]
+    payload = {"schema_version": 1, "origin": "core_acceptance_test_runner", "feature": feature, "executed_at": utc_now(),
+               "test_suite": bindings["test_suite"], "implementation_task_bindings": {"path": str(task_bindings_path.relative_to(root)), "fingerprint": file_fingerprint(task_bindings_path)},
+               "execution_binding_fingerprint": file_fingerprint(execution_bindings_path), "results": results, "passed": all_passed, "history": history[-20:]}
+    if write:
+        write_json(result_path, payload)
+    return {"ok": all_passed, "feature": feature, "path": str(result_path.relative_to(root)), "results": results,
+            "errors": [] if all_passed else ["one or more frozen TC cases failed or are blocked"]}
+
+
+def validate_results(root: Path, feature: str) -> dict[str, Any]:
+    validation, suite, _ = validate(root, feature)
+    errors = list(validation["errors"])
+    _, _, result_path = paths(root, feature)
+    results = read_json(result_path, errors, "acceptance-test results")
+    if suite is None or results is None:
+        return {"ok": False, "feature": feature, "errors": errors}
+    if results.get("origin") != "core_acceptance_test_runner" or results.get("feature") != feature:
+        errors.append("acceptance-test results identity is invalid")
+    meta = suite.get("view_meta") if isinstance(suite.get("view_meta"), dict) else {}
+    if results.get("test_suite") != {"graph_revision": suite.get("graph_revision"), "fingerprint": meta.get("fingerprint")}:
+        errors.append("acceptance-test results are pinned to a stale TC suite")
+    expected = set(validation.get("tc_ids", []))
+    result_entries = results.get("results") if isinstance(results.get("results"), list) else []
+    actual = {item.get("tc_id") for item in result_entries if isinstance(item, dict) and isinstance(item.get("tc_id"), str)}
+    if actual != expected:
+        errors.append("acceptance-test results do not cover exactly the frozen TC suite")
+    failed = sorted(item.get("tc_id", "<unknown>") for item in result_entries if not isinstance(item, dict) or item.get("status") != "passed")
+    if failed:
+        errors.append(f"frozen TC cases not passed: {', '.join(failed)}")
+    return {"ok": not errors, "feature": feature, "errors": errors, "result_path": str(result_path.relative_to(root))}
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("action", choices=("validate", "run", "validate-results"))
+    parser.add_argument("--feature", required=True)
+    parser.add_argument("--write", action="store_true")
+    parser.add_argument("--json", action="store_true")
+    args = parser.parse_args()
+    root = Path.cwd().resolve()
+    if not FEATURE_RE.fullmatch(args.feature):
+        result = {"ok": False, "errors": ["invalid feature"]}
+    elif args.action == "validate":
+        result, _, _ = validate(root, args.feature)
+    elif args.action == "run":
+        result = run(root, args.feature, args.write) if args.write else {"ok": False, "errors": ["--write is required for run"]}
+    else:
+        result = validate_results(root, args.feature)
+    print(json.dumps(result, ensure_ascii=False, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/plugins/devflow-core/skills/workflow-orchestrator/scripts/acceptance_test_suite.rb

@@ -0,0 +1,86 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Core-owned freeze point for a feature's derived TC suite. A skill may propose
+# individual candidates, but it cannot turn them into evidence by confirming
+# them one by one: this command snapshots the complete, fresh suite under CP-3.
+require "json"
+require "open3"
+require "optparse"
+require_relative "control_io"
+require_relative "requirement_graph"
+
+options = { feature: nil, expected_graph_revision: nil, cp_record: nil, write: false, json: false }
+command = ARGV.shift
+OptionParser.new do |parser|
+  parser.on("--feature NAME") { |value| options[:feature] = value }
+  parser.on("--expected-graph-revision REVISION", Integer) { |value| options[:expected_graph_revision] = value }
+  parser.on("--cp-record RECORD") { |value| options[:cp_record] = value }
+  parser.on("--write") { options[:write] = true }
+  parser.on("--json") { options[:json] = true }
+end.parse!
+
+def test_nodes(graph)
+  Array(graph["nodes"]).select { |node| node.is_a?(Hash) && node["kind"] == "test_case" }
+end
+
+def api_test_matrix_check(root, feature, graph)
+  return { "ok" => true, "skipped" => "no_api_nodes" } unless Array(graph["nodes"]).any? { |node| node.is_a?(Hash) && node["kind"] == "api" }
+
+  script = File.join(__dir__, "api_test_matrix.py")
+  stdout, stderr, status = Open3.capture3("python3", script, "validate", "--feature", feature, "--json", chdir: root)
+  payload = JSON.parse(stdout)
+  return payload if status.success? && payload.is_a?(Hash)
+
+  { "ok" => false, "errors" => [stderr.strip.empty? ? "API test matrix checker returned invalid output" : stderr.strip] }
+rescue JSON::ParserError
+  { "ok" => false, "errors" => ["API test matrix checker returned invalid JSON"] }
+end
+
+root = Dir.pwd
+lock = nil
+begin
+  raise ArgumentError, "command must be inspect or freeze" unless %w[inspect freeze].include?(command)
+  raise ArgumentError, "--feature is required" unless RequirementGraph.valid_feature?(options[:feature])
+  lock = DevflowControlIO.acquire_project_lock(root) if command == "freeze"
+  graph, path, relative = RequirementGraph.load_graph(root, options[:feature])
+  cases = test_nodes(graph)
+
+  if command == "inspect"
+    view = RequirementGraph.view_payload(graph, "tests")
+    result = {
+      "ok" => true,
+      "path" => relative,
+      "graph_revision" => graph["revision"],
+      "test_case_ids" => cases.map { |node| node["id"] },
+      "frozen" => view["frozen"],
+      "frozen_graph_revision" => view["frozen_graph_revision"],
+      "stale" => Array(graph["stale"]).select { |entry| cases.any? { |node| node["id"] == entry["node_id"] } }
+    }
+  else
+    raise ArgumentError, "--write is required" unless options[:write]
+    raise ArgumentError, "--expected-graph-revision is required" if options[:expected_graph_revision].nil?
+    raise ArgumentError, "--cp-record is required" if options[:cp_record].to_s.strip.empty?
+    if graph["revision"] != options[:expected_graph_revision]
+      result = { "ok" => false, "reason" => "stale_graph_revision", "expected_revision" => options[:expected_graph_revision], "actual_revision" => graph["revision"] }
+    elsif cases.empty?
+      result = { "ok" => false, "reason" => "test_suite_empty", "message" => "cannot freeze an empty TC suite" }
+    elsif cases.any? { |node| node["status"] == "confirmed" }
+      result = { "ok" => false, "reason" => "test_suite_already_frozen", "message" => "confirmed test cases require a new candidate suite" }
+    elsif Array(graph["stale"]).any?
+      result = { "ok" => false, "reason" => "test_suite_graph_stale", "message" => "cannot freeze TC suite while the requirement graph has stale downstream nodes" }
+    else
+      matrix = api_test_matrix_check(root, options[:feature], graph)
+      result = if matrix["ok"]
+                 RequirementGraph.freeze_test_cases!(root, graph, options[:cp_record])
+               else
+                 { "ok" => false, "reason" => "api_test_matrix_incomplete", "message" => Array(matrix["errors"]).join("; "), "matrix" => matrix }
+               end
+    end
+  end
+rescue ArgumentError, JSON::ParserError => error
+  result = { "ok" => false, "reason" => "acceptance_test_suite_invalid", "message" => error.message }
+ensure
+  DevflowControlIO.release_lock(lock)
+end
+puts JSON.pretty_generate(result)