@suiflex/suitest-mcp 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +77 -0
- package/bin/suitest-mcp.js +123 -0
- package/package.json +50 -0
- package/python/suitest_lifecycle/__init__.py +3 -0
- package/python/suitest_lifecycle/analyzers/__init__.py +1 -0
- package/python/suitest_lifecycle/analyzers/crawl.py +187 -0
- package/python/suitest_lifecycle/analyzers/express.py +226 -0
- package/python/suitest_lifecycle/analyzers/openapi.py +163 -0
- package/python/suitest_lifecycle/analyzers/postman.py +132 -0
- package/python/suitest_lifecycle/analyzers/react.py +107 -0
- package/python/suitest_lifecycle/analyzers/zod_schema.py +131 -0
- package/python/suitest_lifecycle/blackbox/__init__.py +11 -0
- package/python/suitest_lifecycle/blackbox/bootstrap.py +249 -0
- package/python/suitest_lifecycle/blackbox/crawler.py +383 -0
- package/python/suitest_lifecycle/blackbox/detector.py +169 -0
- package/python/suitest_lifecycle/blackbox/generator.py +608 -0
- package/python/suitest_lifecycle/blackbox/graph.py +107 -0
- package/python/suitest_lifecycle/blackbox/mcp.py +546 -0
- package/python/suitest_lifecycle/blackbox/models.py +299 -0
- package/python/suitest_lifecycle/blackbox/prd_ingest.py +108 -0
- package/python/suitest_lifecycle/blackbox/reporter.py +76 -0
- package/python/suitest_lifecycle/blackbox/selector.py +111 -0
- package/python/suitest_lifecycle/cli.py +127 -0
- package/python/suitest_lifecycle/config.py +314 -0
- package/python/suitest_lifecycle/enrich.py +140 -0
- package/python/suitest_lifecycle/exporters/__init__.py +1 -0
- package/python/suitest_lifecycle/exporters/backend.py +345 -0
- package/python/suitest_lifecycle/exporters/frontend.py +459 -0
- package/python/suitest_lifecycle/frontend_runtime.py +77 -0
- package/python/suitest_lifecycle/llm_bridge.py +365 -0
- package/python/suitest_lifecycle/mcp_server.py +187 -0
- package/python/suitest_lifecycle/models.py +166 -0
- package/python/suitest_lifecycle/orchestrator.py +500 -0
- package/python/suitest_lifecycle/paths.py +90 -0
- package/python/suitest_lifecycle/plan.py +366 -0
- package/python/suitest_lifecycle/plan_frontend.py +252 -0
- package/python/suitest_lifecycle/prd.py +92 -0
- package/python/suitest_lifecycle/process.py +111 -0
- package/python/suitest_lifecycle/publish.py +218 -0
- package/python/suitest_lifecycle/readiness.py +83 -0
- package/python/suitest_lifecycle/report.py +179 -0
- package/python/suitest_lifecycle/runner.py +138 -0
- package/python/suitest_lifecycle/serialize.py +131 -0
- package/python/suitest_lifecycle/tcm.py +149 -0
- package/python/suitest_lifecycle/tools.py +217 -0
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"""Readiness detection — never run tests against a server that isn't up.
|
|
2
|
+
|
|
3
|
+
Strategies, tried in order until one succeeds within the timeout:
|
|
4
|
+
1. HTTP probe of the ready URL (any < 500 response counts as "alive").
|
|
5
|
+
2. TCP port-open check (fallback when there's no health route).
|
|
6
|
+
3. Ready-log pattern match (when the process emits a known "listening" line).
|
|
7
|
+
|
|
8
|
+
Returns a :class:`Readiness` verdict with the strategy that succeeded, or
|
|
9
|
+
``ready=False`` with the elapsed time so the caller can mark the run failed and
|
|
10
|
+
attach the startup log.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import socket
|
|
16
|
+
import time
|
|
17
|
+
import urllib.error
|
|
18
|
+
import urllib.request
|
|
19
|
+
from dataclasses import dataclass
|
|
20
|
+
from typing import TYPE_CHECKING
|
|
21
|
+
|
|
22
|
+
if TYPE_CHECKING:
|
|
23
|
+
from collections.abc import Callable
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass(frozen=True)
|
|
27
|
+
class Readiness:
|
|
28
|
+
ready: bool
|
|
29
|
+
strategy: str
|
|
30
|
+
detail: str
|
|
31
|
+
waited_ms: int
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _http_ok(url: str, timeout: float) -> bool:
|
|
35
|
+
req = urllib.request.Request(url, method="GET")
|
|
36
|
+
try:
|
|
37
|
+
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
38
|
+
return int(resp.status) < 500
|
|
39
|
+
except urllib.error.HTTPError as exc:
|
|
40
|
+
return int(exc.code) < 500 # 401/404 still means the server is alive
|
|
41
|
+
except (TimeoutError, urllib.error.URLError, ConnectionError, OSError):
|
|
42
|
+
return False
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _port_open(host: str, port: int, timeout: float) -> bool:
|
|
46
|
+
try:
|
|
47
|
+
with socket.create_connection((host, port), timeout=timeout):
|
|
48
|
+
return True
|
|
49
|
+
except OSError:
|
|
50
|
+
return False
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def wait_until_ready(
|
|
54
|
+
ready_url: str,
|
|
55
|
+
host: str,
|
|
56
|
+
port: int,
|
|
57
|
+
timeout_sec: int,
|
|
58
|
+
*,
|
|
59
|
+
log_reader: Callable[[], str] | None = None,
|
|
60
|
+
ready_log_pattern: str = "",
|
|
61
|
+
poll_interval: float = 0.5,
|
|
62
|
+
monotonic: Callable[[], float] = time.monotonic,
|
|
63
|
+
sleep: Callable[[float], None] = time.sleep,
|
|
64
|
+
) -> Readiness:
|
|
65
|
+
"""Block until the target is ready or the timeout elapses."""
|
|
66
|
+
start = monotonic()
|
|
67
|
+
deadline = start + timeout_sec
|
|
68
|
+
while monotonic() < deadline:
|
|
69
|
+
if _http_ok(ready_url, timeout=min(5.0, poll_interval * 4)):
|
|
70
|
+
return Readiness(True, "http", f"GET {ready_url} alive", _ms(start, monotonic))
|
|
71
|
+
if _port_open(host, port, timeout=min(2.0, poll_interval * 2)):
|
|
72
|
+
return Readiness(True, "port", f"tcp {host}:{port} open", _ms(start, monotonic))
|
|
73
|
+
if ready_log_pattern and log_reader is not None and ready_log_pattern in log_reader():
|
|
74
|
+
return Readiness(True, "log", f"matched '{ready_log_pattern}'", _ms(start, monotonic))
|
|
75
|
+
sleep(poll_interval)
|
|
76
|
+
return Readiness(False, "timeout", f"not ready after {timeout_sec}s", _ms(start, monotonic))
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _ms(start: float, monotonic: Callable[[], float]) -> int:
|
|
80
|
+
return int((monotonic() - start) * 1000)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
__all__ = ["Readiness", "wait_until_ready"]
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
"""Human + machine readable reports.
|
|
2
|
+
|
|
3
|
+
Writes:
|
|
4
|
+
* ``tmp/raw_report.md`` — TestSprite-style per-test report (metadata,
|
|
5
|
+
requirement validation, coverage, gaps/risks).
|
|
6
|
+
* ``reports/summary.json`` — machine summary for CI.
|
|
7
|
+
* ``reports/summary.md`` — cross-mode rollup developers/QA/PO can skim.
|
|
8
|
+
* ``reports/summary.html`` — standalone styled page (no external assets).
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import json
|
|
14
|
+
from typing import TYPE_CHECKING
|
|
15
|
+
|
|
16
|
+
from suitest_lifecycle.models import RunSummary, TestOutcome, TestResult
|
|
17
|
+
from suitest_lifecycle.serialize import summary_to_json
|
|
18
|
+
|
|
19
|
+
if TYPE_CHECKING:
|
|
20
|
+
from suitest_lifecycle.paths import Paths
|
|
21
|
+
|
|
22
|
+
_BADGE = {
|
|
23
|
+
TestOutcome.PASSED: "✅ Passed",
|
|
24
|
+
TestOutcome.FAILED: "❌ Failed",
|
|
25
|
+
TestOutcome.SKIPPED: "⏭️ Skipped",
|
|
26
|
+
TestOutcome.ERROR: "🔥 Error",
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _gaps(summary: RunSummary) -> list[str]:
|
|
31
|
+
gaps: list[str] = []
|
|
32
|
+
if not summary.ready:
|
|
33
|
+
gaps.append(
|
|
34
|
+
f"Target never became ready ({summary.ready_detail}); tests could not run reliably."
|
|
35
|
+
)
|
|
36
|
+
failed = [r for r in summary.results if r.status in (TestOutcome.FAILED, TestOutcome.ERROR)]
|
|
37
|
+
for r in failed:
|
|
38
|
+
first = (r.error or "").splitlines()[-1] if r.error else "no error captured"
|
|
39
|
+
gaps.append(f"{r.test_id} {r.title}: {first}")
|
|
40
|
+
if not gaps:
|
|
41
|
+
gaps.append("No gaps detected — all generated requirements passed.")
|
|
42
|
+
return gaps
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def write_raw_report(summary: RunSummary, paths: Paths, date: str) -> None:
|
|
46
|
+
lines: list[str] = []
|
|
47
|
+
lines.append("# Suitest Testing Report")
|
|
48
|
+
lines.append("")
|
|
49
|
+
lines.append("## 1️⃣ Document Metadata")
|
|
50
|
+
lines.append(f"- **Project:** {summary.project}")
|
|
51
|
+
lines.append(f"- **Mode:** {summary.mode.value}")
|
|
52
|
+
lines.append(f"- **Base URL:** {summary.base_url}")
|
|
53
|
+
lines.append(f"- **Date:** {date}")
|
|
54
|
+
lines.append("- **Prepared by:** Suitest")
|
|
55
|
+
lines.append(
|
|
56
|
+
f"- **Summary:** {summary.total} tests — {summary.passed} passed, "
|
|
57
|
+
f"{summary.failed} failed, {summary.skipped} skipped, {summary.errored} error "
|
|
58
|
+
f"({summary.duration_ms} ms)"
|
|
59
|
+
)
|
|
60
|
+
lines.append(
|
|
61
|
+
f"- **Readiness:** {'ready' if summary.ready else 'NOT READY'} ({summary.ready_detail})"
|
|
62
|
+
)
|
|
63
|
+
lines.append("")
|
|
64
|
+
lines.append("## 2️⃣ Requirement Validation Summary")
|
|
65
|
+
for r in summary.results:
|
|
66
|
+
lines.append("")
|
|
67
|
+
lines.append(f"### {r.test_id} {r.title}")
|
|
68
|
+
lines.append(f"- **Status:** {_BADGE[r.status]}")
|
|
69
|
+
lines.append(f"- **Description:** {r.description}")
|
|
70
|
+
lines.append(f"- **Duration:** {r.duration_ms} ms")
|
|
71
|
+
if r.automation_file:
|
|
72
|
+
lines.append(f"- **Automation:** `{r.automation_file}`")
|
|
73
|
+
if r.error:
|
|
74
|
+
lines.append("- **Error:**")
|
|
75
|
+
lines.append("```")
|
|
76
|
+
lines.extend(r.error.splitlines())
|
|
77
|
+
lines.append("```")
|
|
78
|
+
lines.append("")
|
|
79
|
+
lines.append("## 3️⃣ Coverage & Matching Metrics")
|
|
80
|
+
pct = (summary.passed / summary.total * 100) if summary.total else 0.0
|
|
81
|
+
lines.append(f"- Pass rate: **{pct:.0f}%** ({summary.passed}/{summary.total})")
|
|
82
|
+
by_cat = _coverage_by_outcome(summary.results)
|
|
83
|
+
lines.append("")
|
|
84
|
+
lines.append("| Outcome | Count |")
|
|
85
|
+
lines.append("|---------|-------|")
|
|
86
|
+
for name, count in by_cat:
|
|
87
|
+
lines.append(f"| {name} | {count} |")
|
|
88
|
+
lines.append("")
|
|
89
|
+
lines.append("## 4️⃣ Key Gaps / Risks")
|
|
90
|
+
for g in _gaps(summary):
|
|
91
|
+
lines.append(f"- {g}")
|
|
92
|
+
lines.append("")
|
|
93
|
+
paths.raw_report_md.write_text("\n".join(lines), encoding="utf-8")
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _coverage_by_outcome(results: list[TestResult]) -> list[tuple[str, int]]:
|
|
97
|
+
counts: dict[str, int] = {}
|
|
98
|
+
for r in results:
|
|
99
|
+
counts[r.status.value] = counts.get(r.status.value, 0) + 1
|
|
100
|
+
return sorted(counts.items())
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def write_summary_json(summary: RunSummary, paths: Paths) -> None:
|
|
104
|
+
paths.reports_dir.mkdir(parents=True, exist_ok=True)
|
|
105
|
+
payload = summary_to_json(summary)
|
|
106
|
+
(paths.reports_dir / "summary.json").write_text(json.dumps(payload, indent=2), encoding="utf-8")
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def write_summary_md(summary: RunSummary, paths: Paths) -> None:
|
|
110
|
+
paths.reports_dir.mkdir(parents=True, exist_ok=True)
|
|
111
|
+
pct = (summary.passed / summary.total * 100) if summary.total else 0.0
|
|
112
|
+
lines = [
|
|
113
|
+
f"# Suitest Summary — {summary.project}",
|
|
114
|
+
"",
|
|
115
|
+
f"- Mode: **{summary.mode.value}** · Base URL: `{summary.base_url}`",
|
|
116
|
+
f"- Ready: **{'yes' if summary.ready else 'NO'}** ({summary.ready_detail})",
|
|
117
|
+
f"- Result: **{summary.passed}/{summary.total} passed ({pct:.0f}%)** in {summary.duration_ms} ms",
|
|
118
|
+
f"- Failed: {summary.failed} · Skipped: {summary.skipped} · Error: {summary.errored}",
|
|
119
|
+
"",
|
|
120
|
+
"| Test | Status | ms |",
|
|
121
|
+
"|------|--------|----|",
|
|
122
|
+
]
|
|
123
|
+
for r in summary.results:
|
|
124
|
+
lines.append(f"| {r.test_id} {r.title} | {_BADGE[r.status]} | {r.duration_ms} |")
|
|
125
|
+
(paths.reports_dir / "summary.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def write_summary_html(summary: RunSummary, paths: Paths) -> None:
|
|
129
|
+
paths.reports_dir.mkdir(parents=True, exist_ok=True)
|
|
130
|
+
pct = (summary.passed / summary.total * 100) if summary.total else 0.0
|
|
131
|
+
rows = "\n".join(
|
|
132
|
+
f"<tr class='{r.status.value.lower()}'><td>{r.test_id}</td><td>{_esc(r.title)}</td>"
|
|
133
|
+
f"<td>{r.status.value}</td><td>{r.duration_ms}</td><td><pre>{_esc(r.error)}</pre></td></tr>"
|
|
134
|
+
for r in summary.results
|
|
135
|
+
)
|
|
136
|
+
html = f"""<!doctype html><html><head><meta charset="utf-8">
|
|
137
|
+
<title>Suitest — {_esc(summary.project)}</title>
|
|
138
|
+
<style>
|
|
139
|
+
body{{font-family:ui-sans-serif,system-ui,sans-serif;background:#0a0a0a;color:#fafafa;margin:0;padding:2rem}}
|
|
140
|
+
h1{{font-size:1.4rem}} .meta{{color:#a3a3a3;margin-bottom:1rem}}
|
|
141
|
+
table{{border-collapse:collapse;width:100%;font-size:.85rem}}
|
|
142
|
+
th,td{{border:1px solid #262626;padding:.4rem .6rem;text-align:left;vertical-align:top}}
|
|
143
|
+
th{{background:#161616}} pre{{margin:0;white-space:pre-wrap;color:#f87171;font-size:.75rem}}
|
|
144
|
+
tr.passed td:nth-child(3){{color:#4ade80}} tr.failed td:nth-child(3),tr.error td:nth-child(3){{color:#f87171}}
|
|
145
|
+
.pill{{display:inline-block;padding:.2rem .6rem;border-radius:999px;background:#161616;margin-right:.4rem}}
|
|
146
|
+
</style></head><body>
|
|
147
|
+
<h1>Suitest Report — {_esc(summary.project)} ({summary.mode.value})</h1>
|
|
148
|
+
<div class="meta">
|
|
149
|
+
<span class="pill">Base: {_esc(summary.base_url)}</span>
|
|
150
|
+
<span class="pill">Ready: {"yes" if summary.ready else "NO"}</span>
|
|
151
|
+
<span class="pill">Pass: {summary.passed}/{summary.total} ({pct:.0f}%)</span>
|
|
152
|
+
<span class="pill">{summary.duration_ms} ms</span>
|
|
153
|
+
</div>
|
|
154
|
+
<table><thead><tr><th>ID</th><th>Title</th><th>Status</th><th>ms</th><th>Error</th></tr></thead>
|
|
155
|
+
<tbody>{rows}</tbody></table>
|
|
156
|
+
</body></html>"""
|
|
157
|
+
(paths.reports_dir / "summary.html").write_text(html, encoding="utf-8")
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _esc(text: str) -> str:
|
|
161
|
+
return (
|
|
162
|
+
text.replace("&", "&").replace("<", "<").replace(">", ">").replace('"', """)
|
|
163
|
+
)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def write_all_reports(summary: RunSummary, paths: Paths, date: str) -> None:
|
|
167
|
+
write_raw_report(summary, paths, date)
|
|
168
|
+
write_summary_json(summary, paths)
|
|
169
|
+
write_summary_md(summary, paths)
|
|
170
|
+
write_summary_html(summary, paths)
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
__all__ = [
|
|
174
|
+
"write_all_reports",
|
|
175
|
+
"write_raw_report",
|
|
176
|
+
"write_summary_html",
|
|
177
|
+
"write_summary_json",
|
|
178
|
+
"write_summary_md",
|
|
179
|
+
]
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
"""Execute exported test files and collect structured results.
|
|
2
|
+
|
|
3
|
+
Each ``TCxxx.py`` is runnable standalone (its ``__main__`` calls the test fn), so
|
|
4
|
+
we execute it with the current interpreter, capture stdout/stderr, and map the
|
|
5
|
+
exit code to a :class:`TestResult`. A non-zero exit (e.g. ``AssertionError``)
|
|
6
|
+
becomes ``FAILED`` with the captured traceback as the error message.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import subprocess
|
|
13
|
+
import sys
|
|
14
|
+
import time
|
|
15
|
+
from typing import TYPE_CHECKING
|
|
16
|
+
|
|
17
|
+
from suitest_lifecycle.models import PlanCase, StepResult, TestOutcome, TestResult
|
|
18
|
+
|
|
19
|
+
if TYPE_CHECKING:
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _collect_steps(
|
|
24
|
+
case: PlanCase, test_dir: Path, outcome: TestOutcome
|
|
25
|
+
) -> tuple[list[StepResult], str, str]:
|
|
26
|
+
"""Read the test's ``<TC>.result.json`` sidecar (frontend) or derive steps
|
|
27
|
+
from the plan (backend). Returns (steps, video_path, screenshot_path)."""
|
|
28
|
+
sidecar = test_dir / f"{case.id}.result.json"
|
|
29
|
+
if sidecar.is_file():
|
|
30
|
+
try:
|
|
31
|
+
data = json.loads(sidecar.read_text(encoding="utf-8"))
|
|
32
|
+
except (json.JSONDecodeError, OSError):
|
|
33
|
+
data = {}
|
|
34
|
+
steps = [
|
|
35
|
+
StepResult(
|
|
36
|
+
index=int(s.get("index", i + 1)),
|
|
37
|
+
type=str(s.get("type", "action")),
|
|
38
|
+
description=str(s.get("description", "")),
|
|
39
|
+
status=_as_outcome(str(s.get("status", "PASSED"))),
|
|
40
|
+
screenshot_path=str(s.get("screenshot") or ""),
|
|
41
|
+
)
|
|
42
|
+
for i, s in enumerate(data.get("steps", []) or [])
|
|
43
|
+
if isinstance(s, dict)
|
|
44
|
+
]
|
|
45
|
+
return steps, str(data.get("video") or ""), str(data.get("screenshot") or "")
|
|
46
|
+
# Backend (no sidecar): derive from the plan steps, marking the last failed
|
|
47
|
+
# when the test failed so the Steps panel still shows where it broke.
|
|
48
|
+
steps = []
|
|
49
|
+
n = len(case.steps)
|
|
50
|
+
for i, ps in enumerate(case.steps):
|
|
51
|
+
st = TestOutcome.PASSED
|
|
52
|
+
if outcome in (TestOutcome.FAILED, TestOutcome.ERROR) and i == n - 1:
|
|
53
|
+
st = outcome
|
|
54
|
+
steps.append(StepResult(index=i + 1, type=ps.type, description=ps.description, status=st))
|
|
55
|
+
return steps, "", ""
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _as_outcome(value: str) -> TestOutcome:
|
|
59
|
+
try:
|
|
60
|
+
return TestOutcome(value.upper())
|
|
61
|
+
except ValueError:
|
|
62
|
+
return TestOutcome.PASSED
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _run_one(file_path: Path, python: str, timeout_sec: int) -> tuple[TestOutcome, int, str]:
|
|
66
|
+
start = time.monotonic()
|
|
67
|
+
try:
|
|
68
|
+
proc = subprocess.run(
|
|
69
|
+
[python, str(file_path)],
|
|
70
|
+
capture_output=True,
|
|
71
|
+
text=True,
|
|
72
|
+
timeout=timeout_sec,
|
|
73
|
+
cwd=str(file_path.parent),
|
|
74
|
+
)
|
|
75
|
+
except subprocess.TimeoutExpired:
|
|
76
|
+
return (
|
|
77
|
+
TestOutcome.ERROR,
|
|
78
|
+
int((time.monotonic() - start) * 1000),
|
|
79
|
+
f"timeout after {timeout_sec}s",
|
|
80
|
+
)
|
|
81
|
+
duration_ms = int((time.monotonic() - start) * 1000)
|
|
82
|
+
if proc.returncode == 0:
|
|
83
|
+
return TestOutcome.PASSED, duration_ms, ""
|
|
84
|
+
err = (proc.stderr or "").strip() or (proc.stdout or "").strip() or f"exit {proc.returncode}"
|
|
85
|
+
# Keep the last ~30 lines — enough to see the assertion without flooding the report.
|
|
86
|
+
err_tail = "\n".join(err.splitlines()[-30:])
|
|
87
|
+
return TestOutcome.FAILED, duration_ms, err_tail
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def run_tests(
|
|
91
|
+
cases: list[PlanCase],
|
|
92
|
+
test_dir: Path,
|
|
93
|
+
*,
|
|
94
|
+
selected_ids: list[str] | None = None,
|
|
95
|
+
python: str | None = None,
|
|
96
|
+
timeout_sec: int = 120,
|
|
97
|
+
) -> list[TestResult]:
|
|
98
|
+
interpreter = python or sys.executable
|
|
99
|
+
wanted = set(selected_ids) if selected_ids else None
|
|
100
|
+
results: list[TestResult] = []
|
|
101
|
+
for case in cases:
|
|
102
|
+
if wanted is not None and case.id not in wanted:
|
|
103
|
+
continue
|
|
104
|
+
if not case.automation_file:
|
|
105
|
+
results.append(
|
|
106
|
+
TestResult(
|
|
107
|
+
test_id=case.id,
|
|
108
|
+
title=case.title,
|
|
109
|
+
description=case.description,
|
|
110
|
+
status=TestOutcome.SKIPPED,
|
|
111
|
+
duration_ms=0,
|
|
112
|
+
error="no automation file exported",
|
|
113
|
+
)
|
|
114
|
+
)
|
|
115
|
+
continue
|
|
116
|
+
file_path = test_dir / case.automation_file
|
|
117
|
+
outcome, duration_ms, error = _run_one(file_path, interpreter, timeout_sec)
|
|
118
|
+
steps, video, screenshot = _collect_steps(case, test_dir, outcome)
|
|
119
|
+
artifacts = [p for p in (video, screenshot) if p]
|
|
120
|
+
results.append(
|
|
121
|
+
TestResult(
|
|
122
|
+
test_id=case.id,
|
|
123
|
+
title=case.title,
|
|
124
|
+
description=case.description,
|
|
125
|
+
status=outcome,
|
|
126
|
+
duration_ms=duration_ms,
|
|
127
|
+
error=error,
|
|
128
|
+
automation_file=case.automation_file,
|
|
129
|
+
steps=steps,
|
|
130
|
+
video_path=video,
|
|
131
|
+
screenshot_path=screenshot,
|
|
132
|
+
artifacts=artifacts,
|
|
133
|
+
)
|
|
134
|
+
)
|
|
135
|
+
return results
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
__all__ = ["run_tests"]
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
"""JSON serialisation for lifecycle artifacts (TestSprite-compatible shapes)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import TYPE_CHECKING
|
|
6
|
+
|
|
7
|
+
if TYPE_CHECKING:
|
|
8
|
+
from suitest_lifecycle.models import (
|
|
9
|
+
CodeSummary,
|
|
10
|
+
PlanCase,
|
|
11
|
+
Prd,
|
|
12
|
+
RunSummary,
|
|
13
|
+
TestResult,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
JsonValue = object # documented alias; concrete dicts/lists below are fully typed
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def prd_to_json(prd: Prd) -> dict[str, object]:
|
|
20
|
+
return {
|
|
21
|
+
"meta": {"project": prd.project, "date": prd.date, "prepared_by": prd.prepared_by},
|
|
22
|
+
"product_overview": prd.product_overview,
|
|
23
|
+
"core_goals": list(prd.core_goals),
|
|
24
|
+
"features": [
|
|
25
|
+
{"name": f.name, "description": f.description, "user_flows": list(f.user_flows)}
|
|
26
|
+
for f in prd.features
|
|
27
|
+
],
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def plan_to_json(cases: list[PlanCase]) -> list[dict[str, object]]:
|
|
32
|
+
return [
|
|
33
|
+
{
|
|
34
|
+
"id": c.id,
|
|
35
|
+
"title": c.title,
|
|
36
|
+
"description": c.description,
|
|
37
|
+
"category": c.category,
|
|
38
|
+
"priority": c.priority.value,
|
|
39
|
+
"steps": [{"type": s.type, "description": s.description} for s in c.steps],
|
|
40
|
+
"source_ref": c.source_ref,
|
|
41
|
+
"automation_file": c.automation_file,
|
|
42
|
+
"tags": list(c.tags),
|
|
43
|
+
}
|
|
44
|
+
for c in cases
|
|
45
|
+
]
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def code_summary_to_json(summary: CodeSummary) -> dict[str, object]:
|
|
49
|
+
return {
|
|
50
|
+
"project_name": summary.project_name,
|
|
51
|
+
"mode": summary.mode.value,
|
|
52
|
+
"tech_stack": list(summary.tech_stack),
|
|
53
|
+
"auth_flow": summary.auth_flow,
|
|
54
|
+
"features": list(summary.features),
|
|
55
|
+
"endpoints": [
|
|
56
|
+
{
|
|
57
|
+
"method": e.method,
|
|
58
|
+
"path": e.path,
|
|
59
|
+
"auth_required": e.auth_required,
|
|
60
|
+
"source_file": e.source_file,
|
|
61
|
+
"handler": e.handler,
|
|
62
|
+
}
|
|
63
|
+
for e in summary.endpoints
|
|
64
|
+
],
|
|
65
|
+
"pages": [
|
|
66
|
+
{
|
|
67
|
+
"route": p.route,
|
|
68
|
+
"name": p.name,
|
|
69
|
+
"protected": p.protected,
|
|
70
|
+
"source_file": p.source_file,
|
|
71
|
+
}
|
|
72
|
+
for p in summary.pages
|
|
73
|
+
],
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def result_to_json(r: TestResult) -> dict[str, object]:
|
|
78
|
+
return {
|
|
79
|
+
"testId": r.test_id,
|
|
80
|
+
"title": r.title,
|
|
81
|
+
"description": r.description,
|
|
82
|
+
"status": r.status.value,
|
|
83
|
+
"durationMs": r.duration_ms,
|
|
84
|
+
"error": r.error,
|
|
85
|
+
"automationFile": r.automation_file,
|
|
86
|
+
"artifacts": list(r.artifacts),
|
|
87
|
+
"video": r.video_path,
|
|
88
|
+
"screenshot": r.screenshot_path,
|
|
89
|
+
"steps": [
|
|
90
|
+
{
|
|
91
|
+
"index": s.index,
|
|
92
|
+
"type": s.type,
|
|
93
|
+
"description": s.description,
|
|
94
|
+
"status": s.status.value,
|
|
95
|
+
}
|
|
96
|
+
for s in r.steps
|
|
97
|
+
],
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def results_to_json(results: list[TestResult]) -> list[dict[str, object]]:
|
|
102
|
+
return [result_to_json(r) for r in results]
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def summary_to_json(summary: RunSummary) -> dict[str, object]:
|
|
106
|
+
return {
|
|
107
|
+
"project": summary.project,
|
|
108
|
+
"mode": summary.mode.value,
|
|
109
|
+
"baseUrl": summary.base_url,
|
|
110
|
+
"totals": {
|
|
111
|
+
"total": summary.total,
|
|
112
|
+
"passed": summary.passed,
|
|
113
|
+
"failed": summary.failed,
|
|
114
|
+
"skipped": summary.skipped,
|
|
115
|
+
"errored": summary.errored,
|
|
116
|
+
},
|
|
117
|
+
"durationMs": summary.duration_ms,
|
|
118
|
+
"serverStarted": summary.server_started,
|
|
119
|
+
"ready": summary.ready,
|
|
120
|
+
"readyDetail": summary.ready_detail,
|
|
121
|
+
"results": results_to_json(summary.results),
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
__all__ = [
|
|
126
|
+
"code_summary_to_json",
|
|
127
|
+
"plan_to_json",
|
|
128
|
+
"prd_to_json",
|
|
129
|
+
"results_to_json",
|
|
130
|
+
"summary_to_json",
|
|
131
|
+
]
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
"""TCM — the test-case source of truth.
|
|
2
|
+
|
|
3
|
+
Primary store is a readable JSON mirror under ``suitest-output/tcm/`` so the
|
|
4
|
+
lifecycle runs with zero infrastructure. When the Suitest Postgres stack is
|
|
5
|
+
reachable, :func:`sync_to_db` upserts the same records through the real
|
|
6
|
+
``packages/db`` repositories (best-effort; skipped cleanly when unavailable).
|
|
7
|
+
|
|
8
|
+
Every generated case gets a TCM record; every run updates each case's
|
|
9
|
+
``last_run_result`` / ``last_run_at`` / ``failure_reason`` / ``duration_ms`` —
|
|
10
|
+
satisfying "TCM as source of truth, updated by each run".
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import json
|
|
16
|
+
from dataclasses import dataclass
|
|
17
|
+
from typing import TYPE_CHECKING
|
|
18
|
+
|
|
19
|
+
if TYPE_CHECKING:
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
|
|
22
|
+
from suitest_lifecycle.models import Mode, PlanCase, RunSummary, TestResult
|
|
23
|
+
from suitest_lifecycle.paths import Paths
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass(frozen=True)
|
|
27
|
+
class TcmSyncReport:
|
|
28
|
+
backend: str # "file" | "db" | "db-skipped"
|
|
29
|
+
cases_written: int
|
|
30
|
+
runs_appended: int
|
|
31
|
+
detail: str
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _load(path: Path) -> list[dict[str, object]]:
|
|
35
|
+
if path.is_file():
|
|
36
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
37
|
+
if isinstance(data, list):
|
|
38
|
+
return [d for d in data if isinstance(d, dict)]
|
|
39
|
+
return []
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _case_record(case: PlanCase, mode: Mode) -> dict[str, object]:
|
|
43
|
+
return {
|
|
44
|
+
"id": f"{mode.value}:{case.id}",
|
|
45
|
+
"local_id": case.id,
|
|
46
|
+
"mode": mode.value,
|
|
47
|
+
"title": case.title,
|
|
48
|
+
"description": case.description,
|
|
49
|
+
"category": case.category,
|
|
50
|
+
"priority": case.priority.value,
|
|
51
|
+
"status": "active",
|
|
52
|
+
"tags": [mode.value, case.category.lower()],
|
|
53
|
+
"source_ref": case.source_ref,
|
|
54
|
+
"automation_file": case.automation_file,
|
|
55
|
+
"last_run_result": None,
|
|
56
|
+
"last_run_at": None,
|
|
57
|
+
"failure_reason": None,
|
|
58
|
+
"duration_ms": None,
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def upsert_cases(
|
|
63
|
+
cases: list[PlanCase],
|
|
64
|
+
paths: Paths,
|
|
65
|
+
mode: Mode,
|
|
66
|
+
results: list[TestResult],
|
|
67
|
+
run_at: str,
|
|
68
|
+
) -> int:
|
|
69
|
+
"""Write/merge case records and fold in each case's latest run result."""
|
|
70
|
+
paths.tcm_dir.mkdir(parents=True, exist_ok=True)
|
|
71
|
+
existing = {str(r.get("id")): r for r in _load(paths.tcm_cases_json)}
|
|
72
|
+
result_by_id = {r.test_id: r for r in results}
|
|
73
|
+
|
|
74
|
+
for case in cases:
|
|
75
|
+
rec = _case_record(case, mode)
|
|
76
|
+
prior = existing.get(str(rec["id"]))
|
|
77
|
+
res = result_by_id.get(case.id) # results are keyed by local TC id
|
|
78
|
+
if res is not None:
|
|
79
|
+
rec["last_run_result"] = res.status.value
|
|
80
|
+
rec["last_run_at"] = run_at
|
|
81
|
+
rec["duration_ms"] = res.duration_ms
|
|
82
|
+
rec["failure_reason"] = res.error.splitlines()[-1] if res.error else None
|
|
83
|
+
elif prior is not None:
|
|
84
|
+
for k in ("last_run_result", "last_run_at", "duration_ms", "failure_reason"):
|
|
85
|
+
rec[k] = prior.get(k)
|
|
86
|
+
existing[str(rec["id"])] = rec
|
|
87
|
+
|
|
88
|
+
merged = list(existing.values())
|
|
89
|
+
paths.tcm_cases_json.write_text(json.dumps(merged, indent=2), encoding="utf-8")
|
|
90
|
+
return len(cases)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def record_run(summary: RunSummary, paths: Paths, run_at: str) -> int:
|
|
94
|
+
paths.tcm_dir.mkdir(parents=True, exist_ok=True)
|
|
95
|
+
runs = _load(paths.tcm_runs_json)
|
|
96
|
+
runs.append(
|
|
97
|
+
{
|
|
98
|
+
"run_at": run_at,
|
|
99
|
+
"project": summary.project,
|
|
100
|
+
"mode": summary.mode.value,
|
|
101
|
+
"base_url": summary.base_url,
|
|
102
|
+
"total": summary.total,
|
|
103
|
+
"passed": summary.passed,
|
|
104
|
+
"failed": summary.failed,
|
|
105
|
+
"skipped": summary.skipped,
|
|
106
|
+
"errored": summary.errored,
|
|
107
|
+
"duration_ms": summary.duration_ms,
|
|
108
|
+
"ready": summary.ready,
|
|
109
|
+
}
|
|
110
|
+
)
|
|
111
|
+
paths.tcm_runs_json.write_text(json.dumps(runs, indent=2), encoding="utf-8")
|
|
112
|
+
return 1
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def sync_tcm(
|
|
116
|
+
cases: list[PlanCase],
|
|
117
|
+
summary: RunSummary,
|
|
118
|
+
paths: Paths,
|
|
119
|
+
mode: Mode,
|
|
120
|
+
run_at: str,
|
|
121
|
+
) -> TcmSyncReport:
|
|
122
|
+
written = upsert_cases(cases, paths, mode, summary.results, run_at)
|
|
123
|
+
appended = record_run(summary, paths, run_at)
|
|
124
|
+
db_detail = _try_db_sync(cases, summary, mode)
|
|
125
|
+
return TcmSyncReport(
|
|
126
|
+
backend="file",
|
|
127
|
+
cases_written=written,
|
|
128
|
+
runs_appended=appended,
|
|
129
|
+
detail=f"file mirror at {paths.tcm_dir}; db: {db_detail}",
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _try_db_sync(cases: list[PlanCase], summary: RunSummary, mode: Mode) -> str:
|
|
134
|
+
"""Best-effort upsert through packages/db. Cleanly skips without a DB.
|
|
135
|
+
|
|
136
|
+
Kept import-local so the lifecycle never hard-depends on the API stack.
|
|
137
|
+
"""
|
|
138
|
+
try:
|
|
139
|
+
import importlib
|
|
140
|
+
|
|
141
|
+
importlib.import_module("suitest_db")
|
|
142
|
+
except ImportError:
|
|
143
|
+
return "skipped (suitest_db not installed)"
|
|
144
|
+
# Real DB wiring requires an async session + DATABASE_URL; deferred to the
|
|
145
|
+
# API-side sync service. The file mirror remains the portable source of truth.
|
|
146
|
+
return "available (deferred to API sync service)"
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
__all__ = ["TcmSyncReport", "record_run", "sync_tcm", "upsert_cases"]
|