@topy-ai/maggie 0.7.17 → 0.7.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -5
- package/README.zh-TW.md +12 -2
- package/bin/maggie.js +10 -0
- package/bundled-contracts/maggiedash/README.md +19 -0
- package/bundled-contracts/maggiedash/activity-log-v1.json +27 -0
- package/bundled-contracts/maggiedash/dashboard-runtime-v1.json +33 -0
- package/bundled-contracts/maggiedash/navigation-v1.json +29 -0
- package/bundled-references/universal-booking-adapter.md +36 -1
- package/bundled-skills/maggie-dash/SKILL.md +22 -1
- package/bundled-skills/maggie-deployment/SKILL.md +9 -1
- package/bundled-skills/maggie-feedback/SKILL.md +5 -0
- package/bundled-skills/maggie-qa-workflow/SKILL.md +11 -0
- package/bundled-skills/maggie-service-booking/SKILL.md +19 -1
- package/bundled-tools/clis/maggie_dash.py +26 -5
- package/bundled-tools/clis/maggie_docs.py +151 -0
- package/bundled-tools/clis/maggie_feedback.py +17 -5
- package/bundled-tools/clis/maggie_qa_workflow.py +2 -0
- package/bundled-tools/clis/maggie_release.py +88 -2
- package/bundled-tools/clis/maggie_service_booking.py +42 -3
- package/bundled-tools/runtime/booking_capabilities.py +137 -0
- package/bundled-tools/runtime/maggie_api_contract.py +51 -0
- package/bundled-tools/runtime/maggie_dash_runtime.py +58 -0
- package/bundled-tools/runtime/maggie_schema_audit.py +72 -0
- package/bundled-tools/runtime/service_variants.py +48 -7
- package/package.json +1 -1
- package/references/universal-booking-adapter.md +36 -1
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Audit project documentation for stale trackers and unreferenced files."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import argparse
|
|
7
|
+
import json
|
|
8
|
+
import re
|
|
9
|
+
import subprocess
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
DOCUMENT_EXTENSIONS = {".md", ".mdx", ".json", ".yaml", ".yml"}
|
|
14
|
+
SKIP_PARTS = {".git", "node_modules", "dist", "build", ".next"}
|
|
15
|
+
COMPLETED_RE = re.compile(r"(?i)\b(?:done|complete|completed|shipped|resolved|closed)\b")
|
|
16
|
+
TRACKER_RE = re.compile(r"(?i)(?:plan|task|progress|roadmap|todo)")
|
|
17
|
+
RECORD_RE = re.compile(r"(?i)(?:audit|report|evidence|log|feedback|record)")
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _files(root: Path) -> list[Path]:
|
|
21
|
+
return sorted(
|
|
22
|
+
path for path in root.rglob("*")
|
|
23
|
+
if path.is_file() and path.suffix.casefold() in DOCUMENT_EXTENSIONS
|
|
24
|
+
and not any(part in SKIP_PARTS for part in path.relative_to(root).parts)
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _category(path: Path) -> str:
|
|
29
|
+
if path.suffix.casefold() in {".json", ".yaml", ".yml"}:
|
|
30
|
+
return "data"
|
|
31
|
+
name = path.stem
|
|
32
|
+
if TRACKER_RE.search(name):
|
|
33
|
+
return "plan"
|
|
34
|
+
if RECORD_RE.search(name):
|
|
35
|
+
return "record"
|
|
36
|
+
return "reference"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _identity(path: Path, text: str) -> str:
|
|
40
|
+
heading = next((line.strip().lstrip("#").strip() for line in text.splitlines() if line.lstrip().startswith("#")), "")
|
|
41
|
+
value = heading or path.stem
|
|
42
|
+
return re.sub(r"[^a-z0-9]+", "-", value.casefold()).strip("-")
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _last_change(project: Path, relative: str) -> str | None:
|
|
46
|
+
try:
|
|
47
|
+
result = subprocess.run(
|
|
48
|
+
["git", "-C", str(project), "log", "-1", "--format=%cI", "--", relative],
|
|
49
|
+
capture_output=True, text=True, check=False,
|
|
50
|
+
)
|
|
51
|
+
except OSError:
|
|
52
|
+
return None
|
|
53
|
+
value = result.stdout.strip()
|
|
54
|
+
return value or None
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def audit(project: Path, docs_dir: str = "docs") -> dict:
|
|
58
|
+
root = project.resolve()
|
|
59
|
+
docs = (root / docs_dir).resolve()
|
|
60
|
+
if not docs.is_dir():
|
|
61
|
+
return {"schemaVersion": "maggie-docs-audit.v1", "passed": False, "errors": [f"docs directory does not exist: {docs}"], "files": []}
|
|
62
|
+
files = _files(docs)
|
|
63
|
+
corpus: list[tuple[Path, str]] = []
|
|
64
|
+
for path in _files(root):
|
|
65
|
+
try:
|
|
66
|
+
corpus.append((path, path.read_text(encoding="utf-8", errors="replace")))
|
|
67
|
+
except OSError:
|
|
68
|
+
continue
|
|
69
|
+
entries = []
|
|
70
|
+
for path in files:
|
|
71
|
+
try:
|
|
72
|
+
text = path.read_text(encoding="utf-8", errors="replace")
|
|
73
|
+
except OSError as error:
|
|
74
|
+
entries.append({"path": str(path.relative_to(root)), "status": "unreadable", "error": str(error)})
|
|
75
|
+
continue
|
|
76
|
+
relative = str(path.relative_to(root))
|
|
77
|
+
basename = path.name
|
|
78
|
+
inbound = sum(1 for source, source_text in corpus if source != path and (relative in source_text or basename in source_text))
|
|
79
|
+
category = _category(path)
|
|
80
|
+
completed_tracker = category == "plan" and bool(COMPLETED_RE.search(text))
|
|
81
|
+
findings: list[str] = []
|
|
82
|
+
if completed_tracker:
|
|
83
|
+
findings.append("completed-progress-tracker")
|
|
84
|
+
if inbound == 0 and category in {"reference", "data"}:
|
|
85
|
+
findings.append("unreferenced")
|
|
86
|
+
entries.append({
|
|
87
|
+
"path": relative,
|
|
88
|
+
"category": category,
|
|
89
|
+
"inboundReferences": inbound,
|
|
90
|
+
"lastChanged": _last_change(root, relative),
|
|
91
|
+
"completedTracker": completed_tracker,
|
|
92
|
+
"openTracker": category == "plan" and not completed_tracker,
|
|
93
|
+
"identity": _identity(path, text),
|
|
94
|
+
"findings": findings,
|
|
95
|
+
})
|
|
96
|
+
identities: dict[str, list[dict]] = {}
|
|
97
|
+
for item in entries:
|
|
98
|
+
identities.setdefault(item.get("identity", ""), []).append(item)
|
|
99
|
+
duplicate_groups = [group for key, group in identities.items() if key and len(group) > 1]
|
|
100
|
+
duplicate_paths = {item["path"] for group in duplicate_groups for item in group}
|
|
101
|
+
for item in entries:
|
|
102
|
+
item.pop("identity", None)
|
|
103
|
+
if item["path"] in duplicate_paths:
|
|
104
|
+
item["findings"].append("duplicate-candidate")
|
|
105
|
+
item["staleCandidate"] = bool(item["findings"])
|
|
106
|
+
findings = [item for item in entries if item.get("findings")]
|
|
107
|
+
return {
|
|
108
|
+
"schemaVersion": "maggie-docs-audit.v1",
|
|
109
|
+
"passed": True,
|
|
110
|
+
"project": str(root),
|
|
111
|
+
"docsDirectory": str(docs.relative_to(root)),
|
|
112
|
+
"fileCount": len(entries),
|
|
113
|
+
"findingCount": len(findings),
|
|
114
|
+
"summary": {
|
|
115
|
+
"reference": sum(item.get("category") == "reference" for item in entries),
|
|
116
|
+
"record": sum(item.get("category") == "record" for item in entries),
|
|
117
|
+
"plan": sum(item.get("category") == "plan" for item in entries),
|
|
118
|
+
"data": sum(item.get("category") == "data" for item in entries),
|
|
119
|
+
"completedTrackers": sum(item.get("completedTracker", False) for item in entries),
|
|
120
|
+
"openTrackers": sum(item.get("openTracker", False) for item in entries),
|
|
121
|
+
"unreferenced": sum("unreferenced" in item.get("findings", []) for item in entries),
|
|
122
|
+
"duplicateCandidates": len(duplicate_groups),
|
|
123
|
+
"staleCandidates": sum(item.get("staleCandidate", False) for item in entries),
|
|
124
|
+
},
|
|
125
|
+
"files": entries,
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def main() -> int:
|
|
130
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
131
|
+
parser.add_argument("audit", nargs="?", choices=("audit",), default="audit")
|
|
132
|
+
parser.add_argument("--project", default=".")
|
|
133
|
+
parser.add_argument("--docs-dir", default="docs")
|
|
134
|
+
parser.add_argument("--fail-on-findings", action="store_true")
|
|
135
|
+
parser.add_argument("--output")
|
|
136
|
+
args = parser.parse_args()
|
|
137
|
+
result = audit(Path(args.project), args.docs_dir)
|
|
138
|
+
if args.fail_on_findings and result.get("findingCount"):
|
|
139
|
+
result["passed"] = False
|
|
140
|
+
result.setdefault("errors", []).append("documentation findings require review")
|
|
141
|
+
if args.output:
|
|
142
|
+
output = Path(args.output).expanduser().resolve()
|
|
143
|
+
output.parent.mkdir(parents=True, exist_ok=True)
|
|
144
|
+
output.write_text(json.dumps(result, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
|
145
|
+
result["output"] = str(output)
|
|
146
|
+
print(json.dumps(result, indent=2, ensure_ascii=False))
|
|
147
|
+
return 0 if result.get("passed") else 1
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
if __name__ == "__main__":
|
|
151
|
+
raise SystemExit(main())
|
|
@@ -20,6 +20,12 @@ from urllib.request import Request, urlopen
|
|
|
20
20
|
|
|
21
21
|
DEFAULT_ENDPOINT = "https://feedback.noblox.app/api/feedback"
|
|
22
22
|
SECRET_RE = re.compile(r"(?i)(bearer\s+|(?:api[_-]?key|token|secret|password|npm_config[^=]*|authorization)\s*[=:]\s*)[^\s,;]+")
|
|
23
|
+
LOCAL_PATH_RE = re.compile(
|
|
24
|
+
r"(?<![A-Za-z0-9])(?:~/(?:[^\s,;:)\]}]+/)*[^\s,;:)\]}]+|"
|
|
25
|
+
r"/(?:home|Users|private|tmp|var|mnt|opt|root|srv|workspace|workspaces)(?:/[^\s,;:)\]}]+)+|"
|
|
26
|
+
r"[A-Za-z]:[\\/]+(?:[^\\/\s,;:)\]}]+[\\/]+)*[^\\/\s,;:)\]}]+|"
|
|
27
|
+
r"\\\\[^\\/\s,;:)\]}]+(?:[\\/]+[^\\/\s,;:)\]}]+)+)"
|
|
28
|
+
)
|
|
23
29
|
VERSION_RE = re.compile(r"\d+\.\d+\.\d+(?:[-+][A-Za-z0-9.-]+)?")
|
|
24
30
|
ACKNOWLEDGEMENT_KEYS = ("status", "feedbackId", "requestId")
|
|
25
31
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
@@ -30,7 +36,8 @@ def now() -> str:
|
|
|
30
36
|
|
|
31
37
|
|
|
32
38
|
def safe_text(value: object) -> str:
|
|
33
|
-
|
|
39
|
+
value = SECRET_RE.sub(r"\1[REDACTED]", str(value or ""))
|
|
40
|
+
return LOCAL_PATH_RE.sub("[LOCAL_PATH_REDACTED]", value).strip()
|
|
34
41
|
|
|
35
42
|
|
|
36
43
|
def project_fingerprint(project: Path) -> str:
|
|
@@ -90,6 +97,11 @@ def collect(args: argparse.Namespace) -> int:
|
|
|
90
97
|
skill = args.skill or report.get("skill") or report.get("workflow") or "unknown"
|
|
91
98
|
run_id = args.run_id or report.get("runId") or report.get("jobId") or ""
|
|
92
99
|
actual = args.actual or report.get("actual") or report.get("error") or report.get("message") or ""
|
|
100
|
+
fixed = bool(args.fixed or report.get("fixed") is True or report.get("status") == "fixed")
|
|
101
|
+
resolution = safe_text(args.resolution or report.get("resolution"))
|
|
102
|
+
validation = safe_text(args.validation or report.get("validation"))
|
|
103
|
+
if fixed and (not resolution or not validation):
|
|
104
|
+
raise ValueError("fixed feedback requires non-empty --resolution and --validation")
|
|
93
105
|
context = {"projectFingerprint": project_fingerprint(project)}
|
|
94
106
|
if args.allow_project_context and args.context_note:
|
|
95
107
|
context["note"] = safe_text(args.context_note)
|
|
@@ -103,15 +115,15 @@ def collect(args: argparse.Namespace) -> int:
|
|
|
103
115
|
"skill": skill,
|
|
104
116
|
"runId": run_id,
|
|
105
117
|
"phase": args.phase or report.get("phase") or "",
|
|
106
|
-
"status": report.get("status") or ("fixed" if
|
|
118
|
+
"status": report.get("status") or ("fixed" if fixed else "reported"),
|
|
107
119
|
"summary": safe_text(args.summary),
|
|
108
120
|
"expected": safe_text(args.expected),
|
|
109
121
|
"actual": safe_text(actual),
|
|
110
122
|
"errorFingerprint": safe_text(args.error_fingerprint or report.get("errorFingerprint") or ""),
|
|
111
123
|
"stepsToReproduce": [safe_text(step) for step in args.reproduce],
|
|
112
|
-
"fixed":
|
|
113
|
-
"resolution":
|
|
114
|
-
"validation":
|
|
124
|
+
"fixed": fixed,
|
|
125
|
+
"resolution": resolution,
|
|
126
|
+
"validation": validation,
|
|
115
127
|
"attachments": [attachment(value) for value in args.screenshot],
|
|
116
128
|
"environment": {"os": platform.system().lower(), "python": platform.python_version()},
|
|
117
129
|
"privacy": {"secretsRedacted": True, "projectContextAllowed": bool(args.allow_project_context)},
|
|
@@ -224,6 +224,8 @@ def record(args: argparse.Namespace) -> int:
|
|
|
224
224
|
raise ValueError("a failed scenario must have a recorded fix before retest")
|
|
225
225
|
if phase == "test" and args.status == "fail" and not (args.expected and args.actual and args.error_fingerprint):
|
|
226
226
|
raise ValueError("a failed test requires --expected, --actual and --error-fingerprint")
|
|
227
|
+
if phase in {"test", "retest"} and args.status == "pass" and not args.evidence:
|
|
228
|
+
raise ValueError("a passing test requires --evidence from the browser adapter")
|
|
227
229
|
event = {
|
|
228
230
|
"at": now(),
|
|
229
231
|
"phase": phase,
|
|
@@ -16,7 +16,7 @@ from urllib.error import HTTPError, URLError
|
|
|
16
16
|
from urllib.request import Request, urlopen
|
|
17
17
|
from datetime import datetime, timezone
|
|
18
18
|
from pathlib import Path
|
|
19
|
-
from urllib.parse import urljoin
|
|
19
|
+
from urllib.parse import urljoin, urlsplit
|
|
20
20
|
|
|
21
21
|
|
|
22
22
|
ROOT = Path(__file__).resolve().parent
|
|
@@ -130,14 +130,23 @@ def evidence_gate(project: Path, environment: str) -> dict:
|
|
|
130
130
|
def changed_surface_gate(project: Path) -> dict:
|
|
131
131
|
"""Require visual/runtime evidence when a visitor-facing surface changed."""
|
|
132
132
|
try:
|
|
133
|
-
|
|
133
|
+
changed_output = subprocess.run(
|
|
134
134
|
["git", "-C", str(project), "diff", "--name-only"],
|
|
135
135
|
capture_output=True, text=True, check=True,
|
|
136
136
|
).stdout.splitlines()
|
|
137
|
+
staged_output = subprocess.run(
|
|
138
|
+
["git", "-C", str(project), "diff", "--cached", "--name-only"],
|
|
139
|
+
capture_output=True, text=True, check=True,
|
|
140
|
+
).stdout.splitlines()
|
|
141
|
+
untracked_output = subprocess.run(
|
|
142
|
+
["git", "-C", str(project), "ls-files", "--others", "--exclude-standard", "-z"],
|
|
143
|
+
capture_output=True, text=True, check=True,
|
|
144
|
+
).stdout.split("\0")
|
|
137
145
|
except (OSError, subprocess.CalledProcessError):
|
|
138
146
|
return {"name": "changed-surface-evidence", "passed": True, "exitCode": 0,
|
|
139
147
|
"result": {"passed": True, "changed": False, "reason": "project is not a git worktree"}, "stderr": ""}
|
|
140
148
|
visitor_suffixes = {".astro", ".css", ".scss", ".html", ".jsx", ".tsx", ".js", ".ts", ".svg", ".png", ".jpg", ".jpeg", ".webp"}
|
|
149
|
+
changed = set(changed_output) | set(staged_output) | {path for path in untracked_output if path}
|
|
141
150
|
surfaces = sorted(path for path in changed if Path(path).suffix.lower() in visitor_suffixes)
|
|
142
151
|
if not surfaces:
|
|
143
152
|
return {"name": "changed-surface-evidence", "passed": True, "exitCode": 0,
|
|
@@ -181,6 +190,82 @@ def changed_surface_gate(project: Path) -> dict:
|
|
|
181
190
|
"result": {"passed": not errors, "changed": True, "surfaces": surfaces, "evidence": evidence, "errors": errors}, "stderr": ""}
|
|
182
191
|
|
|
183
192
|
|
|
193
|
+
def _origin(value: object) -> str:
|
|
194
|
+
try:
|
|
195
|
+
parsed = urlsplit(str(value or ""))
|
|
196
|
+
if parsed.scheme and parsed.netloc:
|
|
197
|
+
return f"{parsed.scheme.lower()}://{parsed.netloc.lower()}"
|
|
198
|
+
except ValueError:
|
|
199
|
+
pass
|
|
200
|
+
return ""
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _qa_run_matches(run: dict, manifest_label: str | None, environment: str, base_url: str | None) -> bool:
|
|
204
|
+
if run.get("schemaVersion") != "maggie.qa-run.v1":
|
|
205
|
+
return False
|
|
206
|
+
if run.get("environment") != environment:
|
|
207
|
+
return False
|
|
208
|
+
if manifest_label and run.get("scenarioManifest") != manifest_label:
|
|
209
|
+
return False
|
|
210
|
+
if base_url and _origin(run.get("baseUrl")) != _origin(base_url):
|
|
211
|
+
return False
|
|
212
|
+
return isinstance(run.get("scenarios"), dict) and bool(run.get("scenarios"))
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def qa_gate(project: Path, environment: str, base_url: str | None = None) -> dict:
|
|
216
|
+
"""Consume the latest matching project QA run without running a browser."""
|
|
217
|
+
manifest = project / ".maggie" / "scenario-manifest.json"
|
|
218
|
+
run_dir = project / ".maggie" / "qa-runs"
|
|
219
|
+
run_paths = sorted(run_dir.glob("*.json")) if run_dir.is_dir() else []
|
|
220
|
+
configured = manifest.is_file() or bool(run_paths)
|
|
221
|
+
if not configured:
|
|
222
|
+
return {"name": "scenario-qa", "passed": True, "exitCode": 0,
|
|
223
|
+
"result": {"passed": True, "state": "not-configured", "configured": False,
|
|
224
|
+
"reason": "no scenario manifest or QA run directory"}, "stderr": ""}
|
|
225
|
+
|
|
226
|
+
try:
|
|
227
|
+
manifest_label = str(manifest.relative_to(project)) if manifest.is_file() else None
|
|
228
|
+
except ValueError:
|
|
229
|
+
manifest_label = manifest.name if manifest.is_file() else None
|
|
230
|
+
candidates: list[tuple[Path, dict]] = []
|
|
231
|
+
invalid_count = 0
|
|
232
|
+
for path in run_paths:
|
|
233
|
+
try:
|
|
234
|
+
value = json.loads(path.read_text(encoding="utf-8"))
|
|
235
|
+
if isinstance(value, dict) and _qa_run_matches(value, manifest_label, environment, base_url):
|
|
236
|
+
candidates.append((path, value))
|
|
237
|
+
except (OSError, json.JSONDecodeError):
|
|
238
|
+
invalid_count += 1
|
|
239
|
+
if not candidates:
|
|
240
|
+
return {"name": "scenario-qa", "passed": False, "exitCode": 1,
|
|
241
|
+
"result": {"passed": False, "state": "inconclusive", "configured": True,
|
|
242
|
+
"errors": ["no matching scenario QA run", f"invalidRuns={invalid_count}"]}, "stderr": ""}
|
|
243
|
+
|
|
244
|
+
path, run = max(candidates, key=lambda item: str(item[1].get("updatedAt") or item[1].get("createdAt") or ""))
|
|
245
|
+
errors: list[str] = []
|
|
246
|
+
summary = run.get("summary") if isinstance(run.get("summary"), dict) else {}
|
|
247
|
+
if summary.get("gate") != "pass":
|
|
248
|
+
errors.append("latest matching QA run is not passed")
|
|
249
|
+
for scenario in run.get("scenarios", {}).values():
|
|
250
|
+
if not isinstance(scenario, dict) or scenario.get("status") != "pass":
|
|
251
|
+
errors.append("latest matching QA run contains a non-passing scenario")
|
|
252
|
+
continue
|
|
253
|
+
history = scenario.get("history") if isinstance(scenario.get("history"), list) else []
|
|
254
|
+
last = history[-1] if history else {}
|
|
255
|
+
if last.get("phase") not in {"test", "retest"} or last.get("status") != "pass":
|
|
256
|
+
errors.append("latest matching QA run has a scenario without a passing final test")
|
|
257
|
+
if not isinstance(last.get("evidence"), list) or not last.get("evidence"):
|
|
258
|
+
errors.append("latest matching QA run has a scenario without browser evidence")
|
|
259
|
+
passed = not errors
|
|
260
|
+
try:
|
|
261
|
+
report_path = str(path.relative_to(project))
|
|
262
|
+
except ValueError:
|
|
263
|
+
report_path = path.name
|
|
264
|
+
return {"name": "scenario-qa", "passed": passed, "exitCode": 0 if passed else 1,
|
|
265
|
+
"result": {"passed": passed, "state": "passed" if passed else "failed", "configured": True,
|
|
266
|
+
"run": report_path, "updatedAt": run.get("updatedAt"), "errors": errors}, "stderr": ""}
|
|
267
|
+
|
|
268
|
+
|
|
184
269
|
def editorial_gate(project: Path) -> dict:
|
|
185
270
|
"""Require explicit editorial approval for every launch category.
|
|
186
271
|
|
|
@@ -279,6 +364,7 @@ def main() -> int:
|
|
|
279
364
|
|
|
280
365
|
gates = []
|
|
281
366
|
gates.append(changed_surface_gate(project))
|
|
367
|
+
gates.append(qa_gate(project, args.environment, args.base_url))
|
|
282
368
|
gates.append(build_gate(project))
|
|
283
369
|
for name, command in build_gates(project, args.environment, args.target, not args.skip_compatibility):
|
|
284
370
|
gates.append(run_gate(name, command, project))
|
|
@@ -12,6 +12,7 @@ from urllib.request import Request, urlopen
|
|
|
12
12
|
|
|
13
13
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "runtime"))
|
|
14
14
|
from route_imports import imported_components # noqa: E402
|
|
15
|
+
from booking_capabilities import audit_matrix # noqa: E402
|
|
15
16
|
|
|
16
17
|
NOW = lambda: datetime.now(timezone.utc).isoformat()
|
|
17
18
|
MONEY = re.compile(r"(?:£|GBP\s*)\s*([0-9]+(?:[.,][0-9]{1,2})?)", re.I)
|
|
@@ -76,7 +77,8 @@ def parse(source, provider):
|
|
|
76
77
|
currency=(offer.get("priceCurrency") or ("GBP" if price and ("£" in text or provider=="fresha") else None))
|
|
77
78
|
if price is None and dm is None: continue
|
|
78
79
|
amount=int(round(float(str(price).replace(",",""))*100)) if price is not None else None
|
|
79
|
-
|
|
80
|
+
provider_variant_id = offer.get("sku") or offer.get("@id") or offer.get("id")
|
|
81
|
+
variants.append({"id":str(provider_variant_id or f"{slug(name)}:{dm.group(1) if dm else 'default'}"),"idSource":"provider" if provider_variant_id else "derived","variantSource":"native" if provider_variant_id else "derived-offers","title":offer.get("name") or (f"{dm.group(1)} minutes" if dm else "Standard"),"durationMinutes":int(dm.group(1)) if dm else None,"price":{"amountMinor":amount,"currency":currency,"display":f"£{amount/100:.2f}" if amount is not None and currency=="GBP" else None}})
|
|
80
82
|
url=item.get("url") or item.get("sameAs")
|
|
81
83
|
if not variants and not url: continue
|
|
82
84
|
pid=str(item.get("providerServiceId") or item.get("productID") or item.get("sku") or slug(name))
|
|
@@ -113,7 +115,8 @@ def parse_fresha_embedded(raw, source):
|
|
|
113
115
|
parsed_amount, parsed_currency=money_value(display or "")
|
|
114
116
|
caption=variant.get("caption") or ""
|
|
115
117
|
duration=duration_minutes(caption) or round(float(variant.get("maxInSeconds") or variant.get("minInSeconds") or item.get("maxInSeconds") or item.get("minInSeconds") or 0)/60) or None
|
|
116
|
-
|
|
118
|
+
provider_variant_id = variant.get("id")
|
|
119
|
+
variants.append({"id":str(provider_variant_id or f'{item["serviceId"]}:default'),"idSource":"provider" if provider_variant_id else "derived","variantSource":"native" if provider_variant_id else "single-fallback","title":variant.get("name") or caption or "Standard","durationMinutes":duration,"price":{"amountMinor":parsed_amount if parsed_amount is not None else amount,"currency":parsed_currency or currency,"display":display}})
|
|
117
120
|
if not variants: continue
|
|
118
121
|
booking=item.get("bookingUrl") or f'{source.split("?")[0]}/booking?offerItemId={quote(str(item.get("id") or ""))}'
|
|
119
122
|
record=make_record("fresha",source,str(item["serviceId"]),str(item["name"]),str(item.get("description") or ""),variants,booking,[])
|
|
@@ -157,7 +160,7 @@ def parse_fresha_embedded(raw, source):
|
|
|
157
160
|
for caption, display, vid, vname in variant_pattern.findall(match.group("variants")):
|
|
158
161
|
duration=duration_minutes(caption)
|
|
159
162
|
amount, currency=money_value(display)
|
|
160
|
-
variants.append({"id":vid,"title":vname or caption,"durationMinutes":duration,"price":{"amountMinor":amount,"currency":currency,"display":display}})
|
|
163
|
+
variants.append({"id":vid,"idSource":"provider","variantSource":"native","title":vname or caption,"durationMinutes":duration,"price":{"amountMinor":amount,"currency":currency,"display":display}})
|
|
161
164
|
if not variants: continue
|
|
162
165
|
name=variants[0]["title"]
|
|
163
166
|
records.append(make_record("fresha",source,match.group("pid"),name,description,variants,None,[]))
|
|
@@ -436,6 +439,40 @@ def cmd_category_hash(args):
|
|
|
436
439
|
print(json.dumps(report, indent=2, ensure_ascii=False))
|
|
437
440
|
return 0 if not errors else 1
|
|
438
441
|
|
|
442
|
+
|
|
443
|
+
def cmd_capability_audit(args):
|
|
444
|
+
"""Validate provider declarations and emit a fixture-backed capability matrix."""
|
|
445
|
+
project = root(args)
|
|
446
|
+
def read(path_value):
|
|
447
|
+
path_value = Path(path_value)
|
|
448
|
+
path_value = path_value if path_value.is_absolute() else project / path_value
|
|
449
|
+
return path_value.resolve(), json.loads(path_value.resolve().read_text(encoding="utf-8"))
|
|
450
|
+
try:
|
|
451
|
+
catalogue_path, catalogue = read(args.catalogue)
|
|
452
|
+
declaration_path, declarations = read(args.capabilities_file)
|
|
453
|
+
fixture = None
|
|
454
|
+
fixture_path = None
|
|
455
|
+
fixture_digest = None
|
|
456
|
+
if args.fixture:
|
|
457
|
+
fixture_path, fixture = read(args.fixture)
|
|
458
|
+
fixture_digest = hashlib.sha256(fixture_path.read_bytes()).hexdigest()
|
|
459
|
+
result = audit_matrix(declarations, catalogue, provider=args.provider, fixture=fixture,
|
|
460
|
+
fixture_name=fixture_path.name if fixture_path else None,
|
|
461
|
+
fixture_sha256=fixture_digest)
|
|
462
|
+
except (OSError, ValueError, json.JSONDecodeError) as error:
|
|
463
|
+
print(json.dumps({"schemaVersion": "maggie-provider-variant-capabilities.v1", "passed": False, "errors": [str(error)]}, indent=2))
|
|
464
|
+
return 1
|
|
465
|
+
output = Path(args.output) if args.output else project / "docs" / "provider-variant-capabilities.json"
|
|
466
|
+
if not output.is_absolute():
|
|
467
|
+
output = project / output
|
|
468
|
+
output = output.resolve()
|
|
469
|
+
output.parent.mkdir(parents=True, exist_ok=True)
|
|
470
|
+
result["catalogue"] = catalogue_path.name
|
|
471
|
+
result["declarations"] = declaration_path.name
|
|
472
|
+
output.write_text(json.dumps(result, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
|
473
|
+
print(json.dumps({"status": "passed" if result["passed"] else "failed", "report": str(output), "providers": len(result["providers"]), "errors": result["errors"]}, indent=2, ensure_ascii=False))
|
|
474
|
+
return 0 if result["passed"] else 1
|
|
475
|
+
|
|
439
476
|
def category_audit_report(project, pages_dir, copy_data, rendered_dir=None, environment="staging"):
|
|
440
477
|
errors = []
|
|
441
478
|
copy_path = (project / copy_data).resolve()
|
|
@@ -976,6 +1013,7 @@ def main():
|
|
|
976
1013
|
q=sub.add_parser("category-editorial-review"); q.add_argument("--project",default="."); q.add_argument("--copy-data",default="docs/category-page-copy.json"); q.add_argument("--category",action="append",help="category to review; defaults to all seven launch categories"); q.add_argument("--reviewer"); q.add_argument("--evidence",action="append",default=[],help="review evidence path or URL; repeatable"); q.add_argument("--apply",action="store_true",help="write explicit editorial approval metadata"); q.add_argument("--confirm",action="store_true",help="confirm that the selected records were actually reviewed by a human")
|
|
977
1014
|
q=sub.add_parser("category-hash"); q.add_argument("--project",default="."); q.add_argument("--copy-data",required=True); q.add_argument("--apply",action="store_true",help="backfill deterministic provenance hashes in this generated artifact")
|
|
978
1015
|
q=sub.add_parser("fact-audit"); q.add_argument("--project",default="."); q.add_argument("--backfill-source",action="store_true",help="copy the catalogue sourceUrl into records that have no sourceUrl"); q.add_argument("--overrides",help="JSON of human-approved, evidenced descriptions for provider gaps"); q.add_argument("--apply-overrides",action="store_true",help="apply only approved fact overrides to the draft catalogue")
|
|
1016
|
+
q=sub.add_parser("capability-audit", help="validate provider variant declarations and fixture evidence"); q.add_argument("--project",default="."); q.add_argument("--provider"); q.add_argument("--catalogue",default=".maggie/booking/services.json"); q.add_argument("--capabilities-file",default=".maggie/booking/provider-capabilities.json"); q.add_argument("--fixture",help="sanitized provider fixture JSON"); q.add_argument("--output")
|
|
979
1017
|
q=sub.add_parser("category-context"); q.add_argument("--project",default="."); q.add_argument("--output")
|
|
980
1018
|
q=sub.add_parser("validate-copy"); q.add_argument("--project",default="."); q.add_argument("--service-id",required=True); q.add_argument("--copy-data",required=True,help="AI-authored service copy JSON")
|
|
981
1019
|
q=sub.add_parser("sitemap-audit"); q.add_argument("--project",default="."); q.add_argument("--index",required=True,help="rendered sitemap index XML"); q.add_argument("--sitemap-dir",required=True,help="directory containing rendered child sitemap XML files"); q.add_argument("--base-url",help="reserved for URL evidence and report context"); q.add_argument("--environment",choices=("development","staging","production"),default="staging")
|
|
@@ -997,6 +1035,7 @@ def main():
|
|
|
997
1035
|
if a.command=="category-editorial-review": return cmd_category_editorial_review(a)
|
|
998
1036
|
if a.command=="category-hash": return cmd_category_hash(a)
|
|
999
1037
|
if a.command=="fact-audit": return cmd_fact_audit(a)
|
|
1038
|
+
if a.command=="capability-audit": return cmd_capability_audit(a)
|
|
1000
1039
|
if a.command=="category-context": return cmd_category_context(a)
|
|
1001
1040
|
if a.command=="validate-copy": return cmd_validate_copy(a)
|
|
1002
1041
|
if a.command=="sitemap-audit": return cmd_sitemap_audit(a)
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
"""Provider-neutral service variant capability matrix validation."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from collections import Counter
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
SCHEMA_VERSION = "maggie-provider-variant-capabilities.v1"
|
|
8
|
+
VARIANT_MODES = {"native", "derived-offers", "single-fallback", "blocked"}
|
|
9
|
+
PRICE_SEMANTICS = {"minor-unit", "minor-unit-and-display", "display-only", "unavailable", "unknown"}
|
|
10
|
+
STABLE_ID_CONFIDENCE = {"high", "medium", "low", "unknown", "blocked"}
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _services(value: object) -> list[dict[str, Any]]:
|
|
14
|
+
if isinstance(value, dict) and isinstance(value.get("services"), list):
|
|
15
|
+
return [item for item in value["services"] if isinstance(item, dict)]
|
|
16
|
+
if isinstance(value, list):
|
|
17
|
+
return [item for item in value if isinstance(item, dict)]
|
|
18
|
+
return []
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _fixture_evidence(fixture: object | None, name: str | None, digest: str | None) -> dict[str, Any]:
|
|
22
|
+
services = _services(fixture)
|
|
23
|
+
variants = [variant for service in services for variant in _services({"services": service.get("variants", [])})]
|
|
24
|
+
return {
|
|
25
|
+
"provided": fixture is not None,
|
|
26
|
+
"name": name or None,
|
|
27
|
+
"sha256": digest or None,
|
|
28
|
+
"serviceCount": len(services),
|
|
29
|
+
"variantCounts": [len(service.get("variants") or []) for service in services],
|
|
30
|
+
"allVariantIdsPresent": bool(variants) and all(str(item.get("id") or "").strip() for item in variants),
|
|
31
|
+
"allVariantSourcesPresent": bool(variants) and all(str(item.get("variantSource") or "").strip() for item in variants),
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def audit_provider(declaration: object, catalogue: object, *, fixture: object | None = None,
|
|
36
|
+
fixture_name: str | None = None, fixture_sha256: str | None = None) -> dict[str, Any]:
|
|
37
|
+
"""Validate one provider declaration against catalogue and fixture evidence."""
|
|
38
|
+
item = declaration if isinstance(declaration, dict) else {}
|
|
39
|
+
provider = str(item.get("provider") or "").strip()
|
|
40
|
+
mode = str(item.get("variantMode") or "").strip()
|
|
41
|
+
price_semantics = str(item.get("priceSemantics") or "").strip()
|
|
42
|
+
stable_id_confidence = str(item.get("stableIdConfidence") or "").strip()
|
|
43
|
+
errors: list[str] = []
|
|
44
|
+
if not provider:
|
|
45
|
+
errors.append("provider is required")
|
|
46
|
+
if mode not in VARIANT_MODES:
|
|
47
|
+
errors.append("variantMode must be native, derived-offers, single-fallback, or blocked")
|
|
48
|
+
if price_semantics not in PRICE_SEMANTICS:
|
|
49
|
+
errors.append("priceSemantics is unsupported")
|
|
50
|
+
if stable_id_confidence not in STABLE_ID_CONFIDENCE:
|
|
51
|
+
errors.append("stableIdConfidence is unsupported")
|
|
52
|
+
|
|
53
|
+
services = _services(catalogue)
|
|
54
|
+
active_services = [service for service in services if service.get("status", "active") != "archived"]
|
|
55
|
+
variants = [variant for service in active_services for variant in (service.get("variants") or []) if isinstance(variant, dict)]
|
|
56
|
+
observed_modes = Counter(str(variant.get("variantSource") or "unknown") for variant in variants)
|
|
57
|
+
all_ids = bool(variants) and all(str(variant.get("id") or "").strip() for variant in variants)
|
|
58
|
+
native_id_sources = all(str(variant.get("idSource") or "") in {"provider", "native"} for variant in variants) if variants else False
|
|
59
|
+
structured_prices = bool(variants) and all(
|
|
60
|
+
isinstance(variant.get("price"), dict)
|
|
61
|
+
and isinstance(variant["price"].get("amountMinor"), int)
|
|
62
|
+
and str(variant["price"].get("currency") or "").isalpha()
|
|
63
|
+
for variant in variants
|
|
64
|
+
)
|
|
65
|
+
if not services:
|
|
66
|
+
errors.append("catalogue has no services")
|
|
67
|
+
if mode != "blocked" and not variants:
|
|
68
|
+
errors.append("declared non-blocked provider has no active variants")
|
|
69
|
+
if mode == "native" and variants and not native_id_sources:
|
|
70
|
+
errors.append("native variants require provider/native idSource evidence")
|
|
71
|
+
if mode == "single-fallback" and any(len(service.get("variants") or []) > 1 for service in active_services):
|
|
72
|
+
errors.append("single-fallback provider has a service with multiple variants")
|
|
73
|
+
expected_source = {"native": "native", "derived-offers": "derived-offers", "single-fallback": "single-fallback"}.get(mode)
|
|
74
|
+
if expected_source and variants and any(source != expected_source for source in observed_modes):
|
|
75
|
+
errors.append(f"catalogue variantSource does not match declared {mode} mode")
|
|
76
|
+
if stable_id_confidence == "high" and not native_id_sources:
|
|
77
|
+
errors.append("high stable-ID confidence requires provider/native idSource evidence")
|
|
78
|
+
if fixture is None:
|
|
79
|
+
errors.append("fixture evidence is required")
|
|
80
|
+
|
|
81
|
+
evidence = _fixture_evidence(fixture, fixture_name, fixture_sha256)
|
|
82
|
+
if fixture is not None and evidence["serviceCount"] == 0:
|
|
83
|
+
errors.append("fixture evidence has no services")
|
|
84
|
+
if mode != "blocked" and fixture is not None and not evidence["allVariantIdsPresent"]:
|
|
85
|
+
errors.append("fixture evidence is missing variant IDs")
|
|
86
|
+
if mode != "blocked" and fixture is not None and not evidence["allVariantSourcesPresent"]:
|
|
87
|
+
errors.append("fixture evidence is missing variant sources")
|
|
88
|
+
if price_semantics in {"minor-unit", "minor-unit-and-display"} and variants and not structured_prices:
|
|
89
|
+
errors.append("declared minor-unit price semantics require amountMinor and currency")
|
|
90
|
+
if price_semantics == "display-only" and variants and any(
|
|
91
|
+
isinstance(variant.get("price"), dict) and variant["price"].get("amountMinor") is not None
|
|
92
|
+
for variant in variants
|
|
93
|
+
):
|
|
94
|
+
errors.append("display-only price semantics cannot contain amountMinor")
|
|
95
|
+
if price_semantics == "unavailable" and variants and any(variant.get("price") for variant in variants):
|
|
96
|
+
errors.append("unavailable price semantics cannot contain price data")
|
|
97
|
+
if provider and isinstance(catalogue, dict) and catalogue.get("provider") and catalogue.get("provider") != provider:
|
|
98
|
+
errors.append("catalogue provider does not match declaration")
|
|
99
|
+
|
|
100
|
+
passed = not errors
|
|
101
|
+
return {
|
|
102
|
+
"provider": provider or None,
|
|
103
|
+
"variantMode": mode or None,
|
|
104
|
+
"priceSemantics": price_semantics or None,
|
|
105
|
+
"stableIdConfidence": stable_id_confidence or None,
|
|
106
|
+
"observed": {
|
|
107
|
+
"serviceCount": len(services),
|
|
108
|
+
"activeServiceCount": len(active_services),
|
|
109
|
+
"variantCount": len(variants),
|
|
110
|
+
"variantSources": dict(sorted(observed_modes.items())),
|
|
111
|
+
"allVariantIdsPresent": all_ids,
|
|
112
|
+
"allStructuredPrices": structured_prices,
|
|
113
|
+
},
|
|
114
|
+
"fixtureEvidence": evidence,
|
|
115
|
+
"passed": passed,
|
|
116
|
+
"errors": errors,
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def audit_matrix(declarations: object, catalogue: object, *, provider: str | None = None,
|
|
121
|
+
fixture: object | None = None, fixture_name: str | None = None,
|
|
122
|
+
fixture_sha256: str | None = None) -> dict[str, Any]:
|
|
123
|
+
"""Validate a machine-readable provider declaration file."""
|
|
124
|
+
if isinstance(declarations, dict) and declarations.get("providers"):
|
|
125
|
+
entries = declarations["providers"]
|
|
126
|
+
elif isinstance(declarations, list):
|
|
127
|
+
entries = declarations
|
|
128
|
+
else:
|
|
129
|
+
entries = []
|
|
130
|
+
if not isinstance(entries, list):
|
|
131
|
+
entries = []
|
|
132
|
+
selected = [entry for entry in entries if isinstance(entry, dict) and (not provider or entry.get("provider") == provider)]
|
|
133
|
+
if not selected:
|
|
134
|
+
return {"schemaVersion": SCHEMA_VERSION, "passed": False, "providers": [], "errors": ["no provider declaration selected"]}
|
|
135
|
+
results = [audit_provider(entry, catalogue, fixture=fixture, fixture_name=fixture_name, fixture_sha256=fixture_sha256) for entry in selected]
|
|
136
|
+
return {"schemaVersion": SCHEMA_VERSION, "passed": all(result["passed"] for result in results), "providers": results,
|
|
137
|
+
"errors": [f"{result.get('provider') or 'unknown'}: {error}" for result in results for error in result["errors"]]}
|
|
@@ -4,11 +4,17 @@ from __future__ import annotations
|
|
|
4
4
|
|
|
5
5
|
from collections.abc import Mapping
|
|
6
6
|
from typing import Any
|
|
7
|
+
from urllib.error import HTTPError, URLError
|
|
8
|
+
from urllib.parse import urlparse
|
|
9
|
+
from urllib.request import Request, urlopen
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
import json
|
|
7
12
|
|
|
8
13
|
|
|
9
14
|
SCHEMA = "maggie-api-contract.v1"
|
|
10
15
|
METHODS = {"get", "post", "put", "patch", "delete", "head", "options", "trace"}
|
|
11
16
|
BODY_METHODS = {"post", "put", "patch"}
|
|
17
|
+
MAX_SPEC_BYTES = 2_000_000
|
|
12
18
|
|
|
13
19
|
|
|
14
20
|
def _has_schema(value: object) -> bool:
|
|
@@ -95,3 +101,48 @@ def validate_api_contract(spec: object) -> dict[str, Any]:
|
|
|
95
101
|
operations.append(record)
|
|
96
102
|
return {"schemaVersion": SCHEMA, "passed": not errors, "operations": operations, "errors": errors,
|
|
97
103
|
"operationCount": len(operations), "errorCount": len(errors)}
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _safe_url(source: str) -> tuple[str, str]:
|
|
107
|
+
parsed = urlparse(source)
|
|
108
|
+
if parsed.scheme not in {"https", "http"} or not parsed.netloc:
|
|
109
|
+
raise ValueError("API contract source must be a local file or an absolute HTTPS URL")
|
|
110
|
+
if parsed.username or parsed.password or parsed.query or parsed.fragment:
|
|
111
|
+
raise ValueError("API contract URL must not contain credentials, query parameters, or fragments")
|
|
112
|
+
host = (parsed.hostname or "").casefold()
|
|
113
|
+
if parsed.scheme == "http" and host not in {"localhost", "127.0.0.1", "::1"}:
|
|
114
|
+
raise ValueError("plain HTTP is allowed only for loopback development sources")
|
|
115
|
+
display = f"{parsed.scheme}://{parsed.netloc}{parsed.path or '/'}"
|
|
116
|
+
return source, display
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def load_api_contract_source(source: str | Path, *, timeout: float = 20.0) -> tuple[object, dict[str, Any]]:
|
|
120
|
+
"""Load a local JSON spec or fetch a safe URL without writing the response.
|
|
121
|
+
|
|
122
|
+
The caller receives source metadata suitable for an audit report. Response
|
|
123
|
+
bodies are parsed in memory only, so a live contract check cannot silently
|
|
124
|
+
create a project artifact containing a private API document.
|
|
125
|
+
"""
|
|
126
|
+
value = str(source)
|
|
127
|
+
parsed = urlparse(value)
|
|
128
|
+
if parsed.scheme in {"https", "http"}:
|
|
129
|
+
requested, display = _safe_url(value)
|
|
130
|
+
request = Request(requested, headers={"Accept": "application/json", "User-Agent": "MaggieApiContract/1"})
|
|
131
|
+
try:
|
|
132
|
+
with urlopen(request, timeout=timeout) as response:
|
|
133
|
+
raw = response.read(MAX_SPEC_BYTES + 1)
|
|
134
|
+
except (HTTPError, URLError, TimeoutError) as error:
|
|
135
|
+
raise RuntimeError(f"could not fetch API contract URL: {error}") from error
|
|
136
|
+
if len(raw) > MAX_SPEC_BYTES:
|
|
137
|
+
raise ValueError(f"API contract response exceeds {MAX_SPEC_BYTES} bytes")
|
|
138
|
+
try:
|
|
139
|
+
document = json.loads(raw.decode("utf-8"))
|
|
140
|
+
except (UnicodeDecodeError, json.JSONDecodeError) as error:
|
|
141
|
+
raise ValueError("API contract URL did not return a UTF-8 JSON document") from error
|
|
142
|
+
return document, {"sourceType": "url", "source": display}
|
|
143
|
+
|
|
144
|
+
path = Path(value).expanduser().resolve()
|
|
145
|
+
raw = path.read_bytes()
|
|
146
|
+
if len(raw) > MAX_SPEC_BYTES:
|
|
147
|
+
raise ValueError(f"API contract file exceeds {MAX_SPEC_BYTES} bytes")
|
|
148
|
+
return json.loads(raw.decode("utf-8")), {"sourceType": "file", "source": str(path)}
|