@design-ai/cli 4.55.0

package/tools/audit/doctor_assertions.py

@@ -0,0 +1,256 @@
+"""Shared assertions for design-ai doctor JSON reports used by smoke tests."""
+from __future__ import annotations
+
+import argparse
+
+EXPECTED_DOCTOR_PASS_LABELS = {
+    "Source layout",
+    "Version alignment",
+    "Manifest paths",
+    "Node runtime",
+    "Python runtime",
+    "Audit runner",
+    "Audit scripts",
+    "Doctor assertions helper",
+    "Smoke assertions helper",
+    "Example QA audit",
+    "Package contents check",
+    "Package smoke check",
+    "Registry smoke check",
+    "Installed skills",
+    "Installed agents",
+    "Installed slash commands",
+}
+
+
+def _assert_keys(value: dict, expected: list[str], *, label: str, context: str) -> None:
+    actual = list(value)
+    if actual != expected:
+        raise SystemExit(
+            f"doctor JSON {label} keys changed after {context}: "
+            f"expected {expected}, got {actual}"
+        )
+
+
+def _assert_non_empty_string(value: object, *, label: str, context: str) -> None:
+    if not isinstance(value, str) or not value:
+        raise SystemExit(f"doctor JSON {label} after {context} is missing or not a string")
+
+
+def _assert_non_negative_int(value: object, *, label: str, context: str) -> None:
+    if type(value) is not int or value < 0:
+        raise SystemExit(f"doctor JSON {label} after {context} is missing or not a non-negative integer")
+
+
+def assert_doctor_report_clean(report: dict, *, context: str) -> None:
+    if not isinstance(report, dict):
+        raise SystemExit(f"doctor JSON after {context} is not an object")
+
+    _assert_keys(report, ["context", "checks", "summary", "fix"], label="top-level", context=context)
+
+    report_context = report["context"]
+    if not isinstance(report_context, dict):
+        raise SystemExit(f"doctor JSON context after {context} is not an object")
+    _assert_keys(
+        report_context,
+        ["sourceRoot", "claudeHome", "prefix", "expected"],
+        label="context",
+        context=context,
+    )
+    _assert_non_empty_string(report_context.get("sourceRoot"), label="context.sourceRoot", context=context)
+    _assert_non_empty_string(report_context.get("claudeHome"), label="context.claudeHome", context=context)
+    _assert_non_empty_string(report_context.get("prefix"), label="context.prefix", context=context)
+
+    expected = report_context["expected"]
+    if not isinstance(expected, dict):
+        raise SystemExit(f"doctor JSON context.expected after {context} is not an object")
+    _assert_keys(expected, ["skills", "agents", "commands"], label="context.expected", context=context)
+    for key in ("skills", "agents", "commands"):
+        _assert_non_negative_int(expected.get(key), label=f"context.expected.{key}", context=context)
+
+    checks = report["checks"]
+    if not isinstance(checks, list) or not checks:
+        raise SystemExit(f"doctor JSON checks after {context} is missing or empty")
+    for index, check in enumerate(checks):
+        if not isinstance(check, dict):
+            raise SystemExit(f"doctor JSON checks[{index}] after {context} is not an object")
+        _assert_keys(check, ["status", "label", "detail", "action"], label=f"checks[{index}]", context=context)
+        if check.get("status") not in {"PASS", "WARN", "FAIL"}:
+            raise SystemExit(f"doctor JSON checks[{index}].status after {context} is invalid")
+        _assert_non_empty_string(check.get("label"), label=f"checks[{index}].label", context=context)
+        _assert_non_empty_string(check.get("detail"), label=f"checks[{index}].detail", context=context)
+        if not isinstance(check.get("action"), str):
+            raise SystemExit(f"doctor JSON checks[{index}].action after {context} is not a string")
+
+    summary = report.get("summary", {})
+    if not isinstance(summary, dict):
+        raise SystemExit(f"doctor JSON summary after {context} is not an object")
+    _assert_keys(summary, ["pass", "warn", "fail"], label="summary", context=context)
+    for key in ("pass", "warn", "fail"):
+        _assert_non_negative_int(summary.get(key), label=f"summary.{key}", context=context)
+
+    actual_summary = {
+        "pass": sum(1 for check in checks if check.get("status") == "PASS"),
+        "warn": sum(1 for check in checks if check.get("status") == "WARN"),
+        "fail": sum(1 for check in checks if check.get("status") == "FAIL"),
+    }
+    if summary != actual_summary:
+        raise SystemExit(
+            f"doctor JSON summary after {context} does not match check statuses: "
+            f"expected {actual_summary}, got {summary}"
+        )
+
+    fix = report["fix"]
+    if not isinstance(fix, dict):
+        raise SystemExit(f"doctor JSON fix after {context} is not an object")
+    _assert_keys(fix, ["attempted", "applied", "reason"], label="fix", context=context)
+    if not isinstance(fix.get("attempted"), bool) or not isinstance(fix.get("applied"), bool):
+        raise SystemExit(f"doctor JSON fix booleans after {context} are invalid")
+    if not isinstance(fix.get("reason"), str):
+        raise SystemExit(f"doctor JSON fix.reason after {context} is not a string")
+
+    check_statuses = {
+        check.get("label"): check.get("status")
+        for check in report.get("checks", [])
+        if isinstance(check, dict)
+    }
+    missing_or_failed = sorted(
+        f"{label}={check_statuses.get(label, 'missing')}"
+        for label in EXPECTED_DOCTOR_PASS_LABELS
+        if check_statuses.get(label) != "PASS"
+    )
+    if missing_or_failed:
+        raise SystemExit(
+            f"doctor did not report every required {context} check as PASS: "
+            + ", ".join(missing_or_failed)
+        )
+
+    if summary.get("fail", 0) != 0 or summary.get("warn", 0) != 0:
+        raise SystemExit(f"doctor reported warning(s) or failure(s) after {context}")
+
+
+def _passing_report() -> dict:
+    return {
+        "context": {
+            "sourceRoot": "/tmp/design-ai",
+            "claudeHome": "/tmp/claude",
+            "prefix": "design-",
+            "expected": {
+                "skills": 19,
+                "agents": 4,
+                "commands": 16,
+            },
+        },
+        "checks": [
+            {
+                "status": "PASS",
+                "label": label,
+                "detail": f"{label} fixture passed",
+                "action": "",
+            }
+            for label in sorted(EXPECTED_DOCTOR_PASS_LABELS)
+        ],
+        "summary": {"pass": len(EXPECTED_DOCTOR_PASS_LABELS), "warn": 0, "fail": 0},
+        "fix": {
+            "attempted": False,
+            "applied": False,
+            "reason": "",
+        },
+    }
+
+
+def _expect_assertion_failure(report: dict, *, context: str, expected: str) -> None:
+    try:
+        assert_doctor_report_clean(report, context=context)
+    except SystemExit as error:
+        if expected not in str(error):
+            raise
+    else:
+        raise SystemExit(f"self-test failed: expected doctor assertion failure containing {expected!r}")
+
+
+def run_self_test(*, context: str = "doctor assertion self-test", quiet: bool = False) -> None:
+    assert_doctor_report_clean(_passing_report(), context=context)
+
+    missing_report = _passing_report()
+    missing_report["summary"]["pass"] -= 1
+    missing_report["checks"] = [
+        check
+        for check in missing_report["checks"]
+        if check["label"] != "Registry smoke check"
+    ]
+    _expect_assertion_failure(
+        missing_report,
+        context=context,
+        expected="Registry smoke check=missing",
+    )
+
+    warning_report = _passing_report()
+    warning_report["summary"]["warn"] = 1
+    warning_report["summary"]["pass"] -= 1
+    warning_report["checks"][0]["status"] = "WARN"
+    _expect_assertion_failure(
+        warning_report,
+        context=context,
+        expected="WARN",
+    )
+
+    failed_check_report = _passing_report()
+    for check in failed_check_report["checks"]:
+        if check["label"] == "Node runtime":
+            check["status"] = "FAIL"
+            break
+    failed_check_report["summary"]["pass"] -= 1
+    failed_check_report["summary"]["fail"] = 1
+    _expect_assertion_failure(
+        failed_check_report,
+        context=context,
+        expected="Node runtime=FAIL",
+    )
+
+    missing_fix_report = _passing_report()
+    missing_fix_report.pop("fix")
+    _expect_assertion_failure(
+        missing_fix_report,
+        context=context,
+        expected="top-level keys changed",
+    )
+
+    mismatched_summary_report = _passing_report()
+    mismatched_summary_report["summary"]["pass"] -= 1
+    _expect_assertion_failure(
+        mismatched_summary_report,
+        context=context,
+        expected="does not match check statuses",
+    )
+
+    changed_check_shape_report = _passing_report()
+    changed_check_shape_report["checks"][0].pop("action")
+    _expect_assertion_failure(
+        changed_check_shape_report,
+        context=context,
+        expected="checks[0] keys changed",
+    )
+
+    if not quiet:
+        print("Doctor assertions self-test passed")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--self-test",
+        action="store_true",
+        help="Run local doctor JSON assertion fixtures",
+    )
+    args = parser.parse_args()
+
+    if args.self_test:
+        run_self_test()
+        return
+
+    run_self_test()
+
+
+if __name__ == "__main__":
+    main()

package/tools/audit/example-qa.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+"""Audit the top worked example for every routed design workflow.
+
+This keeps the route/example recommendation surface honest. If routing starts
+pointing a workflow at a weak or incomplete example, this audit fails before the
+package ships.
+"""
+from __future__ import annotations
+
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+CLI = ROOT / "cli" / "bin" / "design-ai.mjs"
+
+
+def run_example_check() -> subprocess.CompletedProcess[str]:
+    return subprocess.run(
+        [
+            "node",
+            str(CLI),
+            "check",
+            "--examples",
+            "--all-routes",
+            "--limit",
+            "1",
+            "--json",
+        ],
+        cwd=ROOT,
+        capture_output=True,
+        text=True,
+    )
+
+
+def issue_lines(report: dict) -> list[str]:
+    lines: list[str] = []
+    for route in report.get("routes", []):
+        if route.get("status") == "pass":
+            continue
+        route_id = route.get("routeId", "(unknown route)")
+        lines.append(f"- {route_id}: {route.get('status', 'unknown')}")
+        for example_report in route.get("examples", []):
+            example = example_report.get("example", {})
+            artifact = example_report.get("report", {})
+            if artifact.get("status") == "pass":
+                continue
+            rel_path = example.get("relPath", "(unknown example)")
+            lines.append(f"  - {rel_path}: {artifact.get('status', 'unknown')}")
+            for result in artifact.get("results", []):
+                if result.get("level") == "pass":
+                    continue
+                evidence = result.get("evidence")
+                suffix = f" — {evidence}" if evidence else ""
+                lines.append(f"    - {result.get('id', 'unknown')}: {result.get('message', '')}{suffix}")
+    return lines
+
+
+def main() -> int:
+    if not CLI.exists():
+        print(f"ERROR: design-ai CLI not found at {CLI}", file=sys.stderr)
+        return 1
+
+    completed = run_example_check()
+    if completed.returncode != 0:
+        print("ERROR: example QA command failed", file=sys.stderr)
+        if completed.stdout.strip():
+            print(completed.stdout.rstrip())
+        if completed.stderr.strip():
+            print(completed.stderr.rstrip(), file=sys.stderr)
+        return 1
+
+    try:
+        report = json.loads(completed.stdout)
+    except json.JSONDecodeError as exc:
+        print(f"ERROR: could not parse example QA JSON: {exc}", file=sys.stderr)
+        if completed.stdout.strip():
+            print(completed.stdout.rstrip())
+        return 1
+
+    status = report.get("status")
+    total_routes = report.get("totalRoutes", 0)
+    passed_routes = report.get("passedRoutes", 0)
+    warned_routes = report.get("warnedRoutes", 0)
+    failed_routes = report.get("failedRoutes", 0)
+    total_examples = report.get("totalExamples", 0)
+    passed_examples = report.get("passedExamples", 0)
+
+    if status != "pass":
+        print(
+            "Example route QA failed: "
+            f"{passed_routes}/{total_routes} routes pass, "
+            f"{warned_routes} warn, {failed_routes} fail"
+        )
+        for line in issue_lines(report):
+            print(line)
+        return 1
+
+    print(
+        "Example route QA passed: "
+        f"{passed_routes}/{total_routes} routes and {passed_examples}/{total_examples} examples pass"
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/tools/audit/frontmatter-check.py

@@ -0,0 +1,199 @@
+#!/usr/bin/env python3
+"""
+Validate YAML frontmatter on every markdown file in knowledge/ and examples/.
+
+Checks:
+  - Frontmatter present (`---\n...\n---`)
+  - Required keys: title, applies_to (or `source` for generated, or `purpose` for PRINCIPLES.md)
+  - For generated files: source and extracted_at present
+  - applies_to is a list (YAML array)
+
+Outputs a report to console; exits 1 if any errors.
+
+Usage:
+  python3 tools/audit/frontmatter-check.py [--strict]
+"""
+from __future__ import annotations
+
+import argparse
+import re
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+
+
+REQUIRED_KEYS = {"title"}
+RECOMMENDED_KEYS = {"applies_to"}
+GENERATED_KEYS = {"source", "extracted_at"}
+# Optional versioning keys (added in v3.11). Validated only if present —
+# missing them is fine for backward compatibility, but if present they
+# should be well-formed.
+VERSIONING_KEYS = {"version", "last_updated", "stability"}
+STABILITY_VALUES = {"stable", "beta", "experimental", "deprecated"}
+
+
+def is_hand_written(text: str) -> bool:
+    return "<!-- hand-written -->" in text
+
+
+def parse_frontmatter(text: str) -> dict[str, object] | None:
+    """Lightweight YAML parser — handles flat keys + simple lists.
+    Returns None if no frontmatter found.
+    Skips a leading HTML comment (e.g., `<!-- hand-written -->`) before frontmatter.
+    """
+    lines = text.splitlines()
+
+    # Skip leading HTML comments (e.g., <!-- hand-written -->) and blank lines
+    cursor = 0
+    while cursor < len(lines):
+        stripped = lines[cursor].strip()
+        if stripped.startswith("<!--") and stripped.endswith("-->"):
+            cursor += 1
+            continue
+        if stripped == "":
+            cursor += 1
+            continue
+        break
+
+    if cursor >= len(lines) or not lines[cursor].strip().startswith("---"):
+        return None
+
+    fm_lines: list[str] = []
+    for line in lines[cursor + 1:]:
+        if line.strip().startswith("---"):
+            break
+        fm_lines.append(line)
+    if not fm_lines:
+        return None
+
+    fm: dict[str, object] = {}
+    current_list_key: str | None = None
+    for line in fm_lines:
+        stripped = line.rstrip()
+        if not stripped:
+            current_list_key = None
+            continue
+
+        # List continuation: "  - item"
+        if current_list_key and stripped.lstrip().startswith("-"):
+            item = stripped.lstrip()[1:].strip()
+            if item:
+                fm[current_list_key] = list(fm.get(current_list_key, [])) + [item]
+            continue
+
+        # New key
+        match = re.match(r"^([a-zA-Z_]+):\s*(.*)$", stripped)
+        if not match:
+            continue
+        key, value = match.group(1), match.group(2).strip()
+        if value == "":
+            current_list_key = key
+            fm[key] = []
+        elif value.startswith("[") and value.endswith("]"):
+            # Inline list
+            inner = value[1:-1]
+            items = [item.strip().strip("\"'") for item in inner.split(",") if item.strip()]
+            fm[key] = items
+            current_list_key = None
+        else:
+            fm[key] = value.strip().strip("\"'")
+            current_list_key = None
+    return fm
+
+
+def validate(path: Path, strict: bool) -> list[str]:
+    text = path.read_text(encoding="utf-8")
+    rel = path.relative_to(ROOT)
+    errors: list[str] = []
+    fm = parse_frontmatter(text)
+
+    if fm is None:
+        errors.append(f"{rel}: no frontmatter found")
+        return errors
+
+    # Required keys
+    missing = REQUIRED_KEYS - set(fm.keys())
+    if missing:
+        errors.append(f"{rel}: missing required keys {missing}")
+
+    # applies_to should be a list
+    if "applies_to" in fm and not isinstance(fm["applies_to"], list):
+        errors.append(f"{rel}: 'applies_to' should be a list, got {type(fm['applies_to']).__name__}")
+
+    # Generated files need source + extracted_at
+    if not is_hand_written(text):
+        # Heuristic: files in knowledge/ are generated unless marked
+        if "knowledge/" in str(rel) and "design-tokens/" not in str(rel) and "/INDEX.md" not in str(rel) and rel.parent.name != "knowledge":
+            # Most other knowledge files are generated; check
+            missing_gen = GENERATED_KEYS - set(fm.keys())
+            if missing_gen and strict:
+                errors.append(f"{rel}: generated file missing {missing_gen}")
+
+    # Recommended keys (warn only in strict)
+    if strict:
+        missing_rec = RECOMMENDED_KEYS - set(fm.keys())
+        if missing_rec:
+            errors.append(f"{rel}: (warn) recommended keys missing {missing_rec}")
+
+    # Versioning keys: validate format if present (added v3.11)
+    version = fm.get("version")
+    if version is not None and not _is_valid_version(version):
+        errors.append(f"{rel}: invalid version '{version}' — expected semver (e.g., 1.0.0)")
+
+    last_updated = fm.get("last_updated")
+    if last_updated is not None and not _is_valid_date_or_yearmonth(last_updated):
+        errors.append(f"{rel}: invalid last_updated '{last_updated}' — expected YYYY-MM or YYYY-MM-DD")
+
+    stability = fm.get("stability")
+    if stability is not None and str(stability) not in STABILITY_VALUES:
+        errors.append(
+            f"{rel}: invalid stability '{stability}' — expected one of {sorted(STABILITY_VALUES)}"
+        )
+
+    return errors
+
+
+def _is_valid_version(value: object) -> bool:
+    """Match semver-like X.Y.Z (with optional -prerelease)."""
+    if not isinstance(value, str):
+        return False
+    return bool(re.match(r"^\d+\.\d+\.\d+(-[A-Za-z0-9.-]+)?$", value))
+
+
+def _is_valid_date_or_yearmonth(value: object) -> bool:
+    if not isinstance(value, str):
+        return False
+    return bool(re.match(r"^\d{4}-\d{2}(-\d{2})?$", value))
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--strict", action="store_true", help="Treat warnings as errors")
+    args = parser.parse_args()
+
+    # Only check knowledge/ — examples/ are prose component specs without
+    # required frontmatter contract.
+    targets: list[Path] = list(ROOT.glob("knowledge/**/*.md"))
+
+    all_errors: list[str] = []
+    files_checked = 0
+    for path in targets:
+        if path.name == "README.md" or path.name == "COVERAGE.md":
+            continue
+        files_checked += 1
+        errors = validate(path, args.strict)
+        all_errors.extend(errors)
+
+    print(f"Checked {files_checked} files")
+    if all_errors:
+        print(f"\n{len(all_errors)} issues:\n")
+        for err in all_errors:
+            print(f"  {err}")
+        sys.exit(1)
+    else:
+        print("All frontmatter valid ✓")
+
+
+if __name__ == "__main__":
+    main()

package/tools/audit/integration-check.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+"""Verify that integration walkthrough docs have the expected sections.
+
+Each walkthrough should include: Prerequisites, Setup, at least 3
+"Walkthrough" sections, Tips, and Next/cross-reference. This catches
+walkthroughs that drift from the standard structure over time.
+
+Usage:
+  python3 tools/audit/integration-check.py
+  python3 tools/audit/integration-check.py --strict
+"""
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+INTEGRATIONS_DIR = ROOT / "docs" / "integrations"
+
+# Walkthrough files (not the MCP-specific ones, which have a different shape).
+WALKTHROUGH_FILES = [
+    "codex-walkthrough.md",
+    "cursor-walkthrough.md",
+    "aider-walkthrough.md",
+    "sdk-walkthrough.md",
+    "vscode-walkthrough.md",
+]
+
+# Required headings (substring match, case-insensitive).
+REQUIRED_SECTIONS = [
+    "prerequisites",
+    "setup",
+    "walkthrough",  # at least one
+    "next",
+]
+
+MIN_WALKTHROUGH_COUNT = 3  # at least 3 numbered "Walkthrough N:" sections
+
+
+def check_file(path: Path) -> list[str]:
+    """Return list of issues for this file."""
+    if not path.exists():
+        return [f"missing: {path.relative_to(ROOT)}"]
+
+    content = path.read_text(encoding="utf-8").lower()
+    issues: list[str] = []
+
+    for section in REQUIRED_SECTIONS:
+        if section not in content:
+            issues.append(f"missing section: '{section}'")
+
+    walkthrough_count = content.count("## walkthrough ")
+    if walkthrough_count < MIN_WALKTHROUGH_COUNT:
+        issues.append(
+            f"only {walkthrough_count} 'Walkthrough N:' sections "
+            f"(min {MIN_WALKTHROUGH_COUNT})"
+        )
+
+    return issues
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--strict", action="store_true", help="Exit 1 on any issue")
+    args = parser.parse_args()
+
+    if not INTEGRATIONS_DIR.exists():
+        print(f"::error::Missing dir: {INTEGRATIONS_DIR.relative_to(ROOT)}")
+        sys.exit(1)
+
+    total_issues = 0
+    for filename in WALKTHROUGH_FILES:
+        path = INTEGRATIONS_DIR / filename
+        issues = check_file(path)
+        if issues:
+            print(f"\n{path.relative_to(ROOT)}:")
+            for issue in issues:
+                print(f"  - {issue}")
+            total_issues += len(issues)
+        else:
+            print(f"✓ {path.relative_to(ROOT)}")
+
+    print()
+    if total_issues == 0:
+        print("All integration walkthroughs valid ✓")
+        sys.exit(0)
+
+    print(f"Found {total_issues} issue(s) across walkthroughs.")
+    if args.strict:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()