@ahmed-g-gad/apothem 0.1.1

package/src/apothem/audit/scan_ai_surfaces_coarse.py

@@ -0,0 +1,261 @@
+# SPDX-License-Identifier: MIT
+
+"""Coarse map of AI-conventions surfaces present in the working tree.
+
+Why this scan exists. The ecosystem ships an AI-conventions surface
+that spans multiple files, each authoritative within its own scope:
+``AGENTS.md`` for the canonical project voice, ``CLAUDE.md`` for the
+Claude Code mirror, ``.github/copilot-instructions.md`` for GitHub Copilot, optional surfaces
+(``site/content/docs/architecture/agents.mdx``, ``.cursorrules``,
+``.windsurfrules``) per the operator's opt-in. The detailed
+section-presence map (``ai-surfaces.json``) is produced by a deeper
+follow-up scan; this coarse pre-scan tells the deeper pass which
+surfaces exist before it begins its content walk.
+
+What this scan covers. Existence-and-presence checks for the
+candidate surface paths plus the four mandatory-behavior blocks
+that should appear inside ``AGENTS.md`` and mirror into ``CLAUDE.md``:
+
+- ``AGENTS.md`` / ``CLAUDE.md`` mandatory blocks: `structured inquiry` discipline,
+  Plans Discipline, Authorship Header, Multi-Surface AI Conventions.
+  Each block's presence is detected by header / phrase heuristics.
+- ``.github/copilot-instructions.md``: present / absent.
+- Optional surfaces: presence flags only (no content scan).
+
+What this scan reports. A single hit per detected absence or per
+detected partial-presence. The output drives the deeper section-
+presence pass and the optional-surface opt-in inquiry.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from dataclasses import asdict
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Final
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+from _scan_lib import (
+    SEVERITY_HIGH,
+    SEVERITY_LOW,
+    SEVERITY_MEDIUM,
+    Hit,
+    load_inventory,
+    read_text_safely,
+)
+
+# Mandatory instruction-surface behavior blocks. Each is detected by a header
+# phrase or canonical paragraph that signals the block's presence.
+# Heuristic-driven; the deeper section-presence scan confirms
+# structural conformance.
+_MANDATORY_BLOCKS: Final[dict[str, list[str]]] = {
+    "structured-inquiry discipline": [
+        "structured inquiry",
+        "interactive-questions",
+        "structured-inquiry",
+    ],
+    "plans-discipline": [
+        "plans-discipline",
+        "Plans Discipline",
+        ".plans/ at",
+        "no-global-plans",
+    ],
+    "authorship-header": [
+        "authorship-header",
+        "Authorship Header",
+        "authorship banner",
+        "five canonical lines",
+    ],
+    "multi-surface-AI-conventions": [
+        "multi-surface",
+        "Multi-Surface",
+        "AI conventions",
+        "copilot-instructions",
+    ],
+}
+
+# Optional opt-in AI-conventions surfaces. Each is a path probed for
+# existence; absence is informational, not a finding. The operator
+# ratifies which subset to materialize.
+_OPTIONAL_SURFACES: Final[list[str]] = [
+    "site/content/docs/architecture/agents.mdx",
+    ".cursorrules",
+    ".windsurfrules",
+]
+
+# The Copilot instructions surface is authored by the dedicated
+# AI-conventions author pass.
+_COPILOT_PATH: Final[str] = ".github/copilot-instructions.md"
+
+# AGENTS.md is the canonical project voice surface; CLAUDE.md is the
+# Claude Code mirror. Absence of either mandatory surface is fatal.
+_AGENTS_PATH: Final[str] = "AGENTS.md"
+_CLAUDE_PATH: Final[str] = "CLAUDE.md"
+
+
+def _scan_instruction_surface(path: str, content: str) -> list[Hit]:
+    """Detect missing mandatory blocks inside an instruction surface."""
+    hits: list[Hit] = []
+    for block_name, signals in _MANDATORY_BLOCKS.items():
+        if not any(signal in content for signal in signals):
+            hits.append(
+                Hit(
+                    file=path,
+                    line=1,
+                    signal=f"instruction-surface-missing-block: {block_name}",
+                    severity=SEVERITY_HIGH,
+                    remediation=(
+                        f"Author the '{block_name}' block in {path} per"
+                        " the spec's mandatory-behavior catalog; the"
+                        " block carries the canonical project directive"
+                        " for that surface."
+                    ),
+                )
+            )
+    return hits
+
+
+def _scan_optional_surfaces(root: Path) -> list[Hit]:
+    """Emit informational hits for opt-in surfaces present on disk."""
+    hits: list[Hit] = []
+    for path in _OPTIONAL_SURFACES:
+        if (root / path).exists():
+            hits.append(
+                Hit(
+                    file=path,
+                    line=0,
+                    signal=f"optional-surface-present: {path}",
+                    severity=SEVERITY_LOW,
+                    remediation=(
+                        "The deeper section-presence scan confirms"
+                        " structure; the opt-in inquiry recovers the"
+                        " operator's ratification for keep / refit /"
+                        " remove."
+                    ),
+                )
+            )
+    return hits
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument(
+        "--inventory",
+        type=Path,
+        default=Path(".audit/inventory.json"),
+    )
+    parser.add_argument("--root", type=Path, default=Path())
+    parser.add_argument(
+        "--output",
+        type=Path,
+        default=Path(".audit/drift-ai-surfaces-coarse.json"),
+    )
+    args = parser.parse_args(argv)
+
+    if not args.inventory.exists():
+        print(
+            f"error: inventory not found at {args.inventory}",
+            file=sys.stderr,
+        )
+        return 1
+
+    _, sha = load_inventory(args.inventory)
+    hits: list[Hit] = []
+
+    # Mandatory instruction-surface presence and block coverage.
+    agents_path = args.root / _AGENTS_PATH
+    claude_path = args.root / _CLAUDE_PATH
+    if not agents_path.exists():
+        hits.append(
+            Hit(
+                file=_AGENTS_PATH,
+                line=0,
+                signal="agents-md-absent",
+                severity=SEVERITY_HIGH,
+                remediation=(
+                    "Author AGENTS.md at the repository root; this is the"
+                    " canonical project instruction surface and its absence"
+                    " disables downstream AI-conventions discipline."
+                ),
+            )
+        )
+    else:
+        hits.extend(
+            _scan_instruction_surface(_AGENTS_PATH, read_text_safely(agents_path))
+        )
+
+    if not claude_path.exists():
+        hits.append(
+            Hit(
+                file=_CLAUDE_PATH,
+                line=0,
+                signal="claude-md-absent",
+                severity=SEVERITY_HIGH,
+                remediation=(
+                    "Author CLAUDE.md as the Claude Code mirror of AGENTS.md;"
+                    " its absence leaves that harness without the shared"
+                    " instruction discipline."
+                ),
+            )
+        )
+    else:
+        hits.extend(
+            _scan_instruction_surface(_CLAUDE_PATH, read_text_safely(claude_path))
+        )
+
+    # Copilot instructions presence.
+    copilot_path = args.root / _COPILOT_PATH
+    if not copilot_path.exists():
+        hits.append(
+            Hit(
+                file=_COPILOT_PATH,
+                line=0,
+                signal="copilot-instructions-absent",
+                severity=SEVERITY_MEDIUM,
+                remediation=(
+                    "The AI-conventions author pass produces"
+                    " .github/copilot-instructions.md per the canonical"
+                    " nine-section structure. Until then the Copilot"
+                    " surface defaults to its built-in heuristics without"
+                    " the ecosystem's directive overlay."
+                ),
+            )
+        )
+
+    # Optional opt-in surfaces.
+    hits.extend(_scan_optional_surfaces(args.root))
+
+    payload = {
+        "generated": datetime.now(timezone.utc).isoformat(),
+        "scanner": "scan_ai_surfaces_coarse",
+        "inventory-source-sha256": sha,
+        "agents-md-present": agents_path.exists(),
+        "claude-md-present": claude_path.exists(),
+        "copilot-instructions-present": copilot_path.exists(),
+        "optional-surfaces-present": [
+            p for p in _OPTIONAL_SURFACES if (args.root / p).exists()
+        ],
+        "hit-count": len(hits),
+        "hits": [asdict(h) for h in hits],
+    }
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    args.output.write_text(
+        json.dumps(payload, indent=2) + "\n",
+        encoding="utf-8",
+    )
+    print(
+        f"scan_ai_surfaces_coarse: AGENTS.md={agents_path.exists()},"
+        f" CLAUDE.md={claude_path.exists()},"
+        f" copilot-instructions={copilot_path.exists()},"
+        f" optional={len(payload['optional-surfaces-present'])} of"
+        f" {len(_OPTIONAL_SURFACES)}; total hits={len(hits)}"
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/src/apothem/audit/scan_drift_features.py

@@ -0,0 +1,143 @@
+# SPDX-License-Identifier: MIT
+
+"""Detect references to deprecated or renamed features.
+
+Why this scan exists. Narrative artifacts (rules, agents, commands,
+hook context, docs) accumulate references to vendor features as the
+upstream surface evolves: model identifiers retire, tool names change,
+flags are renamed, capabilities are deprecated. A reference to a
+retired feature is functionally a broken pointer — it leads readers
+toward behavior the runtime no longer supports. The scan walks every
+narrative surface for tokens drawn from a curated deprecated list and
+emits a finding per occurrence so the refit phases can replace them.
+
+What this scan covers. Deprecated tokens live in
+``src/apothem/audit/deprecated-tokens.txt`` (one token per line; comments
+prefixed with ``#``; case-sensitive substring match). The list is
+intentionally seeded empty: at first run there is no proof any specific
+token is deprecated, and a populated allow-list with no evidence would
+fabricate findings. Tokens are added when the operator confirms a
+specific feature has retired; the file then becomes a living deprecation
+ledger.
+
+What this scan excludes. Source code (``*.py``, ``*.sh``, ``*.ps1``)
+is also walked because deprecated identifiers can appear as string
+literals in tooling scaffolds. Memory and plan-artifact files are
+excluded per the shared narrative-surface filter.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+from typing import Any
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+from _scan_lib import (
+    CONTENT_ROOT,
+    NARRATIVE_CLASSES,
+    SEVERITY_MEDIUM,
+    Hit,
+    WalkCallback,
+    emit_json,
+    load_inventory,
+    walk_narrative_surfaces,
+)
+
+
+def _load_deprecated_tokens(token_file: Path) -> list[str]:
+    """Parse the deprecated-token catalog.
+
+    Lines beginning with ``#`` or empty lines are comments. Returns an
+    empty list when the file is absent — the scan then emits zero hits,
+    which is correct (no deprecated tokens declared, no findings).
+    """
+    if not token_file.exists():
+        return []
+    tokens: list[str] = []
+    for raw in token_file.read_text(encoding="utf-8").splitlines():
+        line = raw.strip()
+        if not line or line.startswith("#"):
+            continue
+        tokens.append(line)
+    return tokens
+
+
+def _scan_file(tokens: list[str]) -> WalkCallback:
+    """Build a callback that records every line containing any token."""
+
+    def _walk(path: Path, record: dict[str, Any], content: str) -> list[Hit]:
+        if not tokens:
+            return []
+        hits: list[Hit] = []
+        for lineno, line in enumerate(content.splitlines(), start=1):
+            for token in tokens:
+                if token in line:
+                    hits.append(
+                        Hit(
+                            file=record["path"],
+                            line=lineno,
+                            signal=f"deprecated-feature-reference: {token}",
+                            severity=SEVERITY_MEDIUM,
+                            remediation=(
+                                f"Replace '{token}' with the current vendor identifier;"
+                                " consult the deprecated-tokens.txt catalog for the"
+                                " replacement guidance."
+                            ),
+                        )
+                    )
+        return hits
+
+    return _walk
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument(
+        "--inventory",
+        type=Path,
+        default=Path(".audit/inventory.json"),
+        help="Path to inventory.json (default: ./.audit/inventory.json)",
+    )
+    parser.add_argument(
+        "--root",
+        type=Path,
+        default=CONTENT_ROOT,
+        help="Content root the inventory paths resolve against "
+        "(default: the src/apothem package).",
+    )
+    parser.add_argument(
+        "--tokens",
+        type=Path,
+        default=Path("src/apothem/audit/deprecated-tokens.txt"),
+        help="Deprecated-token catalog (default: src/apothem/audit/deprecated-tokens.txt)",
+    )
+    parser.add_argument(
+        "--output",
+        type=Path,
+        default=Path(".audit/drift-feature-refs.json"),
+        help="Output JSON path (default: ./.audit/drift-feature-refs.json)",
+    )
+    args = parser.parse_args(argv)
+
+    if not args.inventory.exists():
+        print(f"error: inventory not found at {args.inventory}", file=sys.stderr)
+        return 1
+
+    records, sha = load_inventory(args.inventory)
+    tokens = _load_deprecated_tokens(args.tokens)
+    hits = walk_narrative_surfaces(records, args.root, _scan_file(tokens))
+    emit_json(args.output, "scan_drift_features", hits, sha)
+    narrative_count = sum(1 for r in records if r.get("class") in NARRATIVE_CLASSES)
+    print(
+        f"scan_drift_features: {len(hits)} hit(s) across "
+        f"{narrative_count} narrative files "
+        f"(deprecated-token catalog: {len(tokens)} entries)"
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/src/apothem/audit/scan_frontmatter.py

@@ -0,0 +1,293 @@
+# SPDX-License-Identifier: MIT
+
+"""Scan frontmatter consistency across artifact classes that carry it.
+
+Why this scan exists. Agents, commands, skills, output-styles, and
+several plan-artifact classes carry YAML frontmatter at the head of
+each file. Per-class schemas have not yet been ratified at the time of
+this scan, so the check is heuristic-driven: walk every artifact in a
+class, sample the dominant key set and key order, and flag any sibling
+that diverges. The scan also flags any TOML-style frontmatter
+(``+++...+++``) because the discovered convention across every existing
+file is YAML (``---...---``).
+
+What this scan covers per class.
+
+- **Required-key drift.** Keys present in at least 80% of the class's
+  files are treated as the de-facto required set; siblings missing any
+  of those keys produce a finding.
+- **Key-order drift.** The dominant key sequence is the de-facto order;
+  siblings whose key sequence diverges produce a finding (rearranging a
+  frontmatter block is a near-zero-cost fix that compounds readability).
+- **Format drift.** Any non-YAML frontmatter delimiter (``+++``,
+  ``;;;``, JSON object as first content) is flagged irrespective of
+  class.
+- **Empty required field.** A required key with an empty string /
+  ``null`` value is flagged separately because the absence of value is
+  often more dangerous than the absence of key.
+
+What this scan excludes. Memory and plan-artifact classes are excluded
+from the dominant-pattern computation because they are not refit
+targets at this phase. Files without any frontmatter that belong to a
+class where frontmatter is universally present are flagged as
+``frontmatter-absent``.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from collections import Counter
+from collections.abc import Iterable
+from pathlib import Path
+from typing import Any, Final
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+from _scan_lib import (
+    CONTENT_ROOT,
+    SEVERITY_MEDIUM,
+    Hit,
+    emit_json,
+    load_inventory,
+    read_text_safely,
+)
+
+# Artifact classes that carry frontmatter as a convention. Settings
+# files are JSON; scaffolding files vary; only the listed classes are
+# walked for frontmatter consistency.
+FRONTMATTER_CLASSES: Final[frozenset[str]] = frozenset(
+    {"agent", "command", "skill", "output-style"}
+)
+
+# Frontmatter requires presence in at least this fraction of a class's
+# files to be treated as the de-facto required set.
+REQUIRED_THRESHOLD: Final[float] = 0.8
+
+
+def _extract_frontmatter(content: str) -> tuple[list[str], str | None]:
+    """Return ``(ordered-keys, delimiter)`` for the document's
+    frontmatter, or ``([], None)`` if no frontmatter is present.
+
+    Recognizes YAML (``---``), TOML (``+++``), and a permissive
+    semicolon delimiter (``;;;``). Inside a YAML block, only top-level
+    keys (un-indented ``key:`` lines) are captured; nested keys are not
+    part of the frontmatter contract.
+    """
+    lines = content.splitlines()
+    if not lines:
+        return [], None
+    first = lines[0].strip()
+    if first not in ("---", "+++", ";;;"):
+        return [], None
+    delimiter = first
+    keys: list[str] = []
+    for line in lines[1:]:
+        stripped = line.strip()
+        if stripped == delimiter:
+            return keys, delimiter
+        # Top-level YAML key: un-indented `key:` (no leading whitespace).
+        if line and not line[0].isspace() and ":" in line:
+            key = line.split(":", 1)[0].strip()
+            if key and not key.startswith("#"):
+                keys.append(key)
+    # Closing delimiter not found; treat as malformed frontmatter.
+    return keys, "malformed"
+
+
+def _empty_value_keys(content: str, keys: list[str]) -> list[str]:
+    """Return keys whose value is empty / null / quoted-empty."""
+    if not keys:
+        return []
+    empty: list[str] = []
+    in_block = False
+    for line in content.splitlines():
+        stripped = line.strip()
+        if stripped == "---":
+            if in_block:
+                break
+            in_block = True
+            continue
+        if not in_block or line[:1].isspace() or ":" not in line:
+            continue
+        key, _, value = line.partition(":")
+        key = key.strip()
+        value = value.strip()
+        if key in keys and value in ("", "''", '""', "null", "~"):
+            empty.append(key)
+    return empty
+
+
+def _gather_class_records(
+    records: Iterable[dict[str, Any]], cls: str
+) -> list[dict[str, Any]]:
+    return [r for r in records if r.get("class") == cls]
+
+
+def _compute_class_baseline(
+    class_records: list[dict[str, Any]], root: Path
+) -> tuple[set[str], list[str]]:
+    """Return ``(required-keys, dominant-order)`` for a class."""
+    if not class_records:
+        return set(), []
+    key_presence: Counter[str] = Counter()
+    order_counter: Counter[tuple[str, ...]] = Counter()
+    n = 0
+    for record in class_records:
+        path = root / record["path"]
+        content = read_text_safely(path)
+        if not content:
+            continue
+        keys, delimiter = _extract_frontmatter(content)
+        if not keys or delimiter != "---":
+            continue
+        n += 1
+        for k in keys:
+            key_presence[k] += 1
+        order_counter[tuple(keys)] += 1
+    if n == 0:
+        return set(), []
+    threshold = max(1, int(REQUIRED_THRESHOLD * n))
+    required = {k for k, count in key_presence.items() if count >= threshold}
+    dominant_order = list(order_counter.most_common(1)[0][0]) if order_counter else []
+    return required, dominant_order
+
+
+def _scan_class(cls: str, records: list[dict[str, Any]], root: Path) -> list[Hit]:
+    """Walk a single class and emit hits for divergent siblings."""
+    required, dominant_order = _compute_class_baseline(records, root)
+    hits: list[Hit] = []
+    for record in records:
+        rel = record["path"]
+        path = root / rel
+        content = read_text_safely(path)
+        if not content:
+            continue
+        keys, delimiter = _extract_frontmatter(content)
+        hits.extend(
+            _diagnose_record(rel, content, keys, delimiter, required, dominant_order)
+        )
+    return hits
+
+
+def _diagnose_record(
+    rel: str,
+    content: str,
+    keys: list[str],
+    delimiter: str | None,
+    required: set[str],
+    dominant_order: list[str],
+) -> list[Hit]:
+    """Return all findings for a single file's frontmatter state."""
+    hits: list[Hit] = []
+    if delimiter is None:
+        if required:
+            hits.append(
+                Hit(
+                    file=rel,
+                    line=1,
+                    signal="frontmatter-absent",
+                    severity=SEVERITY_MEDIUM,
+                    remediation=(
+                        "Add the class's required frontmatter keys at the"
+                        " head of the file enclosed by '---' delimiters."
+                    ),
+                )
+            )
+        return hits
+    if delimiter == "+++" or delimiter == ";;;" or delimiter == "malformed":
+        hits.append(
+            Hit(
+                file=rel,
+                line=1,
+                signal=f"frontmatter-non-yaml-or-malformed: {delimiter}",
+                severity=SEVERITY_MEDIUM,
+                remediation=(
+                    "Convert the frontmatter to the YAML form delimited"
+                    " by '---'; non-YAML delimiters break tooling that"
+                    " expects the convention."
+                ),
+            )
+        )
+    missing = required - set(keys)
+    for key in sorted(missing):
+        hits.append(
+            Hit(
+                file=rel,
+                line=1,
+                signal=f"frontmatter-missing-required: {key}",
+                severity=SEVERITY_MEDIUM,
+                remediation=(
+                    f"Add '{key}: <value>' to the frontmatter; the key is"
+                    " present in at least 80% of the class's files."
+                ),
+            )
+        )
+    if dominant_order and keys != dominant_order and not missing:
+        # Only flag order drift when the file has the full required set;
+        # missing-key cases dominate and ordering is secondary.
+        hits.append(
+            Hit(
+                file=rel,
+                line=1,
+                signal="frontmatter-key-order-drift",
+                severity=SEVERITY_MEDIUM,
+                remediation=(
+                    "Reorder the frontmatter keys to match the class's"
+                    f" dominant sequence: {dominant_order}."
+                ),
+            )
+        )
+    for key in _empty_value_keys(content, list(required & set(keys))):
+        hits.append(
+            Hit(
+                file=rel,
+                line=1,
+                signal=f"frontmatter-empty-required-value: {key}",
+                severity=SEVERITY_MEDIUM,
+                remediation=(
+                    f"Populate '{key}' with a non-empty value; the key is"
+                    " required but its value is blank or null."
+                ),
+            )
+        )
+    return hits
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument(
+        "--inventory",
+        type=Path,
+        default=Path(".audit/inventory.json"),
+    )
+    parser.add_argument("--root", type=Path, default=CONTENT_ROOT)
+    parser.add_argument(
+        "--output",
+        type=Path,
+        default=Path(".audit/drift-frontmatter.json"),
+    )
+    args = parser.parse_args(argv)
+
+    if not args.inventory.exists():
+        print(
+            f"error: inventory not found at {args.inventory}",
+            file=sys.stderr,
+        )
+        return 1
+
+    records, sha = load_inventory(args.inventory)
+    hits: list[Hit] = []
+    per_class_counts: dict[str, int] = {}
+    for cls in sorted(FRONTMATTER_CLASSES):
+        cls_records = _gather_class_records(records, cls)
+        per_class_counts[cls] = len(cls_records)
+        hits.extend(_scan_class(cls, cls_records, args.root))
+    emit_json(args.output, "scan_frontmatter", hits, sha)
+    summary = ", ".join(f"{c}={n}" for c, n in per_class_counts.items())
+    print(f"scan_frontmatter: {len(hits)} hit(s) across classes [{summary}]")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())