@deftai/directive-content 0.55.2 → 0.56.0

package/scripts/pack_migrate_rules.py

@@ -0,0 +1,350 @@
+#!/usr/bin/env python3
+"""pack_migrate_rules.py -- one-shot migration: coding/*.md + AGENTS.md +
+main.md -> structured pack.
+
+Builds the canonical structured source ``packs/rules/rules-pack-0.1.json`` (the
+source of truth per ADR-001) by parsing the marker-prefixed RFC2119 directive
+lines out of every ``coding/*.md`` plus the framework directives in AGENTS.md
+and main.md (#1637 s4). This is the #1296 generalization of the #1294 lessons
+pilot + #1295 skills pack: the same render/slice machinery, broadened to the
+framework-rule docs.
+
+What is captured per rule
+-------------------------
+- ``id``     a stable, deterministic ``{domain}-{NNN}`` slug (in-document order).
+- ``tier``   the RFC2119 strength normalized from the coding/* legend marker
+  (per #748): ``!`` -> MUST, ``~`` -> SHOULD, the SHOULD-NOT glyph ->
+  SHOULD_NOT, the MUST-NOT glyph -> MUST_NOT, ``?`` -> MAY. Prose RFC2119
+  bullets (uppercase MUST / SHOULD / ... in a plain ``- `` bullet) are also
+  recognized.
+- ``domain`` the source doc stem (testing, security, hygiene, agents, main...).
+- ``text``   the directive text after the strength marker, verbatim.
+- ``path``   the repo-relative source path (``coding/<doc>.md`` / ``AGENTS.md``
+  / ``main.md``).
+- ``body``   the full source-document body (banner-stripped) for EACH
+  ``coding/*.md`` doc's first rule entry, so every coding doc is regenerated as
+  a banner-marked, drift-checked projection; ``null`` for every other entry AND
+  for ALL AGENTS.md / main.md entries.
+
+Ownership boundary (#1637 s4)
+-----------------------------
+AGENTS.md and main.md are ingested as directive METADATA ONLY (body always
+null). The renderer skips null-body entries, so ``packs:render`` NEVER writes
+AGENTS.md -- it stays owned solely by ``task agents:refresh``. AGENTS.md's
+managed-section block is stripped before extraction (rendered mirror, not
+canonical) to avoid duplicate directives.
+
+Usage::
+
+    uv run python scripts/pack_migrate_rules.py \\
+        [--coding-dir coding] [--extra-source AGENTS.md --extra-source main.md] \\
+        [--out packs/rules/rules-pack-0.1.json]
+
+Exit codes:
+    0 -- migrated successfully
+    1 -- coding dir missing, or no directives discovered
+    2 -- usage error
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+from pathlib import Path
+
+# Repo root resolved from this file's location (scripts/ -> repo root) so the
+# default paths are CWD-independent.
+REPO_ROOT = Path(__file__).resolve().parent.parent
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from _content_root import content_root  # noqa: E402
+
+# Shippable content moved under content/ in the source repo and is
+# flattened to the framework root in a consumer deposit (#1875 C1).
+CONTENT_ROOT = content_root(REPO_ROOT)
+DEFAULT_CODING_DIR = CONTENT_ROOT / "coding"
+DEFAULT_OUT = CONTENT_ROOT / "packs" / "rules" / "rules-pack-0.1.json"
+
+PACK_ID = "rules-pack-0.1"
+PACK_VERSION = "0.1"
+
+# Extra (non-coding) RFC2119 sources ingested as directive METADATA ONLY
+# (#1637 s4): AGENTS.md and main.md contribute id/tier/domain/text/path entries
+# with body=null. They are NEVER rendered as packs:render projections (the
+# renderer skips null-body entries), so AGENTS.md stays solely owned by
+# `task agents:refresh` and the dual-owner collision is structurally
+# impossible. See vbrief DesignNote (1637-s4 ownership boundary).
+DEFAULT_EXTRA_SOURCES = (
+    REPO_ROOT / "AGENTS.md",
+    REPO_ROOT / "main.md",
+)
+
+# AGENTS.md carries a managed section that is a RENDERED MIRROR of
+# templates/agents-entry.md (regenerated by `task agents:refresh`), not a
+# canonical source. It is stripped before directive extraction so the mirrored
+# directives are not ingested twice (once from the maintainer body, once from
+# the managed mirror).
+_MANAGED_SECTION_RE = re.compile(
+    r"<!--\s*deft:managed-section.*?<!--\s*/deft:managed-section\s*-->",
+    re.DOTALL,
+)
+
+# Strength-marker glyphs -> normalized tier. Mirrors the coding/* legend
+# documented at the top of coding/testing.md (and #748's strength axis):
+#   ! = MUST, ~ = SHOULD, the SHOULD-NOT glyph, the MUST-NOT glyph, ? = MAY.
+_SHOULD_NOT_GLYPH = "\u2249"  # the coding/* SHOULD NOT legend glyph
+_MUST_NOT_GLYPH = "\u2297"  # the coding/* MUST NOT legend glyph
+GLYPH_TIER: dict[str, str] = {
+    "!": "MUST",
+    "~": "SHOULD",
+    _SHOULD_NOT_GLYPH: "SHOULD_NOT",
+    _MUST_NOT_GLYPH: "MUST_NOT",
+    "?": "MAY",
+}
+
+# A marker-prefixed directive bullet: optional leading ``- `` then a single
+# strength glyph then the directive text.
+_MARKER_RE = re.compile(
+    rf"^\s*(?:-\s+)?([!~?{_SHOULD_NOT_GLYPH}{_MUST_NOT_GLYPH}])\s+(\S.*)$"
+)
+
+# Prose RFC2119 bullets: a plain ``- `` bullet that spells out the keyword in
+# uppercase (no glyph). Longer keywords are matched first so "MUST NOT" wins
+# over "MUST". The keyword must appear as a standalone token in the bullet.
+_PROSE_TIERS: tuple[tuple[str, str], ...] = (
+    ("MUST NOT", "MUST_NOT"),
+    ("SHOULD NOT", "SHOULD_NOT"),
+    ("MUST", "MUST"),
+    ("SHOULD", "SHOULD"),
+    ("MAY", "MAY"),
+)
+_SLUG_STRIP_RE = re.compile(r"[^a-z0-9]+")
+
+
+def _prose_tier(text: str) -> str | None:
+    """Return the tier for a plain bullet that spells an uppercase RFC2119 word.
+
+    Matches the longest keyword first ("MUST NOT" before "MUST") and requires a
+    word boundary so substrings inside other words are not mistaken for rules.
+    """
+    for keyword, tier in _PROSE_TIERS:
+        if re.search(rf"\b{keyword.replace(' ', r'[ ]')}\b", text):
+            return tier
+    return None
+
+
+def parse_rules(md_text: str, domain: str) -> list[dict]:
+    """Parse a coding doc's marker-prefixed + prose RFC2119 directives.
+
+    Returns one record per directive in document order with ``id`` /
+    ``tier`` / ``domain`` / ``text`` / ``path`` (``body`` is attached by the
+    caller). ``path`` is left for the caller to set.
+    """
+    rules: list[dict] = []
+    seq = 0
+    for raw in md_text.splitlines():
+        line = raw.rstrip()
+        marker = _MARKER_RE.match(line)
+        if marker:
+            tier = GLYPH_TIER[marker.group(1)]
+            text = marker.group(2).strip()
+        else:
+            stripped = line.strip()
+            if not stripped.startswith("- "):
+                continue
+            text = stripped[2:].strip()
+            tier = _prose_tier(text) if text else None
+            if tier is None:
+                continue
+        if not text:
+            continue
+        seq += 1
+        rules.append(
+            {
+                "id": f"{domain}-{seq:03d}",
+                "tier": tier,
+                "domain": domain,
+                "text": text,
+            }
+        )
+    return rules
+
+
+def strip_leading_banner(body: str) -> str:
+    """Strip a leading provenance banner + blank lines from a captured body.
+
+    Makes re-migration idempotent: after the proof doc is regenerated
+    (banner + body), re-running the migration recovers the same body. Only
+    strips a banner block that opens with the renderer's first banner line, so
+    unrelated leading HTML comments survive.
+    """
+    lines = body.split("\n")
+    i = 0
+    while i < len(lines) and lines[i].strip() == "":
+        i += 1
+    if i < len(lines) and lines[i].startswith(
+        "<!-- AUTO-GENERATED by task packs:render"
+    ):
+        while i < len(lines) and lines[i].lstrip().startswith("<!--"):
+            i += 1
+        while i < len(lines) and lines[i].strip() == "":
+            i += 1
+    return "\n".join(lines[i:])
+
+
+def strip_managed_section(md_text: str) -> str:
+    """Remove the AGENTS.md ``<!-- deft:managed-section --> ... <!-- /... -->``
+    block before directive extraction.
+
+    The managed section is a rendered mirror of templates/agents-entry.md
+    (owned by `task agents:refresh`), not a canonical source -- stripping it
+    keeps the maintainer-side directives from being ingested twice. Returns the
+    text unchanged when no managed section is present.
+    """
+    return _MANAGED_SECTION_RE.sub("", md_text)
+
+
+def build_pack(
+    coding_dir: Path, *, extra_sources: tuple[Path, ...] = DEFAULT_EXTRA_SOURCES
+) -> dict:
+    """Scan the coding dir + extra sources and assemble the full pack object.
+
+    Every ``coding/*.md`` doc carries its full body on its FIRST rule entry
+    (regenerate-and-commit projection per ADR-001); the extra sources
+    (AGENTS.md, main.md) are ingested as directive METADATA ONLY (body=null,
+    never rendered) per the #1637 s4 ownership boundary.
+    """
+    base = coding_dir.resolve().parent
+    rules: list[dict] = []
+
+    for md in sorted(coding_dir.glob("*.md")):
+        rel_path = md.resolve().relative_to(base).as_posix()
+        domain = _SLUG_STRIP_RE.sub("-", md.stem.lower()).strip("-")
+        text = md.read_text(encoding="utf-8")
+        doc_rules = parse_rules(text, domain)
+        for idx, rule in enumerate(doc_rules):
+            rule["path"] = rel_path
+            # Attach the full doc body to EACH coding doc's FIRST rule only;
+            # every other entry is metadata-only (body null). The renderer's
+            # documents mode then projects one bodied entry -> one coding/*.md.
+            rule["body"] = strip_leading_banner(text) if idx == 0 else None
+            rules.append(rule)
+
+    # Extra (non-coding) sources: directive metadata only, body always null so
+    # the renderer never writes them (AGENTS.md stays owned by agents:refresh).
+    for src in extra_sources:
+        if not src.is_file():
+            continue
+        try:
+            rel_path = src.resolve().relative_to(base).as_posix()
+        except ValueError:
+            # Post-#1875: the extra harness-entry sources (AGENTS.md, main.md)
+            # stay at the repo/deposit root, while the coding dir moved under
+            # content/ -- so they are not under ``base`` (content/). They are
+            # top-level files, so the consumer-relative path is just the name.
+            rel_path = src.name
+        domain = _SLUG_STRIP_RE.sub("-", src.stem.lower()).strip("-")
+        text = src.read_text(encoding="utf-8")
+        if src.name == "AGENTS.md":
+            text = strip_managed_section(text)
+        for rule in parse_rules(text, domain):
+            rule["path"] = rel_path
+            rule["body"] = None
+            rules.append(rule)
+
+    return {
+        "pack": PACK_ID,
+        "version": PACK_VERSION,
+        "generated_from": (
+            "coding/*.md + AGENTS.md + main.md (marker-prefixed RFC2119 "
+            "directives; AGENTS.md managed-section excluded; coding bodies "
+            "rendered, AGENTS.md/main.md metadata-only)"
+        ),
+        "rules": rules,
+    }
+
+
+def migrate(
+    coding_dir: Path,
+    out: Path,
+    *,
+    extra_sources: tuple[Path, ...] = DEFAULT_EXTRA_SOURCES,
+) -> dict:
+    """Build the pack from ``coding_dir`` (+ extra sources) and write it to ``out``.
+
+    Raises ``FileNotFoundError`` when the coding dir is missing and
+    ``ValueError`` when no directives are discovered.
+    """
+    if not coding_dir.is_dir():
+        raise FileNotFoundError(f"coding directory not found: {coding_dir}")
+
+    pack = build_pack(coding_dir, extra_sources=extra_sources)
+    if not pack["rules"]:
+        raise ValueError(f"no directives discovered under {coding_dir}")
+
+    out.parent.mkdir(parents=True, exist_ok=True)
+    # ensure_ascii=True: the canonical source is serialized as pure ASCII with
+    # \uXXXX escapes (mirrors pack_migrate_lessons / pack_migrate_skills).
+    # Lossless and keeps the JSON clean against `task verify:encoding` (#798)
+    # even though the directive text carries non-ASCII glyphs (RFC2119 symbols,
+    # em dashes, the >= sign).
+    out.write_text(
+        json.dumps(pack, indent=2, ensure_ascii=True) + "\n", encoding="utf-8"
+    )
+    return pack
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="pack_migrate_rules.py",
+        description="Migrate coding/*.md RFC2119 directives into the rules-pack-0.1 source.",
+    )
+    parser.add_argument(
+        "--coding-dir",
+        type=Path,
+        default=DEFAULT_CODING_DIR,
+        help="Directory of coding docs to scan (default: coding/).",
+    )
+    parser.add_argument(
+        "--extra-source",
+        action="append",
+        type=Path,
+        default=None,
+        help=(
+            "Repo-relative non-coding RFC2119 source ingested as metadata only "
+            "(body=null, never rendered). Repeatable. "
+            "Default: AGENTS.md + main.md."
+        ),
+    )
+    parser.add_argument(
+        "--out",
+        type=Path,
+        default=DEFAULT_OUT,
+        help="Output pack JSON path (default: packs/rules/rules-pack-0.1.json).",
+    )
+    args = parser.parse_args(argv)
+
+    extra_sources = (
+        tuple(args.extra_source)
+        if args.extra_source is not None
+        else DEFAULT_EXTRA_SOURCES
+    )
+
+    try:
+        pack = migrate(args.coding_dir, args.out, extra_sources=extra_sources)
+    except FileNotFoundError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 1
+    except ValueError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 1
+
+    bodied = sum(1 for r in pack["rules"] if r["body"] is not None)
+    print(f"Migrated {len(pack['rules'])} rules ({bodied} with body) -> {args.out}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())