@deftai/directive-content 0.55.1 → 0.56.0

package/scripts/pack_migrate_lessons.py

@@ -0,0 +1,336 @@
+#!/usr/bin/env python3
+"""pack_migrate_lessons.py -- one-shot migration: meta/lessons.md -> structured pack.
+
+Parses the hand-authored ``meta/lessons.md`` into the canonical structured
+source ``packs/lessons/lessons-pack-0.1.json`` (the source of truth per
+ADR-001; ``meta/lessons.md`` then becomes a regenerated projection via
+``scripts/pack_render.py``).
+
+Parsing model
+-------------
+The document is split on top-level ``## `` headings. Everything before the
+first ``## `` heading (the ``# Lessons Learned`` title + any authoring
+comment) is document chrome owned by the renderer and is discarded here.
+For each section:
+
+- ``title`` is the full heading text (verbatim, so the projection round-trips).
+- ``date`` is the ``YYYY-MM`` extracted from the heading parenthetical, or null.
+- ``issue_refs`` is every ``#NNN`` reference in the heading, in order.
+- ``source`` is the text of the body's ``**Source:**`` line, or null.
+- ``tags`` is 1-3 tags from the controlled vocabulary, scored from keywords.
+- ``body`` is the full section body verbatim (lossless blob).
+
+Usage::
+
+    uv run python scripts/pack_migrate_lessons.py [--source meta/lessons.md] \\
+        [--out packs/lessons/lessons-pack-0.1.json]
+
+Exit codes:
+    0 -- migrated successfully
+    1 -- source markdown missing or empty
+    2 -- usage error
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+from pathlib import Path
+
+# Repo root resolved from this file's location (scripts/ -> repo root). Used to
+# anchor the default source / output paths so the migration is CWD-independent.
+REPO_ROOT = Path(__file__).resolve().parent.parent
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from _content_root import content_root  # noqa: E402
+
+# Shippable content moved under content/ in the source repo and is
+# flattened to the framework root in a consumer deposit (#1875 C1).
+CONTENT_ROOT = content_root(REPO_ROOT)
+DEFAULT_SOURCE = REPO_ROOT / "meta" / "lessons.md"
+DEFAULT_OUT = CONTENT_ROOT / "packs" / "lessons" / "lessons-pack-0.1.json"
+
+PACK_ID = "lessons-pack-0.1"
+PACK_VERSION = "0.1"
+
+# Controlled tag vocabulary. MUST stay in lockstep with the `enum` and
+# `x-tagVocabulary` in vbrief/schemas/lessons-pack.schema.json.
+TAG_VOCABULARY: tuple[str, ...] = (
+    "windows",
+    "encoding",
+    "review-cycle",
+    "swarm",
+    "release",
+    "github",
+    "context",
+    "debugging",
+    "lifecycle",
+    "powershell",
+    "ci",
+    "agent-orchestration",
+)
+
+# Keyword -> tag scoring table. Matches are lowercase substring tests against
+# the title (weighted heavily) and body (weighted lightly). The tokens are
+# deliberately specific to keep the controlled vocabulary discriminating.
+TAG_KEYWORDS: dict[str, tuple[str, ...]] = {
+    "windows": ("windows", "cp1252", "cp437", "charmap", "win32", "winerror"),
+    "encoding": (
+        "encoding",
+        "utf-8",
+        "utf8",
+        "mojibake",
+        "non-ascii",
+        "u+fffd",
+        " bom",
+        "unicodedecode",
+        "unicodeencode",
+    ),
+    "review-cycle": (
+        "review cycle",
+        "review-cycle",
+        "greptile",
+        "review bot",
+        "checkrun",
+        "check run",
+    ),
+    "swarm": ("swarm", "parallel agent", "worktree", "cohort", "cascade"),
+    "release": ("release", "changelog", "publish", "v0.", "tag time", "cut session"),
+    "github": (
+        "github",
+        "gh api",
+        "gh cli",
+        "gh issue",
+        "gh pr",
+        "graphql",
+        "rest",
+        "pull request",
+        "closingissues",
+    ),
+    "context": ("context engineering", "context rot", "context window", "token", "low-signal"),
+    "debugging": (
+        "debug",
+        "root cause",
+        "root-cause",
+        "investigation",
+        "forensic",
+        "blind spot",
+    ),
+    "lifecycle": (
+        "lifecycle",
+        "vbrief",
+        "scope:",
+        "promote",
+        "activate",
+        "reconcile",
+    ),
+    "powershell": (
+        "powershell",
+        "pwsh",
+        "ps 5.1",
+        "ps5.1",
+        "get-content",
+        "set-content",
+        "here-string",
+    ),
+    "ci": ("pre-commit", "task check", "deterministic gate", " gate ", "pipeline", "self-test"),
+    "agent-orchestration": (
+        "orchestrat",
+        "poller",
+        "dispatch",
+        "sub-agent",
+        "subagent",
+        "agent run",
+        "spawn",
+        "monitor agent",
+    ),
+}
+
+# Fallback tag when no keyword scores -- every entry must carry >= 1 tag.
+FALLBACK_TAG = "agent-orchestration"
+
+_HEADING_RE = re.compile(r"^## (.+)$")
+_DATE_RE = re.compile(r"(\d{4}-\d{2})(?:-\d{2})?")
+_ISSUE_RE = re.compile(r"#(\d+)")
+_SOURCE_RE = re.compile(r"^\*\*Source:\*\*\s*(.+?)\s*$")
+_SLUG_STRIP_RE = re.compile(r"[^a-z0-9]+")
+
+
+def extract_date(title: str) -> str | None:
+    """Return the YYYY-MM date from a heading, or None when absent."""
+    match = _DATE_RE.search(title)
+    return match.group(1) if match else None
+
+
+def extract_issue_refs(title: str) -> list[str]:
+    """Return all ``#NNN`` references in a heading, in order of appearance."""
+    return [f"#{n}" for n in _ISSUE_RE.findall(title)]
+
+
+def extract_source(body: str) -> str | None:
+    """Return the text of the first ``**Source:**`` line in the body, or None."""
+    for line in body.splitlines():
+        match = _SOURCE_RE.match(line.strip())
+        if match:
+            return match.group(1)
+    return None
+
+
+def slugify(title: str, existing: set[str]) -> str:
+    """Derive a stable, unique, lowercase slug from a title.
+
+    Strips a trailing parenthetical date / issue ref before slugifying so the
+    id stays readable, then de-duplicates against ``existing`` by appending a
+    numeric suffix.
+    """
+    base = _SLUG_STRIP_RE.sub("-", title.lower()).strip("-")
+    if not base:
+        base = "lesson"
+    slug = base
+    counter = 2
+    while slug in existing:
+        slug = f"{base}-{counter}"
+        counter += 1
+    existing.add(slug)
+    return slug
+
+
+def assign_tags(title: str, body: str) -> list[str]:
+    """Assign 1-3 controlled-vocabulary tags by keyword scoring.
+
+    Title matches weigh 5x body matches. The top-scoring tags (score > 0) are
+    returned, capped at 3, ordered by score desc then by vocabulary order for
+    determinism. Falls back to a single default tag when nothing scores.
+    """
+    title_l = title.lower()
+    body_l = body.lower()
+    scores: dict[str, int] = {}
+    for tag in TAG_VOCABULARY:
+        score = 0
+        for kw in TAG_KEYWORDS[tag]:
+            if kw in title_l:
+                score += 5
+            score += body_l.count(kw)
+        if score > 0:
+            scores[tag] = score
+
+    if not scores:
+        return [FALLBACK_TAG]
+
+    vocab_order = {tag: i for i, tag in enumerate(TAG_VOCABULARY)}
+    ranked = sorted(scores.items(), key=lambda kv: (-kv[1], vocab_order[kv[0]]))
+    return [tag for tag, _ in ranked[:3]]
+
+
+def parse_lessons(md_text: str) -> list[dict]:
+    """Parse lessons markdown into structured lesson entries.
+
+    Splits on top-level ``## `` headings; content before the first heading is
+    discarded (renderer-owned chrome). Returns entries in document order.
+    """
+    lines = md_text.splitlines()
+    # Collect (heading_text, start_line_index_of_body) for each section.
+    sections: list[tuple[str, int]] = []
+    for idx, line in enumerate(lines):
+        match = _HEADING_RE.match(line)
+        if match:
+            sections.append((match.group(1).strip(), idx))
+
+    entries: list[dict] = []
+    existing_ids: set[str] = set()
+    for s_idx, (title, head_line) in enumerate(sections):
+        end_line = sections[s_idx + 1][1] if s_idx + 1 < len(sections) else len(lines)
+        body = "\n".join(lines[head_line + 1 : end_line]).strip()
+        entries.append(
+            {
+                "id": slugify(title, existing_ids),
+                "title": title,
+                "date": extract_date(title),
+                "issue_refs": extract_issue_refs(title),
+                "tags": assign_tags(title, body),
+                "source": extract_source(body),
+                "body": body,
+            }
+        )
+    return entries
+
+
+def build_pack(md_text: str, generated_from: str) -> dict:
+    """Build the full pack object from the source markdown text."""
+    return {
+        "pack": PACK_ID,
+        "version": PACK_VERSION,
+        "generated_from": generated_from,
+        "lessons": parse_lessons(md_text),
+    }
+
+
+def migrate(source: Path, out: Path) -> dict:
+    """Read ``source`` markdown, build the pack, and write it to ``out``.
+
+    Returns the in-memory pack object. Raises ``FileNotFoundError`` when the
+    source is missing and ``ValueError`` when it is empty.
+    """
+    if not source.is_file():
+        raise FileNotFoundError(f"source markdown not found: {source}")
+    md_text = source.read_text(encoding="utf-8")
+    if not md_text.strip():
+        raise ValueError(f"source markdown is empty: {source}")
+
+    # Record provenance as a repo-relative path when possible.
+    try:
+        generated_from = source.resolve().relative_to(REPO_ROOT).as_posix()
+    except ValueError:
+        generated_from = source.name
+
+    pack = build_pack(md_text, generated_from)
+    out.parent.mkdir(parents=True, exist_ok=True)
+    # ensure_ascii=True: the canonical source is serialized as pure ASCII with
+    # \uXXXX escapes. This is lossless (json.loads reconstructs the exact same
+    # strings, so the rendered projection and slice output are byte-identical),
+    # and it keeps the source clean against `task verify:encoding` (#798): the
+    # lessons content legitimately documents cp1252/cp437 mojibake example
+    # tokens (e.g. the corrupted form of the U+2297 glyph), which the encoding
+    # gate's markdown inline-code stripping exempts in meta/lessons.md but
+    # would flag as a raw bigram in the JSON. Escaping sidesteps that without
+    # mutating the preserved content.
+    out.write_text(json.dumps(pack, indent=2, ensure_ascii=True) + "\n", encoding="utf-8")
+    return pack
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="pack_migrate_lessons.py",
+        description="Migrate meta/lessons.md into the structured lessons-pack-0.1 source.",
+    )
+    parser.add_argument(
+        "--source",
+        type=Path,
+        default=DEFAULT_SOURCE,
+        help="Source markdown to parse (default: meta/lessons.md).",
+    )
+    parser.add_argument(
+        "--out",
+        type=Path,
+        default=DEFAULT_OUT,
+        help="Output pack JSON path (default: packs/lessons/lessons-pack-0.1.json).",
+    )
+    args = parser.parse_args(argv)
+
+    try:
+        pack = migrate(args.source, args.out)
+    except FileNotFoundError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 1
+    except ValueError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 1
+
+    print(f"Migrated {len(pack['lessons'])} lessons -> {args.out}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/scripts/pack_migrate_patterns.py

@@ -0,0 +1,254 @@
+#!/usr/bin/env python3
+"""pack_migrate_patterns.py -- one-shot migration: patterns/*.md -> pack.
+
+Builds the canonical structured source ``packs/patterns/patterns-pack-0.1.json``
+(the source of truth per ADR-001) by scanning every ``patterns/*.md``. This is
+the #1637 generalization of the #1294 lessons pilot + #1295 skills pack + #1296
+rules/strategies packs: the same render/slice machinery, a fifth domain. The
+``patterns/`` directory existed with no pack until #1637 (packs:slice v2).
+
+What is captured per pattern
+----------------------------
+- ``id``          the slugified doc stem (e.g. multi-agent, role-as-overlay).
+- ``title``       the leading ``# `` heading text, verbatim.
+- ``description`` the leading description paragraph after the title, folded to a
+  single normalised string (Legend / See-also / HTML-comment / rule chrome
+  skipped). Empty when the doc has no leading paragraph.
+- ``triggers``    invocation keywords for the pattern. Pattern docs carry no
+  frontmatter and there is no pattern-routing table, so the derivable trigger
+  is the doc stem itself; the list is otherwise empty (mirrors #1296 strategies
+  scope: "if no trigger metadata exists, use an empty list and rely on list").
+- ``path``        the repo-relative ``patterns/<name>.md``.
+- ``body``        the full pattern body (banner-stripped) for the ONE designated
+  proof pattern (``patterns/multi-agent.md``); ``null`` for every other pattern
+  (metadata-only, per the "migrate ONE doc as proof" 0.1-pilot scope).
+
+Usage::
+
+    uv run python scripts/pack_migrate_patterns.py \\
+        [--patterns-dir patterns] [--proof-pattern patterns/multi-agent.md] \\
+        [--out packs/patterns/patterns-pack-0.1.json]
+
+Exit codes:
+    0 -- migrated successfully
+    1 -- patterns dir missing, or no patterns discovered
+    2 -- usage error
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+from pathlib import Path
+
+# Repo root resolved from this file's location (scripts/ -> repo root) so the
+# default paths are CWD-independent.
+REPO_ROOT = Path(__file__).resolve().parent.parent
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from _content_root import content_root  # noqa: E402
+
+# Shippable content moved under content/ in the source repo and is
+# flattened to the framework root in a consumer deposit (#1875 C1).
+CONTENT_ROOT = content_root(REPO_ROOT)
+DEFAULT_PATTERNS_DIR = CONTENT_ROOT / "patterns"
+DEFAULT_OUT = CONTENT_ROOT / "packs" / "patterns" / "patterns-pack-0.1.json"
+
+PACK_ID = "patterns-pack-0.1"
+PACK_VERSION = "0.1"
+
+# The single proof PATTERN whose full body is captured + regenerated as a
+# banner-marked, drift-checked projection.
+DEFAULT_PROOF_PATTERN = "patterns/multi-agent.md"
+
+_H1_RE = re.compile(r"^#\s+(.+?)\s*$")
+_SLUG_STRIP_RE = re.compile(r"[^a-z0-9]+")
+
+# Lines that open the description-paragraph scan but are document chrome, not a
+# description: the RFC2119 legend, "See also" pointers, HTML comments, and
+# horizontal rules.
+_CHROME_PREFIXES = ("legend ", "legend(", "**legend", "**⚠️", "**see also", "<!--")
+
+
+def _is_chrome(line: str) -> bool:
+    """True when a line is document chrome rather than a description paragraph."""
+    low = line.lstrip().lower()
+    if low.startswith(_CHROME_PREFIXES):
+        return True
+    stripped = line.strip()
+    # A horizontal rule (e.g. `---`, `===`) is chrome, not a description.
+    return bool(stripped) and set(stripped) <= {"-", "="}
+
+
+def extract_title(md_text: str) -> str:
+    """Return the leading ``# `` heading text, or '' when absent."""
+    for line in md_text.splitlines():
+        match = _H1_RE.match(line)
+        if match:
+            return match.group(1).strip()
+    return ""
+
+
+def extract_description(md_text: str) -> str:
+    """Return the leading description paragraph after the ``# `` title.
+
+    Scans past the title, skips blank lines and chrome lines (Legend, See-also,
+    HTML comments, horizontal rules), then collects the first contiguous block
+    of non-blank, non-chrome lines and folds it to a single normalised string.
+    A leading blockquote marker (``> ``) is stripped so redirect/superseded
+    notes still yield a readable description.
+    """
+    lines = md_text.splitlines()
+    i = 0
+    n = len(lines)
+    # Advance to just past the first H1.
+    while i < n and not _H1_RE.match(lines[i]):
+        i += 1
+    if i < n:
+        i += 1  # skip the title line itself
+    # Skip leading blanks / chrome before the description.
+    while i < n and (lines[i].strip() == "" or _is_chrome(lines[i])):
+        i += 1
+    # Collect the first contiguous non-blank block. A markdown heading is a
+    # section boundary, not a description -- a doc whose first content after the
+    # title is a `## ` heading has no leading description paragraph.
+    block: list[str] = []
+    while i < n and lines[i].strip() != "" and not lines[i].lstrip().startswith("#"):
+        stripped = lines[i].strip()
+        if stripped.startswith(">"):
+            stripped = stripped.lstrip(">").strip()
+        if stripped:
+            block.append(stripped)
+        i += 1
+    return " ".join(block)
+
+
+def strip_leading_banner(body: str) -> str:
+    """Strip a leading provenance banner + blank lines from a captured body.
+
+    Makes re-migration idempotent: after the proof pattern is regenerated
+    (banner + body), re-running the migration recovers the same body. Only
+    strips a banner block that opens with the renderer's first banner line, so
+    unrelated leading HTML comments survive.
+    """
+    lines = body.split("\n")
+    i = 0
+    while i < len(lines) and lines[i].strip() == "":
+        i += 1
+    if i < len(lines) and lines[i].startswith(
+        "<!-- AUTO-GENERATED by task packs:render"
+    ):
+        while i < len(lines) and lines[i].lstrip().startswith("<!--"):
+            i += 1
+        while i < len(lines) and lines[i].strip() == "":
+            i += 1
+    return "\n".join(lines[i:])
+
+
+def build_pattern_entry(
+    md: Path, patterns_dir: Path, *, capture_body: bool
+) -> dict:
+    """Build one pattern entry from its markdown file."""
+    rel_path = md.resolve().relative_to(patterns_dir.resolve().parent).as_posix()
+    stem_slug = _SLUG_STRIP_RE.sub("-", md.stem.lower()).strip("-")
+    text = md.read_text(encoding="utf-8")
+    return {
+        "id": stem_slug,
+        "title": extract_title(text),
+        "description": extract_description(text),
+        "triggers": [stem_slug] if stem_slug else [],
+        "path": rel_path,
+        "body": strip_leading_banner(text) if capture_body else None,
+    }
+
+
+def build_pack(patterns_dir: Path, *, proof_pattern: str) -> dict:
+    """Scan the patterns dir and assemble the full pack object."""
+    patterns: list[dict] = []
+    for md in sorted(patterns_dir.glob("*.md")):
+        rel_path = md.resolve().relative_to(patterns_dir.resolve().parent).as_posix()
+        patterns.append(
+            build_pattern_entry(
+                md, patterns_dir, capture_body=(rel_path == proof_pattern)
+            )
+        )
+
+    return {
+        "pack": PACK_ID,
+        "version": PACK_VERSION,
+        "generated_from": "patterns/*.md",
+        "patterns": patterns,
+    }
+
+
+def migrate(patterns_dir: Path, out: Path, *, proof_pattern: str) -> dict:
+    """Build the pack from ``patterns_dir`` and write it to ``out``.
+
+    Raises ``FileNotFoundError`` when the dir is missing and ``ValueError`` when
+    no patterns are discovered.
+    """
+    if not patterns_dir.is_dir():
+        raise FileNotFoundError(f"patterns directory not found: {patterns_dir}")
+
+    pack = build_pack(patterns_dir, proof_pattern=proof_pattern)
+    if not pack["patterns"]:
+        raise ValueError(f"no patterns discovered under {patterns_dir}")
+
+    out.parent.mkdir(parents=True, exist_ok=True)
+    # ensure_ascii=True: the canonical source is serialized as pure ASCII with
+    # \uXXXX escapes (mirrors the other pack migrations). Lossless and keeps the
+    # JSON clean against `task verify:encoding` (#798) even when a pattern body
+    # carries non-ASCII glyphs (em dashes, RFC2119 symbols, emoji in diagrams).
+    out.write_text(
+        json.dumps(pack, indent=2, ensure_ascii=True) + "\n", encoding="utf-8"
+    )
+    return pack
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="pack_migrate_patterns.py",
+        description="Migrate patterns/*.md into the patterns-pack-0.1 source.",
+    )
+    parser.add_argument(
+        "--patterns-dir",
+        type=Path,
+        default=DEFAULT_PATTERNS_DIR,
+        help="Directory of pattern docs to scan (default: patterns/).",
+    )
+    parser.add_argument(
+        "--proof-pattern",
+        default=DEFAULT_PROOF_PATTERN,
+        help="Repo-relative path of the one pattern whose full body is captured.",
+    )
+    parser.add_argument(
+        "--out",
+        type=Path,
+        default=DEFAULT_OUT,
+        help="Output pack JSON path (default: packs/patterns/patterns-pack-0.1.json).",
+    )
+    args = parser.parse_args(argv)
+
+    try:
+        pack = migrate(
+            args.patterns_dir, args.out, proof_pattern=args.proof_pattern
+        )
+    except FileNotFoundError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 1
+    except ValueError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 1
+
+    bodied = sum(1 for s in pack["patterns"] if s["body"] is not None)
+    print(
+        f"Migrated {len(pack['patterns'])} patterns ({bodied} with body) "
+        f"-> {args.out}"
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())