devcouncil 0.2.0 → 0.3.0

package/src/devcouncil/knowledge/fetch.py

@@ -0,0 +1,223 @@
+"""Materialize an OKF bundle *source* into a local directory for ingest.
+
+``dev okf ingest`` historically accepted only a local bundle directory. Bundles travel,
+though — as ``.tar.gz``/``.zip`` archives or behind a git URL — so this module resolves
+any of those forms to a concrete on-disk directory the existing read/validate/copy logic
+can consume unchanged:
+
+* an existing local directory → returned as-is (no temp dir, nothing to clean up);
+* a local archive (``.tar.gz``/``.tgz``/``.zip``) → extracted into a temp dir, with a
+  **path-traversal guard** that rejects entries (or link targets) escaping the target;
+* a git URL (``http(s)://``, ``git@``, ``ssh://``, or ``*.git``) → ``git clone --depth 1``
+  into a temp dir (best-effort; a clear error is raised if git is missing or the clone
+  fails).
+
+Callers are responsible for invoking :meth:`FetchedBundle.cleanup` (in a ``finally``) to
+remove any temp dir once the bundle has been read/copied.
+"""
+
+from __future__ import annotations
+
+import shutil
+import subprocess
+import tarfile
+import tempfile
+import zipfile
+from dataclasses import dataclass
+from pathlib import Path
+
+_ARCHIVE_SUFFIXES = (".tar.gz", ".tgz", ".zip")
+
+
+class UnsafeArchiveError(ValueError):
+    """An archive entry (or link target) resolves outside the extraction directory.
+
+    Subclasses :class:`ValueError` so callers can catch either; raised by the extraction
+    guard before any unsafe member is written to disk (a path-traversal / Zip-Slip block).
+    """
+
+
+@dataclass
+class FetchedBundle:
+    """The resolved local bundle directory plus any temp dir that must be cleaned up.
+
+    ``directory`` is the bundle root to read. ``cleanup_dir`` is the temp directory created
+    for archives/git (``None`` for a pre-existing local directory, which is returned as-is
+    and must NOT be deleted). ``suggested_name`` is a sensible default ingest subfolder name
+    derived from the original source (its temp dir name would otherwise be random).
+    """
+
+    directory: Path
+    cleanup_dir: Path | None
+    suggested_name: str = ""
+
+    def cleanup(self) -> None:
+        """Remove the temp dir if one was created; safe to call when there is none."""
+        if self.cleanup_dir is not None:
+            shutil.rmtree(self.cleanup_dir, ignore_errors=True)
+
+
+def is_git_url(source: str) -> bool:
+    """Whether ``source`` looks like a git remote we should ``git clone``."""
+    s = source.strip()
+    return (
+        s.startswith("http://")
+        or s.startswith("https://")
+        or s.startswith("git@")
+        or s.startswith("ssh://")
+        or s.endswith(".git")
+    )
+
+
+def _archive_stem(name: str) -> str:
+    """The base name of an archive file with its (possibly two-part) suffix removed."""
+    lower = name.lower()
+    for suffix in _ARCHIVE_SUFFIXES:
+        if lower.endswith(suffix):
+            return name[: -len(suffix)] or "bundle"
+    return Path(name).stem or "bundle"
+
+
+def _git_repo_name(url: str) -> str:
+    tail = url.rstrip("/").split("/")[-1].split(":")[-1]
+    if tail.endswith(".git"):
+        tail = tail[:-4]
+    return tail or "bundle"
+
+
+def _within(base: Path, target: Path) -> bool:
+    """Whether resolved ``target`` is ``base`` itself or lives beneath it."""
+    try:
+        target.relative_to(base)
+        return True
+    except ValueError:
+        # relative_to only raises when target is neither base nor beneath it, so an
+        # escaping path is unambiguously outside the extraction root.
+        return False
+
+
+def _safe_extract_tar(archive: Path, dest: Path) -> None:
+    """Extract a tar archive into ``dest``, rejecting any path-escaping member/link."""
+    dest_resolved = dest.resolve()
+    with tarfile.open(archive, "r:*") as tf:
+        members = tf.getmembers()
+        for member in members:
+            target = (dest / member.name).resolve()
+            if not _within(dest_resolved, target):
+                raise UnsafeArchiveError(
+                    f"unsafe archive entry {member.name!r} escapes the extraction directory"
+                )
+            # A symlink/hardlink could still point outside even if its own path is safe.
+            # Symlink ``linkname`` is relative to the link's own directory; hardlink
+            # ``linkname`` is relative to the archive root — resolve each against the
+            # correct base, else a hardlink escaping via the root is mis-validated.
+            if member.issym() or member.islnk():
+                base = target.parent if member.issym() else dest
+                link_target = (base / member.linkname).resolve()
+                if not _within(dest_resolved, link_target):
+                    raise UnsafeArchiveError(
+                        f"unsafe link target {member.linkname!r} in entry {member.name!r}"
+                    )
+        # ``filter="data"`` is the safe extraction default on Python 3.12+ (defense in depth
+        # alongside the explicit guard above); fall back gracefully on older interpreters.
+        try:
+            tf.extractall(dest, members=members, filter="data")
+        except TypeError:  # pragma: no cover - Python < 3.12 has no filter kwarg
+            tf.extractall(dest, members=members)
+
+
+def _safe_extract_zip(archive: Path, dest: Path) -> None:
+    """Extract a zip archive into ``dest``, rejecting any path-escaping member (Zip-Slip)."""
+    dest_resolved = dest.resolve()
+    with zipfile.ZipFile(archive) as zf:
+        for name in zf.namelist():
+            target = (dest / name).resolve()
+            if not _within(dest_resolved, target):
+                raise UnsafeArchiveError(
+                    f"unsafe archive entry {name!r} escapes the extraction directory"
+                )
+        zf.extractall(dest)
+
+
+def _resolve_bundle_root(extracted: Path) -> Path:
+    """Descend into a lone top-level directory.
+
+    Archives produced with ``tar czf x.tgz somedir`` (or a git repo whose bundle lives in
+    a subdir) nest everything under one directory; collapsing it makes the returned path the
+    actual bundle root. If the markdown already sits at the top level, the dir is used as-is.
+    Purely best-effort: :func:`read_bundle` recurses anyway, so a wrong guess is harmless.
+    """
+    entries = [p for p in extracted.iterdir() if not p.name.startswith(".")]
+    if any(p.is_file() and p.suffix == ".md" for p in entries):
+        return extracted
+    if len(entries) == 1 and entries[0].is_dir():
+        return entries[0]
+    return extracted
+
+
+def fetch_bundle(source: str) -> FetchedBundle:
+    """Resolve ``source`` (local dir, local archive, or git URL) to a local bundle dir.
+
+    Raises :class:`UnsafeArchiveError` for a path-escaping archive entry, :class:`FileNotFoundError`
+    for a non-existent local path that isn't a git URL, and :class:`RuntimeError` if a git
+    clone is required but git is missing or the clone fails.
+    """
+    raw = source.strip()
+    local = Path(raw).expanduser()
+
+    # (a) existing local directory — use it directly; nothing to clean up.
+    if local.is_dir():
+        resolved = local.resolve()
+        return FetchedBundle(directory=resolved, cleanup_dir=None, suggested_name=resolved.name)
+
+    # (b) local archive — extract into a temp dir behind the traversal guard.
+    if local.is_file() and local.name.lower().endswith(_ARCHIVE_SUFFIXES):
+        tmp = Path(tempfile.mkdtemp(prefix="okf-archive-"))
+        try:
+            if local.name.lower().endswith(".zip"):
+                _safe_extract_zip(local, tmp)
+            else:
+                _safe_extract_tar(local, tmp)
+        except BaseException:
+            shutil.rmtree(tmp, ignore_errors=True)
+            raise
+        return FetchedBundle(
+            directory=_resolve_bundle_root(tmp),
+            cleanup_dir=tmp,
+            suggested_name=_archive_stem(local.name),
+        )
+
+    # (c) git URL — shallow clone into a temp dir.
+    if is_git_url(raw):
+        return _clone_git(raw)
+
+    raise FileNotFoundError(
+        f"bundle source not found: {source!r} (expected a directory, a .tar.gz/.tgz/.zip "
+        "archive, or a git URL)"
+    )
+
+
+def _clone_git(url: str) -> FetchedBundle:
+    """``git clone --depth 1`` ``url`` into a fresh temp dir (best-effort)."""
+    if shutil.which("git") is None:
+        raise RuntimeError("git is not installed; cannot clone bundle from a git URL")
+    parent = Path(tempfile.mkdtemp(prefix="okf-git-"))
+    target = parent / "clone"
+    try:
+        result = subprocess.run(
+            ["git", "clone", "--depth", "1", url, str(target)],
+            capture_output=True,
+            text=True,
+        )
+    except OSError as exc:  # pragma: no cover - git present but unexecutable
+        shutil.rmtree(parent, ignore_errors=True)
+        raise RuntimeError(f"git clone failed to start: {exc}") from exc
+    if result.returncode != 0:
+        shutil.rmtree(parent, ignore_errors=True)
+        detail = (result.stderr or result.stdout or "").strip()
+        raise RuntimeError(f"git clone of {url!r} failed: {detail}")
+    return FetchedBundle(
+        directory=_resolve_bundle_root(target),
+        cleanup_dir=parent,
+        suggested_name=_git_repo_name(url),
+    )

package/src/devcouncil/knowledge/frontmatter.py

@@ -0,0 +1,51 @@
+"""Markdown + YAML frontmatter: the single split/build implementation.
+
+Both the skills library (:mod:`devcouncil.skills.registry`) and the knowledge formats
+(OKF, design.md) store structured metadata in a leading ``---`` YAML block followed by a
+markdown body. Keeping one parser/serializer here means a fix to frontmatter handling
+applies everywhere rather than drifting between copies.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import yaml
+
+
+def split_frontmatter(text: str) -> tuple[dict[str, Any], str]:
+    """Split ``text`` into (frontmatter dict, body).
+
+    Returns ``({}, text)`` when there is no leading ``---`` block or the block does not
+    parse to a mapping. Mirrors the historical behavior of
+    ``skills.registry._split_frontmatter`` so existing skill files keep parsing.
+    """
+    if text.startswith("---"):
+        parts = text.split("---", 2)
+        if len(parts) == 3:
+            try:
+                meta = yaml.safe_load(parts[1]) or {}
+            except yaml.YAMLError:
+                meta = {}
+            return (meta if isinstance(meta, dict) else {}), parts[2].lstrip("\r\n")
+    return {}, text
+
+
+def build_frontmatter_markdown(meta: dict[str, Any], body: str) -> str:
+    """Render a ``---`` YAML frontmatter block above ``body``.
+
+    Empty/``None`` values are dropped so the frontmatter stays minimal (OKF and design.md
+    both favor only-what-you-have metadata). Key order is preserved as given by the caller
+    (``sort_keys=False``); Unicode is kept literal rather than escaped.
+    """
+    clean = {k: v for k, v in meta.items() if v not in (None, "", [], {})}
+    front = yaml.safe_dump(
+        clean,
+        sort_keys=False,
+        default_flow_style=False,
+        allow_unicode=True,
+    ).strip()
+    body = body.strip()
+    if not front:
+        return f"{body}\n" if body else ""
+    return f"---\n{front}\n---\n\n{body}\n" if body else f"---\n{front}\n---\n"

package/src/devcouncil/knowledge/okf.py

@@ -0,0 +1,202 @@
+"""Open Knowledge Format (OKF) v0.1 — model, bundle I/O, and validation.
+
+OKF (Google Cloud) formalizes the "LLM-wiki" pattern: a directory of markdown files,
+each carrying a small YAML frontmatter header, cross-linked with plain markdown links to
+form a portable, vendor-neutral knowledge graph. The only required frontmatter field is
+``type``; everything else is producer-defined.
+
+DevCouncil uses this module in both directions:
+
+* **Export** — :mod:`devcouncil.reporting.okf_bundle_writer` builds an :class:`OKFBundle`
+  from the artifact graph and calls :func:`write_bundle`.
+* **Ingest** — :func:`read_bundle` parses an external bundle so it can be surfaced as
+  planning context (:mod:`devcouncil.knowledge.sources`).
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+from devcouncil.knowledge.frontmatter import build_frontmatter_markdown, split_frontmatter
+
+# Markdown inline links: [text](target). We only resolve relative, non-anchor, non-URL
+# targets into intra-bundle edges; external resources live in the `resource` field.
+_LINK_RE = re.compile(r"\[(?P<text>[^\]]+)\]\((?P<target>[^)]+)\)")
+_URL_RE = re.compile(r"^[a-zA-Z][a-zA-Z0-9+.-]*://")
+
+
+class OKFDocument(BaseModel):
+    """A single OKF document: YAML frontmatter header + markdown body.
+
+    ``rel_path`` is the document's POSIX path relative to the bundle root (e.g.
+    ``tasks/TASK-001.md``); it is the node identity used when resolving links. ``links``
+    are resolved intra-bundle edges (relative link targets normalized to bundle-relative
+    POSIX paths), computed by :func:`read_bundle`.
+    """
+
+    type: str
+    title: str = ""
+    description: str = ""
+    resource: str = ""
+    tags: list[str] = Field(default_factory=list)
+    timestamp: str = ""
+    body: str = ""
+    rel_path: str = ""
+    links: list[str] = Field(default_factory=list)
+
+    def to_markdown(self) -> str:
+        """Render this document as OKF markdown (frontmatter + body)."""
+        meta: dict[str, Any] = {
+            "type": self.type,
+            "title": self.title,
+            "description": self.description,
+            "resource": self.resource,
+            "tags": self.tags,
+            "timestamp": self.timestamp,
+        }
+        return build_frontmatter_markdown(meta, self.body)
+
+    @classmethod
+    def from_markdown(cls, text: str, rel_path: str = "") -> "OKFDocument":
+        """Parse OKF markdown into a document (links are resolved by :func:`read_bundle`)."""
+        meta, body = split_frontmatter(text)
+        tags = meta.get("tags") or []
+        if isinstance(tags, str):
+            tags = [tags]
+        return cls(
+            type=str(meta.get("type") or ""),
+            title=str(meta.get("title") or ""),
+            description=str(meta.get("description") or ""),
+            resource=str(meta.get("resource") or ""),
+            tags=[str(t) for t in tags],
+            timestamp=str(meta.get("timestamp") or ""),
+            body=body.strip(),
+            rel_path=rel_path,
+        )
+
+
+class OKFBundle(BaseModel):
+    """A collection of OKF documents keyed by bundle-relative path."""
+
+    documents: list[OKFDocument] = Field(default_factory=list)
+
+    def by_path(self) -> dict[str, OKFDocument]:
+        return {doc.rel_path: doc for doc in self.documents if doc.rel_path}
+
+
+def _resolve_link(source_rel_path: str, target: str) -> str | None:
+    """Resolve a markdown link target found in ``source_rel_path`` to a bundle-relative
+    POSIX path, or ``None`` if it is external (URL), an in-page anchor, a non-document
+    asset, or escapes root.
+
+    Only ``.md`` targets are treated as intra-bundle document edges: a bundle's document
+    set is markdown-only, so links to images (``![alt](x.png)``) or other assets must not
+    be recorded as edges — otherwise ``validate_bundle`` would flag every such link as a
+    broken intra-bundle reference.
+    """
+    target = target.strip()
+    if not target or target.startswith("#") or _URL_RE.match(target) or target.startswith("mailto:"):
+        return None
+    target = target.split("#", 1)[0].strip()  # drop any anchor fragment
+    if not target or not target.endswith(".md"):
+        return None
+    source_dir = PurePosix(source_rel_path).parent
+    try:
+        resolved = (source_dir / target).resolve_relative()
+    except ValueError:
+        return None
+    return resolved
+
+
+class PurePosix:
+    """Tiny relative-POSIX-path helper.
+
+    ``pathlib.PurePosixPath`` does not collapse ``..`` segments (it has no filesystem to
+    resolve against), so this resolves ``a/b/../c`` → ``a/c`` purely lexically and rejects
+    paths that escape the bundle root. Kept local to avoid pulling in os.path semantics
+    that differ on Windows.
+    """
+
+    def __init__(self, raw: str) -> None:
+        self.parts = [p for p in raw.replace("\\", "/").split("/") if p not in ("", ".")]
+
+    @property
+    def parent(self) -> "PurePosix":
+        p = PurePosix("")
+        p.parts = self.parts[:-1]
+        return p
+
+    def __truediv__(self, other: str) -> "PurePosix":
+        p = PurePosix("")
+        p.parts = self.parts + [seg for seg in other.replace("\\", "/").split("/") if seg not in ("", ".")]
+        return p
+
+    def resolve_relative(self) -> str:
+        out: list[str] = []
+        for seg in self.parts:
+            if seg == "..":
+                if not out:
+                    raise ValueError("path escapes bundle root")
+                out.pop()
+            else:
+                out.append(seg)
+        return "/".join(out)
+
+
+def read_bundle(bundle_dir: Path) -> OKFBundle:
+    """Read an OKF bundle from ``bundle_dir``: parse every ``*.md`` file and resolve
+    intra-bundle markdown links into :attr:`OKFDocument.links` edges."""
+    bundle_dir = bundle_dir.expanduser().resolve()
+    docs: list[OKFDocument] = []
+    for path in sorted(bundle_dir.rglob("*.md")):
+        rel = path.relative_to(bundle_dir).as_posix()
+        doc = OKFDocument.from_markdown(path.read_text(encoding="utf-8"), rel_path=rel)
+        links: list[str] = []
+        seen_links: set[str] = set()
+        for match in _LINK_RE.finditer(doc.body):
+            resolved = _resolve_link(rel, match.group("target"))
+            if resolved and resolved not in seen_links:
+                seen_links.add(resolved)
+                links.append(resolved)
+        doc.links = links
+        docs.append(doc)
+    return OKFBundle(documents=docs)
+
+
+def write_bundle(bundle: OKFBundle, bundle_dir: Path) -> list[Path]:
+    """Write every document in ``bundle`` to ``bundle_dir`` at its ``rel_path``.
+
+    Returns the list of written file paths. Documents without a ``rel_path`` are skipped.
+    """
+    bundle_dir = bundle_dir.expanduser().resolve()
+    written: list[Path] = []
+    for doc in bundle.documents:
+        if not doc.rel_path:
+            continue
+        target = bundle_dir / doc.rel_path
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_text(doc.to_markdown(), encoding="utf-8")
+        written.append(target)
+    return written
+
+
+def validate_bundle(bundle: OKFBundle) -> list[str]:
+    """Return human-readable validation problems for ``bundle`` (empty list == valid).
+
+    Checks the OKF invariants DevCouncil relies on: every document declares a ``type``,
+    and every intra-bundle link resolves to a document actually present in the bundle.
+    """
+    problems: list[str] = []
+    present = set(bundle.by_path().keys())
+    for doc in bundle.documents:
+        where = doc.rel_path or doc.title or "<unknown>"
+        if not doc.type.strip():
+            problems.append(f"{where}: missing required 'type' frontmatter field")
+        for link in doc.links:
+            if link not in present:
+                problems.append(f"{where}: broken link to '{link}' (no such document in bundle)")
+    return problems

package/src/devcouncil/knowledge/skill_bridge.py

@@ -0,0 +1,96 @@
+"""Single source of truth for Skill <-> OKF document interconversion.
+
+DevCouncil skills (:class:`devcouncil.skills.registry.Skill`) and the Open Knowledge
+Format (:class:`devcouncil.knowledge.okf.OKFDocument`) describe the same kind of
+artifact from two angles: a skill is "guidance that fires on triggers", an OKF document
+is "a typed, portable markdown node". This module is the one place that maps between
+them, so exporting skills into an OKF bundle and ingesting an OKF bundle back into skills
+stay symmetric and don't drift apart across the codebase.
+
+Skill documents are marked with the OKF ``type`` value :data:`SKILL_OKF_TYPE`; that type
+tag is what lets :func:`okf_document_to_skill` tell skill nodes apart from other OKF nodes
+(BigQuery tables, tasks, requirements, ...) in a mixed bundle.
+
+Import-cycle note: :mod:`devcouncil.skills.registry` imports this module, so ``Skill`` /
+``SkillTriggers`` are imported *lazily* inside :func:`okf_document_to_skill` rather than at
+module top. ``OKFDocument`` is safe to import at top because ``knowledge.okf`` does not
+import the skills package.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from devcouncil.knowledge.okf import OKFDocument
+
+if TYPE_CHECKING:
+    from devcouncil.skills.registry import Skill
+
+# The OKF `type` frontmatter value carried by every skill document. Used both when
+# emitting skills (export) and when filtering a mixed bundle back into skills (ingest).
+SKILL_OKF_TYPE = "Engineering Skill"
+
+# Fallback name derivation when a document has no rel_path to take a stem from.
+_SLUG_RE = re.compile(r"[^a-z0-9]+")
+
+
+def _slug(text: str) -> str:
+    """Lowercase, hyphen-joined slug of ``text`` (used to name a skill that lacks a path)."""
+    return _SLUG_RE.sub("-", text.strip().lower()).strip("-") or "skill"
+
+
+def skill_to_okf_document(skill: "Skill", rel_dir: str = "skills") -> OKFDocument:
+    """Render a :class:`Skill` as an OKF document for inclusion in a bundle.
+
+    Keyword triggers become OKF ``tags`` (sorted + deduped for stable, diff-friendly
+    output); ``timestamp`` is left empty because a skill is library content, not a
+    timestamped artifact. The document lands at ``<rel_dir>/<skill.name>.md``.
+    """
+    return OKFDocument(
+        type=SKILL_OKF_TYPE,
+        title=skill.title or skill.name,
+        description=skill.description,
+        tags=sorted(set(skill.triggers.keywords)),
+        timestamp="",
+        body=skill.body,
+        rel_path=f"{rel_dir.rstrip('/')}/{skill.name}.md",
+    )
+
+
+def is_skill_document(doc: OKFDocument) -> bool:
+    """Whether ``doc`` is a skill node, i.e. its OKF ``type`` is :data:`SKILL_OKF_TYPE`.
+
+    Comparison is case-insensitive and whitespace-trimmed so hand-edited bundles still
+    round-trip.
+    """
+    return doc.type.strip().lower() == SKILL_OKF_TYPE.lower()
+
+
+def okf_document_to_skill(doc: OKFDocument) -> "Skill | None":
+    """Reconstruct a :class:`Skill` from an OKF document, or ``None`` if it isn't a skill.
+
+    Non-skill-typed nodes (BigQuery tables, tasks, ...) return ``None`` so callers can
+    map over a mixed bundle and keep only the skill nodes. The skill ``name`` comes from
+    the document's ``rel_path`` stem when present, else a slug of its title. The stem is
+    slugged too, so a foreign bundle whose file is ``skills/Foo Bar.md`` yields the skill
+    name ``foo-bar`` (a clean identifier that scaffolds to a sane ``.claude/skills`` dir),
+    not ``Foo Bar``. ``always`` is ``False`` and ``globs`` empty because OKF tags only carry
+    keyword triggers; ``source_path`` is ``None`` since the skill originates from a bundle.
+    """
+    if not is_skill_document(doc):
+        return None
+    # Lazy import to avoid a registry <-> skill_bridge import cycle (see module docstring).
+    from devcouncil.skills.registry import Skill, SkillTriggers
+
+    name = _slug(Path(doc.rel_path).stem) if doc.rel_path else _slug(doc.title)
+    return Skill(
+        name=name,
+        title=doc.title,
+        description=doc.description,
+        always=False,
+        triggers=SkillTriggers(keywords=list(doc.tags), globs=[]),
+        body=doc.body,
+        source_path=None,
+    )