devcouncil 0.1.1 → 0.3.0

package/src/devcouncil/knowledge/okf.py

@@ -0,0 +1,202 @@
+"""Open Knowledge Format (OKF) v0.1 — model, bundle I/O, and validation.
+
+OKF (Google Cloud) formalizes the "LLM-wiki" pattern: a directory of markdown files,
+each carrying a small YAML frontmatter header, cross-linked with plain markdown links to
+form a portable, vendor-neutral knowledge graph. The only required frontmatter field is
+``type``; everything else is producer-defined.
+
+DevCouncil uses this module in both directions:
+
+* **Export** — :mod:`devcouncil.reporting.okf_bundle_writer` builds an :class:`OKFBundle`
+  from the artifact graph and calls :func:`write_bundle`.
+* **Ingest** — :func:`read_bundle` parses an external bundle so it can be surfaced as
+  planning context (:mod:`devcouncil.knowledge.sources`).
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+from devcouncil.knowledge.frontmatter import build_frontmatter_markdown, split_frontmatter
+
+# Markdown inline links: [text](target). We only resolve relative, non-anchor, non-URL
+# targets into intra-bundle edges; external resources live in the `resource` field.
+_LINK_RE = re.compile(r"\[(?P<text>[^\]]+)\]\((?P<target>[^)]+)\)")
+_URL_RE = re.compile(r"^[a-zA-Z][a-zA-Z0-9+.-]*://")
+
+
+class OKFDocument(BaseModel):
+    """A single OKF document: YAML frontmatter header + markdown body.
+
+    ``rel_path`` is the document's POSIX path relative to the bundle root (e.g.
+    ``tasks/TASK-001.md``); it is the node identity used when resolving links. ``links``
+    are resolved intra-bundle edges (relative link targets normalized to bundle-relative
+    POSIX paths), computed by :func:`read_bundle`.
+    """
+
+    type: str
+    title: str = ""
+    description: str = ""
+    resource: str = ""
+    tags: list[str] = Field(default_factory=list)
+    timestamp: str = ""
+    body: str = ""
+    rel_path: str = ""
+    links: list[str] = Field(default_factory=list)
+
+    def to_markdown(self) -> str:
+        """Render this document as OKF markdown (frontmatter + body)."""
+        meta: dict[str, Any] = {
+            "type": self.type,
+            "title": self.title,
+            "description": self.description,
+            "resource": self.resource,
+            "tags": self.tags,
+            "timestamp": self.timestamp,
+        }
+        return build_frontmatter_markdown(meta, self.body)
+
+    @classmethod
+    def from_markdown(cls, text: str, rel_path: str = "") -> "OKFDocument":
+        """Parse OKF markdown into a document (links are resolved by :func:`read_bundle`)."""
+        meta, body = split_frontmatter(text)
+        tags = meta.get("tags") or []
+        if isinstance(tags, str):
+            tags = [tags]
+        return cls(
+            type=str(meta.get("type") or ""),
+            title=str(meta.get("title") or ""),
+            description=str(meta.get("description") or ""),
+            resource=str(meta.get("resource") or ""),
+            tags=[str(t) for t in tags],
+            timestamp=str(meta.get("timestamp") or ""),
+            body=body.strip(),
+            rel_path=rel_path,
+        )
+
+
+class OKFBundle(BaseModel):
+    """A collection of OKF documents keyed by bundle-relative path."""
+
+    documents: list[OKFDocument] = Field(default_factory=list)
+
+    def by_path(self) -> dict[str, OKFDocument]:
+        return {doc.rel_path: doc for doc in self.documents if doc.rel_path}
+
+
+def _resolve_link(source_rel_path: str, target: str) -> str | None:
+    """Resolve a markdown link target found in ``source_rel_path`` to a bundle-relative
+    POSIX path, or ``None`` if it is external (URL), an in-page anchor, a non-document
+    asset, or escapes root.
+
+    Only ``.md`` targets are treated as intra-bundle document edges: a bundle's document
+    set is markdown-only, so links to images (``![alt](x.png)``) or other assets must not
+    be recorded as edges — otherwise ``validate_bundle`` would flag every such link as a
+    broken intra-bundle reference.
+    """
+    target = target.strip()
+    if not target or target.startswith("#") or _URL_RE.match(target) or target.startswith("mailto:"):
+        return None
+    target = target.split("#", 1)[0].strip()  # drop any anchor fragment
+    if not target or not target.endswith(".md"):
+        return None
+    source_dir = PurePosix(source_rel_path).parent
+    try:
+        resolved = (source_dir / target).resolve_relative()
+    except ValueError:
+        return None
+    return resolved
+
+
+class PurePosix:
+    """Tiny relative-POSIX-path helper.
+
+    ``pathlib.PurePosixPath`` does not collapse ``..`` segments (it has no filesystem to
+    resolve against), so this resolves ``a/b/../c`` → ``a/c`` purely lexically and rejects
+    paths that escape the bundle root. Kept local to avoid pulling in os.path semantics
+    that differ on Windows.
+    """
+
+    def __init__(self, raw: str) -> None:
+        self.parts = [p for p in raw.replace("\\", "/").split("/") if p not in ("", ".")]
+
+    @property
+    def parent(self) -> "PurePosix":
+        p = PurePosix("")
+        p.parts = self.parts[:-1]
+        return p
+
+    def __truediv__(self, other: str) -> "PurePosix":
+        p = PurePosix("")
+        p.parts = self.parts + [seg for seg in other.replace("\\", "/").split("/") if seg not in ("", ".")]
+        return p
+
+    def resolve_relative(self) -> str:
+        out: list[str] = []
+        for seg in self.parts:
+            if seg == "..":
+                if not out:
+                    raise ValueError("path escapes bundle root")
+                out.pop()
+            else:
+                out.append(seg)
+        return "/".join(out)
+
+
+def read_bundle(bundle_dir: Path) -> OKFBundle:
+    """Read an OKF bundle from ``bundle_dir``: parse every ``*.md`` file and resolve
+    intra-bundle markdown links into :attr:`OKFDocument.links` edges."""
+    bundle_dir = bundle_dir.expanduser().resolve()
+    docs: list[OKFDocument] = []
+    for path in sorted(bundle_dir.rglob("*.md")):
+        rel = path.relative_to(bundle_dir).as_posix()
+        doc = OKFDocument.from_markdown(path.read_text(encoding="utf-8"), rel_path=rel)
+        links: list[str] = []
+        seen_links: set[str] = set()
+        for match in _LINK_RE.finditer(doc.body):
+            resolved = _resolve_link(rel, match.group("target"))
+            if resolved and resolved not in seen_links:
+                seen_links.add(resolved)
+                links.append(resolved)
+        doc.links = links
+        docs.append(doc)
+    return OKFBundle(documents=docs)
+
+
+def write_bundle(bundle: OKFBundle, bundle_dir: Path) -> list[Path]:
+    """Write every document in ``bundle`` to ``bundle_dir`` at its ``rel_path``.
+
+    Returns the list of written file paths. Documents without a ``rel_path`` are skipped.
+    """
+    bundle_dir = bundle_dir.expanduser().resolve()
+    written: list[Path] = []
+    for doc in bundle.documents:
+        if not doc.rel_path:
+            continue
+        target = bundle_dir / doc.rel_path
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_text(doc.to_markdown(), encoding="utf-8")
+        written.append(target)
+    return written
+
+
+def validate_bundle(bundle: OKFBundle) -> list[str]:
+    """Return human-readable validation problems for ``bundle`` (empty list == valid).
+
+    Checks the OKF invariants DevCouncil relies on: every document declares a ``type``,
+    and every intra-bundle link resolves to a document actually present in the bundle.
+    """
+    problems: list[str] = []
+    present = set(bundle.by_path().keys())
+    for doc in bundle.documents:
+        where = doc.rel_path or doc.title or "<unknown>"
+        if not doc.type.strip():
+            problems.append(f"{where}: missing required 'type' frontmatter field")
+        for link in doc.links:
+            if link not in present:
+                problems.append(f"{where}: broken link to '{link}' (no such document in bundle)")
+    return problems

package/src/devcouncil/knowledge/skill_bridge.py

@@ -0,0 +1,96 @@
+"""Single source of truth for Skill <-> OKF document interconversion.
+
+DevCouncil skills (:class:`devcouncil.skills.registry.Skill`) and the Open Knowledge
+Format (:class:`devcouncil.knowledge.okf.OKFDocument`) describe the same kind of
+artifact from two angles: a skill is "guidance that fires on triggers", an OKF document
+is "a typed, portable markdown node". This module is the one place that maps between
+them, so exporting skills into an OKF bundle and ingesting an OKF bundle back into skills
+stay symmetric and don't drift apart across the codebase.
+
+Skill documents are marked with the OKF ``type`` value :data:`SKILL_OKF_TYPE`; that type
+tag is what lets :func:`okf_document_to_skill` tell skill nodes apart from other OKF nodes
+(BigQuery tables, tasks, requirements, ...) in a mixed bundle.
+
+Import-cycle note: :mod:`devcouncil.skills.registry` imports this module, so ``Skill`` /
+``SkillTriggers`` are imported *lazily* inside :func:`okf_document_to_skill` rather than at
+module top. ``OKFDocument`` is safe to import at top because ``knowledge.okf`` does not
+import the skills package.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from devcouncil.knowledge.okf import OKFDocument
+
+if TYPE_CHECKING:
+    from devcouncil.skills.registry import Skill
+
+# The OKF `type` frontmatter value carried by every skill document. Used both when
+# emitting skills (export) and when filtering a mixed bundle back into skills (ingest).
+SKILL_OKF_TYPE = "Engineering Skill"
+
+# Fallback name derivation when a document has no rel_path to take a stem from.
+_SLUG_RE = re.compile(r"[^a-z0-9]+")
+
+
+def _slug(text: str) -> str:
+    """Lowercase, hyphen-joined slug of ``text`` (used to name a skill that lacks a path)."""
+    return _SLUG_RE.sub("-", text.strip().lower()).strip("-") or "skill"
+
+
+def skill_to_okf_document(skill: "Skill", rel_dir: str = "skills") -> OKFDocument:
+    """Render a :class:`Skill` as an OKF document for inclusion in a bundle.
+
+    Keyword triggers become OKF ``tags`` (sorted + deduped for stable, diff-friendly
+    output); ``timestamp`` is left empty because a skill is library content, not a
+    timestamped artifact. The document lands at ``<rel_dir>/<skill.name>.md``.
+    """
+    return OKFDocument(
+        type=SKILL_OKF_TYPE,
+        title=skill.title or skill.name,
+        description=skill.description,
+        tags=sorted(set(skill.triggers.keywords)),
+        timestamp="",
+        body=skill.body,
+        rel_path=f"{rel_dir.rstrip('/')}/{skill.name}.md",
+    )
+
+
+def is_skill_document(doc: OKFDocument) -> bool:
+    """Whether ``doc`` is a skill node, i.e. its OKF ``type`` is :data:`SKILL_OKF_TYPE`.
+
+    Comparison is case-insensitive and whitespace-trimmed so hand-edited bundles still
+    round-trip.
+    """
+    return doc.type.strip().lower() == SKILL_OKF_TYPE.lower()
+
+
+def okf_document_to_skill(doc: OKFDocument) -> "Skill | None":
+    """Reconstruct a :class:`Skill` from an OKF document, or ``None`` if it isn't a skill.
+
+    Non-skill-typed nodes (BigQuery tables, tasks, ...) return ``None`` so callers can
+    map over a mixed bundle and keep only the skill nodes. The skill ``name`` comes from
+    the document's ``rel_path`` stem when present, else a slug of its title. The stem is
+    slugged too, so a foreign bundle whose file is ``skills/Foo Bar.md`` yields the skill
+    name ``foo-bar`` (a clean identifier that scaffolds to a sane ``.claude/skills`` dir),
+    not ``Foo Bar``. ``always`` is ``False`` and ``globs`` empty because OKF tags only carry
+    keyword triggers; ``source_path`` is ``None`` since the skill originates from a bundle.
+    """
+    if not is_skill_document(doc):
+        return None
+    # Lazy import to avoid a registry <-> skill_bridge import cycle (see module docstring).
+    from devcouncil.skills.registry import Skill, SkillTriggers
+
+    name = _slug(Path(doc.rel_path).stem) if doc.rel_path else _slug(doc.title)
+    return Skill(
+        name=name,
+        title=doc.title,
+        description=doc.description,
+        always=False,
+        triggers=SkillTriggers(keywords=list(doc.tags), globs=[]),
+        body=doc.body,
+        source_path=None,
+    )

package/src/devcouncil/knowledge/sources.py

@@ -0,0 +1,239 @@
+"""Knowledge sources: discover and select OKF / design.md context for prompts.
+
+A :class:`KnowledgeSource` is the prompt-facing view of an ingested knowledge file. It
+mirrors :class:`devcouncil.skills.registry.Skill` — same frontmatter contract, same
+trigger-based selection and relevance ranking — so OKF bundles and a project design system
+flow into planning/council/task prompts through the existing budget-aware machinery.
+
+On-disk layout (under the project root)::
+
+    .devcouncil/knowledge/
+        design/design.md        # one design system, always selected
+        okf/*.md                # ingested OKF documents, selected by trigger/keyword
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+from devcouncil.knowledge.frontmatter import split_frontmatter
+from devcouncil.skills.registry import SkillTriggers, _keyword_in_text
+
+KNOWLEDGE_DIR = ".devcouncil/knowledge"
+
+Kind = Literal["okf", "design"]
+
+
+class KnowledgeSource(BaseModel):
+    name: str
+    kind: Kind
+    description: str = ""
+    always: bool = False
+    triggers: SkillTriggers = Field(default_factory=SkillTriggers)
+    body: str = ""
+    priority: int = 50
+    source_path: Path | None = None
+
+    def _match_and_score(self, goal_lower: str) -> tuple[bool, int]:
+        """Whether this source applies to ``goal_lower`` and its relevance rank, in one
+        keyword scan. Unlike a Skill, ``matches`` is NOT equivalent to ``score > 0`` here:
+        OKF sources have a nonzero ``priority`` floor, so the two must be returned together."""
+        if self.always:
+            return True, 1_000_000 + self.priority
+        hits = sum(1 for kw in self.triggers.keywords if _keyword_in_text(kw, goal_lower))
+        return hits > 0, self.priority + 5 * hits
+
+    def matches(self, goal: str) -> bool:
+        """True if this source applies to the given goal text.
+
+        Design systems are always-on (a coding agent should always honor them); OKF
+        knowledge is matched on goal keywords like a domain skill.
+        """
+        return self._match_and_score(goal.lower())[0]
+
+    def relevance_score(self, goal: str) -> int:
+        return self._match_and_score(goal.lower())[1]
+
+    def render(self) -> str:
+        """A titled markdown block for inclusion in a prompt preamble."""
+        header = f"## {self.description or self.name}".rstrip()
+        return f"{header}\n\n{self.body.strip()}".strip()
+
+
+# Parsed-source cache keyed on (resolved path, mtime_ns, kind, always, priority). Knowledge
+# discovery runs once per task during planning (and repeatedly via MCP), so without this an
+# N-task plan re-reads + re-parses every knowledge file N times. Keyed on mtime so an edited
+# or freshly-ingested file is re-parsed; the args are in the key because they shape the
+# resulting source. Cached sources are read-only (callers only match/score/render them).
+_source_cache: dict[tuple[str, int, str, bool, int], KnowledgeSource] = {}
+_SOURCE_CACHE_MAX = 512
+
+# Directory-scan cache for discover_knowledge_sources(). The glob/rglob filesystem walk
+# repeats 5-30x per planning session (once per task, plus MCP calls) over a directory tree
+# that almost never changes mid-session, while per-file PARSING is already memoized by
+# _source_cache. This caches the discovered source *list* keyed on the call's shaping args.
+# Each entry also stores a signature: the sorted set of (resolved-path, mtime_ns) for every
+# scanned file. On the next call we recompute that signature (a cheap stat per file vs. the
+# directory walk + frontmatter parse) and invalidate if it differs, so a freshly written,
+# edited, added or removed file is picked up. Keying on the path set (not just mtimes) means
+# add/remove is detected even if the filesystem's mtime granularity is too coarse to notice
+# an in-place rewrite within the same tick. Cached lists are read-only to callers.
+_DiscoverKey = tuple[str, str, bool, int, int]
+_DiscoverSig = tuple[tuple[str, int], ...]
+_discover_cache: dict[_DiscoverKey, tuple[_DiscoverSig, list[KnowledgeSource]]] = {}
+_DISCOVER_CACHE_MAX = 256
+
+
+def clear_knowledge_caches() -> None:
+    """Drop the cached parsed knowledge sources (useful in long-running processes/tests)."""
+    _source_cache.clear()
+    _discover_cache.clear()
+
+
+def _source_from_file(path: Path, kind: Kind, always: bool, priority: int) -> KnowledgeSource:
+    try:
+        key: tuple[str, int, str, bool, int] | None = (
+            str(path.resolve()), path.stat().st_mtime_ns, kind, always, priority,
+        )
+    except OSError:
+        key = None
+    if key is not None:
+        cached = _source_cache.get(key)
+        if cached is not None:
+            return cached
+    source = _parse_source_file(path, kind, always, priority)
+    if key is not None:
+        if len(_source_cache) >= _SOURCE_CACHE_MAX:
+            _source_cache.clear()
+        _source_cache[key] = source
+    return source
+
+
+def _parse_source_file(path: Path, kind: Kind, always: bool, priority: int) -> KnowledgeSource:
+    meta, body = split_frontmatter(path.read_text(encoding="utf-8"))
+    triggers = meta.get("triggers") or {}
+    # Derive keywords from explicit triggers and, for OKF, the document's tags — so an
+    # OKF doc tagged [sales, revenue] fires on goals mentioning those domains for free.
+    keywords = list(triggers.get("keywords") or [])
+    if kind == "okf":
+        tags = meta.get("tags") or []
+        if isinstance(tags, str):
+            tags = [tags]
+        keywords.extend(str(t) for t in tags)
+    description = str(meta.get("description") or meta.get("title") or meta.get("type") or path.stem)
+    return KnowledgeSource(
+        name=str(meta.get("name") or path.stem),
+        kind=kind,
+        description=description,
+        always=always,
+        triggers=SkillTriggers(keywords=keywords, globs=list(triggers.get("globs") or [])),
+        body=body.strip(),
+        priority=priority,
+        source_path=path,
+    )
+
+
+def discover_knowledge_sources(
+    project_root: Path,
+    directory: str = KNOWLEDGE_DIR,
+    design_always: bool = True,
+    design_priority: int = 80,
+    okf_priority: int = 50,
+) -> list[KnowledgeSource]:
+    """Find ingested knowledge under ``<project_root>/<directory>/{design,okf}``."""
+    base = project_root / directory
+    # Collect the files to ingest (the directory walk we want to cache), as
+    # (path, kind, always, priority) in stable order, before parsing any of them.
+    scanned: list[tuple[Path, Kind, bool, int]] = []
+    design_dir = base / "design"
+    if design_dir.exists():
+        for path in sorted(design_dir.glob("*.md")):
+            scanned.append((path, "design", design_always, design_priority))
+    okf_dir = base / "okf"
+    if okf_dir.exists():
+        for path in sorted(okf_dir.rglob("*.md")):
+            # Skip OKF index files: they are navigation, not knowledge worth injecting.
+            if path.name.lower() == "index.md":
+                continue
+            scanned.append((path, "okf", False, okf_priority))
+
+    # Signature of the current scan: (resolved path, mtime_ns) for each file. A miss here
+    # (file added/removed/edited) invalidates the cached list for this key.
+    sig_parts: list[tuple[str, int]] = []
+    for path, _kind, _always, _priority in scanned:
+        try:
+            sig_parts.append((str(path.resolve()), path.stat().st_mtime_ns))
+        except OSError:
+            # Vanished between glob and stat: skip from the signature; parsing handles it.
+            continue
+    signature: _DiscoverSig = tuple(sig_parts)
+
+    key: _DiscoverKey = (
+        str(project_root.resolve()), directory, design_always, design_priority, okf_priority,
+    )
+    cached = _discover_cache.get(key)
+    if cached is not None and cached[0] == signature:
+        return cached[1]
+
+    sources = [
+        _source_from_file(path, kind, always, priority)
+        for path, kind, always, priority in scanned
+    ]
+    if len(_discover_cache) >= _DISCOVER_CACHE_MAX:
+        _discover_cache.clear()
+    _discover_cache[key] = (signature, sources)
+    return sources
+
+
+def select_knowledge_sources(
+    goal: str = "",
+    project_root: Path | None = None,
+    directory: str = KNOWLEDGE_DIR,
+    design_always: bool = True,
+) -> list[KnowledgeSource]:
+    """Select and rank the knowledge sources that apply to ``goal``.
+
+    Always-on design sources sort first; OKF sources follow, ranked by relevance. Returns
+    an empty list when nothing is ingested.
+    """
+    if project_root is None:
+        return []
+    sources = discover_knowledge_sources(project_root, directory, design_always=design_always)
+    # One goal.lower() + one keyword scan per source (match and rank computed together).
+    goal_lower = goal.lower()
+    scored: list[tuple[KnowledgeSource, int]] = []
+    for source in sources:
+        ok, score = source._match_and_score(goal_lower)
+        if ok:
+            scored.append((source, score))
+    scored.sort(key=lambda item: (not item[0].always, -item[1], item[0].name))
+    return [source for source, _ in scored]
+
+
+def render_knowledge_preamble(
+    sources: list[KnowledgeSource],
+    max_chars: int = 6000,
+    kind: Kind | None = None,
+) -> str:
+    """Concatenate source bodies (optionally filtered to one ``kind``) into a single
+    preamble, bounded to ``max_chars`` total. Sources are taken in the order given (so the
+    most relevant survive the budget); the first source is always included."""
+    separator = "\n\n---\n\n"
+    chosen = [s for s in sources if kind is None or s.kind == kind]
+    blocks: list[str] = []
+    total = 0
+    for source in chosen:
+        block = source.render()
+        if not block:
+            continue
+        # Count the separator that will join this block to the previous one, so the budget
+        # bounds the *rendered* preamble length, not just the sum of block bodies.
+        extra = len(separator) if blocks else 0
+        if blocks and total + extra + len(block) > max_chars:
+            break
+        blocks.append(block)
+        total += extra + len(block)
+    return separator.join(blocks).strip()