devcouncil 0.2.0 → 0.3.0

package/src/devcouncil/reporting/report_builder.py

@@ -1,10 +1,12 @@
+from pathlib import Path
+
 from devcouncil.artifacts.graph import ArtifactGraph
 from devcouncil.reporting.markdown_report import MarkdownReportGenerator
 from devcouncil.reporting.json_report import JsonReportGenerator
 
 class ReportBuilder:
     """Builds reports in various formats from the artifact graph."""
-    
+
     @staticmethod
     def build_markdown(graph: ArtifactGraph, live_review: dict | None = None) -> str:
         return MarkdownReportGenerator.generate(graph, live_review=live_review)
@@ -12,3 +14,18 @@ class ReportBuilder:
     @staticmethod
     def build_json(graph: ArtifactGraph, live_review: dict | None = None) -> str:
         return JsonReportGenerator.generate(graph, live_review=live_review)
+
+    @staticmethod
+    def build_okf_bundle(
+        graph: ArtifactGraph,
+        output_dir: Path,
+        repo_map=None,
+        project_name: str = "DevCouncil Project",
+        timestamp: str = "",
+    ) -> list[Path]:
+        """Export the artifact graph as an Open Knowledge Format bundle on disk."""
+        from devcouncil.reporting.okf_bundle_writer import OKFBundleWriter
+
+        return OKFBundleWriter.generate(
+            graph, output_dir, repo_map=repo_map, project_name=project_name, timestamp=timestamp
+        )

package/src/devcouncil/skills/registry.py

@@ -10,13 +10,15 @@ scaffolded into a target repo's ``.claude/skills/`` directory.
 from __future__ import annotations
 
 import fnmatch
+import functools
 import os
 import re
 from pathlib import Path
 
-import yaml
 from pydantic import BaseModel, Field
 
+from devcouncil.knowledge.frontmatter import build_frontmatter_markdown, split_frontmatter
+
 LIBRARY_DIR = Path(__file__).resolve().parent / "library"
 
 
@@ -90,26 +92,18 @@ class Skill(BaseModel):
 
     def to_skill_md(self) -> str:
         """Render as a Claude-Code-style SKILL.md (name + description frontmatter + body)."""
-        front = yaml.safe_dump(
+        return build_frontmatter_markdown(
             {"name": self.name, "description": self.description},
-            sort_keys=False,
-            default_flow_style=False,
-            allow_unicode=True,
-        ).strip()
-        return f"---\n{front}\n---\n\n{self.body.strip()}\n"
+            self.body,
+        )
 
 
-def _split_frontmatter(text: str) -> tuple[dict, str]:
-    if text.startswith("---"):
-        parts = text.split("---", 2)
-        if len(parts) == 3:
-            meta = yaml.safe_load(parts[1]) or {}
-            return (meta if isinstance(meta, dict) else {}), parts[2].lstrip("\n")
-    return {}, text
+# Frontmatter parsing lives in devcouncil.knowledge.frontmatter so skills and the OKF /
+# design.md formats share one implementation; kept aliased here for existing callers.
+_split_frontmatter = split_frontmatter
 
 
-def _skill_from_file(path: Path) -> Skill:
-    meta, body = _split_frontmatter(path.read_text(encoding="utf-8"))
+def _skill_from_meta(path: Path, meta: dict, body: str) -> Skill:
     triggers = meta.get("triggers") or {}
     return Skill(
         name=str(meta.get("name") or path.stem),
@@ -130,13 +124,19 @@ def _skill_from_file(path: Path) -> Skill:
 REPO_SKILL_DIRS = (".claude/skills", ".devcouncil/skills")
 
 
-def _is_skill_file(path: Path) -> bool:
-    """A markdown file is a skill only if it has frontmatter with a name."""
+def _try_skill_from_file(path: Path) -> Skill | None:
+    """Parse a markdown file into a Skill in a single read, or None if it isn't a skill.
+
+    A markdown file is a skill only if its frontmatter carries a ``name``; plain docs
+    (e.g. a contributor README) are ignored. Reads the file once — previously callers
+    read it twice (an ``_is_skill_file`` check followed by a separate parse)."""
     try:
-        meta, _ = _split_frontmatter(path.read_text(encoding="utf-8"))
+        meta, body = _split_frontmatter(path.read_text(encoding="utf-8"))
     except OSError:
-        return False
-    return bool(meta.get("name"))
+        return None
+    if not meta.get("name"):
+        return None
+    return _skill_from_meta(path, meta, body)
 
 
 def discover_repo_skills(project_root: Path) -> list[Skill]:
@@ -154,16 +154,72 @@ def discover_repo_skills(project_root: Path) -> list[Skill]:
         candidates = sorted(base.rglob("SKILL.md")) + sorted(base.glob("*.md"))
         for path in candidates:
             resolved = path.resolve()
-            if resolved in seen or not _is_skill_file(path):
+            if resolved in seen:
+                continue
+            skill = _try_skill_from_file(path)
+            if skill is None:
                 continue
             seen.add(resolved)
-            found.append(_skill_from_file(path))
+            found.append(skill)
     return found
 
 
-def load_skills(library_dir: Path = LIBRARY_DIR, project_root: Path | None = None) -> list[Skill]:
+def load_okf_skills(project_root: Path, directory: str = ".devcouncil/knowledge") -> list[Skill]:
+    """Load skills from ingested OKF documents typed as engineering skills.
+
+    Reads every ``*.md`` under ``<project_root>/<directory>/okf/`` (recursively), parses
+    each as an :class:`~devcouncil.knowledge.okf.OKFDocument`, and keeps the ones whose
+    ``type`` marks them as a skill (the OKF->Skill conversion returns ``None`` for any
+    other node type — BigQuery tables, tasks, ...). This is how an OKF bundle ingested
+    from another repo contributes its skills to selection alongside the packaged library.
+
+    ``index.md`` files are skipped: a bundle index is navigation scaffolding, not a node.
+    The document's ``rel_path`` is set relative to the okf dir so its skill ``name`` derives
+    from the file stem (matching how the export side names ``skills/<name>.md``).
+    """
+    # Lazy import to avoid a registry <-> skill_bridge import cycle (see that module);
+    # OKFDocument is cycle-safe (knowledge.okf doesn't import skills) but kept here too
+    # to keep the OKF-ingest dependency local to the one function that uses it.
+    from devcouncil.knowledge.okf import OKFDocument
+    from devcouncil.knowledge.skill_bridge import okf_document_to_skill
+
+    okf_dir = project_root / directory / "okf"
+    if not okf_dir.exists():
+        return []
+    skills: list[Skill] = []
+    for path in sorted(okf_dir.rglob("*.md")):
+        if path.name == "index.md":
+            continue
+        rel = path.relative_to(okf_dir).as_posix()
+        doc = OKFDocument.from_markdown(path.read_text(encoding="utf-8"), rel_path=rel)
+        skill = okf_document_to_skill(doc)
+        if skill is not None:
+            skills.append(skill)
+    return skills
+
+
+@functools.lru_cache(maxsize=32)
+def load_skills(
+    library_dir: Path = LIBRARY_DIR,
+    project_root: Path | None = None,
+    include_okf: bool = True,
+) -> list[Skill]:
     """Load skills: the packaged library plus, when ``project_root`` is given, the
-    repo's own skills. Repo-local skills override packaged ones with the same name.
+    repo's own skills and (when ``include_okf``) skills from ingested OKF documents.
+    Repo-local skills override packaged ones with the same name.
+
+    Result is cached per (library_dir, project_root, include_okf) for the lifetime of
+    the process, mirroring the repo-basename cache below: a ``dev e2e``/``repair-all``
+    run calls ``select_skills`` once per task, and re-reading+parsing the whole skill
+    tree (library glob, repo ``SKILL.md`` discovery, OKF markdown) every time is pure
+    waste since the on-disk skills don't change during a run. The returned list is
+    shared and must be treated read-only by callers (all current callers only iterate
+    it); ``clear_skill_caches()`` drops the cache when a refresh is needed.
+
+    OKF-derived skills are merged in last and only for names not already taken, so a
+    packaged library skill or a repo-local skill always wins a name conflict over an
+    ingested bundle node (the local definition is authoritative and carries richer
+    selection metadata like globs that OKF tags can't represent).
 
     Always-on skills come first, then alphabetical. Markdown files without skill
     frontmatter (e.g. a contributor README) are ignored.
@@ -171,8 +227,8 @@ def load_skills(library_dir: Path = LIBRARY_DIR, project_root: Path | None = Non
     by_name: dict[str, Skill] = {}
     if library_dir.exists():
         for path in sorted(library_dir.glob("*.md")):
-            if _is_skill_file(path):
-                skill = _skill_from_file(path)
+            skill = _try_skill_from_file(path)
+            if skill is not None:
                 by_name[skill.name] = skill
     if project_root is not None:
         for skill in discover_repo_skills(project_root):
@@ -190,6 +246,21 @@ def load_skills(library_dir: Path = LIBRARY_DIR, project_root: Path | None = Non
                     "triggers": skill.triggers if has_own_triggers else base.triggers,
                 })
             by_name[skill.name] = skill  # repo-local wins on name conflict
+        if include_okf:
+            # Honor a custom knowledge.directory: `dev okf ingest` and knowledge-source
+            # discovery both write/read under the configured dir, so the skill-ingest read
+            # path must too — otherwise ingested OKF skills land somewhere this never looks
+            # and silently never get selected. Best-effort; lazy import keeps app.config out
+            # of the skills package's module-load graph.
+            knowledge_dir = ".devcouncil/knowledge"
+            try:
+                from devcouncil.app.config import load_config
+                knowledge_dir = load_config(project_root).knowledge.directory
+            except Exception:
+                pass
+            for skill in load_okf_skills(project_root, directory=knowledge_dir):
+                # Only fill gaps: library + repo-local skills win on name conflict.
+                by_name.setdefault(skill.name, skill)
     skills = list(by_name.values())
     skills.sort(key=lambda s: (not s.always, s.name))
     return skills
@@ -211,8 +282,10 @@ _BASENAME_CACHE_MAX = 32
 
 
 def clear_skill_caches() -> None:
-    """Drop the cached repo file scans (useful in long-running processes/tests)."""
+    """Drop the cached repo file scans and loaded-skill sets (useful in long-running
+    processes/tests). Fully resets module-level skill state so test isolation holds."""
     _basename_cache.clear()
+    load_skills.cache_clear()
 
 
 def _walk_repo_basenames(project_root: Path) -> set[str]:
@@ -261,11 +334,16 @@ def select_skills(
     """
     skills = load_skills(library_dir, project_root)
     repo_files = _collect_repo_basenames(project_root) if project_root else set()
-    matched = [s for s in skills if s.matches(goal, repo_files)]
-    # Rank by relevance so the most applicable domain skill survives the inline budget
-    # on a polyglot repo, instead of whichever happened to load first / sort alphabetically.
-    # always-on skills keep their leading position (highest score); ties break by name.
-    scored = [(s, s.relevance_score(goal, repo_files)) for s in matched]
+    # Score each skill once and keep the ones that apply: for a Skill, matches() is exactly
+    # relevance_score() > 0 (always-on -> 1_000_000; otherwise a positive score requires a
+    # keyword/glob hit, which is what matches() tests), so a single pass replaces the old
+    # match-then-score double walk. (This equivalence is Skill-specific — do NOT copy it to
+    # KnowledgeSource, whose nonzero priority floor breaks it.)
+    # Rank by relevance so the most applicable domain skill survives the inline budget on a
+    # polyglot repo; always-on skills keep their leading position; ties break by name.
+    scored = [
+        (s, score) for s in skills if (score := s.relevance_score(goal, repo_files)) > 0
+    ]
     scored.sort(key=lambda item: (not item[0].always, -item[1], item[0].name))
     return [skill for skill, _ in scored]
 

package/src/devcouncil/storage/db.py

@@ -15,7 +15,37 @@ from devcouncil.storage.models import SchemaVersionModel
 # v4: machine-routable gap columns (file, line, suggested_command,
 #     acceptance_criterion_id) so the repair contract survives a reload.
 # v5: partial unique index enforcing one ACTIVE lease per task (single-writer).
-SCHEMA_VERSION = 5
+# v6: gap column expected_verification_method (repair loop tells an executor-remediable
+#     "incomplete" from a manual/llm one across a reload).
+SCHEMA_VERSION = 6
+
+
+# Module-level caches keyed by *resolved* paths so distinct project roots stay
+# independent (important for test isolation, where every test uses a fresh
+# tmp_path). `_db_instances` returns the same Database (and its single engine)
+# for a given project root instead of rebuilding the engine + re-running the
+# schema check on every get_db() call. `_dedup_done` records db paths whose
+# active-lease dedup migration has already run this process, so the one-time
+# table scan does not repeat on subsequent opens of the same database.
+_db_instances: dict[Path, "Database"] = {}
+_dedup_done: set[Path] = set()
+
+
+def reset_db_cache() -> None:
+    """Drop all cached Database instances and per-path guards.
+
+    Disposes pooled engine connections so the underlying SQLite files can be
+    safely removed/recreated. Intended for tests (or long-lived processes) that
+    rebuild a project's .devcouncil database under a path already opened this
+    process; without this a cached engine could point at a stale/deleted file.
+    """
+    for db in _db_instances.values():
+        try:
+            db.engine.dispose()
+        except Exception:
+            pass
+    _db_instances.clear()
+    _dedup_done.clear()
 
 
 class Database:
@@ -62,8 +92,17 @@ class Database:
         # already has two ACTIVE leases. Older databases predate the constraint, so
         # collapse any duplicates first — keep the newest active lease per task, mark the
         # rest stale — making index creation succeed and restoring single-writer state.
+        #
+        # This only needs to run once per database per process: after the first pass the
+        # partial unique index (created immediately after, in _create_missing_indexes)
+        # prevents any new duplicate active leases, so re-scanning on every open is wasted
+        # work. Guard on the resolved path so distinct databases remain independent.
+        key = self.db_path.resolve()
+        if key in _dedup_done:
+            return
         inspector = inspect(self.engine)
         if "task_leases" not in inspector.get_table_names():
+            # Table not materialized yet; don't record as done so a later open retries.
             return
         with self.engine.begin() as conn:
             rows = conn.execute(
@@ -85,6 +124,7 @@ class Database:
                         text("UPDATE task_leases SET status = 'stale', released_at = :ts WHERE id = :id"),
                         {"ts": now, "id": lease_id},
                     )
+        _dedup_done.add(key)
 
     def _create_missing_columns(self):
         # create_all never alters an existing table, so columns added to a model
@@ -139,9 +179,25 @@ class Database:
 def get_db(project_root: Path = Path(".")) -> Optional[Database]:
     dev_dir = project_root / ".devcouncil"
     if not dev_dir.exists():
+        # Checked before the cache so a not-yet-initialized (or removed) project
+        # never yields a stale cached Database.
         return None
-    
+
+    key = project_root.resolve()
     db_path = dev_dir / "state.sqlite"
+    cached = _db_instances.get(key)
+    # Only reuse the cached instance while its underlying file still exists. If the
+    # .devcouncil dir was wiped and recreated under the same path within one process,
+    # the cached engine points at a deleted file — dispose its pool and rebuild fresh.
+    if cached is not None:
+        if db_path.exists():
+            return cached
+        try:
+            cached.engine.dispose()
+        except Exception:
+            pass
+
     db = Database(db_path)
     db.ensure_schema_version()
+    _db_instances[key] = db
     return db

package/src/devcouncil/storage/models.py

@@ -69,6 +69,10 @@ class GapModel(SQLModel, table=True):
     line: Optional[int] = None
     suggested_command: Optional[str] = None
     acceptance_criterion_id: Optional[str] = None
+    # Verification method the criterion expects (unit_test/manual/llm_review/...). Persisted
+    # so the repair loop can tell an executor-remediable "incomplete" (an automatable check
+    # that did not run) from one a human must close (manual/llm_review) after a reload.
+    expected_verification_method: Optional[str] = None
 
 class CritiqueFindingModel(SQLModel, table=True):
     __tablename__ = "critique_findings"

package/src/devcouncil/storage/native.py

@@ -142,16 +142,14 @@ class TaskLeaseRepository:
         ttl_seconds: Optional[int] = None,
         force: bool = False,
     ) -> TaskLeaseRecord:
-        active = self.active_for_task(task_id)
+        active = self._active_model_for_task(task_id)
         if active is not None and not force:
             raise ValueError(f"Active lease already exists for task {task_id}")
         if active is not None and force:
-            existing = self.session.get(TaskLeaseModel, active.id)
-            if existing is not None:
-                existing.status = "stale"
-                existing.released_at = _utc_now()
-                self.session.add(existing)
-                self.session.commit()
+            active.status = "stale"
+            active.released_at = _utc_now()
+            self.session.add(active)
+            self.session.commit()
 
         lease_id = str(uuid.uuid4())
         token = secrets.token_urlsafe(32)
@@ -185,11 +183,8 @@ class TaskLeaseRepository:
         return _lease_from_model(model)
 
     def release(self, task_id: str, lease_token: str, status: str = "released") -> bool:
-        active = self.active_for_task(task_id)
-        if active is None or active.lease_token != lease_token:
-            return False
-        model = self.session.get(TaskLeaseModel, active.id)
-        if model is None:
+        model = self._active_model_for_task(task_id)
+        if model is None or model.lease_token != lease_token:
             return False
         model.status = status
         model.released_at = _utc_now()
@@ -197,7 +192,11 @@ class TaskLeaseRepository:
         self.session.commit()
         return True
 
-    def active_for_task(self, task_id: str) -> TaskLeaseRecord | None:
+    def _active_model_for_task(self, task_id: str) -> Optional[TaskLeaseModel]:
+        """Return the live ACTIVE lease *model* for a task (or None), lazily expiring a
+        lease that is past its TTL. Callers that need a Record wrap the result via
+        _lease_from_model — this avoids re-fetching the same row by primary key after a
+        Record-returning lookup."""
         statement = (
             select(TaskLeaseModel)
             .where(TaskLeaseModel.task_id == task_id)
@@ -216,6 +215,12 @@ class TaskLeaseRepository:
                 self.session.add(model)
                 self.session.commit()
                 return None
+        return model
+
+    def active_for_task(self, task_id: str) -> TaskLeaseRecord | None:
+        model = self._active_model_for_task(task_id)
+        if model is None:
+            return None
         return _lease_from_model(model)
 
     def validate(self, task_id: str, lease_token: str) -> bool:
@@ -225,11 +230,8 @@ class TaskLeaseRepository:
     def renew(self, task_id: str, lease_token: str, ttl_seconds: int) -> TaskLeaseRecord | None:
         """Push the lease's expiry out by ``ttl_seconds`` from now. Returns the updated
         record, or None when the token is invalid / the lease already expired."""
-        active = self.active_for_task(task_id)
-        if active is None or active.lease_token != lease_token:
-            return None
-        model = self.session.get(TaskLeaseModel, active.id)
-        if model is None:
+        model = self._active_model_for_task(task_id)
+        if model is None or model.lease_token != lease_token:
             return None
         model.expires_at = (datetime.now(timezone.utc) + timedelta(seconds=ttl_seconds)).isoformat()
         self.session.add(model)

package/src/devcouncil/storage/repositories.py

@@ -150,27 +150,43 @@ class GapRepository:
     def __init__(self, session: Session):
         self.session = session
 
+    @staticmethod
+    def _to_gap(m: GapModel) -> Gap:
+        return Gap.model_validate({
+            "id": m.id,
+            "severity": m.severity,
+            "gap_type": m.gap_type,
+            "requirement_id": m.requirement_id,
+            "task_id": m.task_id,
+            "description": m.description,
+            "evidence": json.loads(m.evidence_json),
+            "recommended_fix": m.recommended_fix,
+            "blocking": m.blocking,
+            "file": m.file,
+            "line": m.line,
+            "suggested_command": m.suggested_command,
+            "acceptance_criterion_id": m.acceptance_criterion_id,
+            "expected_verification_method": m.expected_verification_method,
+        })
+
     def get_all(self) -> List[Gap]:
         statement = select(GapModel)
         models = self.session.exec(statement).all()
-        results = []
-        for m in models:
-            results.append(Gap.model_validate({
-                "id": m.id,
-                "severity": m.severity,
-                "gap_type": m.gap_type,
-                "requirement_id": m.requirement_id,
-                "task_id": m.task_id,
-                "description": m.description,
-                "evidence": json.loads(m.evidence_json),
-                "recommended_fix": m.recommended_fix,
-                "blocking": m.blocking,
-                "file": m.file,
-                "line": m.line,
-                "suggested_command": m.suggested_command,
-                "acceptance_criterion_id": m.acceptance_criterion_id,
-            }))
-        return results
+        return [self._to_gap(m) for m in models]
+
+    def get_for_task(self, task_id: str) -> List[Gap]:
+        statement = select(GapModel).where(col(GapModel.task_id) == task_id)
+        models = self.session.exec(statement).all()
+        return [self._to_gap(m) for m in models]
+
+    def get_blocking_for_task(self, task_id: str) -> List[Gap]:
+        statement = (
+            select(GapModel)
+            .where(col(GapModel.task_id) == task_id)
+            .where(col(GapModel.blocking) == True)  # noqa: E712 - SQL boolean comparison
+        )
+        models = self.session.exec(statement).all()
+        return [self._to_gap(m) for m in models]
 
     def save(self, gap: Gap):
         model = GapModel(
@@ -187,6 +203,7 @@ class GapRepository:
             line=gap.line,
             suggested_command=gap.suggested_command,
             acceptance_criterion_id=gap.acceptance_criterion_id,
+            expected_verification_method=gap.expected_verification_method,
         )
         self.session.merge(model)
         self.session.commit()