devcouncil 0.3.1 → 0.4.1

package/src/devcouncil/verification/verify_setup.py

@@ -0,0 +1,176 @@
+"""Verify-task setup and teardown helpers extracted from Verifier."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from dataclasses import dataclass, field
+from typing import Any, List, Optional, TYPE_CHECKING
+
+from devcouncil.app.config import load_config
+from devcouncil.domain.requirement import Requirement
+from devcouncil.domain.task import Task
+from devcouncil.telemetry.stages import log_step
+from devcouncil.utils.git_snapshot import GitWorktreeSnapshot
+from devcouncil.verification.difficulty import resolve_rigor_policy
+
+if TYPE_CHECKING:
+    from devcouncil.verification.verifier import Verifier
+
+logger = logging.getLogger(__name__)
+
+_AD_HOC_TASK_ID = "CHECK"
+
+
+@dataclass
+class VerifyRunContext:
+    """Configuration and diff state primed at the start of verify_task."""
+
+    config: Any | None
+    rigor: Any
+    ac_samples: int = 1
+    ac_repair_attempts: int = 1
+    ac_per_criterion: bool = False
+    command_timeout: int = 300
+    changed_files: List[str] = field(default_factory=list)
+    diff_content: str = ""
+    diff_empty: bool = True
+
+
+def prime_verify_memos(verifier: Verifier) -> None:
+    """Batch git plumbing and prime per-call caches."""
+    snapshot = GitWorktreeSnapshot.capture(verifier.project_root)
+    verifier._git_fallback.git_snapshot = snapshot
+    verifier._git_fallback.untracked_cache = snapshot.untracked_files
+
+
+def resolve_verify_context(
+    verifier: Verifier,
+    task: Task,
+    requirements: List[Requirement],
+) -> VerifyRunContext:
+    """Load config, rigor policy, and diff snapshot for a verify run."""
+    ac_samples, ac_repair_attempts = 1, 1
+    ac_per_criterion = False
+    cfg = None
+    command_timeout = 300
+    try:
+        from devcouncil.app.config import role_runs_on_local_provider
+
+        cfg = load_config(verifier.project_root)
+        command_timeout = cfg.execution.command_timeout
+        local_monitor = role_runs_on_local_provider(cfg, "implementation_reviewer")
+        ac_samples, ac_repair_attempts, ac_per_criterion = (
+            cfg.verification.acceptance_checks.resolved(local_monitor)
+        )
+        # Explicit overrides are honored but never silently: single-shot acceptance
+        # checks on a local monitor rubber-stamped real defects in calibration probes.
+        # Once per process — verify runs per task, and 20 identical lines are spam.
+        from devcouncil.telemetry.logging_setup import warn_once
+
+        for warning in cfg.verification.acceptance_checks.unsafe_override_warnings(local_monitor):
+            warn_once(logger, warning)
+    except Exception as exc:
+        # Without config we cannot detect a local monitor, so this falls back to
+        # single-shot cloud defaults — the exact mode that is unsafe if the monitor
+        # IS local. Say so instead of degrading silently.
+        from devcouncil.telemetry.logging_setup import warn_once
+
+        warn_once(
+            logger,
+            f"Verification config unavailable ({exc}); acceptance checks fall back to "
+            "single-shot defaults (samples=1). If the monitor runs on a local model, "
+            "fix the config load — single-shot local checks miss real defects.",
+        )
+        cfg = None
+    rigor = resolve_rigor_policy(task, requirements, config=cfg)
+    if rigor.min_acceptance_samples > 1:
+        ac_samples = max(ac_samples, rigor.min_acceptance_samples)
+    log_step(
+        "verify/1: resolved rigor policy",
+        project_root=verifier.project_root,
+        task_id=task.id,
+        difficulty=rigor.difficulty,
+        rigor_applied=list(rigor.applied),
+    )
+    verifier._command_timeout_cache = command_timeout
+    is_ad_hoc_check = task.id == _AD_HOC_TASK_ID
+    # CHECK represents the live working tree, including edits recorded in the global
+    # baseline. Baseline/checkpoint subtraction only applies to persisted workflow tasks.
+    changed_files = (
+        verifier.get_changed_files()
+        if is_ad_hoc_check
+        else verifier.get_task_changed_files(task.id)
+    )
+    diff_content = verifier.get_diff()
+    if not is_ad_hoc_check:
+        committed_diff = verifier._committed_task_diff(task.id)
+        if committed_diff.strip():
+            diff_content = committed_diff
+    diff_empty = not bool(diff_content.strip())
+    log_step(
+        "verify/2: collected diff and changed files",
+        project_root=verifier.project_root,
+        task_id=task.id,
+        changed_files=len(changed_files),
+        diff_empty=diff_empty,
+    )
+    return VerifyRunContext(
+        config=cfg,
+        rigor=rigor,
+        ac_samples=ac_samples,
+        ac_repair_attempts=ac_repair_attempts,
+        ac_per_criterion=ac_per_criterion,
+        command_timeout=command_timeout,
+        changed_files=changed_files,
+        diff_content=diff_content,
+        diff_empty=diff_empty,
+    )
+
+
+def start_verify_futures(
+    verifier: Verifier,
+    *,
+    task: Task,
+    requirements: List[Requirement],
+    diff_content: str,
+    ac_samples: int,
+    ac_per_criterion: bool,
+) -> tuple[Optional[asyncio.Task[Any]], Optional[asyncio.Task[Any]]]:
+    """Launch acceptance-compiler and implementation-reviewer LLM passes."""
+    compile_future: Optional[asyncio.Task[Any]] = None
+    if verifier.acceptance_compiler and diff_content and task.acceptance_criterion_ids:
+        if hasattr(verifier.acceptance_compiler, "compile_candidates"):
+            compile_coro = verifier.acceptance_compiler.compile_candidates(
+                task, requirements, diff_content, samples=ac_samples,
+                per_criterion=ac_per_criterion,
+            )
+        else:
+            compile_coro = verifier.acceptance_compiler.compile(task, requirements, diff_content)
+        compile_future = asyncio.create_task(compile_coro)
+    review_future: Optional[asyncio.Task[Any]] = None
+    if verifier.reviewer and diff_content:
+        review_future = asyncio.create_task(
+            verifier.reviewer.review_changes(task, requirements, diff_content)
+        )
+    return compile_future, review_future
+
+
+async def cleanup_verify_futures(
+    verifier: Verifier,
+    compile_future: Optional[asyncio.Task[Any]],
+    review_future: Optional[asyncio.Task[Any]],
+) -> None:
+    """Drain background LLM tasks and clear per-call memos."""
+    for fut in (compile_future, review_future):
+        if fut is not None:
+            if not fut.done():
+                fut.cancel()
+            try:
+                await fut
+            except (asyncio.CancelledError, Exception):
+                pass
+    verifier._git_fallback.git_snapshot = None
+    verifier._git_fallback.untracked_cache = None
+    verifier._command_timeout_cache = None
+    verifier._project_deps_cache = None

package/src/devcouncil/verification/wiki_refresh.py

@@ -0,0 +1,208 @@
+"""Post-verify wiki-freshness trigger for large refactors.
+
+The agent-facing codebase wiki (``dev wiki``) drifts as the code changes. A big
+refactor — one that spans several subsystems or touches many files — is exactly the kind
+of change that leaves the wiki stale. This module decides, after a verify run, whether a
+change was "large" and either FLAGS the pages that a refresh would rewrite (cheap, the
+default) or actually triggers ``dev wiki update --no-llm`` as a post-step
+(``verification.wiki_refresh.auto_update``).
+
+Deliberately best-effort and non-blocking: it is an orientation aid, never a gate. Any
+failure (no map, no wiki, config error) degrades to "not considered" and is logged, not
+raised. Flagging is free of model calls; auto-update runs the deterministic skeleton.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TYPE_CHECKING, List, Mapping, Optional, Sequence
+
+from devcouncil.indexing.subsystem_map import areas_touched
+
+if TYPE_CHECKING:
+    from devcouncil.domain.gap import Gap
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class WikiRefreshOutcome:
+    considered: bool = False       # was the change large enough to act on?
+    triggered: bool = False        # did we actually run an update?
+    subsystems_touched: int = 0
+    files_touched: int = 0
+    stale_pages: List[str] = field(default_factory=list)
+    reason: str = ""
+
+    def to_dict(self) -> dict:
+        return {
+            "considered": self.considered,
+            "triggered": self.triggered,
+            "subsystems_touched": self.subsystems_touched,
+            "files_touched": self.files_touched,
+            "stale_pages": list(self.stale_pages),
+            "reason": self.reason,
+        }
+
+
+def _resolve_config(project_root: Path, config):
+    if config is not None:
+        return config
+    try:
+        from devcouncil.app.config import load_config
+
+        return load_config(project_root).verification.wiki_refresh
+    except Exception as exc:
+        logger.debug("wiki refresh: config unavailable: %s", exc)
+        return None
+
+
+def is_large_change(
+    *, files_touched: int, subsystems_touched: int, min_files: int, min_subsystems: int
+) -> bool:
+    """A change is "large" when it spans many subsystems OR touches many files."""
+    return subsystems_touched >= min_subsystems or files_touched >= min_files
+
+
+def _stale_pages(project_root: Path) -> List[str]:
+    """Pages a refresh would rewrite, or ``[]`` when the wiki/map is unavailable."""
+    try:
+        from devcouncil.indexing.repo_mapper import RepoMap
+        from devcouncil.knowledge.wiki import wiki_dir_for, wiki_stale_pages
+
+        map_path = project_root / ".devcouncil" / "repo_map.json"
+        wiki_dir = wiki_dir_for(project_root)
+        if not map_path.is_file() or not (wiki_dir / "index.md").is_file():
+            return []
+        repo_map = RepoMap.model_validate_json(map_path.read_text(encoding="utf-8"))
+        return sorted(wiki_stale_pages(project_root, repo_map, wiki_dir).keys())
+    except Exception as exc:
+        logger.debug("wiki refresh: stale-page detection failed: %s", exc)
+        return []
+
+
+def _run_update(project_root: Path) -> bool:
+    """Run the deterministic wiki refresh (no model calls). True on success."""
+    try:
+        from devcouncil.knowledge.wiki import refresh_wiki
+
+        refresh_wiki(project_root, llm=False, force=False, remap=False)
+        return True
+    except Exception as exc:
+        logger.warning("wiki refresh: auto-update failed: %s", exc)
+        return False
+
+
+def wiki_refresh_advisory_gap(
+    outcome: WikiRefreshOutcome,
+    *,
+    task_id: str,
+    gap_id: str,
+) -> "Gap | None":
+    """Return a non-blocking advisory gap when stale wiki pages were flagged."""
+    if not (outcome.considered and not outcome.triggered and outcome.stale_pages):
+        return None
+    from devcouncil.domain.gap import Gap
+
+    preview = ", ".join(outcome.stale_pages[:5])
+    if len(outcome.stale_pages) > 5:
+        preview = f"{preview} (+{len(outcome.stale_pages) - 5} more)"
+    return Gap(
+        id=gap_id,
+        severity="low",
+        gap_type="architecture_drift",
+        task_id=task_id,
+        description=(
+            f"Codebase wiki may be stale after this large change: "
+            f"{len(outcome.stale_pages)} page(s) need refresh ({preview})."
+        ),
+        evidence=list(outcome.stale_pages[:10]),
+        recommended_fix="Run `dev wiki update` to refresh agent-facing documentation.",
+        blocking=False,
+    )
+
+
+def evaluate_wiki_refresh(
+    project_root: Path,
+    changed_files: Sequence[str],
+    *,
+    repo_map: Optional[Mapping] = None,
+    config=None,
+) -> WikiRefreshOutcome:
+    """Decide whether the change warrants a wiki refresh and act per config.
+
+    Returns a :class:`WikiRefreshOutcome`. When the change is large: with
+    ``auto_update`` it runs ``dev wiki update --no-llm`` and reports ``triggered``;
+    otherwise it reports the ``stale_pages`` a manual ``dev wiki update`` would rewrite.
+    """
+    root = Path(project_root)
+    changed = [p for p in changed_files if p and p.strip()]
+    cfg = _resolve_config(root, config)
+    if cfg is not None and not getattr(cfg, "enabled", True):
+        return WikiRefreshOutcome(reason="wiki refresh disabled", files_touched=len(changed))
+
+    min_files = int(getattr(cfg, "min_files", 8)) if cfg is not None else 8
+    min_subsystems = int(getattr(cfg, "min_subsystems", 3)) if cfg is not None else 3
+    auto_update = bool(getattr(cfg, "auto_update", False)) if cfg is not None else False
+
+    if repo_map is None:
+        try:
+            from devcouncil.utils.json_persist import read_json
+
+            map_path = root / ".devcouncil" / "repo_map.json"
+            repo_map = read_json(map_path) if map_path.is_file() else None
+        except Exception:
+            repo_map = None
+
+    areas = areas_touched(changed, repo_map) if repo_map else []
+    files_touched = len(changed)
+    subsystems_touched = len(areas)
+
+    if not is_large_change(
+        files_touched=files_touched,
+        subsystems_touched=subsystems_touched,
+        min_files=min_files,
+        min_subsystems=min_subsystems,
+    ):
+        return WikiRefreshOutcome(
+            considered=False,
+            files_touched=files_touched,
+            subsystems_touched=subsystems_touched,
+            reason=(
+                f"change below threshold ({files_touched} file(s), "
+                f"{subsystems_touched} subsystem(s); need >={min_files} files "
+                f"or >={min_subsystems} subsystems)"
+            ),
+        )
+
+    outcome = WikiRefreshOutcome(
+        considered=True,
+        files_touched=files_touched,
+        subsystems_touched=subsystems_touched,
+    )
+    if auto_update:
+        outcome.triggered = _run_update(root)
+        outcome.reason = (
+            "large change — ran `dev wiki update --no-llm`"
+            if outcome.triggered else "large change — auto-update attempted but failed"
+        )
+        logger.info(
+            "wiki refresh: large change (%d files / %d subsystems) — auto-update %s",
+            files_touched, subsystems_touched, "ok" if outcome.triggered else "failed",
+        )
+    else:
+        outcome.stale_pages = _stale_pages(root)
+        outcome.reason = (
+            f"large change — {len(outcome.stale_pages)} stale wiki page(s); "
+            "run `dev wiki update`"
+            if outcome.stale_pages
+            else "large change — wiki may be stale; run `dev wiki update`"
+        )
+        logger.info(
+            "wiki refresh: large change (%d files / %d subsystems) — %d stale page(s) "
+            "flagged (run `dev wiki update`)",
+            files_touched, subsystems_touched, len(outcome.stale_pages),
+        )
+    return outcome

package/src/semantic_layer/__init__.py

@@ -0,0 +1,58 @@
+"""Semantic layer for local LLM pipelines — cache, routing, and RAG compression."""
+
+from __future__ import annotations
+
+from .benchmark import benchmark_cache_path, run_benchmark
+from .cache import CacheEntry, CacheLookupResult, SemanticCache
+from .compressor import CompressedContext, SemanticCompressor
+from .config import (
+    CacheConfig,
+    CompressorConfig,
+    EmbeddingConfig,
+    LLMConfig,
+    ModelTier,
+    RouterConfig,
+    SemanticLayerConfig,
+)
+from .embeddings import EmbeddingService
+from .llm_backends import (
+    HuggingFaceBackend,
+    LLMBackend,
+    LlamaCppBackend,
+    OllamaBackend,
+    create_backend,
+)
+from .pipeline import PipelineResponse, SemanticPipeline
+from .router import RouteDecision, SemanticRouter
+from .tuner import LookupEvent, ThresholdAutoTuner
+
+__all__ = [
+    "CacheConfig",
+    "CacheEntry",
+    "CacheLookupResult",
+    "CompressedContext",
+    "CompressorConfig",
+    "EmbeddingConfig",
+    "EmbeddingService",
+    "HuggingFaceBackend",
+    "LLMBackend",
+    "LLMConfig",
+    "LlamaCppBackend",
+    "LookupEvent",
+    "ModelTier",
+    "OllamaBackend",
+    "PipelineResponse",
+    "RouteDecision",
+    "RouterConfig",
+    "SemanticCache",
+    "SemanticCompressor",
+    "SemanticLayerConfig",
+    "SemanticPipeline",
+    "SemanticRouter",
+    "ThresholdAutoTuner",
+    "benchmark_cache_path",
+    "create_backend",
+    "run_benchmark",
+]
+
+__version__ = "1.0.0"

package/src/semantic_layer/benchmark.py

@@ -0,0 +1,75 @@
+"""Benchmark harness for semantic layer latency and cache hit rate."""
+
+from __future__ import annotations
+
+import statistics
+import time
+from typing import Sequence
+
+from .config import SemanticLayerConfig
+from .pipeline import SemanticPipeline
+
+
+def benchmark_cache_path(
+    pipeline: SemanticPipeline,
+    queries: Sequence[str],
+    warmup: int = 5,
+) -> dict[str, float]:
+    """Measure semantic-layer-only latency (no LLM) for cache hits."""
+    from .embeddings import EmbeddingService
+
+    embedder = EmbeddingService.get_instance()
+    latencies: list[float] = []
+
+    for q in list(queries)[:warmup]:
+        embedder.embed_one(q)
+
+    for q in queries:
+        t0 = time.perf_counter()
+        vec = embedder.embed_one(q)
+        pipeline.cache.lookup(q, vec)
+        latencies.append((time.perf_counter() - t0) * 1000)
+
+    if not latencies:
+        return {
+            "p50_ms": 0.0,
+            "p95_ms": 0.0,
+            "p99_ms": 0.0,
+            "mean_ms": 0.0,
+            "hit_rate": pipeline.cache.hit_rate,
+        }
+
+    latencies.sort()
+    last = len(latencies) - 1
+    return {
+        "p50_ms": latencies[len(latencies) // 2],
+        "p95_ms": latencies[int(last * 0.95)],
+        "p99_ms": latencies[int(last * 0.99)],
+        "mean_ms": statistics.mean(latencies),
+        "hit_rate": pipeline.cache.hit_rate,
+    }
+
+
+def run_benchmark(queries: list[str], documents: list[str] | None = None) -> None:
+    config = SemanticLayerConfig()
+    pipeline = SemanticPipeline(config)
+
+    # Prime cache
+    for q in queries[:10]:
+        pipeline.run(q, documents)
+
+    stats = benchmark_cache_path(pipeline, queries)
+    print("Semantic layer cache-path benchmark:")
+    for k, v in stats.items():
+        print(f"  {k}: {v:.2f}" if isinstance(v, float) else f"  {k}: {v}")
+
+
+if __name__ == "__main__":
+    sample_queries = [
+        "What is Python?",
+        "Explain Python programming language",
+        "Define Python language",
+        "How do I write a for loop in Python?",
+        "Implement a distributed cache with consistent hashing",
+    ] * 20
+    run_benchmark(sample_queries)