@deftai/directive-content 0.55.2 → 0.56.0

package/scripts/codebase_default_extractor.py

@@ -0,0 +1,495 @@
+#!/usr/bin/env python3
+"""Dependency-free default extractor for #1595 codebase-map artifacts.
+
+The default path is intentionally modest: it walks the repository, applies the
+authored ``codeStructure`` module globs when available, and uses import-line
+heuristics for coarse coupling. It does not parse ASTs, call the network, or
+ask a model to summarize code. Richer providers can replace this artifact via
+``scripts/codebase_provider.py`` once they satisfy the provider contract.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import re
+import sys
+from collections import Counter, defaultdict
+from contextlib import suppress
+from pathlib import Path
+from typing import Any
+
+import code_structure_validate
+from codebase_projection_registry import (
+    CODEBASE_MAP_FORMAT_VERSION,
+    CODEBASE_MAP_KIND,
+    CODEBASE_PROVIDER_CONTRACT_VERSION,
+)
+
+DEFAULT_PROVIDER_NAME = "directive-default-extractor"
+DEFAULT_PROVIDER_VERSION = "0.1"
+MAX_IMPORT_SCAN_BYTES = 262_144
+MAX_FILES_PER_MODULE = 100
+MAX_EVIDENCE_PER_EDGE = 5
+
+SKIP_DIRS = {
+    ".git",
+    ".mypy_cache",
+    ".pytest_cache",
+    ".ruff_cache",
+    ".venv",
+    "__pycache__",
+    "build",
+    "dist",
+    "node_modules",
+    "swarm-worktrees",
+}
+
+LANGUAGE_BY_SUFFIX = {
+    ".go": "Go",
+    ".js": "JavaScript",
+    ".jsx": "JavaScript",
+    ".json": "JSON",
+    ".md": "Markdown",
+    ".py": "Python",
+    ".sh": "Shell",
+    ".ts": "TypeScript",
+    ".tsx": "TypeScript",
+    ".yaml": "YAML",
+    ".yml": "YAML",
+}
+
+ENTRYPOINT_NAMES = {
+    "__main__.py",
+    "cli.py",
+    "cmd.py",
+    "index.js",
+    "index.ts",
+    "main.go",
+    "main.py",
+    "run",
+    "run.py",
+}
+
+IMPORT_PATTERNS = [
+    re.compile(r"^\s*import\s+([A-Za-z_][\w.]*)"),
+    re.compile(r"^\s*from\s+([A-Za-z_][\w.]*)\s+import\s+"),
+    re.compile(r"^\s*import\s+.*?\s+from\s+[\"']([^\"']+)[\"']"),
+    re.compile(r"^\s*(?:const|let|var)\s+.*?=\s*require\([\"']([^\"']+)[\"']\)"),
+    re.compile(r"^\s*import\s+[\"']([^\"']+)[\"']"),
+]
+
+
+def _posix(path: Path) -> str:
+    return path.as_posix()
+
+
+def _relative_file(path: Path, project_root: Path) -> str:
+    return _posix(path.relative_to(project_root))
+
+
+def file_sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def _source_content_hash(project_root: Path, rel_paths: list[str]) -> dict[str, Any]:
+    digest = hashlib.sha256()
+    file_count = 0
+    for rel_path in sorted(set(rel_paths)):
+        path = project_root / rel_path
+        if not path.is_file():
+            continue
+        file_count += 1
+        digest.update(rel_path.encode("utf-8"))
+        digest.update(b"\0")
+        digest.update(file_sha256(path).encode("ascii"))
+        digest.update(b"\0")
+    return {
+        "algorithm": "sha256",
+        "scope": "codeStructure-and-module-files",
+        "value": digest.hexdigest(),
+        "fileCount": file_count,
+    }
+
+
+def default_code_structure_path(project_root: Path, code_structure_path: Path | None) -> Path:
+    """Return the authored codeStructure source path used by the default extractor."""
+    return code_structure_path or project_root / "vbrief" / "PROJECT-DEFINITION.vbrief.json"
+
+
+def _stable_id(value: str) -> str:
+    slug = re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-")
+    return slug or "root"
+
+
+def _repo_files(project_root: Path) -> list[Path]:
+    files: list[Path] = []
+    for dirpath, dirnames, filenames in os.walk(project_root):
+        dirnames[:] = [dirname for dirname in dirnames if dirname not in SKIP_DIRS]
+        for filename in filenames:
+            files.append(Path(dirpath) / filename)
+    return sorted(files, key=lambda item: _relative_file(item, project_root))
+
+
+def _glob_files(project_root: Path, globs: list[str]) -> list[Path]:
+    files: dict[str, Path] = {}
+    for glob_value in globs:
+        for match in project_root.glob(glob_value):
+            if not match.is_file():
+                continue
+            rel_parts = match.relative_to(project_root).parts
+            if any(part in SKIP_DIRS for part in rel_parts):
+                continue
+            files[_relative_file(match, project_root)] = match
+    return [files[key] for key in sorted(files)]
+
+
+def _load_authored_code_structure(
+    project_root: Path, code_structure_path: Path | None
+) -> tuple[dict[str, Any] | None, str | None]:
+    path = default_code_structure_path(project_root, code_structure_path)
+    if not path.exists():
+        return None, None
+    data = code_structure_validate.load_json_file(path)
+    extracted = code_structure_validate.extract_code_structure(data)
+    if extracted is None:
+        return None, None
+    return extracted.record, extracted.home
+
+
+def _module_prefixes(module: dict[str, Any], files: list[Path], project_root: Path) -> set[str]:
+    prefixes: set[str] = set()
+    for glob_value in module.get("pathGlobs", []):
+        if not isinstance(glob_value, str):
+            continue
+        first = glob_value.split("/", maxsplit=1)[0]
+        if (
+            first
+            and not code_structure_validate._has_glob_magic(first)
+            and first not in {".", "**"}
+        ):
+            prefixes.add(first.removesuffix(".py"))
+    for file_path in files:
+        rel_parts = file_path.relative_to(project_root).parts
+        if len(rel_parts) > 1:
+            prefixes.add(rel_parts[0])
+        elif file_path.suffix == ".py":
+            prefixes.add(file_path.stem)
+    return {prefix for prefix in prefixes if prefix}
+
+
+def _curated_modules(
+    project_root: Path, code_structure: dict[str, Any]
+) -> tuple[list[dict[str, Any]], dict[str, str], dict[str, set[str]], list[dict[str, str]]]:
+    artifacts: list[dict[str, Any]] = []
+    file_to_module: dict[str, str] = {}
+    prefixes_by_module: dict[str, set[str]] = {}
+    degraded: list[dict[str, str]] = []
+
+    for raw_module in code_structure.get("modules", []):
+        if not isinstance(raw_module, dict):
+            continue
+        module_id = str(raw_module.get("id", "unknown"))
+        globs = [value for value in raw_module.get("pathGlobs", []) if isinstance(value, str)]
+        files = _glob_files(project_root, globs)
+        rel_files = [_relative_file(path, project_root) for path in files]
+        for rel_path in rel_files:
+            file_to_module.setdefault(rel_path, module_id)
+        if len(rel_files) > MAX_FILES_PER_MODULE:
+            degraded.append(
+                {
+                    "code": "MODULE-FILES-TRUNCATED",
+                    "module": module_id,
+                    "message": (
+                        f"Module file list was truncated to {MAX_FILES_PER_MODULE} "
+                        "deterministic entries."
+                    ),
+                }
+            )
+        artifacts.append(
+            {
+                "id": module_id,
+                "name": raw_module.get("name"),
+                "purpose": raw_module.get("purpose"),
+                "pathGlobs": globs,
+                "fileCount": len(rel_files),
+                "files": rel_files[:MAX_FILES_PER_MODULE],
+                "derivedFrom": {
+                    "intent": "codeStructure.modules[]",
+                    "files": "repository-glob-walk",
+                },
+            }
+        )
+        prefixes_by_module[module_id] = _module_prefixes(raw_module, files, project_root)
+    return artifacts, file_to_module, prefixes_by_module, degraded
+
+
+def _directory_modules(
+    project_root: Path,
+) -> tuple[list[dict[str, Any]], dict[str, str], dict[str, set[str]], list[dict[str, str]]]:
+    grouped: dict[str, list[Path]] = defaultdict(list)
+    for file_path in _repo_files(project_root):
+        parts = file_path.relative_to(project_root).parts
+        if not parts:
+            continue
+        top = parts[0] if len(parts) > 1 else "root-files"
+        grouped[top].append(file_path)
+
+    modules: list[dict[str, Any]] = []
+    file_to_module: dict[str, str] = {}
+    prefixes_by_module: dict[str, set[str]] = {}
+    degraded_markers: list[dict[str, str]] = [
+        {
+            "code": "NO-CODESTRUCTURE",
+            "message": (
+                "No authored codeStructure metadata was found; modules were derived from "
+                "top-level repository paths."
+            ),
+        }
+    ]
+    for top in sorted(grouped):
+        module_id = _stable_id(top)
+        rel_files = [_relative_file(path, project_root) for path in sorted(grouped[top])]
+        for rel_path in rel_files:
+            file_to_module.setdefault(rel_path, module_id)
+        if len(rel_files) > MAX_FILES_PER_MODULE:
+            degraded_markers.append(
+                {
+                    "code": "MODULE-FILES-TRUNCATED",
+                    "module": module_id,
+                    "message": (
+                        f"Module file list was truncated to {MAX_FILES_PER_MODULE} "
+                        "deterministic entries."
+                    ),
+                }
+            )
+        modules.append(
+            {
+                "id": module_id,
+                "name": top,
+                "purpose": None,
+                "pathGlobs": [f"{top}/**/*" if top != "root-files" else "*"],
+                "fileCount": len(rel_files),
+                "files": rel_files[:MAX_FILES_PER_MODULE],
+                "derivedFrom": {
+                    "intent": "directory-derived-fallback",
+                    "files": "repository-tree-walk",
+                },
+            }
+        )
+        prefixes_by_module[module_id] = {top} if top != "root-files" else set()
+
+    return modules, file_to_module, prefixes_by_module, degraded_markers
+
+
+def _read_imports(path: Path) -> list[tuple[int, str]]:
+    if path.suffix not in {".go", ".js", ".jsx", ".py", ".ts", ".tsx"}:
+        return []
+    try:
+        if path.stat().st_size > MAX_IMPORT_SCAN_BYTES:
+            return []
+        lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
+    except OSError:
+        return []
+
+    imports: list[tuple[int, str]] = []
+    in_go_import_block = False
+    for line_number, line in enumerate(lines, start=1):
+        stripped = line.strip()
+        if path.suffix == ".go":
+            if stripped == "import (":
+                in_go_import_block = True
+                continue
+            if in_go_import_block and stripped == ")":
+                in_go_import_block = False
+                continue
+            if in_go_import_block:
+                match = re.search(r'"([^"]+)"', stripped)
+                if match:
+                    imports.append((line_number, match.group(1)))
+                continue
+            match = re.match(r'^\s*import\s+"([^"]+)"', line)
+            if match:
+                imports.append((line_number, match.group(1)))
+            continue
+
+        for pattern in IMPORT_PATTERNS:
+            match = pattern.match(line)
+            if match:
+                imports.append((line_number, match.group(1)))
+                break
+    return imports
+
+
+def _import_targets(ref: str, prefixes_by_module: dict[str, set[str]]) -> set[str]:
+    if ref.startswith("."):
+        return set()
+    normalized = ref.removeprefix("@")
+    first_segment = re.split(r"[/.]", normalized, maxsplit=1)[0]
+    targets: set[str] = set()
+    for module_id, prefixes in prefixes_by_module.items():
+        if first_segment in prefixes or ref in prefixes:
+            targets.add(module_id)
+    return targets
+
+
+def _coupling_edges(
+    project_root: Path, file_to_module: dict[str, str], prefixes_by_module: dict[str, set[str]]
+) -> list[dict[str, Any]]:
+    edges: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list)
+    for rel_path, source_module in sorted(file_to_module.items()):
+        path = project_root / rel_path
+        for line_number, import_ref in _read_imports(path):
+            for target_module in _import_targets(import_ref, prefixes_by_module):
+                if target_module == source_module:
+                    continue
+                evidence = edges[(source_module, target_module)]
+                if len(evidence) < MAX_EVIDENCE_PER_EDGE:
+                    evidence.append(
+                        {"path": rel_path, "line": line_number, "import": import_ref}
+                    )
+
+    return [
+        {
+            "from": source,
+            "to": target,
+            "derivedFrom": "import-line-heuristic",
+            "confidence": "heuristic",
+            "evidence": evidence,
+        }
+        for (source, target), evidence in sorted(edges.items())
+    ]
+
+
+def _entry_points(file_to_module: dict[str, str]) -> list[dict[str, str]]:
+    entries: list[dict[str, str]] = []
+    for rel_path, module_id in sorted(file_to_module.items()):
+        path = Path(rel_path)
+        if path.name in ENTRYPOINT_NAMES or path.parts[:1] == ("cmd",):
+            entries.append(
+                {
+                    "path": rel_path,
+                    "module": module_id,
+                    "derivedFrom": "filename-heuristic",
+                    "confidence": "heuristic",
+                }
+            )
+    return entries
+
+
+def _language_distribution(file_to_module: dict[str, str]) -> list[dict[str, Any]]:
+    counts = Counter(
+        LANGUAGE_BY_SUFFIX.get(Path(rel_path).suffix, "Other") for rel_path in file_to_module
+    )
+    return [
+        {"language": language, "files": count, "derivedFrom": "extension-heuristic"}
+        for language, count in sorted(counts.items())
+    ]
+
+
+def build_codebase_map(
+    project_root: Path,
+    *,
+    code_structure_path: Path | None = None,
+    fallback_reason: str | None = None,
+) -> dict[str, Any]:
+    """Build a deterministic tier-1 codebase-map artifact."""
+    project_root = project_root.resolve()
+    code_structure, source_home = _load_authored_code_structure(project_root, code_structure_path)
+    source_path = default_code_structure_path(project_root, code_structure_path)
+
+    if code_structure is not None:
+        modules, file_to_module, prefixes_by_module, degraded = _curated_modules(
+            project_root, code_structure
+        )
+    else:
+        modules, file_to_module, prefixes_by_module, degraded = _directory_modules(project_root)
+
+    content_hash_paths = list(file_to_module)
+    with suppress(ValueError):
+        content_hash_paths.append(source_path.resolve().relative_to(project_root).as_posix())
+
+    degraded.append(
+        {
+            "code": "AST-FREE-HEURISTICS",
+            "message": (
+                "Default extractor uses repository walking and import-line heuristics only; "
+                "no AST or language parser provider was configured."
+            ),
+        }
+    )
+    if fallback_reason:
+        degraded.append({"code": "PROVIDER-FALLBACK", "message": fallback_reason})
+
+    return {
+        "formatVersion": CODEBASE_MAP_FORMAT_VERSION,
+        "contractVersion": CODEBASE_PROVIDER_CONTRACT_VERSION,
+        "kind": CODEBASE_MAP_KIND,
+        "provider": {
+            "name": DEFAULT_PROVIDER_NAME,
+            "version": DEFAULT_PROVIDER_VERSION,
+            "mode": "default",
+            "degraded": True,
+            **({"fallbackReason": fallback_reason} if fallback_reason else {}),
+        },
+        "source": {
+            "projectRoot": str(project_root),
+            "codeStructurePath": str(source_path),
+            "codeStructureHome": source_home,
+            "contentHash": _source_content_hash(project_root, content_hash_paths),
+        },
+        "modules": modules,
+        "coupling": _coupling_edges(project_root, file_to_module, prefixes_by_module),
+        "entryPoints": _entry_points(file_to_module),
+        "languageDistribution": _language_distribution(file_to_module),
+        "degraded": degraded,
+    }
+
+
+def config_error_to_dict(
+    path: Path, error: code_structure_validate.CodeStructureConfigError
+) -> dict[str, Any]:
+    """Return the structured config-error shape shared by codebase CLIs."""
+    return {
+        "path": str(path),
+        "ok": False,
+        "errors": [{"code": "CS-CONFIG", "message": str(error), "location": str(path)}],
+        "warnings": [],
+    }
+
+
+def main(argv: list[str] | None = None) -> int:
+    """CLI entry point."""
+    parser = argparse.ArgumentParser(description="Emit the default codebase-map artifact.")
+    parser.add_argument("--project-root", default=".", help="Repository root to inspect.")
+    parser.add_argument("--path", help="Explicit vBRIEF containing codeStructure metadata.")
+    args = parser.parse_args(argv)
+
+    project_root = Path(args.project_root)
+    code_structure_path = Path(args.path) if args.path else None
+    try:
+        artifact = build_codebase_map(project_root, code_structure_path=code_structure_path)
+    except code_structure_validate.CodeStructureConfigError as exc:
+        print(
+            json.dumps(
+                config_error_to_dict(
+                    default_code_structure_path(project_root, code_structure_path), exc
+                ),
+                indent=2,
+                sort_keys=True,
+            ),
+            file=sys.stderr,
+        )
+        return 2
+    print(json.dumps(artifact, indent=2, sort_keys=True))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())