@ngocsangairvds/vsaf 3.2.13 → 3.2.15

package/tools/vds-scripts/audit_orchestrator/src/vds_audit_orchestrator/collectors/material_cache.py

@@ -1,1252 +0,0 @@
-"""Shared cache for extracted materials under the effective VDS cache root."""
-
-from __future__ import annotations
-
-import json
-import os
-import shutil
-from datetime import UTC, datetime, timedelta
-from pathlib import Path
-from typing import Any, Literal
-
-from structlog import get_logger
-from vds_cli_common.paths import get_vds_cache_root
-
-logger = get_logger()
-
-
-DEFAULT_TTL_HOURS = 24
-MANIFEST_VERSION = 2
-CONFLUENCE_INDEX_SCHEMA_VERSION = 27
-DEFAULT_PROJECT_NAMESPACE = "global"
-CacheMode = Literal["read", "read-write", "refresh", "off"]
-
-
-class MaterialCache:
-    """Shared cache for extracted materials."""
-
-    def __init__(self, cache_dir: Path | None = None, ttl_hours: int | None = None) -> None:
-        env_ttl = os.getenv("VDS_MATERIAL_CACHE_TTL_HOURS")
-        resolved_ttl = ttl_hours
-        if resolved_ttl is None and env_ttl:
-            try:
-                resolved_ttl = int(env_ttl)
-            except ValueError:
-                logger.warning("material_cache_ttl_invalid", value=env_ttl)
-        if resolved_ttl is None:
-            resolved_ttl = DEFAULT_TTL_HOURS
-
-        self.ttl_hours = resolved_ttl
-        default_root = get_vds_cache_root()
-        self.cache_dir = cache_dir or default_root / "confluence"
-        try:
-            self.cache_dir.mkdir(parents=True, exist_ok=True)
-        except (OSError, PermissionError) as exc:
-            logger.warning(
-                "material_cache_dir_creation_failed",
-                cache_dir=str(self.cache_dir),
-                error=str(exc),
-            )
-            # Fall back to a temp directory if default fails
-            import tempfile
-
-            self.cache_dir = Path(tempfile.gettempdir()) / "vds_cache_fallback"
-            self.cache_dir.mkdir(parents=True, exist_ok=True)
-        self.manifest_path = self.cache_dir / "manifest.json"
-        self._diagnostics = self._empty_diagnostics()
-        self._manifest = self._load_manifest()
-
-    def _load_manifest(self) -> dict[str, Any]:
-        if self.manifest_path.exists():
-            try:
-                data = json.loads(self.manifest_path.read_text(encoding="utf-8"))
-            except Exception as exc:
-                logger.warning("material_cache_manifest_read_failed", error=str(exc))
-                data = {}
-        else:
-            data = {}
-
-        self._ensure_manifest_schema(data)
-        self._rebuild_indexes(data)
-        return data
-
-    def _save_manifest(self) -> None:
-        self._ensure_manifest_schema(self._manifest)
-        self._rebuild_indexes(self._manifest)
-        self._manifest["ttl_hours"] = self.ttl_hours
-        self._manifest["updated_at"] = self._now_iso()
-        try:
-            self.manifest_path.write_text(
-                json.dumps(self._manifest, indent=2, default=str),
-                encoding="utf-8",
-            )
-        except Exception as exc:
-            logger.warning("material_cache_manifest_write_failed", error=str(exc))
-
-    def _ensure_manifest_schema(self, data: dict[str, Any]) -> None:
-        data.setdefault("version", 1)
-        data.setdefault("manifest_version", MANIFEST_VERSION)
-        data.setdefault("index_schema_version", CONFLUENCE_INDEX_SCHEMA_VERSION)
-        data.setdefault("ttl_hours", self.ttl_hours)
-        data.setdefault("updated_at", self._now_iso())
-        data.setdefault("pages", {})
-        data.setdefault("attachments", {})
-        data.setdefault("bitbucket", {})
-        indexes = data.setdefault("indexes", {})
-        confluence = indexes.setdefault("confluence", {})
-        confluence["schema_version"] = CONFLUENCE_INDEX_SCHEMA_VERSION
-        confluence.setdefault("pages_by_project", {})
-        confluence.setdefault("attachments_by_project", {})
-
-    def _rebuild_indexes(self, data: dict[str, Any]) -> None:
-        indexes = data.setdefault("indexes", {})
-        confluence = indexes.setdefault("confluence", {})
-        confluence["schema_version"] = CONFLUENCE_INDEX_SCHEMA_VERSION
-        pages_by_project: dict[str, dict[str, str]] = {}
-        attachments_by_project: dict[str, dict[str, str]] = {}
-
-        for entry_key, entry in data.get("pages", {}).items():
-            if not isinstance(entry, dict):
-                continue
-            page_id = entry.get("page_id")
-            if not page_id:
-                continue
-            namespace = self._entry_project_key(entry)
-            pages_by_project.setdefault(namespace, {})[str(page_id)] = str(entry_key)
-
-        for entry_key, entry in data.get("attachments", {}).items():
-            if not isinstance(entry, dict):
-                continue
-            page_id = entry.get("page_id")
-            attachment_id = entry.get("attachment_id")
-            if not page_id or not attachment_id:
-                continue
-            namespace = self._entry_project_key(entry)
-            index_key = f"{page_id}:{attachment_id}"
-            attachments_by_project.setdefault(namespace, {})[index_key] = str(entry_key)
-
-        confluence["pages_by_project"] = pages_by_project
-        confluence["attachments_by_project"] = attachments_by_project
-
-    def _empty_diagnostics(self) -> dict[str, int]:
-        return {
-            "page_hits": 0,
-            "page_misses": 0,
-            "attachment_hits": 0,
-            "attachment_misses": 0,
-            "bitbucket_hits": 0,
-            "bitbucket_misses": 0,
-            "cache_skipped": 0,
-            "cache_skipped_writes": 0,
-            "bytes_saved": 0,
-            # Phase 126 Package D TSK-126.14: Fetch reason tracking
-            "fetch_cache_hit": 0,
-            "fetch_signal_refresh": 0,
-            "fetch_fresh_download": 0,
-        }
-
-    def reset_diagnostics(self) -> None:
-        self._diagnostics = self._empty_diagnostics()
-
-    def get_diagnostics(self) -> dict[str, int]:
-        return dict(self._diagnostics)
-
-    def _diag_inc(self, field: str, amount: int = 1) -> None:
-        self._diagnostics[field] = self._diagnostics.get(field, 0) + max(0, int(amount))
-
-    # Phase 126 Package D TSK-126.14: Fetch reason tracking helpers
-    def _track_fetch_reason(self, reason: str) -> None:
-        """Track fetch reason in diagnostics.
-
-        Maps fetch reason strings to diagnostic counters for cache health
-        monitoring and optimization insights.
-
-        Args:
-            reason: The fetch reason string (e.g., "cache_hit", "signal_refresh",
-                   "fresh_download", or other custom reasons)
-        """
-        if reason == "cache_hit":
-            self._diag_inc("fetch_cache_hit")
-        elif reason == "signal_refresh":
-            self._diag_inc("fetch_signal_refresh")
-        elif reason == "fresh_download":
-            self._diag_inc("fetch_fresh_download")
-
-    def _infer_fetch_reason_from_entry(self, entry: dict[str, Any]) -> str:
-        """Infer fetch reason from a cached entry.
-
-        Examines entry metadata to determine why this entry would be returned
-        from cache. Used for materialization flow tagging.
-
-        Args:
-            entry: Cache entry dict with metadata
-
-        Returns:
-            Fetch reason string: "cache_hit", "signal_refresh", or "cache_reuse"
-        """
-        if not isinstance(entry, dict):
-            return "cache_hit"
-
-        # Check for explicit status indicating cache reuse
-        status = entry.get("status")
-        if status == "not_modified":
-            return "cache_hit"
-
-        # Check for fetch_reason metadata
-        existing_reason = entry.get("fetch_reason")
-        if existing_reason:
-            if "version_match" in str(existing_reason):
-                return "cache_hit"
-            if "changed" in str(existing_reason):
-                return "signal_refresh"
-
-        # Check age for heuristic classification
-        cached_at = self._parse_ts(entry.get("cached_at"))
-        if cached_at:
-            age_hours = (datetime.now(UTC) - cached_at).total_seconds() / 3600
-            if age_hours < self.ttl_hours:
-                return "cache_hit"
-            # Entries past TTL but still returned indicate a signal-based refresh
-            return "signal_refresh"
-
-        return "cache_hit"
-
-    def _now_iso(self) -> str:
-        return datetime.now(UTC).isoformat()
-
-    def _parse_ts(self, value: str | None) -> datetime | None:
-        if not value:
-            return None
-        try:
-            return datetime.fromisoformat(value)
-        except ValueError:
-            return None
-
-    def _is_expired(self, entry: dict[str, Any]) -> bool:
-        if self.ttl_hours <= 0:
-            return True
-        cached_at = self._parse_ts(entry.get("cached_at"))
-        if not cached_at:
-            return True
-        return datetime.now(UTC) - cached_at > timedelta(hours=self.ttl_hours)
-
-    def _normalize_cache_mode(self, cache_mode: CacheMode | str | None) -> CacheMode:
-        if cache_mode is None:
-            return "read-write"
-        normalized = str(cache_mode).strip().lower()
-        if normalized in {"read", "read-write", "refresh", "off"}:
-            return normalized  # type: ignore[return-value]
-        logger.warning("material_cache_mode_invalid", cache_mode=cache_mode, fallback="read-write")
-        return "read-write"
-
-    def _cache_read_enabled(self, cache_mode: CacheMode | str | None) -> bool:
-        mode = self._normalize_cache_mode(cache_mode)
-        return mode in {"read", "read-write"}
-
-    def _cache_write_enabled(self, cache_mode: CacheMode | str | None) -> bool:
-        mode = self._normalize_cache_mode(cache_mode)
-        return mode in {"read-write", "refresh"}
-
-    def _normalize_project_key(self, project_key: str | None) -> str:
-        value = (project_key or "").strip()
-        return value if value else DEFAULT_PROJECT_NAMESPACE
-
-    def _entry_project_key(self, entry: dict[str, Any]) -> str:
-        namespace = entry.get("namespace")
-        if isinstance(namespace, dict):
-            value = namespace.get("project_key")
-            if isinstance(value, str) and value.strip():
-                return value.strip()
-        value = entry.get("project_key")
-        if isinstance(value, str) and value.strip():
-            return value.strip()
-        return DEFAULT_PROJECT_NAMESPACE
-
-    def _entry_matches_filters(
-        self,
-        entry: dict[str, Any],
-        *,
-        project_key: str | None = None,
-        path_contains: str | None = None,
-        max_age_hours: int | None = None,
-    ) -> bool:
-        if project_key:
-            expected = self._normalize_project_key(project_key)
-            if self._entry_project_key(entry) != expected:
-                return False
-
-        if max_age_hours is not None:
-            cached_at = self._parse_ts(entry.get("cached_at"))
-            if not cached_at:
-                return False
-            age = datetime.now(UTC) - cached_at
-            if age < timedelta(hours=max_age_hours):
-                return False
-
-        if path_contains:
-            needle = path_contains.strip()
-            if needle:
-                cache_paths = entry.get("cache_paths", {})
-                if not isinstance(cache_paths, dict):
-                    return False
-                if not any(needle in str(rel_path) for rel_path in cache_paths.values()):
-                    return False
-
-        return True
-
-    def _remove_cache_files(self, entry: dict[str, Any], *, dry_run: bool = False) -> int:
-        removed_files = 0
-        cache_paths = entry.get("cache_paths", {})
-        if not isinstance(cache_paths, dict):
-            return removed_files
-        for rel_path in cache_paths.values():
-            full_path = self._cache_path(str(rel_path))
-            if dry_run or not full_path.exists():
-                continue
-            full_path.unlink()
-            removed_files += 1
-        return removed_files
-
-    def _namespace_path_component(self, project_key: str | None) -> str | None:
-        normalized = self._normalize_project_key(project_key)
-        return normalized.replace("/", "_").replace("\\", "_")
-
-    def _page_cache_key(self, page_id: str, project_key: str | None = None) -> str:
-        normalized = self._normalize_project_key(project_key)
-        return f"{normalized}::{page_id}"
-
-    def _attachment_cache_key(self, attachment_id: str, project_key: str | None = None) -> str:
-        normalized = self._normalize_project_key(project_key)
-        return f"{normalized}::{attachment_id}"
-
-    def _find_page_entry(self, page_id: str, project_key: str | None = None) -> tuple[str, dict[str, Any]] | None:
-        pages = self._manifest.get("pages", {})
-        if not isinstance(pages, dict):
-            return None
-        normalized = self._normalize_project_key(project_key)
-        candidate_key = self._page_cache_key(str(page_id), normalized)
-        candidate = pages.get(candidate_key)
-        if isinstance(candidate, dict):
-            return candidate_key, candidate
-        for key, entry in pages.items():
-            if (
-                isinstance(entry, dict)
-                and str(entry.get("page_id")) == str(page_id)
-                and self._entry_project_key(entry) == normalized
-            ):
-                return str(key), entry
-        return None
-
-    def prune(
-        self,
-        *,
-        max_age_hours: int | None = None,
-        project_key: str | None = None,
-        path_contains: str | None = None,
-        dry_run: bool = False,
-    ) -> dict[str, int]:
-        """Remove expired entries from cache and disk.
-
-        Returns:
-            Dict with counts of removed items by type.
-        """
-        removed = {"pages": 0, "attachments": 0, "bitbucket": 0, "files": 0}
-
-        def _selected(entry: dict[str, Any]) -> bool:
-            if max_age_hours is None and project_key is None and path_contains is None:
-                return self._is_expired(entry)
-            return self._entry_matches_filters(
-                entry,
-                project_key=project_key,
-                path_contains=path_contains,
-                max_age_hours=max_age_hours,
-            )
-
-        # Prune pages
-        page_ids_to_remove = []
-        for pid, entry in self._manifest.get("pages", {}).items():
-            if _selected(entry):
-                page_ids_to_remove.append(pid)
-                removed["files"] += self._remove_cache_files(entry, dry_run=dry_run)
-
-        for pid in page_ids_to_remove:
-            if not dry_run:
-                del self._manifest["pages"][pid]
-            removed["pages"] += 1
-
-        # Prune attachments
-        att_ids_to_remove = []
-        for aid, entry in self._manifest.get("attachments", {}).items():
-            if _selected(entry):
-                att_ids_to_remove.append(aid)
-                removed["files"] += self._remove_cache_files(entry, dry_run=dry_run)
-
-        for aid in att_ids_to_remove:
-            if not dry_run:
-                del self._manifest["attachments"][aid]
-            removed["attachments"] += 1
-
-        # Prune bitbucket
-        bb_keys_to_remove = []
-        for key, entry in self._manifest.get("bitbucket", {}).items():
-            if _selected(entry):
-                bb_keys_to_remove.append(key)
-                removed["files"] += self._remove_cache_files(entry, dry_run=dry_run)
-
-        for key in bb_keys_to_remove:
-            if not dry_run:
-                del self._manifest["bitbucket"][key]
-            removed["bitbucket"] += 1
-
-        if not dry_run and (removed["pages"] + removed["attachments"] + removed["bitbucket"] > 0):
-            self._save_manifest()
-
-        logger.info(
-            "cache_pruned",
-            removed=removed,
-            dry_run=dry_run,
-            max_age_hours=max_age_hours,
-            project_key=project_key,
-            path_contains=path_contains,
-        )
-        return removed
-
-    def verify_integrity(self) -> dict[str, list[str]]:
-        """Verify integrity of cached files against manifest checksums.
-
-        Returns:
-            Dict mapping type to list of corrupted/missing entry IDs.
-        """
-        corrupted: dict[str, list[str]] = {"pages": [], "attachments": [], "bitbucket": []}
-
-        # Check pages
-        for pid, entry in self._manifest.get("pages", {}).items():
-            if not self._page_entry_valid(entry):
-                corrupted["pages"].append(pid)
-
-        # Check attachments
-        for aid, entry in self._manifest.get("attachments", {}).items():
-            if not self._attachment_entry_valid(entry):
-                corrupted["attachments"].append(aid)
-
-        # Check bitbucket
-        for key, entry in self._manifest.get("bitbucket", {}).items():
-            if not self._bitbucket_entry_valid(entry):
-                corrupted["bitbucket"].append(key)
-
-        return corrupted
-
-    def _file_sha256(self, path: Path) -> str:
-        import hashlib
-
-        digest = hashlib.sha256()
-        with path.open("rb") as handle:
-            for chunk in iter(lambda: handle.read(1024 * 1024), b""):
-                digest.update(chunk)
-        return digest.hexdigest()
-
-    def _cache_path(self, relative: str) -> Path:
-        return self.cache_dir / relative
-
-    def _entry_file_ok(self, path: Path | None, sha256: str | None) -> bool:
-        if not path or not sha256:
-            return False
-        if not path.exists():
-            return False
-        return self._file_sha256(path) == sha256
-
-    def _page_entry_valid(self, entry: dict[str, Any]) -> bool:
-        if self._is_expired(entry):
-            return False
-        cache_paths = entry.get("cache_paths", {})
-        hashes = entry.get("hashes", {})
-        html_ok = self._entry_file_ok(
-            self._cache_path(cache_paths.get("html")) if cache_paths.get("html") else None,
-            hashes.get("html_sha256"),
-        )
-        if not html_ok:
-            return False
-        md_path = cache_paths.get("markdown")
-        if not md_path:
-            return True
-        return self._entry_file_ok(self._cache_path(md_path), hashes.get("markdown_sha256"))
-
-    def _attachment_entry_valid(self, entry: dict[str, Any]) -> bool:
-        if self._is_expired(entry):
-            return False
-        cache_paths = entry.get("cache_paths", {})
-        hashes = entry.get("hashes", {})
-        raw_ok = self._entry_file_ok(
-            self._cache_path(cache_paths.get("file")) if cache_paths.get("file") else None,
-            hashes.get("file_sha256"),
-        )
-        if not raw_ok:
-            return False
-        md_path = cache_paths.get("markdown")
-        if not md_path:
-            return True
-        return self._entry_file_ok(self._cache_path(md_path), hashes.get("markdown_sha256"))
-
-    def _bitbucket_entry_valid(self, entry: dict[str, Any]) -> bool:
-        if self._is_expired(entry):
-            return False
-        cache_paths = entry.get("cache_paths", {})
-        hashes = entry.get("hashes", {})
-        return self._entry_file_ok(
-            self._cache_path(cache_paths.get("file")) if cache_paths.get("file") else None,
-            hashes.get("file_sha256"),
-        )
-
-    def validate_local_page_entry(self, entry: dict[str, Any] | None) -> bool:
-        if not entry:
-            return False
-        html_path = entry.get("html_path")
-        if not isinstance(html_path, str) or not html_path:
-            return False
-        html_ok = self._entry_file_ok(
-            Path(html_path),
-            entry.get("html_sha256"),
-        )
-        if not html_ok:
-            return False
-        markdown_path = entry.get("markdown_path")
-        if not isinstance(markdown_path, str) or not markdown_path:
-            return True
-        return self._entry_file_ok(Path(markdown_path), entry.get("markdown_sha256"))
-
-    def validate_local_attachment_entry(self, entry: dict[str, Any] | None) -> bool:
-        if not entry:
-            return False
-        file_path = entry.get("file_path")
-        if not isinstance(file_path, str) or not file_path:
-            return False
-        raw_ok = self._entry_file_ok(
-            Path(file_path),
-            entry.get("file_sha256"),
-        )
-        if not raw_ok:
-            return False
-        markdown_path = entry.get("markdown_path")
-        if not isinstance(markdown_path, str) or not markdown_path:
-            return True
-        return self._entry_file_ok(Path(markdown_path), entry.get("markdown_sha256"))
-
-    def validate_local_bitbucket_entry(self, entry: dict[str, Any] | None) -> bool:
-        if not entry:
-            return False
-        path_value = entry.get("path")
-        if not isinstance(path_value, str) or not path_value:
-            return False
-        return self._entry_file_ok(
-            Path(path_value),
-            entry.get("sha256"),
-        )
-
-    def get_page_entry(self, page_id: str, *, project_key: str | None = None) -> dict[str, Any] | None:
-        resolved = self._find_page_entry(page_id, project_key=project_key)
-        if not resolved:
-            return None
-        _, entry = resolved
-        if not self._page_entry_valid(entry):
-            return None
-        return entry
-
-    def get_child_entries(
-        self,
-        parent_page_id: str,
-        *,
-        project_key: str | None = None,
-    ) -> list[dict[str, Any]]:
-        results = []
-        namespace = self._normalize_project_key(project_key) if project_key else None
-        for entry in self._manifest.get("pages", {}).values():
-            if entry.get("parent_page_id") != parent_page_id:
-                continue
-            if namespace and self._entry_project_key(entry) != namespace:
-                continue
-            if self._page_entry_valid(entry):
-                results.append(entry)
-        return results
-
-    def get_attachment_entries(
-        self,
-        page_id: str,
-        *,
-        project_key: str | None = None,
-    ) -> list[dict[str, Any]]:
-        results = []
-        namespace = self._normalize_project_key(project_key) if project_key else None
-        for entry in self._manifest.get("attachments", {}).values():
-            if str(entry.get("page_id")) != str(page_id):
-                continue
-            if namespace and self._entry_project_key(entry) != namespace:
-                continue
-            if self._attachment_entry_valid(entry):
-                results.append(entry)
-        return results
-
-    def get_bitbucket_entries(self, project_key: str) -> list[dict[str, Any]]:
-        results = []
-        for entry in self._manifest.get("bitbucket", {}).values():
-            if entry.get("project") != project_key:
-                continue
-            if self._bitbucket_entry_valid(entry):
-                results.append(entry)
-        return results
-
-    def _copy_if_needed(self, src: Path, dest: Path, sha256: str | None = None) -> tuple[bool, int]:
-        if dest.exists() and sha256 and self._file_sha256(dest) == sha256:
-            try:
-                return False, dest.stat().st_size
-            except OSError:
-                return False, 0
-        dest.parent.mkdir(parents=True, exist_ok=True)
-        shutil.copy2(src, dest)
-        return True, 0
-
-    def materialize_page(
-        self,
-        page_id: str,
-        output_dir: Path,
-        *,
-        project_key: str | None = None,
-        cache_mode: CacheMode | str | None = None,
-    ) -> dict[str, Any] | None:
-        if not self._cache_read_enabled(cache_mode):
-            self._diag_inc("cache_skipped")
-            self._diag_inc("page_misses")
-            logger.info("material_cache_decision", entity="page", decision="skip_read_disabled", page_id=str(page_id))
-            return None
-        entry = self.get_page_entry(page_id, project_key=project_key)
-        if not entry:
-            self._diag_inc("page_misses")
-            logger.info("material_cache_decision", entity="page", decision="miss", page_id=str(page_id))
-            return None
-
-        title = entry.get("title") or f"page-{page_id}"
-        cache_paths = entry.get("cache_paths", {})
-        hashes = entry.get("hashes", {})
-
-        html_cache_rel = cache_paths.get("html")
-        markdown_cache_rel = cache_paths.get("markdown")
-        if not html_cache_rel:
-            return None
-
-        html_cache = self._cache_path(html_cache_rel)
-        html_output = output_dir / f"{page_id}_{title}.html"
-        html_copied, html_saved = self._copy_if_needed(html_cache, html_output, hashes.get("html_sha256"))
-        if not html_copied:
-            self._diag_inc("cache_skipped_writes")
-            self._diag_inc("bytes_saved", html_saved)
-
-        markdown_output = None
-        if markdown_cache_rel:
-            markdown_cache = self._cache_path(markdown_cache_rel)
-            markdown_output = output_dir / f"{title}.md"
-            md_copied, md_saved = self._copy_if_needed(markdown_cache, markdown_output, hashes.get("markdown_sha256"))
-            if not md_copied:
-                self._diag_inc("cache_skipped_writes")
-                self._diag_inc("bytes_saved", md_saved)
-
-        entry["last_accessed"] = self._now_iso()
-        entry["last_seen_at"] = self._now_iso()
-        self._save_manifest()
-        self._diag_inc("page_hits")
-        logger.info("material_cache_decision", entity="page", decision="hit", page_id=str(page_id))
-
-        # Phase 126 Package D TSK-126.14: Add fetch_reason to materialization output
-        fetch_reason = self._infer_fetch_reason_from_entry(entry)
-        self._track_fetch_reason(fetch_reason)
-
-        return {
-            "page_id": page_id,
-            "title": title,
-            "page_url": entry.get("page_url"),
-            "status": entry.get("status"),
-            "markdown_path": str(markdown_output) if markdown_output else None,
-            "html_path": str(html_output),
-            "html_size_bytes": entry.get("sizes", {}).get("html_size_bytes"),
-            "markdown_size_bytes": entry.get("sizes", {}).get("markdown_size_bytes"),
-            "html_sha256": hashes.get("html_sha256"),
-            "markdown_sha256": hashes.get("markdown_sha256"),
-            "converter": entry.get("converter", "markitdown"),
-            "format": entry.get("format"),
-            "source_format": entry.get("source_format"),
-            "cached": True,
-            "cached_from": "shared_cache",
-            "fetch_reason": fetch_reason,
-        }
-
-    def materialize_child_pages(
-        self,
-        parent_page_id: str,
-        output_dir: Path,
-        *,
-        project_key: str | None = None,
-        cache_mode: CacheMode | str | None = None,
-    ) -> list[dict[str, Any]]:
-        if not self._cache_read_enabled(cache_mode):
-            self._diag_inc("cache_skipped")
-            logger.info(
-                "material_cache_decision",
-                entity="child_pages",
-                decision="skip_read_disabled",
-                parent_page_id=str(parent_page_id),
-            )
-            return []
-        results = []
-        for entry in self.get_child_entries(parent_page_id, project_key=project_key):
-            page_id = entry.get("page_id")
-            if not page_id:
-                continue
-            meta = self.materialize_page(page_id, output_dir, project_key=project_key, cache_mode=cache_mode)
-            if meta:
-                meta["depth"] = entry.get("depth")
-                results.append(meta)
-        return results
-
-    def materialize_attachments(
-        self,
-        page_id: str,
-        output_dir: Path,
-        *,
-        project_key: str | None = None,
-        cache_mode: CacheMode | str | None = None,
-    ) -> list[dict[str, Any]]:
-        if not self._cache_read_enabled(cache_mode):
-            self._diag_inc("cache_skipped")
-            self._diag_inc("attachment_misses")
-            logger.info(
-                "material_cache_decision", entity="attachment", decision="skip_read_disabled", page_id=str(page_id)
-            )
-            return []
-        results = []
-        attachments_dir = output_dir / "attachments"
-        attachments_dir.mkdir(parents=True, exist_ok=True)
-
-        for entry in self.get_attachment_entries(page_id, project_key=project_key):
-            attachment_id = entry.get("attachment_id")
-            title = entry.get("title") or str(attachment_id)
-            cache_paths = entry.get("cache_paths", {})
-            hashes = entry.get("hashes", {})
-
-            raw_rel = cache_paths.get("file")
-            if not raw_rel:
-                continue
-
-            raw_cache = self._cache_path(raw_rel)
-            raw_output = attachments_dir / title
-            raw_copied, raw_saved = self._copy_if_needed(raw_cache, raw_output, hashes.get("file_sha256"))
-            if not raw_copied:
-                self._diag_inc("cache_skipped_writes")
-                self._diag_inc("bytes_saved", raw_saved)
-
-            markdown_output = None
-            markdown_rel = cache_paths.get("markdown")
-            if markdown_rel:
-                markdown_cache = self._cache_path(markdown_rel)
-                markdown_output = attachments_dir / f"{Path(title).stem}.md"
-                md_copied, md_saved = self._copy_if_needed(
-                    markdown_cache, markdown_output, hashes.get("markdown_sha256")
-                )
-                if not md_copied:
-                    self._diag_inc("cache_skipped_writes")
-                    self._diag_inc("bytes_saved", md_saved)
-
-            entry["last_accessed"] = self._now_iso()
-            entry["last_seen_at"] = self._now_iso()
-
-            # Phase 126 Package D TSK-126.14: Add fetch_reason to attachment materialization
-            fetch_reason = self._infer_fetch_reason_from_entry(entry)
-            self._track_fetch_reason(fetch_reason)
-
-            results.append(
-                {
-                    "attachment_id": attachment_id,
-                    "title": title,
-                    "file_path": str(raw_output),
-                    "markdown_path": str(markdown_output) if markdown_output else None,
-                    "file_sha256": hashes.get("file_sha256"),
-                    "markdown_sha256": hashes.get("markdown_sha256"),
-                    "status": entry.get("status"),
-                    "converter": entry.get("converter", "markitdown"),
-                    "format": entry.get("format"),
-                    "source_format": entry.get("source_format"),
-                    "cached": True,
-                    "cached_from": "shared_cache",
-                    "fetch_reason": fetch_reason,
-                }
-            )
-
-        if results:
-            self._diag_inc("attachment_hits", len(results))
-            self._save_manifest()
-            logger.info(
-                "material_cache_decision",
-                entity="attachment",
-                decision="hit",
-                page_id=str(page_id),
-                count=len(results),
-            )
-        else:
-            self._diag_inc("attachment_misses")
-            logger.info("material_cache_decision", entity="attachment", decision="miss", page_id=str(page_id))
-        return results
-
-    def materialize_bitbucket(
-        self,
-        project_key: str,
-        output_dir: Path,
-        *,
-        cache_mode: CacheMode | str | None = None,
-    ) -> list[dict[str, Any]]:
-        if not self._cache_read_enabled(cache_mode):
-            self._diag_inc("cache_skipped")
-            self._diag_inc("bitbucket_misses")
-            logger.info(
-                "material_cache_decision", entity="bitbucket", decision="skip_read_disabled", project=project_key
-            )
-            return []
-        results = []
-        for entry in self.get_bitbucket_entries(project_key):
-            project = entry.get("project")
-            repo = entry.get("repo")
-            file_name = entry.get("file") or "README.md"
-            if not project or not repo:
-                continue
-
-            cache_paths = entry.get("cache_paths", {})
-            hashes = entry.get("hashes", {})
-            file_rel = cache_paths.get("file")
-            if not file_rel:
-                continue
-
-            cache_file = self._cache_path(file_rel)
-            output_file = output_dir / "bitbucket" / project / repo / file_name
-            copied, saved = self._copy_if_needed(cache_file, output_file, hashes.get("file_sha256"))
-            if not copied:
-                self._diag_inc("cache_skipped_writes")
-                self._diag_inc("bytes_saved", saved)
-
-            entry["last_accessed"] = self._now_iso()
-            entry["last_seen_at"] = self._now_iso()
-
-            # Phase 126 Package D TSK-126.14: Add fetch_reason to Bitbucket materialization
-            fetch_reason = self._infer_fetch_reason_from_entry(entry)
-            self._track_fetch_reason(fetch_reason)
-
-            results.append(
-                {
-                    "project": project,
-                    "repo": repo,
-                    "file": file_name,
-                    "path": str(output_file),
-                    "sha256": hashes.get("file_sha256"),
-                    "default_branch": entry.get("default_branch"),
-                    "last_commit": entry.get("last_commit"),
-                    "status": entry.get("status"),
-                    "cached": True,
-                    "cached_from": "shared_cache",
-                    "fetch_reason": fetch_reason,
-                }
-            )
-
-        if results:
-            self._diag_inc("bitbucket_hits", len(results))
-            self._save_manifest()
-            logger.info(
-                "material_cache_decision",
-                entity="bitbucket",
-                decision="hit",
-                project=project_key,
-                count=len(results),
-            )
-        else:
-            self._diag_inc("bitbucket_misses")
-            logger.info("material_cache_decision", entity="bitbucket", decision="miss", project=project_key)
-        return results
-
-    def store_page(
-        self,
-        page_meta: dict[str, Any],
-        *,
-        parent_page_id: str | None = None,
-        project_key: str | None = None,
-        cache_mode: CacheMode | str | None = None,
-    ) -> None:
-        if not self._cache_write_enabled(cache_mode):
-            self._diag_inc("cache_skipped")
-            logger.info(
-                "material_cache_decision",
-                entity="page",
-                decision="skip_write_disabled",
-                page_id=str(page_meta.get("page_id")),
-            )
-            return
-        page_id = page_meta.get("page_id")
-        if not page_id:
-            return
-
-        html_path = page_meta.get("html_path")
-        markdown_path = page_meta.get("markdown_path")
-        html_sha256 = page_meta.get("html_sha256")
-        markdown_sha256 = page_meta.get("markdown_sha256")
-
-        cache_paths: dict[str, str] = {}
-        hashes: dict[str, str | None] = {}
-
-        namespace_path = self._namespace_path_component(project_key)
-        page_dir = f"pages/{namespace_path}/{page_id}"
-        if html_path and html_sha256 and Path(html_path).exists():
-            cache_html_rel = f"{page_dir}/page.html"
-            cache_html = self._cache_path(cache_html_rel)
-            copied, saved = self._copy_if_needed(Path(html_path), cache_html, html_sha256)
-            if not copied:
-                self._diag_inc("cache_skipped_writes")
-                self._diag_inc("bytes_saved", saved)
-            cache_paths["html"] = cache_html_rel
-            hashes["html_sha256"] = html_sha256
-
-        if markdown_path and markdown_sha256 and Path(markdown_path).exists():
-            cache_md_rel = f"{page_dir}/page.md"
-            cache_md = self._cache_path(cache_md_rel)
-            copied, saved = self._copy_if_needed(Path(markdown_path), cache_md, markdown_sha256)
-            if not copied:
-                self._diag_inc("cache_skipped_writes")
-                self._diag_inc("bytes_saved", saved)
-            cache_paths["markdown"] = cache_md_rel
-            hashes["markdown_sha256"] = markdown_sha256
-
-        normalized_project_key = self._normalize_project_key(project_key)
-        entry = {
-            "page_id": str(page_id),
-            "title": page_meta.get("title"),
-            "page_url": page_meta.get("page_url"),
-            "status": page_meta.get("status"),
-            "converter": page_meta.get("converter", "markitdown"),
-            "format": page_meta.get("format"),
-            "source_format": page_meta.get("source_format"),
-            "cached_at": self._now_iso(),
-            "parent_page_id": parent_page_id,
-            "depth": page_meta.get("depth"),
-            "cache_paths": cache_paths,
-            "hashes": hashes,
-            "sizes": {
-                "html_size_bytes": page_meta.get("html_size_bytes"),
-                "markdown_size_bytes": page_meta.get("markdown_size_bytes"),
-            },
-            "source": {
-                "page_version": page_meta.get("page_version"),
-                "page_updated": page_meta.get("page_updated"),
-            },
-            "version": page_meta.get("page_version"),
-            "updated": page_meta.get("page_updated"),
-            "sha256": html_sha256,
-            "last_seen_at": self._now_iso(),
-            "namespace": {
-                "kind": "confluence",
-                "project_key": normalized_project_key,
-            },
-        }
-
-        manifest_key = self._page_cache_key(str(page_id), project_key)
-        self._manifest.setdefault("pages", {})[manifest_key] = entry
-        self._save_manifest()
-        logger.info("material_cache_decision", entity="page", decision="refresh_write", page_id=str(page_id))
-
-    def store_attachments(
-        self,
-        page_id: str,
-        attachments_meta: list[dict[str, Any]],
-        *,
-        project_key: str | None = None,
-        cache_mode: CacheMode | str | None = None,
-    ) -> None:
-        if not self._cache_write_enabled(cache_mode):
-            self._diag_inc("cache_skipped")
-            logger.info(
-                "material_cache_decision",
-                entity="attachment",
-                decision="skip_write_disabled",
-                page_id=str(page_id),
-            )
-            return
-        if not attachments_meta:
-            return
-
-        for entry in attachments_meta:
-            attachment_id = entry.get("attachment_id")
-            if not attachment_id:
-                continue
-
-            file_path = entry.get("file_path")
-            markdown_path = entry.get("markdown_path")
-            file_sha256 = entry.get("file_sha256")
-            markdown_sha256 = entry.get("markdown_sha256")
-
-            cache_paths: dict[str, str] = {}
-            hashes: dict[str, str | None] = {}
-
-            namespace_path = self._namespace_path_component(project_key)
-            attachment_dir = f"attachments/{namespace_path}/{attachment_id}"
-            if file_path and file_sha256 and Path(file_path).exists():
-                file_name = Path(file_path).name
-                cache_file_rel = f"{attachment_dir}/{file_name}"
-                cache_file = self._cache_path(cache_file_rel)
-                copied, saved = self._copy_if_needed(Path(file_path), cache_file, file_sha256)
-                if not copied:
-                    self._diag_inc("cache_skipped_writes")
-                    self._diag_inc("bytes_saved", saved)
-                cache_paths["file"] = cache_file_rel
-                hashes["file_sha256"] = file_sha256
-
-            if markdown_path and markdown_sha256 and Path(markdown_path).exists():
-                cache_md_rel = f"{attachment_dir}/attachment.md"
-                cache_md = self._cache_path(cache_md_rel)
-                copied, saved = self._copy_if_needed(Path(markdown_path), cache_md, markdown_sha256)
-                if not copied:
-                    self._diag_inc("cache_skipped_writes")
-                    self._diag_inc("bytes_saved", saved)
-                cache_paths["markdown"] = cache_md_rel
-                hashes["markdown_sha256"] = markdown_sha256
-
-            normalized_project_key = self._normalize_project_key(project_key)
-            stored = {
-                "attachment_id": str(attachment_id),
-                "page_id": page_id,
-                "title": entry.get("title"),
-                "status": entry.get("status"),
-                "converter": entry.get("converter", "markitdown"),
-                "format": entry.get("format"),
-                "source_format": entry.get("source_format"),
-                "cached_at": self._now_iso(),
-                "cache_paths": cache_paths,
-                "hashes": hashes,
-                "sizes": {
-                    "file_size_bytes": entry.get("file_size_bytes"),
-                    "markdown_size_bytes": entry.get("markdown_size_bytes"),
-                },
-                "source": {
-                    "attachment_version": entry.get("attachment_version"),
-                    "attachment_size": entry.get("attachment_size"),
-                },
-                "version": entry.get("attachment_version"),
-                "updated": None,
-                "sha256": file_sha256,
-                "last_seen_at": self._now_iso(),
-                "namespace": {
-                    "kind": "confluence",
-                    "project_key": normalized_project_key,
-                },
-            }
-            manifest_key = self._attachment_cache_key(str(attachment_id), project_key)
-            self._manifest.setdefault("attachments", {})[manifest_key] = stored
-
-        self._save_manifest()
-        logger.info(
-            "material_cache_decision",
-            entity="attachment",
-            decision="refresh_write",
-            page_id=str(page_id),
-            count=len(attachments_meta),
-        )
-
-    def store_bitbucket(
-        self,
-        bitbucket_meta: list[dict[str, Any]],
-        *,
-        cache_mode: CacheMode | str | None = None,
-    ) -> None:
-        if not self._cache_write_enabled(cache_mode):
-            self._diag_inc("cache_skipped")
-            logger.info("material_cache_decision", entity="bitbucket", decision="skip_write_disabled")
-            return
-        if not bitbucket_meta:
-            return
-
-        for entry in bitbucket_meta:
-            project = entry.get("project")
-            repo = entry.get("repo")
-            file_name = entry.get("file") or "README.md"
-            path = entry.get("path")
-            sha256 = entry.get("sha256")
-            if not project or not repo:
-                continue
-
-            cache_paths: dict[str, str] = {}
-            hashes: dict[str, str | None] = {}
-
-            if path and sha256 and Path(path).exists():
-                cache_file_rel = f"bitbucket/{project}/{repo}/{file_name}"
-                cache_file = self._cache_path(cache_file_rel)
-                copied, saved = self._copy_if_needed(Path(path), cache_file, sha256)
-                if not copied:
-                    self._diag_inc("cache_skipped_writes")
-                    self._diag_inc("bytes_saved", saved)
-                cache_paths["file"] = cache_file_rel
-                hashes["file_sha256"] = sha256
-
-            cache_key = f"{project}/{repo}/{file_name}"
-            stored = {
-                "project": project,
-                "repo": repo,
-                "file": file_name,
-                "status": entry.get("status"),
-                "cached_at": self._now_iso(),
-                "cache_paths": cache_paths,
-                "hashes": hashes,
-                "default_branch": entry.get("default_branch"),
-                "last_commit": entry.get("last_commit"),
-            }
-            self._manifest.setdefault("bitbucket", {})[cache_key] = stored
-
-        self._save_manifest()
-        logger.info("material_cache_decision", entity="bitbucket", decision="refresh_write", count=len(bitbucket_meta))
-
-    def get_project_metrics(self, project_key: str) -> dict[str, Any]:
-        """Return project-specific cache statistics (TSK-126.4, AC-126.2.6).
-
-        Computes metrics for materials belonging to a specific project namespace,
-        including counts by type, freshness distribution, and size information.
-
-        Args:
-            project_key: Project namespace (e.g., "88718940" or "CEP")
-
-        Returns:
-            Dict with project-specific cache statistics:
-            - total_pages: Number of cached pages for this project
-            - total_attachments: Number of cached attachments for this project
-            - total_repos: Number of cached Bitbucket repos for this project
-            - fresh_count: Entries cached within TTL
-            - aging_count: Entries past TTL but not yet stale
-            - stale_count: Entries marked as stale (expired)
-            - unknown_count: Entries with indeterminate freshness
-            - total_size_bytes: Total disk size for project entries
-            - oldest_entry_age_hours: Age of oldest project entry
-            - newest_entry_age_hours: Age of newest project entry
-        """
-        normalized_key = self._normalize_project_key(project_key)
-
-        # Initialize counters
-        metrics = {
-            "total_pages": 0,
-            "total_attachments": 0,
-            "total_repos": 0,
-            "fresh_count": 0,
-            "aging_count": 0,
-            "stale_count": 0,
-            "unknown_count": 0,
-            "total_size_bytes": 0,
-            "oldest_entry_age_hours": 0.0,
-            "newest_entry_age_hours": 0.0,
-            "cache_hit_rate": 0.0,
-            "project_key": normalized_key,
-        }
-
-        now = datetime.now(UTC)
-        ages: list[float] = []
-
-        # Count pages
-        for entry in self._manifest.get("pages", {}).values():
-            if not isinstance(entry, dict):
-                continue
-            if self._entry_project_key(entry) != normalized_key:
-                continue
-
-            metrics["total_pages"] += 1
-
-            # Count by freshness
-            cached_at = self._parse_ts(entry.get("cached_at"))
-            if cached_at:
-                age_hours = (now - cached_at).total_seconds() / 3600.0
-                ages.append(age_hours)
-
-                # Estimate size from cache paths
-                _cache_paths = entry.get("cache_paths", {})
-                html_size = entry.get("sizes", {}).get("html_size_bytes", 0)
-                md_size = entry.get("sizes", {}).get("markdown_size_bytes", 0)
-                metrics["total_size_bytes"] += html_size + md_size
-
-                # Special handling for ttl_hours=0: all entries are immediately stale
-                if age_hours <= 0 or self.ttl_hours <= 0:
-                    metrics["stale_count"] += 1
-                elif age_hours < self.ttl_hours:
-                    metrics["fresh_count"] += 1
-                elif age_hours < self.ttl_hours * 2:
-                    metrics["aging_count"] += 1
-                else:
-                    metrics["stale_count"] += 1
-            else:
-                metrics["unknown_count"] += 1
-
-        # Count attachments
-        for entry in self._manifest.get("attachments", {}).values():
-            if not isinstance(entry, dict):
-                continue
-            if self._entry_project_key(entry) != normalized_key:
-                continue
-
-            metrics["total_attachments"] += 1
-
-            cached_at = self._parse_ts(entry.get("cached_at"))
-            if cached_at:
-                age_hours = (now - cached_at).total_seconds() / 3600.0
-                ages.append(age_hours)
-
-                file_size = entry.get("sizes", {}).get("file_size_bytes", 0)
-                md_size = entry.get("sizes", {}).get("markdown_size_bytes", 0)
-                metrics["total_size_bytes"] += file_size + md_size
-
-                # Special handling for ttl_hours=0: all entries are immediately stale
-                if age_hours <= 0 or self.ttl_hours <= 0:
-                    metrics["stale_count"] += 1
-                elif age_hours < self.ttl_hours:
-                    metrics["fresh_count"] += 1
-                elif age_hours < self.ttl_hours * 2:
-                    metrics["aging_count"] += 1
-                else:
-                    metrics["stale_count"] += 1
-            else:
-                metrics["unknown_count"] += 1
-
-        # Count repos
-        for entry in self._manifest.get("bitbucket", {}).values():
-            if not isinstance(entry, dict):
-                continue
-            if entry.get("project") != normalized_key:
-                # For bitbucket, project field is the project key
-                continue
-
-            metrics["total_repos"] += 1
-
-            cached_at = self._parse_ts(entry.get("cached_at"))
-            if cached_at:
-                age_hours = (now - cached_at).total_seconds() / 3600.0
-                ages.append(age_hours)
-
-                # Special handling for ttl_hours=0: all entries are immediately stale
-                if age_hours <= 0 or self.ttl_hours <= 0:
-                    metrics["stale_count"] += 1
-                elif age_hours < self.ttl_hours:
-                    metrics["fresh_count"] += 1
-                elif age_hours < self.ttl_hours * 2:
-                    metrics["aging_count"] += 1
-                else:
-                    metrics["stale_count"] += 1
-            else:
-                metrics["unknown_count"] += 1
-
-        # Compute age stats
-        if ages:
-            metrics["oldest_entry_age_hours"] = round(max(ages), 2)
-            metrics["newest_entry_age_hours"] = round(min(ages), 2)
-
-        # Compute cache hit rate (percentage of entries with valid cached_at)
-        total_entries = metrics["total_pages"] + metrics["total_attachments"] + metrics["total_repos"]
-        if total_entries > 0:
-            valid_entries = total_entries - metrics["unknown_count"]
-            metrics["cache_hit_rate"] = round(valid_entries / total_entries * 100, 2)
-        else:
-            metrics["cache_hit_rate"] = 100.0
-
-        return metrics