@ngocsangairvds/vsaf 3.2.14 → 3.2.16

package/tools/vds-scripts/audit_orchestrator/src/vds_audit_orchestrator/parse_cmds.py

@@ -1,608 +0,0 @@
-"""Parse commands for vds-audit CLI (Phase 91 decomposition)."""
-
-from __future__ import annotations
-
-import asyncio
-import json
-
-# pyright: reportUnusedVariable=false
-import re
-import tempfile
-from collections.abc import Awaitable, Callable
-from pathlib import Path
-from typing import TYPE_CHECKING, Any, cast
-
-import typer
-import yaml
-
-from vds_audit_orchestrator.cli_common import _STATE_DSN_ENV_VAR, _cli_state, app, console
-from vds_audit_orchestrator.cli_impl import (
-    _apply_parse_registry_scope_guidance,
-    _build_git_manifest_payload,
-    _build_grouped_bitbucket_projects_contract,
-    _build_parse_registry_output_payload,
-    _build_registry_state_payload,
-    _build_repo_coverage_summary,
-    _emit_json,
-    _emit_parse_payload,
-    _normalize_parse_project_selector,
-    _parse_confluence_ref,
-    _parse_registry_metadata,
-    _project_storage_key_from_payload,
-    _raise_project_selector_error,
-    _record_state_discovery,
-    _require_consumer_state_dsn,
-    _require_producer_state_dsn,
-    _resolve_confluence_page_id,
-    _resolve_parse_manifest_payload,
-    _seed_evidence_project_from_payload,
-    _seed_scope_targets_from_payload,
-    _sort_repo_entries,
-    _validate_required_credentials,
-    _wire_registry_adaptive_depth_kwargs,
-)
-from vds_audit_orchestrator.clients.confluence_cli_client import ConfluenceCliClient
-from vds_audit_orchestrator.collectors.registry_parser import RegistryPageParser
-from vds_audit_orchestrator.seed import select_project, select_project_by_key, select_repos
-
-if TYPE_CHECKING:
-    from vds_audit_orchestrator.models.registry import ProjectRegistry
-
-
-@app.command(name="parse-registry")
-def parse_registry(
-    registry_page: str = typer.Option(
-        ...,
-        "--registry-page",
-        help=(
-            "Confluence project page, department/container page, or registry/root page URL or ID "
-            "(numeric ID, ?pageId=, or /display/<SPACE>/<Title>)"
-        ),
-    ),
-    confluence_server: str | None = typer.Option(
-        None,
-        "--confluence-server",
-        help="Confluence server key: internal|external. If omitted, inferred from URL when possible.",
-    ),
-    checklist_page: str | None = typer.Option(
-        None,
-        "--checklist-page",
-        help="Explicit checklist page ID or URL override (bypasses auto-detection)",
-    ),
-    recursive_depth: int = typer.Option(
-        1,
-        "--recursive-depth",
-        help=(
-            "Recursive child-page discovery depth. Project-page entry is atomic, "
-            "department/container entry recursively aggregates child projects, "
-            "and root entry recursively aggregates departments."
-        ),
-    ),
-    parse_concurrency: int = typer.Option(
-        3,
-        "--parse-concurrency",
-        min=1,
-        max=RegistryPageParser.MAX_PARSE_CONCURRENCY,
-        help="Concurrency for parsing candidate project pages from the registry tree",
-    ),
-    timeout: int = typer.Option(
-        120,
-        "--timeout",
-        help="Timeout (seconds) for Confluence vds-cli calls",
-    ),
-    retries: int = typer.Option(
-        2,
-        "--retries",
-        help="Number of retries for transient Confluence errors",
-    ),
-    backoff: float = typer.Option(
-        1.0,
-        "--backoff",
-        help="Base backoff (seconds) for retries; exponential with attempt number",
-    ),
-    max_projects: int | None = typer.Option(
-        None,
-        "--max-projects",
-        help="Maximum number of projects to parse (budget limit). If omitted, no limit is applied.",
-    ),
-    step_timeout: float | None = typer.Option(
-        None,
-        "--step-timeout",
-        help="Per-step timeout (seconds) for individual page parsing. If omitted, no per-step timeout.",
-    ),
-    adaptive_depth: bool = typer.Option(
-        True,
-        "--adaptive-depth/--no-adaptive-depth",
-        help="Enable adaptive child-page traversal depth during registry parsing.",
-    ),
-    adaptive_max_depth: int = typer.Option(
-        2,
-        "--adaptive-max-depth",
-        min=1,
-        help="Maximum depth for adaptive traversal when --adaptive-depth is enabled.",
-    ),
-    prefetch_dir: Path | None = typer.Option(
-        None,
-        "--prefetch-dir",
-        help="Directory containing crawl-tree artifacts (crawl_manifest.json) to use instead of network calls.",
-    ),
-    staged_crawl_mode: str | None = typer.Option(
-        None,
-        "--staged-crawl-mode",
-        help=(
-            "Optional staged crawl mode for pre-parse crawl-tree bootstrap. "
-            "When set, parse-registry prefetches crawl_manifest.json via crawl-tree before parsing."
-        ),
-    ),
-    prefetch_depth: int = typer.Option(
-        2,
-        "--prefetch-depth",
-        min=1,
-        help="Traversal depth for crawl-tree prefetch bootstrap when --staged-crawl-mode is enabled.",
-    ),
-    exclude_empty_projects: bool = typer.Option(
-        False,
-        "--exclude-empty-projects/--include-empty-projects",
-        help="Exclude parsed projects that have no discovered document/confluence/bitbucket links.",
-    ),
-    state_dsn: str | None = typer.Option(
-        None,
-        "--state-dsn",
-        envvar=_STATE_DSN_ENV_VAR,
-        help=f"Persist discovery state to backend DSN (fallback: {_STATE_DSN_ENV_VAR}).",
-    ),
-) -> None:
-    """Parse project, department/container, or root registry structure without downloading materials.
-
-    Useful for validating parsing logic and discovering project structure
-    quickly. Outputs the ProjectRegistry model as JSON.
-
-    Registry parsing is deterministic checklist-first and lineage-aware:
-    - project-page entry parses one atomic project and reconstructs lineage upward
-    - department/container entry recursively aggregates child project parsing
-    - root entry recursively aggregates department/container parsing
-    - Root child classification uses 3.1/3.2 title patterns
-
-    Budget controls (Phase 24 TSK-243):
-    - --max-projects: Limit the number of projects parsed
-    - --step-timeout: Per-step timeout for individual page parsing
-
-    Adaptive depth controls (Phase 35):
-    - --adaptive-depth / --no-adaptive-depth: Toggle adaptive traversal heuristics
-    - --adaptive-max-depth: Cap adaptive traversal depth (default: 2)
-
-    Staged prefetch bootstrap:
-    - --staged-crawl-mode: Run a crawl-tree prefetch before parsing to generate local artifacts
-    - --prefetch-depth: Depth used by crawl-tree prefetch (default: 2)
-
-    The parser returns partial results when some pages fail or timeout,
-    with diagnostics in the output JSON indicating what failed.
-    """
-    resolved_state_dsn = _require_producer_state_dsn(state_dsn, command="parse-registry")
-
-    # Resolve checklist page ID if URL provided.
-    checklist_page_id: str | None = None
-    if checklist_page:
-        ref = _parse_confluence_ref(checklist_page)
-        checklist_page_id = ref.page_id or checklist_page
-    staged_crawl_mode = (staged_crawl_mode or "").strip() or None
-    resolved_prefetch_dir = prefetch_dir.resolve() if prefetch_dir else None
-
-    # Existing prefetch artifacts can run offline, but staged prefetch bootstrap always needs Confluence credentials.
-    if resolved_prefetch_dir is None or staged_crawl_mode is not None:
-        _validate_required_credentials(command="parse-registry", require_confluence=True)
-
-    # Compatibility hook for decomposed CLI tests that monkeypatch symbols on `vds_audit_orchestrator.cli`.
-    # Keep command implementation using those patched symbols when provided.
-    def _compat_cli_symbol(name: str, default: Any) -> Any:
-        try:
-            import vds_audit_orchestrator.cli as cli_module
-        except Exception:
-            return default
-        return getattr(cli_module, name, default)
-
-    confluence_client_cls = cast(
-        "type[ConfluenceCliClient]", _compat_cli_symbol("ConfluenceCliClient", ConfluenceCliClient)
-    )
-    registry_parser_cls = cast("type[RegistryPageParser]", _compat_cli_symbol("RegistryPageParser", RegistryPageParser))
-    resolve_page_id_fn = cast(
-        "Callable[[Any, str], Awaitable[str]]",
-        _compat_cli_symbol("_resolve_confluence_page_id", _resolve_confluence_page_id),
-    )
-
-    async def _run_parse(active_prefetch_dir: Path | None) -> ProjectRegistry:
-        inferred_server = _parse_confluence_ref(registry_page).server
-        # Let RegistryPageParser own retry/backoff policy for this command to avoid
-        # retry multiplication (client retries x parser retries).
-        confluence = confluence_client_cls.for_registry_discovery(
-            server=confluence_server or inferred_server,
-            timeout=timeout,
-        )
-        registry_id = await resolve_page_id_fn(confluence, registry_page)
-
-        if staged_crawl_mode:
-            if active_prefetch_dir is None:
-                raise RuntimeError("Internal error: staged crawl prefetch requires a prefetch directory.")
-            active_prefetch_dir.mkdir(parents=True, exist_ok=True)
-            crawl_tree = getattr(confluence, "crawl_tree", None)
-            if not callable(crawl_tree):
-                raise RuntimeError(
-                    "Staged crawl prefetch unavailable: Confluence client does not implement crawl_tree()."
-                )
-            crawl_tree_fn = cast("Callable[..., Awaitable[dict[str, Any]]]", crawl_tree)
-            await crawl_tree_fn(
-                registry_id,
-                active_prefetch_dir,
-                depth=prefetch_depth,
-                include_attachments=False,
-                emit_corpus=False,
-                emit_link_graph=False,
-                emit_tables=False,
-                emit_mentions=False,
-                body_format="storage",
-                cache_mode="off",
-                incremental=False,
-                manifest_path=active_prefetch_dir / "crawl_manifest.json",
-                staged_crawl_mode=staged_crawl_mode,
-            )
-        parser_kwargs: dict[str, Any] = {
-            "recursive_depth": recursive_depth,
-            "retries": retries,
-            "backoff": backoff,
-            "parse_concurrency": parse_concurrency,
-            "checklist_page": checklist_page_id,
-            "max_projects": max_projects,
-            "step_timeout": step_timeout,
-            "prefetch_dir": active_prefetch_dir,
-        }
-        parser_kwargs = _wire_registry_adaptive_depth_kwargs(
-            parser_kwargs,
-            adaptive_depth=adaptive_depth,
-            adaptive_max_depth=adaptive_max_depth,
-        )
-        parser = registry_parser_cls(confluence, **parser_kwargs)
-        return await parser.parse_registry(registry_id)
-
-    try:
-        tmp_prefetch_dir: tempfile.TemporaryDirectory[str] | None = None
-        if staged_crawl_mode and resolved_prefetch_dir is None:
-            tmp_prefetch_dir = tempfile.TemporaryDirectory(prefix="vds-audit-registry-prefetch-")
-            resolved_prefetch_dir = Path(tmp_prefetch_dir.name)
-        try:
-            registry = asyncio.run(_run_parse(resolved_prefetch_dir))
-        finally:
-            if tmp_prefetch_dir is not None:
-                tmp_prefetch_dir.cleanup()
-        if exclude_empty_projects:
-            registry.projects = [
-                project
-                for project in registry.projects
-                if project.documents or project.confluence_links or project.bitbucket_links
-            ]
-        scope_guidance = _apply_parse_registry_scope_guidance(
-            registry=registry,
-            registry_selector=registry_page,
-        )
-        _record_state_discovery(
-            state_dsn=resolved_state_dsn,
-            payload=_build_registry_state_payload(registry),
-        )
-
-        diagnostics = registry.diagnostics
-        result_status = diagnostics.result_status if diagnostics else "success"
-        parse_output = _build_parse_registry_output_payload(registry)
-
-        if _cli_state.get("json_only"):
-            _emit_json(parse_output)
-            if result_status == "fail_fast":
-                raise typer.Exit(2)
-            return
-
-        console.print_json(json.dumps(parse_output, indent=2, default=str))
-
-        count = len(registry.projects)
-        console.print(f"[green]Found {count} projects in registry.[/green]")
-        if scope_guidance:
-            console.print(f"[yellow]{scope_guidance}[/yellow]")
-        if diagnostics and diagnostics.result_message:
-            if result_status == "partial_success":
-                console.print(f"[yellow]{diagnostics.result_message}[/yellow]")
-            elif result_status == "fail_fast":
-                console.print(f"[red]{diagnostics.result_message}[/red]")
-                raise typer.Exit(2)
-
-    except typer.Exit:
-        raise
-    except Exception as exc:
-        message = f"Registry parsing failed: {exc}"
-        if _cli_state.get("json_only"):
-            _emit_json({"error": message})
-        else:
-            console.print(f"[red]{message}[/red]")
-        raise typer.Exit(1) from exc
-
-
-@app.command(name="parse-project")
-def parse_project(
-    project: str = typer.Option(
-        ...,
-        "--project",
-        help=("Project selector to resolve (numeric Confluence page ID or Confluence URL containing pageId=<ID>)."),
-    ),
-    state_dsn: str | None = typer.Option(
-        None,
-        "--state-dsn",
-        envvar=_STATE_DSN_ENV_VAR,
-        help=f"State backend DSN (fallback: {_STATE_DSN_ENV_VAR}).",
-    ),
-) -> None:
-    """Return the canonical project payload derived from centralized state."""
-
-    resolved_state_dsn = _require_consumer_state_dsn(state_dsn, command="parse-project")
-    manifest_payload, projects = _resolve_parse_manifest_payload(state_dsn=resolved_state_dsn, command="parse-project")
-    try:
-        normalized_selector = _normalize_parse_project_selector(project)
-        project_payload = select_project(projects, normalized_selector)
-    except (ValueError, LookupError) as exc:
-        _raise_project_selector_error(selector=project, projects=projects, exc=exc)
-
-    project_output = dict(project_payload)
-    # Phase 90 hard-cut: parse outputs should not expose legacy flat repo arrays.
-    project_output.pop("bitbucket", None)
-    project_output["evidence_project"] = _seed_evidence_project_from_payload(project_payload)
-    project_output["project_scope_targets"] = _seed_scope_targets_from_payload(project_payload)
-    project_repo_entries = (
-        [entry for entry in project_payload.get("bitbucket", []) if isinstance(entry, dict)]
-        if isinstance(project_payload.get("bitbucket"), list)
-        else []
-    )
-    project_storage_key = _project_storage_key_from_payload(project_payload)
-    project_output["project_storage_key"] = project_storage_key
-    bitbucket_contract = _build_grouped_bitbucket_projects_contract(
-        project_repo_entries,
-        project_storage_key=project_storage_key,
-    )
-
-    output = {
-        "registry": _parse_registry_metadata(manifest_payload),
-        "project": project_output,
-        "repo_coverage": _build_repo_coverage_summary(project_repo_entries),
-        "bitbucket_projects": bitbucket_contract["bitbucket_projects"],
-        "bitbucket_quarantine": bitbucket_contract["quarantine"],
-    }
-    _emit_parse_payload(output)
-
-
-@app.command(name="parse-repo")
-def parse_repo(
-    project: str = typer.Option(
-        ...,
-        "--project",
-        help=("Project selector to resolve (numeric Confluence page ID or Confluence URL containing pageId=<ID>)."),
-    ),
-    repo: str | None = typer.Option(
-        None,
-        "--repo",
-        help="Bitbucket repo key (slug), composite storage key ({project}:{repo}), or repo URL; required unless --all-repos.",
-    ),
-    all_repos: bool = typer.Option(
-        False,
-        "--all-repos",
-        help="Return seeds for every repository in the selected project.",
-    ),
-    bitbucket_project: str | None = typer.Option(
-        None,
-        "--bitbucket-project",
-        help="Restrict repository selection to a Bitbucket project key/group.",
-    ),
-    state_dsn: str | None = typer.Option(
-        None,
-        "--state-dsn",
-        envvar=_STATE_DSN_ENV_VAR,
-        help=f"State backend DSN (fallback: {_STATE_DSN_ENV_VAR}).",
-    ),
-) -> None:
-    """Return repository seed payload(s) derived from centralized state."""
-
-    if all_repos and repo:
-        raise typer.BadParameter("--repo cannot be combined with --all-repos.")
-    if not all_repos and not repo:
-        raise typer.BadParameter("--repo is required unless --all-repos is set.")
-    resolved_state_dsn = _require_consumer_state_dsn(state_dsn, command="parse-repo")
-    manifest_payload, projects = _resolve_parse_manifest_payload(state_dsn=resolved_state_dsn, command="parse-repo")
-    try:
-        normalized_selector = _normalize_parse_project_selector(project)
-        project_payload = select_project(projects, normalized_selector)
-    except (ValueError, LookupError) as exc:
-        _raise_project_selector_error(selector=project, projects=projects, exc=exc)
-
-    try:
-        if all_repos:
-            repo_entries = select_repos(
-                project_payload,
-                include_all=True,
-                bitbucket_project=bitbucket_project,
-            )
-        else:
-            assert repo is not None
-            repo_entries = select_repos(
-                project_payload,
-                repo_identifier=repo,
-                bitbucket_project=bitbucket_project,
-            )
-    except (ValueError, LookupError) as exc:
-        raise typer.BadParameter(str(exc)) from exc
-
-    ordered_repo_entries = _sort_repo_entries(repo_entries)
-    project_repo_entries = (
-        [entry for entry in project_payload.get("bitbucket", []) if isinstance(entry, dict)]
-        if isinstance(project_payload.get("bitbucket"), list)
-        else []
-    )
-    project_storage_key = _project_storage_key_from_payload(project_payload)
-    project_bitbucket_contract = _build_grouped_bitbucket_projects_contract(
-        project_repo_entries,
-        project_storage_key=project_storage_key,
-    )
-    selected_bitbucket_contract = _build_grouped_bitbucket_projects_contract(
-        ordered_repo_entries,
-        project_storage_key=project_storage_key,
-    )
-    selected_repo_payloads = [
-        dict(repo_payload)
-        for project_group in selected_bitbucket_contract["bitbucket_projects"]
-        if isinstance(project_group, dict)
-        for repo_payload in project_group.get("repos", [])
-        if isinstance(repo_payload, dict)
-    ]
-    output = {
-        "registry": _parse_registry_metadata(manifest_payload),
-        "project": {
-            "project_storage_key": project_storage_key,
-            "project_name": project_payload.get("project_name"),
-            "project_name_en": project_payload.get("project_name_en"),
-            "department": project_payload.get("department"),
-            "page_id": project_payload.get("page_id"),
-            "documents": project_payload.get("documents"),
-            "confluence_links": project_payload.get("confluence_links"),
-            "bitbucket_links": project_payload.get("bitbucket_links"),
-            "evidence_project": _seed_evidence_project_from_payload(project_payload),
-            "project_scope_targets": _seed_scope_targets_from_payload(project_payload),
-        },
-        "repos": selected_repo_payloads,
-        "repo_coverage": _build_repo_coverage_summary(project_repo_entries),
-        "selected_repo_coverage": _build_repo_coverage_summary(ordered_repo_entries),
-        "project_bitbucket_projects": project_bitbucket_contract["bitbucket_projects"],
-        "project_bitbucket_quarantine": project_bitbucket_contract["quarantine"],
-        "bitbucket_projects": selected_bitbucket_contract["bitbucket_projects"],
-        "bitbucket_quarantine": selected_bitbucket_contract["quarantine"],
-    }
-
-    # Phase 157 P157-B: persist scope snapshot for deterministic re-execution
-    scope_id = _persist_scope_snapshot(
-        state_dsn=resolved_state_dsn,
-        project_key=project_storage_key,
-        repo_entries=ordered_repo_entries,
-        bitbucket_contract=selected_bitbucket_contract,
-    )
-    if scope_id:
-        output["scope_id"] = scope_id
-
-    _emit_parse_payload(output)
-
-
-def _scope_snapshot_repo_keys(
-    *,
-    project_key: str,
-    repo_entries: list[dict[str, Any]],
-) -> list[str]:
-    from vds_audit_orchestrator.cli_impl import _repo_storage_key_from_repo_entry
-
-    repo_keys: list[str] = []
-    for entry in repo_entries:
-        if not isinstance(entry, dict):
-            continue
-        repo_storage_key = _repo_storage_key_from_repo_entry(entry, project_storage_key=project_key)
-        if repo_storage_key:
-            repo_keys.append(repo_storage_key)
-            continue
-        repo_slug = str(entry.get("repo_slug") or entry.get("repo_key") or "").strip()
-        if repo_slug:
-            fallback_repo_key = re.sub(r"\s+", "-", repo_slug.strip().lower())
-            repo_keys.append(f"{project_key}-{fallback_repo_key}")
-    return sorted({key for key in repo_keys if key})
-
-
-def _persist_scope_snapshot(
-    *,
-    state_dsn: str,
-    project_key: str,
-    repo_entries: list[dict[str, Any]],
-    bitbucket_contract: dict[str, Any],
-) -> str | None:
-    """Persist a scope snapshot to Postgres and return the scope_id.
-
-    Returns None if the store is unreachable (best-effort — parse-repo
-    output is still usable without a scope_id).
-    """
-    from vds_audit_orchestrator.state.store import PostgresStateStore
-
-    repo_keys = _scope_snapshot_repo_keys(project_key=project_key, repo_entries=repo_entries)
-    if not repo_keys:
-        return None
-
-    bb_project_keys: list[str] = []
-    partition_filters: dict[str, str] = {}
-    for group in bitbucket_contract.get("bitbucket_projects", []):
-        if isinstance(group, dict):
-            bb_key = str(group.get("bitbucket_project_key") or "").strip()
-            if bb_key:
-                bb_project_keys.append(bb_key)
-            repo_filter = str(group.get("repo_filter") or "").strip()
-            if bb_key and repo_filter:
-                partition_filters[bb_key] = repo_filter
-
-    try:
-        store = PostgresStateStore(state_dsn)
-        try:
-            scope_id = store.create_scope_snapshot(
-                project_key=project_key,
-                repo_keys=sorted(repo_keys),
-                bitbucket_project_keys=bb_project_keys or None,
-                partition_filters=partition_filters or None,
-            )
-            return scope_id
-        finally:
-            store.close()
-    except Exception:
-        return None
-
-
-@app.command(name="export-git-manifest")
-def export_git_manifest(
-    output: Path = typer.Option(
-        ...,
-        "--output",
-        help="Output path for vds-git compatible repository manifest YAML.",
-    ),
-    state_dsn: str | None = typer.Option(
-        None,
-        "--state-dsn",
-        envvar=_STATE_DSN_ENV_VAR,
-        help=f"State backend DSN (fallback: {_STATE_DSN_ENV_VAR}).",
-    ),
-    project_storage_key: str | None = typer.Option(
-        None,
-        "--project-storage-key",
-        help="Optional project storage key to export a single project.",
-    ),
-    workspace_root: Path | None = typer.Option(
-        None,
-        "--workspace-root",
-        help="Optional workspace root used to relativize repository paths.",
-    ),
-) -> None:
-    """Export vds-git compatible manifest YAML from centralized state metadata."""
-
-    resolved_state_dsn = _require_consumer_state_dsn(state_dsn, command="export-git-manifest")
-    _, projects = _resolve_parse_manifest_payload(state_dsn=resolved_state_dsn, command="export-git-manifest")
-
-    if project_storage_key:
-        try:
-            projects = [select_project_by_key(projects, project_storage_key)]
-        except (ValueError, LookupError) as exc:
-            raise typer.BadParameter(str(exc)) from exc
-
-    resolved_workspace_root = workspace_root.expanduser().resolve() if workspace_root is not None else None
-    payload = _build_git_manifest_payload(projects=projects, workspace_root=resolved_workspace_root)
-
-    output.parent.mkdir(parents=True, exist_ok=True)
-    output.write_text(
-        yaml.safe_dump(payload, sort_keys=False, allow_unicode=False),
-        encoding="utf-8",
-    )
-
-    if _cli_state.get("json_only"):
-        _emit_json({"output": str(output), "repositories": payload["repositories"]})
-        return
-    console.print(f"[green]Exported git manifest with {len(payload['repositories'])} repositories to {output}[/green]")