@ngocsangairvds/vsaf 3.1.27 → 3.2.1

package/tools/vds-scripts/confluence_orchestrator/src/confluence_orchestrator/orchestration.py

@@ -0,0 +1,165 @@
+"""Batch polling and webhook management utilities for Confluence Server/DC."""
+
+from __future__ import annotations
+
+import builtins
+import time
+from collections.abc import Iterable, Iterator, Sequence
+from dataclasses import dataclass, field
+from typing import Any
+
+from .http import ConfluenceClient
+
+
+@dataclass(slots=True)
+class BatchMetrics:
+    """Execution metrics captured during a batch scan."""
+
+    scanned: int = 0
+    pages_processed: int = 0
+    requests: int = 0
+    started_at: float = field(default_factory=time.time)
+    finished_at: float | None = None
+
+    def mark_request(self) -> None:
+        self.requests += 1
+
+    def mark_processed(self, count: int) -> None:
+        self.pages_processed += count
+
+    def mark_finished(self) -> None:
+        self.finished_at = time.time()
+
+    @property
+    def duration_seconds(self) -> float | None:
+        if self.finished_at is None:
+            return None
+        return round(self.finished_at - self.started_at, 3)
+
+
+class BatchRunner:
+    """Polling utilities for change detection using REST v1 CQL queries."""
+
+    def __init__(
+        self,
+        http_client: ConfluenceClient,
+        *,
+        expand: Sequence[str] | None = None,
+    ) -> None:
+        self._http = http_client
+        self._expand = expand
+        self._last_metrics: BatchMetrics | None = None
+
+    def scan(
+        self,
+        cql: str,
+        *,
+        limit: int = 25,
+        max_results: int | None = None,
+        start: int = 0,
+        expand: Sequence[str] | None = None,
+    ) -> Iterator[dict[str, Any]]:
+        """Yield individual content results for the given CQL query."""
+
+        metrics = BatchMetrics()
+        self._last_metrics = metrics
+        applied_expand = expand if expand is not None else self._expand
+        fetched = 0
+        current_start = start
+
+        while True:
+            remaining = None if max_results is None else max_results - fetched
+            page_limit = limit if remaining is None else min(limit, remaining)
+            params = {
+                "cql": cql,
+                "limit": page_limit,
+                "start": current_start,
+            }
+            if applied_expand:
+                params["expand"] = ",".join(applied_expand)
+
+            metrics.mark_request()
+            # Use SDK's cql method instead of raw request
+            payload = self._http.search_cql(cql, limit=page_limit, expand=applied_expand, start=current_start)
+            results = payload.get("results", [])
+            count = len(results)
+            if not results:
+                break
+            metrics.mark_processed(count)
+            metrics.scanned += count
+            for item in results:
+                fetched += 1
+                yield item
+
+            current_start = payload.get("start", current_start) + count
+            if max_results is not None and fetched >= max_results:
+                break
+
+        metrics.mark_finished()
+        self._last_metrics = metrics
+
+    def snapshot(
+        self,
+        cql: str,
+        *,
+        limit: int = 25,
+        max_results: int | None = None,
+        expand: Sequence[str] | None = None,
+    ) -> dict[str, Any]:
+        """Return an aggregated snapshot for reporting pipelines."""
+
+        results: list[dict[str, Any]] = []
+        for item in self.scan(cql, limit=limit, max_results=max_results, expand=expand):
+            results.append(item)
+        return {
+            "cql": cql,
+            "limit": limit,
+            "maxResults": max_results,
+            "count": len(results),
+            "results": results,
+        }
+
+    @property
+    def last_metrics(self) -> BatchMetrics | None:
+        return self._last_metrics
+
+
+class WebhookClient:
+    """Manage Confluence Server/DC webhooks via REST v1."""
+
+    _BASE_PATH = "/rest/webhooks/1.0/webhook"
+
+    def __init__(self, http_client: ConfluenceClient) -> None:
+        self._http = http_client
+
+    def create(
+        self,
+        *,
+        name: str,
+        url: str,
+        events: Iterable[str],
+        active: bool = True,
+        filters: dict[str, Any] | None = None,
+    ) -> dict[str, Any]:
+        payload = {
+            "name": name,
+            "url": url,
+            "events": list(events),
+            "excludeBody": False,
+            "active": active,
+        }
+        if filters:
+            payload["filters"] = filters
+        response = self._http.request("POST", self._BASE_PATH, json=payload)
+        return response.json()
+
+    def list(self) -> builtins.list[dict[str, Any]]:
+        response = self._http.request("GET", self._BASE_PATH)
+        payload = response.json()
+        return payload if isinstance(payload, list) else payload.get("results", [])
+
+    def delete(self, webhook_id: str) -> None:
+        self._http.request("DELETE", f"{self._BASE_PATH}/{webhook_id}")
+
+
+__all__ = ["BatchRunner", "BatchMetrics", "WebhookClient"]

package/tools/vds-scripts/confluence_orchestrator/src/confluence_orchestrator/reporting.py

@@ -0,0 +1,78 @@
+"""Run reporting utilities for the Confluence orchestrator."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+
+TIMESTAMP_FMT = "%Y-%m-%dT%H-%M-%SZ"
+
+
+@dataclass(slots=True)
+class RunSummary:
+    command: str
+    args: list[str]
+    server: str
+    exit_code: int
+    duration_ms: int
+    started_at: datetime
+    finished_at: datetime
+
+    @property
+    def success(self) -> bool:
+        return self.exit_code == 0
+
+
+def _ensure_directory(base: Path, timestamp: datetime) -> Path:
+    day_dir = base / str(timestamp.year) / f"{timestamp.month:02d}" / f"{timestamp.day:02d}"
+    day_dir.mkdir(parents=True, exist_ok=True)
+    return day_dir
+
+
+def write_reports(
+    summary: RunSummary,
+    *,
+    base_dir: Path,
+    include_markdown: bool = True,
+) -> Path:
+    """Persist JSON (and optional Markdown) reports and return JSON path."""
+
+    timestamp = summary.finished_at.astimezone(UTC)
+    directory = _ensure_directory(base_dir, timestamp)
+    slug = f"{timestamp.strftime(TIMESTAMP_FMT)}_{summary.command}"
+
+    json_path = directory / f"{slug}.json"
+    json_payload = {
+        "command": summary.command,
+        "args": summary.args,
+        "server": summary.server,
+        "exit_code": summary.exit_code,
+        "duration_ms": summary.duration_ms,
+        "started_at": summary.started_at.astimezone(UTC).isoformat().replace("+00:00", "Z"),
+        "finished_at": summary.finished_at.astimezone(UTC).isoformat().replace("+00:00", "Z"),
+        "success": summary.success,
+    }
+    json_path.write_text(json.dumps(json_payload, indent=2, sort_keys=True))
+
+    if include_markdown:
+        md_path = directory / f"{slug}.md"
+        status = "✅ Success" if summary.success else "❌ Failure"
+        md_lines = [
+            f"# Confluence Run Report — {summary.command}",
+            "",
+            f"- **Status**: {status} (exit code {summary.exit_code})",
+            f"- **Server**: {summary.server}",
+            f"- **Duration**: {summary.duration_ms} ms",
+            f"- **Started**: {json_payload['started_at']}",
+            f"- **Finished**: {json_payload['finished_at']}",
+        ]
+        if summary.args:
+            md_lines.append(f"- **Arguments**: {' '.join(summary.args)}")
+        md_path.write_text("\n".join(md_lines) + "\n")
+
+    return json_path
+
+
+__all__ = ["RunSummary", "write_reports"]

package/tools/vds-scripts/confluence_orchestrator/src/confluence_orchestrator/tree.py

@@ -0,0 +1,121 @@
+"""Confluence page tree traversal and export utilities."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+import structlog
+
+from .http import ConfluenceClient
+
+
+@dataclass
+class TreeNode:
+    """Represents a Confluence page in the tree."""
+
+    page_id: str
+    title: str
+    space_key: str
+    url: str
+    children: list[TreeNode] = field(default_factory=list)
+    attachments: list[dict[str, Any]] = field(default_factory=list)
+    body: str | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert tree node to dictionary."""
+        return {
+            "id": self.page_id,
+            "title": self.title,
+            "space": self.space_key,
+            "url": self.url,
+            "body": self.body,
+            "attachments": self.attachments,
+            "children": [child.to_dict() for child in self.children],
+        }
+
+
+class TreeExporter:
+    """Recursively fetch Confluence page tree."""
+
+    def __init__(self, client: ConfluenceClient) -> None:
+        self._client = client
+        self._log = structlog.get_logger(__name__)
+
+    def build_tree(
+        self,
+        root_id: str,
+        *,
+        depth: int = -1,
+        include_content: bool = False,
+        include_attachments: bool = False,
+    ) -> TreeNode:
+        """
+        Recursively build a tree of pages.
+
+        Args:
+            root_id: The root page ID to start from.
+            depth: How deep to recurse. -1 for infinite, 0 for just root.
+            include_content: Whether to fetch page body (storage format).
+            include_attachments: Whether to fetch attachment metadata.
+        """
+        self._log.info("fetching_page", page_id=root_id)
+
+        # Prepare expand fields
+        expand = ["space"]
+        if include_content:
+            expand.append("body.storage")
+
+        # Fetch page details
+        page = self._client.get_page(root_id, expand=expand)
+        title = page.get("title", "Unknown")
+        space_key = page.get("space", {}).get("key", "Unknown")
+        links = page.get("_links", {})
+        base_url = links.get("base", "")
+        web_ui = links.get("webui", "")
+        full_url = f"{base_url}{web_ui}" if base_url and web_ui else web_ui
+
+        body = None
+        if include_content:
+            body = page.get("body", {}).get("storage", {}).get("value")
+
+        attachments = []
+        if include_attachments:
+            try:
+                # This fetches metadata, not content
+                attachments = self._client.get_attachments(root_id)
+            except Exception as e:
+                self._log.warning("failed_fetching_attachments", page_id=root_id, error=str(e))
+
+        node = TreeNode(
+            page_id=root_id,
+            title=title,
+            space_key=space_key,
+            url=full_url,
+            children=[],
+            attachments=attachments,
+            body=body,
+        )
+
+        # Recurse if depth allows
+        if depth != 0:
+            try:
+                # Fetch immediate children summaries
+                children_summaries = self._client.get_child_pages(root_id)
+
+                next_depth = depth - 1 if depth > 0 else -1
+
+                for child_summary in children_summaries:
+                    child_id = child_summary.get("id")
+                    if child_id:
+                        child_node = self.build_tree(
+                            child_id,
+                            depth=next_depth,
+                            include_content=include_content,
+                            include_attachments=include_attachments,
+                        )
+                        node.children.append(child_node)
+            except Exception as e:
+                self._log.warning("failed_fetching_children", page_id=root_id, error=str(e))
+
+        return node

package/tools/vds-scripts/confluence_orchestrator/sync_pdfs_from_markdown.py

@@ -0,0 +1,213 @@
+#!/usr/bin/env python3
+"""
+Sync PDF files from markdown sources.
+
+This script:
+1. Scans directory for markdown files
+2. Checks if corresponding PDF exists
+3. Compares modification times
+4. Regenerates PDFs using vds-cli pdf md2pdf when markdown is newer
+5. Reports summary of regenerated/skipped files
+"""
+
+from __future__ import annotations
+
+import subprocess
+import sys
+from pathlib import Path
+from typing import Any
+
+import typer
+from rich.console import Console
+from rich.progress import Progress, SpinnerColumn, TextColumn
+from rich.table import Table
+
+app = typer.Typer(help="Sync PDF files from markdown sources")
+console = Console()
+
+# Path to vds-scripts for running vds-cli (relative to script location)
+VDS_SCRIPTS = Path(__file__).resolve().parent.parent
+
+
+def find_markdown_files(directory: Path, pattern: str | None = None, recursive: bool = True) -> list[Path]:
+    """Find all markdown files in directory, optionally matching pattern."""
+    if recursive:
+        md_files = sorted(directory.rglob("*.md"))
+    else:
+        md_files = sorted(directory.glob("*.md"))
+    if pattern:
+        md_files = [f for f in md_files if pattern.lower() in f.name.lower()]
+    return md_files
+
+
+def get_corresponding_pdf(md_file: Path) -> Path:
+    """Get corresponding PDF path for a markdown file."""
+    return md_file.with_suffix(".pdf")
+
+
+def needs_regeneration(md_file: Path, pdf_file: Path, force: bool = False) -> bool:
+    """Check if PDF needs regeneration based on modification times."""
+    if force:
+        return True
+    if not pdf_file.exists():
+        return True
+    # Regenerate if markdown is newer than PDF
+    md_mtime = md_file.stat().st_mtime
+    pdf_mtime = pdf_file.stat().st_mtime
+    return md_mtime > pdf_mtime
+
+
+def regenerate_pdf(md_file: Path, pdf_file: Path, dry_run: bool = False) -> tuple[bool, str]:
+    """Regenerate PDF from markdown using vds-cli pdf md2pdf."""
+    if dry_run:
+        return True, "Would regenerate (dry-run)"
+
+    try:
+        # Ensure output directory exists
+        pdf_file.parent.mkdir(parents=True, exist_ok=True)
+
+        # Call vds-cli pdf md2pdf
+        cmd = [
+            "uv",
+            "run",
+            "--project",
+            "vds_cli",
+            "vds-cli",
+            "pdf",
+            "--",
+            "md2pdf",
+            str(md_file),
+            "--output",
+            str(pdf_file),
+        ]
+
+        result = subprocess.run(
+            cmd,
+            cwd=str(VDS_SCRIPTS),
+            capture_output=True,
+            text=True,
+            timeout=300,  # 5 minute timeout
+        )
+
+        if result.returncode == 0:
+            return True, "Regenerated successfully"
+        else:
+            error_msg = result.stderr.strip() or result.stdout.strip()
+            return False, f"Error: {error_msg}"
+
+    except subprocess.TimeoutExpired:
+        return False, "Timeout after 5 minutes"
+    except Exception as e:
+        return False, f"Exception: {str(e)}"
+
+
+@app.command()
+def sync(
+    directory: Path = typer.Argument(
+        Path("."), help="Directory to scan for markdown files"
+    ),
+    pattern: str | None = typer.Option(None, "--pattern", "-p", help="Filter markdown files by pattern"),
+    dry_run: bool = typer.Option(False, "--dry-run", help="Preview changes without regenerating PDFs"),
+    force: bool = typer.Option(False, "--force", help="Regenerate all PDFs regardless of timestamps"),
+    recursive: bool = typer.Option(True, "--recursive/--no-recursive", help="Scan subdirectories recursively"),
+) -> None:
+    """Sync PDFs from markdown sources."""
+
+    directory = directory.resolve()
+    if not directory.exists() or not directory.is_dir():
+        console.print(f"[red]Error: Directory not found: {directory}[/red]")
+        raise typer.Exit(code=1) from None
+
+    try:
+        # Find markdown files
+        console.print(f"[bold]Scanning markdown files in:[/bold] {directory}")
+        md_files = find_markdown_files(directory, pattern, recursive)
+
+        if not md_files:
+            console.print("[yellow]No markdown files found[/yellow]")
+            raise typer.Exit(code=0) from None
+
+        console.print(f"Found [green]{len(md_files)}[/green] markdown file(s)")
+
+        # Check each markdown file
+        to_regenerate: list[tuple[Path, Path]] = []
+        skipped: list[tuple[Path, Path]] = []
+        errors: list[tuple[Path, str]] = []
+
+        console.print("\n[bold]Checking PDF status...[/bold]")
+        for md_file in md_files:
+            pdf_file = get_corresponding_pdf(md_file)
+            if needs_regeneration(md_file, pdf_file, force):
+                to_regenerate.append((md_file, pdf_file))
+            else:
+                skipped.append((md_file, pdf_file))
+
+        # Display sync plan
+        table = Table(title="Sync Plan", show_header=True, header_style="bold magenta")
+        table.add_column("Markdown File", style="cyan")
+        table.add_column("PDF File", style="green")
+        table.add_column("Action", style="yellow")
+
+        for md_file, pdf_file in to_regenerate:
+            status = "REGENERATE" if pdf_file.exists() else "CREATE"
+            table.add_row(
+                str(md_file.relative_to(directory)),
+                str(pdf_file.relative_to(directory)),
+                f"[yellow]{status}[/yellow]",
+            )
+
+        for md_file, pdf_file in skipped:
+            table.add_row(
+                str(md_file.relative_to(directory)),
+                str(pdf_file.relative_to(directory)),
+                "[green]SKIP (in sync)[/green]",
+            )
+
+        if to_regenerate:
+            console.print(table)
+            console.print(f"\n[bold]Summary:[/bold] {len(to_regenerate)} to regenerate, {len(skipped)} skipped")
+
+            if dry_run:
+                console.print("\n[yellow]DRY RUN: No PDFs regenerated[/yellow]")
+                raise typer.Exit(code=0) from None
+
+            # Regenerate PDFs
+            console.print("\n[bold]Regenerating PDFs...[/bold]")
+
+            with Progress(
+                SpinnerColumn(),
+                TextColumn("[progress.description]{task.description}"),
+                console=console,
+            ) as progress:
+                for md_file, pdf_file in to_regenerate:
+                    task = progress.add_task(f"Regenerating {md_file.name}...", total=None)
+
+                    success, message = regenerate_pdf(md_file, pdf_file, dry_run=False)
+                    progress.update(task, completed=True)
+
+                    if success:
+                        console.print(f"  [green]✓[/green] {md_file.name} → {pdf_file.name}")
+                    else:
+                        console.print(f"  [red]✗[/red] {md_file.name}: {message}")
+                        errors.append((md_file, message))
+
+            if errors:
+                console.print(f"\n[yellow]Completed with {len(errors)} error(s)[/yellow]")
+                for md_file, error_msg in errors:
+                    console.print(f"  • {md_file.name}: {error_msg}")
+                raise typer.Exit(code=2) from None
+            else:
+                console.print(f"\n[bold green]✓ All PDFs regenerated successfully![/bold green]")
+        else:
+            console.print("\n[green]All PDFs are in sync with markdown sources[/green]")
+
+    except Exception as exc:
+        console.print(f"[red]Error: {exc}[/red]")
+        raise typer.Exit(code=1) from None
+
+
+if __name__ == "__main__":
+    app()
+
+
+