@ngocsangairvds/vsaf 4.1.8 → 4.1.9

package/skills/vds-skill/vds-scripts/confluence_orchestrator/src/confluence_orchestrator/tree_copier.py

@@ -1,431 +0,0 @@
-"""Confluence page-tree copy orchestrator.
-
-Three-phase implementation:
-  1. Crawl  — BFS walk of source tree → dict[page_id, PageNode]
-  2. Copy   — Create/update pages in BFS order, copy attachments + labels
-  3. Remap  — Rewrite internal links in all copied pages via regex
-"""
-
-from __future__ import annotations
-
-import io
-import re
-from collections import deque
-from dataclasses import dataclass, field
-from typing import TYPE_CHECKING
-
-import structlog
-
-if TYPE_CHECKING:
-    from .content import ContentClient
-
-_log = structlog.get_logger(__name__)
-
-_100MB = 100 * 1024 * 1024  # NFR-05: warn before downloading attachments > 100 MB
-
-# Pattern 1: module-level constant — matches any ri:content-id attribute.
-# Filtering against page_map is done in code (not the regex itself).
-_CONTENT_ID_RE = re.compile(r'ri:content-id="(\d+)"')
-
-
-# ---------------------------------------------------------------------------
-# Data structures
-# ---------------------------------------------------------------------------
-
-
-@dataclass(frozen=True)
-class PageNode:
-    """Immutable snapshot of one source page."""
-
-    page_id: str
-    title: str
-    body_storage: str
-    children: tuple[str, ...]  # child page IDs
-    attachments: tuple[AttachmentMeta, ...]
-    labels: tuple[str, ...]
-
-
-@dataclass(frozen=True)
-class AttachmentMeta:
-    """Metadata for one attachment (no content bytes — fetched on demand)."""
-
-    attachment_id: str
-    filename: str
-    content_type: str
-    size: int  # bytes
-
-
-@dataclass(frozen=True)
-class CopyOptions:
-    """Configuration for the copy operation."""
-
-    skip_attachments: bool = False
-    skip_labels: bool = False
-    skip_link_remap: bool = False
-    dry_run: bool = False
-    max_depth: int | None = None  # None = unlimited; ceiling enforced in CLI (NFR-05)
-
-
-@dataclass
-class CopyResult:
-    """Mutable result accumulator."""
-
-    page_map: dict[str, str] = field(default_factory=dict)  # source_id → target_id
-    pages_created: int = 0
-    pages_updated: int = 0  # root page is updated, not created
-    attachments_copied: int = 0
-    labels_copied: int = 0
-    links_remapped: int = 0
-    errors: list[str] = field(default_factory=list)
-
-
-# ---------------------------------------------------------------------------
-# TreeCopier
-# ---------------------------------------------------------------------------
-
-
-class TreeCopier:
-    """Copies a Confluence page tree from one space/server to another."""
-
-    def __init__(
-        self,
-        source_client: ContentClient,
-        target_client: ContentClient,
-        target_space_key: str,
-        source_space_key: str,
-        options: CopyOptions | None = None,
-    ) -> None:
-        self._source = source_client
-        self._target_client = target_client
-        self._target_space_key = target_space_key
-        self._source_space_key = source_space_key
-        self._options = options or CopyOptions()
-        self._log = _log.bind(
-            source_space=source_space_key,
-            target_space=target_space_key,
-        )
-        # Base URLs for Pattern 4 link remapping (absolute URL rewrite)
-        self._source_base_url = source_client.base_url
-        self._target_base_url = target_client.base_url
-
-    # ------------------------------------------------------------------
-    # Main entry point
-    # ------------------------------------------------------------------
-
-    def copy_tree(
-        self,
-        source_root_id: str,
-        target_root_id: str,
-    ) -> CopyResult:
-        """Main entry point. Runs all 3 phases."""
-        self._log.info("copy_tree_start", source_root=source_root_id, target_root=target_root_id)
-        page_tree = self._crawl_source(source_root_id)
-
-        if self._options.dry_run:
-            total_size = sum(att.size for node in page_tree.values() for att in node.attachments)
-            self._log.info(
-                "dry_run_summary",
-                pages=len(page_tree),
-                total_attachment_bytes=total_size,
-            )
-            return CopyResult(
-                page_map=dict.fromkeys(page_tree, ""),
-                pages_created=len(page_tree) - 1,  # root is updated, not created
-                attachments_copied=sum(len(n.attachments) for n in page_tree.values()),
-            )
-
-        result = self._copy_pages(page_tree, source_root_id, target_root_id)
-        if not self._options.skip_link_remap:
-            self._remap_links(result, page_tree)
-        self._log.info(
-            "copy_tree_done",
-            pages_created=result.pages_created,
-            pages_updated=result.pages_updated,
-            attachments_copied=result.attachments_copied,
-            labels_copied=result.labels_copied,
-            links_remapped=result.links_remapped,
-            errors=len(result.errors),
-        )
-        return result
-
-    # ------------------------------------------------------------------
-    # Phase 1: Crawl
-    # ------------------------------------------------------------------
-
-    def _crawl_source(self, root_id: str) -> dict[str, PageNode]:
-        """BFS walk of source tree. Returns {page_id: PageNode}."""
-        tree: dict[str, PageNode] = {}
-        # Queue items: (page_id, depth)
-        queue: deque[tuple[str, int]] = deque([(root_id, 0)])
-        while queue:
-            page_id, depth = queue.popleft()
-            if page_id in tree:
-                continue
-            if self._options.max_depth is not None and depth > self._options.max_depth:
-                continue
-            node = self._fetch_page_node(page_id)
-            tree[page_id] = node
-            self._log.debug("page_crawled", page_id=page_id, title=node.title, depth=depth)
-            for child_id in node.children:
-                queue.append((child_id, depth + 1))
-        self._log.info("crawl_complete", total_pages=len(tree))
-        return tree
-
-    def _fetch_page_node(self, page_id: str) -> PageNode:
-        """Fetch one page's content, attachments, and labels."""
-        page = self._source.get_page(page_id, expand=["body.storage", "space"])
-        body_storage = page.get("body", {}).get("storage", {}).get("value", "")
-
-        # Children
-        children_raw = self._source.get_child_pages(page_id)
-        children = tuple(str(c["id"]) for c in children_raw if isinstance(c, dict) and "id" in c)
-
-        # Attachments — list_attachments() returns {"results": [...]}
-        raw_atts = self._source.list_attachments(page_id).get("results", [])
-        attachments = tuple(
-            AttachmentMeta(
-                attachment_id=str(att["id"]),
-                filename=str(att.get("title", f"attachment_{att['id']}")),
-                content_type=str(att.get("metadata", {}).get("mediaType", "application/octet-stream")),
-                size=int(att.get("extensions", {}).get("fileSize", 0)),
-            )
-            for att in raw_atts
-            if isinstance(att, dict) and "id" in att
-        )
-
-        # Labels
-        labels = tuple(self._source.get_page_labels(page_id))
-
-        return PageNode(
-            page_id=page_id,
-            title=str(page.get("title", "")),
-            body_storage=body_storage,
-            children=children,
-            attachments=attachments,
-            labels=labels,
-        )
-
-    # ------------------------------------------------------------------
-    # Phase 2: Copy
-    # ------------------------------------------------------------------
-
-    def _copy_pages(
-        self,
-        page_tree: dict[str, PageNode],
-        source_root_id: str,
-        target_root_id: str,
-    ) -> CopyResult:
-        """Create/update pages in BFS order. Copy attachments and labels."""
-        result = CopyResult()
-        result.page_map[source_root_id] = target_root_id  # seed root mapping
-
-        # Queue items: (source_id, target_parent_id)
-        # For root the target_parent_id is unused — the target page ID comes from page_map.
-        queue: deque[tuple[str, str]] = deque([(source_root_id, "")])
-
-        while queue:
-            source_id, target_parent_id = queue.popleft()
-            node = page_tree.get(source_id)
-            if node is None:
-                continue
-
-            is_root = source_id == source_root_id
-            existing_target_id = result.page_map.get(source_id) if is_root else None
-
-            try:
-                new_target_id = self._copy_single_page(
-                    node,
-                    target_parent_id=target_parent_id,
-                    is_root=is_root,
-                    target_page_id=existing_target_id,
-                )
-            except Exception as exc:
-                msg = f"Failed to copy page {source_id!r} ({node.title!r}): {exc}"
-                self._log.error(
-                    "page_copy_failed",
-                    source_id=source_id,
-                    title=node.title,
-                    exc_info=True,
-                )
-                result.errors.append(msg)
-                # Skip entire subtree when a page fails (duplicate title, auth error, etc.)
-                continue
-
-            result.page_map[source_id] = new_target_id
-            if is_root:
-                result.pages_updated += 1
-            else:
-                result.pages_created += 1
-
-            # Attachments
-            if not self._options.skip_attachments and node.attachments:
-                result.attachments_copied += self._copy_attachments(node, new_target_id, result)
-
-            # Labels
-            if not self._options.skip_labels and node.labels:
-                result.labels_copied += self._copy_labels(node, new_target_id, result)
-
-            # Enqueue children (only those present in the crawled tree)
-            for child_id in node.children:
-                if child_id in page_tree:
-                    queue.append((child_id, new_target_id))
-
-        return result
-
-    def _copy_single_page(
-        self,
-        node: PageNode,
-        target_parent_id: str,
-        is_root: bool,
-        target_page_id: str | None,
-    ) -> str:
-        """Copy one page. Returns the target page ID."""
-        if is_root and target_page_id is not None:
-            # Update root page body; preserve the target page's own title
-            self._target_client.update_page(
-                target_page_id,
-                body=node.body_storage,
-                representation="storage",
-            )
-            self._log.info("page_updated", source_id=node.page_id, target_id=target_page_id)
-            return target_page_id
-
-        created = self._target_client.create_page(
-            space_key=self._target_space_key,
-            title=node.title,
-            body=node.body_storage,
-            parent_id=target_parent_id,
-            representation="storage",
-        )
-        new_id = str(created["id"])
-        self._log.info("page_created", source_id=node.page_id, target_id=new_id, title=node.title)
-        return new_id
-
-    def _copy_attachments(
-        self,
-        node: PageNode,
-        target_page_id: str,
-        result: CopyResult,
-    ) -> int:
-        """Download from source, upload to target. Returns count of copied attachments."""
-        copied = 0
-        for att in node.attachments:
-            if att.size > _100MB:
-                self._log.warning(
-                    "large_attachment",
-                    attachment_id=att.attachment_id,
-                    filename=att.filename,
-                    size_bytes=att.size,
-                )
-            try:
-                content = self._source._http.get_attachment_content(att.attachment_id)
-                self._target_client.upload_attachment(
-                    target_page_id,
-                    io.BytesIO(content),
-                    filename=att.filename,
-                    content_type=att.content_type,
-                )
-                copied += 1
-            except Exception as exc:
-                msg = f"Attachment {att.attachment_id!r} ({att.filename!r}) failed: {exc}"
-                self._log.error(
-                    "attachment_copy_failed",
-                    attachment_id=att.attachment_id,
-                    filename=att.filename,
-                    exc_info=True,
-                )
-                result.errors.append(msg)
-        return copied
-
-    def _copy_labels(
-        self,
-        node: PageNode,
-        target_page_id: str,
-        result: CopyResult,
-    ) -> int:
-        """Copy labels to target page. Returns count of successfully applied labels."""
-        copied = 0
-        for label in node.labels:
-            try:
-                self._target_client.set_page_label(target_page_id, label)
-                copied += 1
-            except Exception as exc:
-                msg = f"Label {label!r} on page {node.page_id!r} failed: {exc}"
-                self._log.warning(
-                    "label_copy_failed",
-                    label=label,
-                    source_id=node.page_id,
-                    target_page_id=target_page_id,
-                    exc_info=True,
-                )
-                result.errors.append(msg)
-        return copied
-
-    # ------------------------------------------------------------------
-    # Phase 3: Remap links
-    # ------------------------------------------------------------------
-
-    def _remap_links(
-        self,
-        result: CopyResult,
-        page_tree: dict[str, PageNode],
-    ) -> None:
-        """Rewrite internal links in all copied pages.
-
-        Four pattern classes (FR-01.5):
-          1. ri:content-id="<old_id>" → remapped ID (only IDs in page_map)
-          2. ri:space-key="<source_space>" → target space key
-          3. /display/<source_space>/ relative paths → target space
-          4. <source_base_url>/display/<source_space>/ absolute URLs → target
-        """
-        # Patterns 2–4 are compiled at runtime because they embed source_space_key
-        # and source_base_url which are only known at runtime.  They MUST NOT be
-        # pre-compiled as module-level constants — re.Pattern has no .format().
-        escaped_space = re.escape(self._source_space_key)
-        escaped_base_url = re.escape(self._source_base_url)
-
-        space_key_re = re.compile(rf'ri:space-key="{escaped_space}"')
-        display_url_re = re.compile(rf"/display/{escaped_space}/")
-        abs_url_re = re.compile(rf"{escaped_base_url}/display/{escaped_space}/")
-
-        def _remap_id(m: re.Match[str]) -> str:
-            mapped = result.page_map.get(m.group(1))
-            return f'ri:content-id="{mapped}"' if mapped else m.group(0)
-
-        for _source_id, target_id in result.page_map.items():
-            if not target_id:  # dry-run placeholder
-                continue
-            try:
-                page = self._target_client.get_page(target_id, expand=["body.storage"])
-                body = page.get("body", {}).get("storage", {}).get("value", "")
-                new_body = body
-
-                new_body, n1 = _CONTENT_ID_RE.subn(_remap_id, new_body)
-                new_body, n2 = space_key_re.subn(f'ri:space-key="{self._target_space_key}"', new_body)
-                # Pattern 4 must run before Pattern 3: abs URL contains /display/space/ as a
-                # substring. If Pattern 3 fires first it rewrites the space portion, leaving the
-                # source base URL intact and making Pattern 4 unable to match.
-                new_body, n4 = abs_url_re.subn(f"{self._target_base_url}/display/{self._target_space_key}/", new_body)
-                new_body, n3 = display_url_re.subn(f"/display/{self._target_space_key}/", new_body)
-
-                result.links_remapped += n1 + n2 + n3 + n4
-                if new_body != body:
-                    self._target_client.update_page(target_id, body=new_body)
-                    self._log.debug(
-                        "links_remapped",
-                        target_id=target_id,
-                        replacements=n1 + n2 + n3 + n4,
-                    )
-            except Exception as exc:
-                msg = f"Link remap failed for target {target_id!r}: {exc}"
-                self._log.error("remap_failed", target_id=target_id, exc_info=True)
-                result.errors.append(msg)
-
-
-__all__ = [
-    "AttachmentMeta",
-    "CopyOptions",
-    "CopyResult",
-    "PageNode",
-    "TreeCopier",
-]

package/skills/vds-skill/vds-scripts/confluence_orchestrator/sync_pdfs_from_markdown.py

@@ -1,203 +0,0 @@
-#!/usr/bin/env python3
-"""
-Sync PDF files from markdown sources.
-
-This script:
-1. Scans directory for markdown files
-2. Checks if corresponding PDF exists
-3. Compares modification times
-4. Regenerates PDFs using vds-cli pdf md2pdf when markdown is newer
-5. Reports summary of regenerated/skipped files
-"""
-
-from __future__ import annotations
-
-import subprocess
-from pathlib import Path
-
-import typer
-from rich.console import Console
-from rich.progress import Progress, SpinnerColumn, TextColumn
-from rich.table import Table
-
-app = typer.Typer(help="Sync PDF files from markdown sources")
-console = Console()
-
-# Path to vds-scripts for running vds-cli (relative to script location)
-VDS_SCRIPTS = Path(__file__).resolve().parent.parent
-
-
-def find_markdown_files(directory: Path, pattern: str | None = None, recursive: bool = True) -> list[Path]:
-    """Find all markdown files in directory, optionally matching pattern."""
-    md_files = sorted(directory.rglob("*.md")) if recursive else sorted(directory.glob("*.md"))
-    if pattern:
-        md_files = [f for f in md_files if pattern.lower() in f.name.lower()]
-    return md_files
-
-
-def get_corresponding_pdf(md_file: Path) -> Path:
-    """Get corresponding PDF path for a markdown file."""
-    return md_file.with_suffix(".pdf")
-
-
-def needs_regeneration(md_file: Path, pdf_file: Path, force: bool = False) -> bool:
-    """Check if PDF needs regeneration based on modification times."""
-    if force:
-        return True
-    if not pdf_file.exists():
-        return True
-    # Regenerate if markdown is newer than PDF
-    md_mtime = md_file.stat().st_mtime
-    pdf_mtime = pdf_file.stat().st_mtime
-    return md_mtime > pdf_mtime
-
-
-def regenerate_pdf(md_file: Path, pdf_file: Path, dry_run: bool = False) -> tuple[bool, str]:
-    """Regenerate PDF from markdown using vds-cli pdf md2pdf."""
-    if dry_run:
-        return True, "Would regenerate (dry-run)"
-
-    try:
-        # Ensure output directory exists
-        pdf_file.parent.mkdir(parents=True, exist_ok=True)
-
-        # Call vds-cli pdf md2pdf
-        cmd = [
-            "uv",
-            "run",
-            "--project",
-            "vds_cli",
-            "vds-cli",
-            "pdf",
-            "--",
-            "md2pdf",
-            str(md_file),
-            "--output",
-            str(pdf_file),
-        ]
-
-        result = subprocess.run(
-            cmd,
-            cwd=str(VDS_SCRIPTS),
-            capture_output=True,
-            text=True,
-            timeout=300,  # 5 minute timeout
-        )
-
-        if result.returncode == 0:
-            return True, "Regenerated successfully"
-        else:
-            error_msg = result.stderr.strip() or result.stdout.strip()
-            return False, f"Error: {error_msg}"
-
-    except subprocess.TimeoutExpired:
-        return False, "Timeout after 5 minutes"
-    except Exception as e:
-        return False, f"Exception: {e!s}"
-
-
-@app.command()
-def sync(
-    directory: Path = typer.Argument(Path(), help="Directory to scan for markdown files"),
-    pattern: str | None = typer.Option(None, "--pattern", "-p", help="Filter markdown files by pattern"),
-    dry_run: bool = typer.Option(False, "--dry-run", help="Preview changes without regenerating PDFs"),
-    force: bool = typer.Option(False, "--force", help="Regenerate all PDFs regardless of timestamps"),
-    recursive: bool = typer.Option(True, "--recursive/--no-recursive", help="Scan subdirectories recursively"),
-) -> None:
-    """Sync PDFs from markdown sources."""
-
-    directory = directory.resolve()
-    if not directory.exists() or not directory.is_dir():
-        console.print(f"[red]Error: Directory not found: {directory}[/red]")
-        raise typer.Exit(code=1) from None
-
-    try:
-        # Find markdown files
-        console.print(f"[bold]Scanning markdown files in:[/bold] {directory}")
-        md_files = find_markdown_files(directory, pattern, recursive)
-
-        if not md_files:
-            console.print("[yellow]No markdown files found[/yellow]")
-            raise typer.Exit(code=0) from None
-
-        console.print(f"Found [green]{len(md_files)}[/green] markdown file(s)")
-
-        # Check each markdown file
-        to_regenerate: list[tuple[Path, Path]] = []
-        skipped: list[tuple[Path, Path]] = []
-        errors: list[tuple[Path, str]] = []
-
-        console.print("\n[bold]Checking PDF status...[/bold]")
-        for md_file in md_files:
-            pdf_file = get_corresponding_pdf(md_file)
-            if needs_regeneration(md_file, pdf_file, force):
-                to_regenerate.append((md_file, pdf_file))
-            else:
-                skipped.append((md_file, pdf_file))
-
-        # Display sync plan
-        table = Table(title="Sync Plan", show_header=True, header_style="bold magenta")
-        table.add_column("Markdown File", style="cyan")
-        table.add_column("PDF File", style="green")
-        table.add_column("Action", style="yellow")
-
-        for md_file, pdf_file in to_regenerate:
-            status = "REGENERATE" if pdf_file.exists() else "CREATE"
-            table.add_row(
-                str(md_file.relative_to(directory)),
-                str(pdf_file.relative_to(directory)),
-                f"[yellow]{status}[/yellow]",
-            )
-
-        for md_file, pdf_file in skipped:
-            table.add_row(
-                str(md_file.relative_to(directory)),
-                str(pdf_file.relative_to(directory)),
-                "[green]SKIP (in sync)[/green]",
-            )
-
-        if to_regenerate:
-            console.print(table)
-            console.print(f"\n[bold]Summary:[/bold] {len(to_regenerate)} to regenerate, {len(skipped)} skipped")
-
-            if dry_run:
-                console.print("\n[yellow]DRY RUN: No PDFs regenerated[/yellow]")
-                raise typer.Exit(code=0) from None
-
-            # Regenerate PDFs
-            console.print("\n[bold]Regenerating PDFs...[/bold]")
-
-            with Progress(
-                SpinnerColumn(),
-                TextColumn("[progress.description]{task.description}"),
-                console=console,
-            ) as progress:
-                for md_file, pdf_file in to_regenerate:
-                    task = progress.add_task(f"Regenerating {md_file.name}...", total=None)
-
-                    success, message = regenerate_pdf(md_file, pdf_file, dry_run=False)
-                    progress.update(task, completed=True)
-
-                    if success:
-                        console.print(f"  [green]✓[/green] {md_file.name} → {pdf_file.name}")
-                    else:
-                        console.print(f"  [red]✗[/red] {md_file.name}: {message}")
-                        errors.append((md_file, message))
-
-            if errors:
-                console.print(f"\n[yellow]Completed with {len(errors)} error(s)[/yellow]")
-                for md_file, error_msg in errors:
-                    console.print(f"  • {md_file.name}: {error_msg}")
-                raise typer.Exit(code=2) from None
-            else:
-                console.print("\n[bold green]✓ All PDFs regenerated successfully![/bold green]")
-        else:
-            console.print("\n[green]All PDFs are in sync with markdown sources[/green]")
-
-    except Exception as exc:
-        console.print(f"[red]Error: {exc}[/red]")
-        raise typer.Exit(code=1) from None
-
-
-if __name__ == "__main__":
-    app()