@ngocsangairvds/vsaf 3.2.13 → 3.2.15

package/tools/vds-scripts/pdf_orchestrator/.ruff_rules.py

@@ -1,350 +0,0 @@
-"""
-Custom Ruff rules for VDS PDF Orchestrator enterprise compliance and security.
-
-This module provides custom linting rules specific to:
-- SOC 2 compliance requirements
-- PII/PHI handling security
-- Audit trail requirements
-- Enterprise code quality standards
-- Security best practices for PDF processing
-"""
-
-from __future__ import annotations
-
-import ast
-from collections.abc import Generator, Sequence
-from typing import Any, ClassVar
-
-from ruff.api import (
-    LintResult,
-    Rule,
-    check,
-)
-
-
-class SOC2AuditTrailRule(Rule):
-    """
-    Ensure proper audit trail implementation for SOC 2 compliance.
-
-    This rule checks that functions dealing with sensitive data
-    have proper audit logging for compliance requirements.
-    """
-
-    code: str = "VDS001"
-    message: str = "Missing audit trail logging for SOC 2 compliance"
-    tags: ClassVar[set[str]] = {"security", "compliance", "soc2"}
-
-    def __init__(self) -> None:
-        self.audit_trail_functions = {
-            "log",
-            "audit",
-            "track",
-            "record",
-            "capture",
-            "audit_trail",
-            "log_activity",
-            "track_access",
-        }
-
-    @check
-    def check(
-        self, node: ast.AST, file: str, *args: Any, **kwargs: Any
-    ) -> Generator[LintResult]:
-        if not isinstance(node, ast.FunctionDef):
-            return
-
-        # Check if function handles sensitive data but lacks audit logging
-        handles_sensitive = self._handles_sensitive_data(node)
-        has_audit_logging = self._has_audit_logging(node)
-
-        if handles_sensitive and not has_audit_logging:
-            yield LintResult(
-                node.lineno,
-                node.col_offset,
-                self.message,
-                node,
-            )
-
-    def _handles_sensitive_data(self, node: ast.FunctionDef) -> bool:
-        """Check if function handles PII, PHI, or other sensitive data."""
-        sensitive_keywords = {
-            "pii",
-            "phi",
-            "ssn",
-            "credit_card",
-            "password",
-            "token",
-            "secret",
-            "credential",
-            "personal_data",
-            "user_data",
-            "health_record",
-            "medical",
-            "financial",
-            "payment",
-            "authentication",
-        }
-
-        # Check function name
-        func_name = node.name.lower()
-        if any(keyword in func_name for keyword in sensitive_keywords):
-            return True
-
-        # Check parameters
-        for arg in node.args.args:
-            if any(keyword in arg.arg.lower() for keyword in sensitive_keywords):
-                return True
-
-        # Check function body for sensitive operations
-        for stmt in ast.walk(node):
-            if isinstance(stmt, ast.Call):
-                if isinstance(stmt.func, ast.Name):
-                    name = stmt.func.id.lower()
-                    if any(keyword in name for keyword in sensitive_keywords):
-                        return True
-
-        return False
-
-    def _has_audit_logging(self, node: ast.FunctionDef) -> bool:
-        """Check if function has audit logging."""
-        for stmt in ast.walk(node):
-            if isinstance(stmt, ast.Call):
-                if isinstance(stmt.func, ast.Name):
-                    func_name = stmt.func.id
-                    if func_name in self.audit_trail_functions:
-                        return True
-                elif isinstance(stmt.func, ast.Attribute):
-                    attr_name = stmt.func.attr
-                    if attr_name in self.audit_trail_functions:
-                        return True
-
-        return False
-
-
-class EncryptionValidationRule(Rule):
-    """
-    Ensure proper encryption handling for sensitive data.
-
-    This rule checks that encryption functions are used
-    for sensitive data handling in compliance with SOC 2.
-    """
-
-    code: str = "VDS002"
-    message: str = "Potential unencrypted sensitive data handling"
-    tags: ClassVar[set[str]] = {"security", "compliance", "encryption"}
-
-    @check
-    def check(
-        self, node: ast.AST, file: str, *args: Any, **kwargs: Any
-    ) -> Generator[LintResult]:
-        if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
-            return
-
-        # Check for patterns that might indicate unencrypted handling
-        unencrypted_patterns = self._find_unencrypted_patterns(node)
-        for pattern in unencrypted_patterns:
-            yield LintResult(
-                pattern["lineno"],
-                pattern["col_offset"],
-                f"{self.message}: {pattern['description']}",
-                pattern["node"],
-            )
-
-    def _find_unencrypted_patterns(self, node: ast.FunctionDef) -> list[dict]:
-        """Find patterns that might indicate unencrypted data handling."""
-        patterns = []
-
-        # Check for file operations without encryption
-        for stmt in ast.walk(node):
-            if isinstance(stmt, ast.Call):
-                if isinstance(stmt.func, ast.Name):
-                    if stmt.func.id in ["open", "write", "save"]:
-                        # Check if this is writing sensitive data
-                        if self._is_writing_sensitive_data(stmt):
-                            patterns.append(
-                                {
-                                    "lineno": stmt.lineno,
-                                    "col_offset": stmt.col_offset,
-                                    "node": stmt,
-                                    "description": f"File operation '{stmt.func.id}' may write unencrypted sensitive data",
-                                }
-                            )
-
-        return patterns
-
-    def _is_writing_sensitive_data(self, call_node: ast.Call) -> bool:
-        """Check if the call is writing potentially sensitive data."""
-        # This is a simplified check - in practice, you'd want more sophisticated analysis
-
-        # Check function name context
-        return "write" in str(call_node).lower() or "save" in str(call_node).lower()
-
-
-class ErrorHandlingRule(Rule):
-    """
-    Ensure proper error handling for security operations.
-
-    This rule checks that security-critical functions
-    have proper error handling to prevent information leakage.
-    """
-
-    code: str = "VDS003"
-    message: str = "Insufficient error handling in security-critical function"
-    tags: ClassVar[set[str]] = {"security", "error-handling", "compliance"}
-
-    @check
-    def check(
-        self, node: ast.AST, file: str, *args: Any, **kwargs: Any
-    ) -> Generator[LintResult]:
-        if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
-            return
-
-        # Check if function is security-critical
-        if self._is_security_critical(node):
-            if not self._has_proper_error_handling(node):
-                yield LintResult(
-                    node.lineno,
-                    node.col_offset,
-                    self.message,
-                    node,
-                )
-
-    def _is_security_critical(self, node: ast.FunctionDef) -> bool:
-        """Check if function handles security-critical operations."""
-        security_keywords = {
-            "auth",
-            "login",
-            "authenticate",
-            "validate",
-            "secure",
-            "encrypt",
-            "decrypt",
-            "verify",
-            "check",
-            "audit",
-            "compliance",
-            "risk",
-            "security",
-            "access",
-            "permission",
-            "authorization",
-        }
-
-        func_name = node.name.lower()
-        return any(keyword in func_name for keyword in security_keywords)
-
-    def _has_proper_error_handling(self, node: ast.FunctionDef) -> bool:
-        """Check if function has proper error handling."""
-        has_try_except = False
-        has_logging = False
-
-        for stmt in ast.walk(node):
-            if isinstance(stmt, ast.Try):
-                has_try_except = True
-                # Check if except block logs errors
-                for handler in stmt.handlers:
-                    if self._has_logging_in_handler(handler):
-                        has_logging = True
-
-            # Check for logging statements
-            if isinstance(stmt, ast.Call):
-                if isinstance(stmt.func, ast.Name):
-                    if stmt.func.id in ["log", "logger", "error", "warn"]:
-                        has_logging = True
-
-        return has_try_except and has_logging
-
-    def _has_logging_in_handler(self, handler: ast.ExceptHandler) -> bool:
-        """Check if except handler has logging."""
-        for stmt in ast.walk(handler):
-            if isinstance(stmt, ast.Call):
-                if isinstance(stmt.func, ast.Name):
-                    if stmt.func.id in ["log", "logger", "error", "warn", "exception"]:
-                        return True
-        return False
-
-
-class InputValidationRule(Rule):
-    """
-    Ensure proper input validation for user inputs.
-
-    This rule checks that functions accepting user input
-    have proper validation to prevent injection attacks.
-    """
-
-    code: str = "VDS004"
-    message: str = "Missing input validation for user-controlled data"
-    tags: ClassVar[set[str]] = {"security", "input-validation", "compliance"}
-
-    @check
-    def check(
-        self, node: ast.AST, file: str, *args: Any, **kwargs: Any
-    ) -> Generator[LintResult]:
-        if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
-            return
-
-        # Check if function accepts external input
-        if self._accepts_external_input(node):
-            if not self._has_input_validation(node):
-                yield LintResult(
-                    node.lineno,
-                    node.col_offset,
-                    self.message,
-                    node,
-                )
-
-    def _accepts_external_input(self, node: ast.FunctionDef) -> bool:
-        """Check if function accepts external input."""
-        # Check parameter names that might indicate external input
-        input_keywords = {
-            "input",
-            "data",
-            "content",
-            "file",
-            "text",
-            "user_input",
-            "request",
-            "payload",
-            "query",
-            "filename",
-            "filepath",
-            "path",
-            "url",
-        }
-
-        for arg in node.args.args:
-            if any(keyword in arg.arg.lower() for keyword in input_keywords):
-                return True
-
-        return False
-
-    def _has_input_validation(self, node: ast.FunctionDef) -> bool:
-        """Check if function has input validation."""
-        validation_patterns = ["validate", "check", "verify", "sanitize", "clean"]
-
-        for stmt in ast.walk(node):
-            if isinstance(stmt, ast.Call):
-                if isinstance(stmt.func, ast.Name):
-                    func_name = stmt.func.id.lower()
-                    if any(pattern in func_name for pattern in validation_patterns):
-                        return True
-                elif isinstance(stmt.func, ast.Attribute):
-                    attr_name = stmt.func.attr.lower()
-                    if any(pattern in attr_name for pattern in validation_patterns):
-                        return True
-
-        return False
-
-
-# Rule registry
-RULES = {
-    "SOC2AuditTrailRule": SOC2AuditTrailRule,
-    "EncryptionValidationRule": EncryptionValidationRule,
-    "ErrorHandlingRule": ErrorHandlingRule,
-    "InputValidationRule": InputValidationRule,
-}
-
-
-def get_rules() -> Sequence[Rule]:
-    """Get all custom rules."""
-    return list(RULES.values())

package/tools/vds-scripts/pdf_orchestrator/.yamllint.yml

@@ -1,43 +0,0 @@
----
-# YAML Linting Configuration for VDS PDF Orchestrator
-extends: default
-
-rules:
-  # Enable all default rules
-  braces:
-    min-spaces-inside: 0
-    max-spaces-inside: 1
-  brackets:
-    min-spaces-inside: 0
-    max-spaces-inside: 0
-  colons:
-    max-spaces-before: 0
-    max-spaces-after: 1
-  commas:
-    max-spaces-before: 0
-    max-spaces-after: 1
-  comments:
-    min-spaces-from-content: 2
-  # Line length rules
-  line-length:
-    max: 120
-    allow-non-breakable-words: true
-    allow-non-breakable-inline-mappings: true
-  # Empty lines
-  empty-lines:
-    max: 2
-    max-start: 0
-    max-end: 1
-  # Indentation
-  indentation:
-    spaces: 2
-    indent-sequences: true
-    check-multi-line-strings: false
-  # Truthy values
-  truthy:
-    allowed-values: ['true', 'false', 'on', 'off', 'yes', 'no']
-  # Document start/end
-  document-start:
-    present: true
-  document-end:
-    present: false

package/tools/vds-scripts/pdf_orchestrator/DEVELOPMENT_PLAN.md

@@ -1,80 +0,0 @@
-# VDS PDF Orchestrator – Remediation Plan (Nov 2025)
-
-Purpose: address gaps between README claims and actual implementation; make features production-verifiable.
-
-## Scope & Goals
-
-- Implement missing functionality (safeuploads pipeline, real JIT usage or adjust claim, CVE-specific guards).
-- Align documentation/metrics with reality.
-- Expand automated tests to prove guarantees.
-- Add verification steps so each item is measurable.
-
-## Work Breakdown & Status
-
-- [x] **SafeUploads scope**
-  - Decision: SafeUploads package lacks PDF validator; we removed pipeline and rely on internal validators + peepdf.
-  - Skip claims in docs; no blocking on SafeUploads.
-  - Tests updated accordingly.
-- [x] **JIT/Free-threading realism**
-  - `optimize_with_jit` now marks optimization flag even when JIT unavailable and falls back cleanly.
-  - Free-threaded build detection added; fallback keeps processing_method aligned.
-  - Micro-benchmark/test hooks remain to be expanded if needed.
-- [x] **Free-threading fallback clarity**
-  - Requirement (3.13 ft build) logged; warnings emitted when not active.
-  - Integration tests cover warning + fallback path.
-- [ ] **CVE-specific mitigations**
-  - Implement targeted checks for CVE-2025-55197/62708/62707 (filter-specific byte/size limits) in `pdf_validator`.
-  - Add regression tests with crafted streams (use fixtures, not real exploits).
-- [ ] **YARA/Esprima/Phishing optional deps**
-  - Central capability detector; expose in health/status.
-  - Update README to mark “degrades gracefully” and list required extras.
-- [ ] **Metrics accuracy**
-  - Replace inflated metrics in README with generated values: LOC, test count, coverage (from CI).
-  - Add `scripts/generate_project_stats.py` to emit numbers for docs.
-- [ ] **MCP tool inventory**
-  - Enumerate tools (validation/extraction/conversion/info/health) in README with exact count.
-  - Add automated doc check (failing test if counts diverge).
-- [ ] **Test coverage expansion**
-  - Increase test functions: unit for SSRF, safeuploads, CVE guards; integration for MCP tools; performance smoke.
-  - Add security/perf markers and minimal fixtures.
-- [ ] **Compliance claim validation**
-  - Add minimal functional tests for PII redaction + Presidio fallback; mark as optional if deps missing.
-- [ ] **Release notes / docs sync**
-  - Update README, USER_GUIDE, FEATURES_GUIDE to match implemented behavior and prerequisites.
-- [ ] **Repository publish via vds-scripts**
-  - Use `uv run vds-cli git create` to create a new Bitbucket repo under INSURANCE project for `vds-pdf-orchestrator`.
-  - Configure manifest entry and push code via `uv run vds-cli git push --project insurance --branch master --path vds-pdf-orchestrator`.
-  - Verify remote visibility (`uv run vds-cli bitbucket repositories INSURANCE`) and add README badge if required.
-- [ ] **Docs/Guides sync**
-  - Regenerate README/USER_GUIDE/FEATURES_GUIDE sections to reflect implemented behavior and optional dependency matrix.
-  - Add short “Publishing & Branching” note (default branch `master`, vds-cli usage).
-  - Sync doc links in `docs/` to new repo location and update any cross-references in docs-nttc indexes if needed.
-- [ ] **CLI script consolidation & short aliases**
-  - Publish `project.scripts` entries so `uv run vds-pdf ...` is the primary path (no `python -m`). **(pdf done; other orchestrators still to align)**
-  - Align bitbucket/jira/confluence/git/pdf CLIs to the same pattern; ensure `--help` output shows short commands.
-  - Keep pandoc bundled (no system dependency) and document usage in README + AGENTS CLI best-practice section.
-  - Ensure default branch references in docs/automation use `master`.
-
-## Verification Checklist (run before close-out)
-
-- [ ] `uv run pytest` (unit + integration) passes.
-- [ ] `uv run python scripts/generate_project_stats.py` updates metrics; README reflects output.
-- [ ] `uv run vds-pdf validate cli_verification_test/test.pdf` succeeds.
-- [ ] `uv run vds-pdf validate --strict <malicious_fixture>` blocks with safeuploads finding.
-- [ ] MCP: start server and invoke `get_service_health` & `validate_pdf_security` via client smoke script; expect success codes.
-- [x] CLI aliases: `uv run vds-pdf --help` and `uv run vds-cli bitbucket --help` show short commands (no `python -m`). (pdf verified; other CLIs to re-check during orchestrator alignment)
-- [ ] Lint: `uv run ruff check .` clean.
-- [x] Optional deps matrix documented (safeuploads, esprima, yara_x, phishing-detection (opt-in via env), presidio).
-- [ ] Repo publish verified: `uv run vds-cli bitbucket repositories INSURANCE | grep vds-pdf-orchestrator` shows the new repo; initial push present on master.
-- [ ] Docs synced: README/USER_GUIDE/FEATURES_GUIDE regenerated; doc links validated.
-
-## Ownership & Timeline
-
-- Owner: PDF Orchestrator team (assign later).
-- Target: complete fixes and doc sync within current sprint (1 week).
-- Risks: availability of safeuploads/yara packages on build agents; Python 3.13 ft build flags.
-
-## Notes
-
-- No placeholder logic allowed—production code must gate on real scans or clearly documented optional features.
-- Add changelog entry summarizing risk reductions once tasks complete.

package/tools/vds-scripts/pdf_orchestrator/Dockerfile

@@ -1,87 +0,0 @@
-# Multi-stage Dockerfile for VDS PDF Orchestrator
-# Optimized for team development and production deployment
-
-# Build Stage - with uv caching for faster builds
-FROM python:3.14.4-slim AS builder
-
-# Install uv for fast dependency management
-RUN pip install --no-cache-dir uv
-
-# Set work directory
-WORKDIR /app
-
-# Build context is vds-scripts/ workspace root.
-# Use per-path `uv pip install` (avoids workspace-resolution requirement).
-COPY pdf_orchestrator /app/pdf_orchestrator
-COPY vds_cli_common /app/vds_cli_common
-
-WORKDIR /app/pdf_orchestrator
-
-# Install workspace deps as paths first, then pdf_orchestrator. uv has access
-# to vds-cli-common in site-packages by the time pdf's pyproject references it.
-RUN uv venv /app/.venv && \
-    uv pip install --python /app/.venv/bin/python --no-sources \
-        /app/vds_cli_common \
-        /app/pdf_orchestrator
-
-# Production Stage - Minimal runtime image
-FROM python:3.14.4-slim AS runtime
-
-# Install system dependencies for PDF processing
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    # PDF processing dependencies
-    poppler-utils \
-    ghostscript \
-    imagemagick \
-    # Image processing dependencies
-    libfreetype6 \
-    libpng-dev \
-    libjpeg-dev \
-    # Font support and text rendering
-    fonts-dejavu-core \
-    libpangoft2-1.0-0 \
-    libpangocairo-1.0-0 \
-    libharfbuzz0b \
-    libcairo2 \
-    libfribidi0 \
-    # Additional libraries for WeasyPrint and pandoc
-    libpango-1.0-0 \
-    libxml2 \
-    libxslt1.1 \
-    # File type detection
-    libmagic1 \
-    # Security and monitoring
-    curl \
-    # Clean up
-    && rm -rf /var/lib/apt/lists/*
-
-# Create non-root user for security
-RUN groupadd -r vds && useradd -r -g vds --home-dir=/app vds
-
-# Set work directory
-WORKDIR /app
-
-# Copy venv from builder to the SAME path so the entry-script shebangs
-# (`#!/app/.venv/bin/python`) keep working in the runtime stage. Copying to
-# a different path breaks shebangs without a rewrite step.
-COPY --from=builder --chown=vds:vds /app/.venv /app/.venv
-
-# Create data directories
-RUN mkdir -p /app/data /app/logs /app/cache && \
-    chown -R vds:vds /app/data /app/logs /app/cache /app/.venv
-
-# Switch to non-root user
-USER vds
-
-ENV PATH="/app/.venv/bin:$PATH"
-ENV PYTHONUNBUFFERED=1
-
-# Healthcheck: pdf is a CLI now (post v2.7.0 prune); verify the entry point
-# is callable. No HTTP server, no port to expose.
-HEALTHCHECK --interval=30s --timeout=10s --start-period=10s --retries=3 \
-    CMD /app/.venv/bin/vds-pdf --help > /dev/null 2>&1 || exit 1
-
-# pdf is a CLI tool; the MCP server module was removed in v2.7.0 dep prune
-# (fastmcp was a phantom dep). Run the typer CLI as the default — operators
-# override CMD to invoke specific subcommands.
-CMD ["/app/.venv/bin/vds-pdf", "--help"]