@ngocsangairvds/vsaf 4.1.9 → 4.1.11

package/skills/vds-skill/runtime/platform_core/pyproject.toml

@@ -0,0 +1,49 @@
+[project]
+name = "vds-platform-core"
+version = "0.1.0"
+description = "Platform-level error hierarchy, severity registry, and logging factory for VDS"
+requires-python = ">=3.12"
+license = { text = "MIT" }
+authors = [{ name = "VDS Team" }]
+dependencies = [
+    "structlog>=24.4.0",
+    "pydantic>=2",
+    "pydantic-settings>=2",
+    "stamina>=24.2",
+    "httpx>=0.27",
+    "alembic>=1.18.4,<2",
+    "sqlalchemy>=2.0,<3",
+]
+
+[dependency-groups]
+dev = [
+    "pytest>=7.0.0",
+    "pytest-asyncio>=0.23",
+    "pytest-cov>=4.0.0",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/vds_platform_core"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["src"]
+addopts = "-v --tb=short"
+asyncio_mode = "auto"
+
+[tool.coverage.run]
+source = ["src/vds_platform_core"]
+branch = true
+
+[tool.coverage.report]
+exclude_lines = [
+    "pragma: no cover",
+    "if TYPE_CHECKING:",
+    "if __name__ == .__main__.:",
+    "@abstractmethod",
+]
+fail_under = 80

package/skills/vds-skill/runtime/platform_core/src/vds_platform_core/__init__.py

@@ -0,0 +1,16 @@
+"""VDS Platform Core — shared error hierarchy, severity registry, and logging factory."""
+
+from __future__ import annotations
+
+__version__ = "0.1.0"
+
+from vds_platform_core.config import HTTPClientSettings
+from vds_platform_core.protocols import CredentialResolver, HTTPClient, RetryPolicy
+
+__all__ = [
+    "CredentialResolver",
+    "HTTPClient",
+    "HTTPClientSettings",
+    "RetryPolicy",
+    "__version__",
+]

package/skills/vds-skill/runtime/platform_core/src/vds_platform_core/alembic/__init__.py

@@ -0,0 +1,18 @@
+"""vds_platform_core.alembic — DRY Alembic env.py helper (ADR-INF-12).
+
+Exposes the three-function public API used by each service's alembic/env.py.
+Landed in WP-5.2; all 4 service env.py files delegate to these helpers,
+reducing per-service boilerplate from 138-170 LOC to ~25 LOC.
+"""
+
+from vds_platform_core.alembic.runtime import (
+    resolve_database_url,
+    run_migrations_offline,
+    run_migrations_online,
+)
+
+__all__ = [
+    "resolve_database_url",
+    "run_migrations_offline",
+    "run_migrations_online",
+]

package/skills/vds-skill/runtime/platform_core/src/vds_platform_core/alembic/runtime.py

@@ -0,0 +1,139 @@
+"""DRY Alembic env.py runtime helpers (WP-5.2 / ADR-INF-12).
+
+Centralises the ~80% boilerplate shared across 4 service alembic/env.py files
+(memory, audit, progress, scheduler). Each service's env.py is ~25 LOC of
+declarative configuration.
+
+API:
+    resolve_database_url  — read DSN from a single canonical env var
+    run_migrations_offline — configure Alembic context for offline SQL output
+    run_migrations_online  — create engine, configure context, run migrations
+
+Design constraints (per ADR-INF-11):
+  - NEVER call connection.execute(text("SET search_path ...")) here.
+    Under SQLAlchemy 2.0 that statement opens an implicit autocommit
+    transaction that closes before Alembic's own begin_transaction() runs,
+    silently breaking the alembic_version CREATE/INSERT. Per-app search_path
+    is handled at the orchestrator level (psycopg `_connect()` helper for
+    progress; SQLAlchemy `connect` event listener for memory) — see
+    references/connection-pooling.md F21/D1 caveats.
+  - NullPool is always used — no connection reuse across CLI invocations.
+  - include_schemas=True is always set — required for cross-schema management.
+
+v2.15.12: Removed `force_psycopg_v3` param + URL-rewriting / `?options=`
+stripping logic. Per-stanza connect_query in pgbouncer.ini handles search_path
+at backend startup; URLs no longer carry `?options=` fragments. Function
+signature simplified from a fallback-list `env_vars: Sequence[str]` to a
+single canonical `env_var: str` per the no-fallback directive.
+"""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Callable
+from typing import Any
+
+from alembic import context
+from sqlalchemy import MetaData, engine_from_config, pool
+
+# Type alias matching alembic's include_name callback signature.
+IncludeNameCallback = Callable[[str | None, str, dict[str, Any]], bool]
+
+
+def resolve_database_url(*, env_var: str) -> str:
+    """Return the database URL for Alembic migrations.
+
+    Reads a single canonical env var. No fallback chain; if unset, raises.
+    Post-v2.15.12 each service's env.py declares ONE env var (e.g., audit uses
+    ``VDS_AUDIT_STATE_DSN``, memory/progress/scheduler use ``DATABASE_URL``).
+    Mixed compose-vs-host-mode operator workflows must set the canonical var
+    explicitly — there is no implicit fallback to a typed legacy variable.
+
+    Raises:
+        RuntimeError: if the env var is unset or empty.
+    """
+    url = os.environ.get(env_var)
+    if not url:
+        raise RuntimeError(
+            f"Required environment variable is not set: {env_var}. Set it before running alembic or pass -x db=<url>."
+        )
+    return url
+
+
+def run_migrations_offline(
+    *,
+    target_metadata: MetaData,
+    schema: str,
+    url: str,
+    include_name: IncludeNameCallback,
+    version_table_schema: str | None = None,
+) -> None:
+    """Configure Alembic context for offline mode and run migrations.
+
+    Offline mode produces SQL script output rather than executing against a
+    live database.  Useful for generating migration SQL for review/approval.
+
+    *version_table_schema* defaults to *schema* when not provided, implementing
+    the in-schema alembic_version policy from ADR-INF-11.  Pass an explicit
+    value (e.g. ``version_table_schema="sched"``) to preserve WP-5.1's
+    explicit override for scheduler.
+    """
+    effective_vts = version_table_schema if version_table_schema is not None else schema
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+        include_schemas=True,
+        include_name=include_name,
+        version_table="alembic_version",
+        version_table_schema=effective_vts,
+    )
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online(
+    *,
+    target_metadata: MetaData,
+    schema: str,
+    url: str,
+    include_name: IncludeNameCallback,
+    version_table_schema: str | None = None,
+    alembic_config: Any,
+) -> None:
+    """Create a synchronous engine and run migrations in online mode.
+
+    *alembic_config* must be the ``context.config`` object from the caller's
+    env.py (used to read the [alembic] ini section and inject sqlalchemy.url).
+
+    *version_table_schema* defaults to *schema* — same policy as offline mode.
+
+    IMPORTANT: this function NEVER calls connection.execute(text("SET search_path
+    ...")).  Doing so outside an explicit SA2.0 transaction causes the
+    alembic_version CREATE/INSERT to silently fail.  Search_path is set
+    out-of-band via DSN options + ALTER DATABASE (per ADR-INF-11 §SA2.0
+    Implicit-TX Gotcha).
+    """
+    effective_vts = version_table_schema if version_table_schema is not None else schema
+
+    configuration = alembic_config.get_section(alembic_config.config_ini_section, {})
+    configuration["sqlalchemy.url"] = url
+
+    connectable = engine_from_config(
+        configuration,
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata,
+            include_schemas=True,
+            include_name=include_name,
+            version_table="alembic_version",
+            version_table_schema=effective_vts,
+        )
+        with context.begin_transaction():
+            context.run_migrations()

package/skills/vds-skill/runtime/platform_core/src/vds_platform_core/config.py

@@ -0,0 +1,88 @@
+"""Shared settings helpers for VDS orchestrators."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+from pydantic_settings.sources import DotEnvSettingsSource, PydanticBaseSettingsSource
+
+from vds_platform_core.env import DEFAULT_ENV_PATHS
+
+if TYPE_CHECKING:
+    from vds_platform_core.protocols import RetryPolicy
+
+VDS_BASE_SETTINGS_CONFIG = {
+    "env_file": "~/.vds/.env",
+    "env_file_encoding": "utf-8",
+    "extra": "ignore",
+    "dotenv_filtering": "match_prefix",
+    "env_nested_delimiter": "__",
+    "case_sensitive": False,
+    "populate_by_name": True,
+}
+
+
+def vds_settings_customise_sources(
+    settings_cls: type[BaseSettings],
+    init_settings: PydanticBaseSettingsSource,
+    env_settings: PydanticBaseSettingsSource,
+    dotenv_settings: PydanticBaseSettingsSource,
+    file_secret_settings: PydanticBaseSettingsSource,
+    *,
+    env_paths: Iterable[Path] | None = None,
+) -> tuple[PydanticBaseSettingsSource, ...]:
+    """Return a common settings source chain for VDS packages."""
+    sources: list[PydanticBaseSettingsSource] = [init_settings, env_settings]
+    candidates = tuple(env_paths) if env_paths is not None else DEFAULT_ENV_PATHS
+
+    for env_path in candidates:
+        resolved = env_path.expanduser()
+        if resolved.exists():
+            sources.append(
+                DotEnvSettingsSource(
+                    settings_cls,
+                    env_file=resolved,
+                    env_file_encoding="utf-8",
+                )
+            )
+
+    return tuple(sources)
+
+
+class HTTPClientSettings(BaseSettings):
+    """Pydantic-settings mixin providing HTTP timeout + retry fields.
+
+    Compose with a concrete settings class:
+        class MySettings(HTTPClientSettings, BaseSettings):
+            model_config = SettingsConfigDict(**VDS_BASE_SETTINGS_CONFIG, env_prefix="MY_")
+    """
+
+    model_config = SettingsConfigDict(**VDS_BASE_SETTINGS_CONFIG)
+
+    connect_timeout: float = Field(5.0, ge=0)
+    read_timeout: float = Field(30.0, ge=0)
+    write_timeout: float = Field(30.0, ge=0)
+    pool_timeout: float = Field(5.0, ge=0)
+    max_retries: int = Field(3, ge=0, le=20)
+    retry_backoff_initial: float = Field(0.5, ge=0)
+    retry_backoff_multiplier: float = Field(2.0, ge=1)
+    retry_max_wait: float = Field(60.0, ge=0)
+    rate_limit_aware: bool = True
+
+    def to_retry_policy(self) -> RetryPolicy:
+        """Return an exponential_backoff RetryPolicy configured from these fields."""
+        from vds_platform_core.http.retry import exponential_backoff
+
+        return exponential_backoff(  # type: ignore[return-value]
+            max_attempts=self.max_retries,
+            initial_wait=self.retry_backoff_initial,
+            multiplier=self.retry_backoff_multiplier,
+            max_wait=self.retry_max_wait,
+        )
+
+
+__all__ = ["VDS_BASE_SETTINGS_CONFIG", "HTTPClientSettings", "vds_settings_customise_sources"]

package/skills/vds-skill/runtime/platform_core/src/vds_platform_core/credentials.py

@@ -0,0 +1,40 @@
+"""Credential helpers for VDS orchestrators."""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+
+from pydantic import SecretStr
+
+
+def resolve_secret(value: SecretStr | str | None) -> str | None:
+    """Unwrap SecretStr to plain str; pass through str; None/empty -> None.
+
+    Idempotent: ``resolve_secret(resolve_secret(x)) == resolve_secret(x)``.
+    """
+    if value is None:
+        return None
+    if isinstance(value, SecretStr):
+        value = value.get_secret_value()
+    return value or None
+
+
+@dataclass
+class HeaderAuthResolver:
+    """Header-based credential resolver -- default CredentialResolver binding.
+
+    Reads *token_field* from the process environment on each :meth:`resolve`
+    call.  *header_name* carries the intended HTTP header (e.g.
+    ``"Authorization"``).
+    """
+
+    token_field: str
+    header_name: str
+
+    def resolve(self) -> str | None:
+        """Return the token from env, or ``None`` if absent/empty."""
+        return os.getenv(self.token_field) or None
+
+
+__all__ = ["HeaderAuthResolver", "resolve_secret"]

package/skills/vds-skill/runtime/platform_core/src/vds_platform_core/env.py

@@ -0,0 +1,24 @@
+"""Platform-level env-path defaults — TSK-7.12b.
+
+Provides DEFAULT_ENV_PATHS so platform_core.config can resolve the primary
+VDS .env file without importing from vds_cli_common (which sits one layer up).
+Respects the same VDS_ENV_FILE / VDS_CONFIG_DIR env overrides as vds_cli_common.
+"""
+
+import os
+from pathlib import Path
+
+
+def _vds_env_path() -> Path:
+    """Resolve the primary VDS .env file, honoring env-var overrides."""
+    if raw := (os.getenv("VDS_ENV_FILE") or "").strip():
+        return Path(raw).expanduser().resolve()
+    config_dir = (os.getenv("VDS_CONFIG_DIR") or "").strip()
+    root = Path(config_dir).expanduser().resolve() if config_dir else Path.home() / ".vds"
+    return root / ".env"
+
+
+# Evaluated once at import time — same semantics as vds_cli_common.paths.get_default_env_paths()
+DEFAULT_ENV_PATHS: tuple[Path, ...] = (_vds_env_path(),)
+
+__all__ = ["DEFAULT_ENV_PATHS"]

package/skills/vds-skill/runtime/platform_core/src/vds_platform_core/errors.py

@@ -0,0 +1,127 @@
+"""VDS platform error hierarchy.
+
+Provides a structured exception tree rooted at VDSError(RuntimeError) so that
+all platform exceptions are distinguishable from stdlib exceptions and from each
+other, while remaining catchable via a single VDSError guard.
+
+Hierarchy::
+
+    RuntimeError
+    └── VDSError                     (base; carries optional context dict)
+        ├── VDSClientError           (remote / SDK operation errors)
+        │   ├── AuthError
+        │   ├── ClientPermissionError (NOT PermissionError)
+        │   ├── TransportError
+        │   │   └── ClientTimeoutError (NOT TimeoutError)
+        │   ├── RetryError
+        │   ├── ResponseError
+        │   ├── NotFoundError
+        │   ├── RateLimitError
+        │   └── ConflictError
+        └── SettingsError            (local config / env errors)
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+class VDSError(RuntimeError):
+    """Root of the VDS exception hierarchy.
+
+    Args:
+        message: Human-readable error description.
+        context: Optional structured diagnostic payload (repo name, attempt
+            count, HTTP status, etc.).  Stored as ``self.context``.
+        **kwargs: Forwarded to ``RuntimeError.__init__`` for cooperative MRO
+            compatibility (required when VDSError participates in diamond
+            inheritance).
+    """
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        context: dict[str, Any] | None = None,
+        **kwargs: Any,
+    ) -> None:
+        super().__init__(message, **kwargs)
+        self.context: dict[str, Any] = context or {}
+
+
+# ---------------------------------------------------------------------------
+# Client errors — remote / SDK operation failures
+# ---------------------------------------------------------------------------
+
+
+class VDSClientError(VDSError):
+    """Base for all errors originating from remote calls or SDK operations."""
+
+
+class AuthError(VDSClientError):
+    """Authentication failure (invalid credentials, expired token, etc.)."""
+
+
+class ClientPermissionError(VDSClientError):
+    """Authorisation failure — caller lacks permission for the operation.
+
+    Named ``ClientPermissionError`` rather than ``PermissionError`` to avoid
+    shadowing the stdlib built-in.
+    """
+
+
+class TransportError(VDSClientError):
+    """Network / transport-layer failure (connection refused, DNS error, etc.)."""
+
+
+class ClientTimeoutError(TransportError):
+    """Request exceeded its deadline.
+
+    Named ``ClientTimeoutError`` rather than ``TimeoutError`` to avoid
+    shadowing the stdlib built-in.
+    """
+
+
+class RetryError(VDSClientError):
+    """All retry attempts exhausted without a successful response."""
+
+
+class ResponseError(VDSClientError):
+    """Server returned an unexpected or malformed response."""
+
+
+class NotFoundError(VDSClientError):
+    """Requested resource does not exist on the remote."""
+
+
+class RateLimitError(VDSClientError):
+    """Rate limit / quota exceeded on the remote API."""
+
+
+class ConflictError(VDSClientError):
+    """Operation conflicts with the current state of the remote resource."""
+
+
+# ---------------------------------------------------------------------------
+# Settings errors — local configuration / environment failures
+# ---------------------------------------------------------------------------
+
+
+class SettingsError(VDSError):
+    """Configuration or environment setup error (missing env var, bad config file, etc.)."""
+
+
+__all__ = [
+    "AuthError",
+    "ClientPermissionError",
+    "ClientTimeoutError",
+    "ConflictError",
+    "NotFoundError",
+    "RateLimitError",
+    "ResponseError",
+    "RetryError",
+    "SettingsError",
+    "TransportError",
+    "VDSClientError",
+    "VDSError",
+]

package/skills/vds-skill/runtime/platform_core/src/vds_platform_core/http/__init__.py

@@ -0,0 +1,18 @@
+"""platform_core.http — HTTP middleware (retry, auth, errors, pagination, stack)."""
+
+from vds_platform_core.http.auth import with_auth
+from vds_platform_core.http.errors import map_http_error, register_http_error_mapping
+from vds_platform_core.http.pagination import paginate
+from vds_platform_core.http.retry import exponential_backoff, with_retry
+from vds_platform_core.http.stack import build_client, vds_default_stack
+
+__all__ = [
+    "build_client",
+    "exponential_backoff",
+    "map_http_error",
+    "paginate",
+    "register_http_error_mapping",
+    "vds_default_stack",
+    "with_auth",
+    "with_retry",
+]

package/skills/vds-skill/runtime/platform_core/src/vds_platform_core/http/auth.py

@@ -0,0 +1,32 @@
+"""Auth middleware: inject CredentialResolver into HTTPClient headers."""
+
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from vds_platform_core.protocols import CredentialResolver, HTTPClient
+
+
+class _AuthWrappedClient:
+    def __init__(self, client: HTTPClient, resolver: CredentialResolver, header_name: str) -> None:
+        self._client, self._resolver, self._header_name = client, resolver, header_name
+
+    def _inject(self, kw: dict[str, Any]) -> dict[str, Any]:
+        if (token := self._resolver.resolve()) is None:
+            return kw
+        headers: dict[str, str] = dict(kw.get("headers") or {})
+        headers.setdefault(self._header_name, token)
+        return {**kw, "headers": headers}
+
+    async def get(self, url: str, **kw: Any) -> Any:
+        return await self._client.get(url, **self._inject(kw))
+
+    async def post(self, url: str, **kw: Any) -> Any:
+        return await self._client.post(url, **self._inject(kw))
+
+    async def aclose(self) -> None:
+        await self._client.aclose()
+
+
+def with_auth(client: HTTPClient, resolver: CredentialResolver, *, header_name: str = "Authorization") -> HTTPClient:
+    """Return *client* wrapped to inject *resolver*'s credential as a request header."""
+    return _AuthWrappedClient(client, resolver, header_name)  # type: ignore[return-value]

package/skills/vds-skill/runtime/platform_core/src/vds_platform_core/http/errors.py

@@ -0,0 +1,47 @@
+"""HTTP-status → VDSClientError mapper — TSK-7.4."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from vds_platform_core.errors import (
+    AuthError,
+    ClientPermissionError,
+    ConflictError,
+    NotFoundError,
+    RateLimitError,
+    TransportError,
+    VDSClientError,
+)
+
+_DEFAULT_MAP: dict[int, type[VDSClientError]] = {
+    401: AuthError,
+    403: ClientPermissionError,
+    404: NotFoundError,
+    409: ConflictError,
+    429: RateLimitError,
+}
+_SERVICE_MAP: dict[str, dict[int, type[VDSClientError]]] = {}
+
+
+def register_http_error_mapping(service: str, status: int, error_cls: type[VDSClientError]) -> None:
+    """Register a service-specific HTTP status → exception mapping (overrides defaults)."""
+    _SERVICE_MAP.setdefault(service, {})[status] = error_cls
+
+
+def map_http_error(status: int, body: Any, *, service: str | None = None) -> VDSClientError:
+    """Convert HTTP *status* + *body* into a VDSClientError.
+
+    Lookup order: service override → default map → 5xx → generic.
+    """
+    message = str(body)
+    if service and (cls := _SERVICE_MAP.get(service, {}).get(status)):
+        return cls(message)
+    if cls := _DEFAULT_MAP.get(status):
+        return cls(message)
+    if 500 <= status < 600:
+        return TransportError(message)
+    return VDSClientError(message)
+
+
+__all__ = ["map_http_error", "register_http_error_mapping"]

package/skills/vds-skill/runtime/platform_core/src/vds_platform_core/http/pagination.py

@@ -0,0 +1,65 @@
+"""Generic async paginator — platform_core.http.pagination (TSK-7.17)."""
+
+from __future__ import annotations
+
+from collections.abc import AsyncIterator, Callable
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from vds_platform_core.protocols import HTTPClient
+
+
+async def paginate(
+    client: HTTPClient,
+    url: str,
+    *,
+    params: dict[str, Any] | None = None,
+    page_param: str = "page",
+    size_param: str = "page_size",
+    page_size: int = 100,
+    items_extractor: Callable[[dict[str, Any]], list[dict[str, Any]]] = lambda r: r.get("items", []),
+    next_extractor: Callable[[dict[str, Any]], str | None] | None = None,
+    max_pages: int = 10000,
+) -> AsyncIterator[dict[str, Any]]:
+    """Async generator that pages through an HTTP resource.
+
+    Two styles:
+    - Page-number (default): increments ``page_param`` each call until
+      ``items_extractor`` returns an empty list.
+    - Cursor-based: if ``next_extractor`` is provided, follows its returned
+      URL/token until it returns ``None``.
+
+    Raises RuntimeError after ``max_pages`` iterations (runaway API guard).
+    """
+    base_params: dict[str, Any] = dict(params or {})
+    page_count = 0
+    current_url: str | None = url
+
+    if next_extractor is not None:
+        # --- Cursor-based pagination ---
+        while current_url is not None:
+            if page_count >= max_pages:
+                raise RuntimeError(f"paginate: max_pages={max_pages} exceeded on {url!r}")
+            response: dict[str, Any] = await client.get(current_url, params=base_params)
+            for item in items_extractor(response):
+                yield item
+            current_url = next_extractor(response)
+            page_count += 1
+    else:
+        # --- Page-number pagination ---
+        page = 1
+        while True:
+            if page_count >= max_pages:
+                raise RuntimeError(f"paginate: max_pages={max_pages} exceeded on {url!r}")
+            req_params = {**base_params, page_param: page, size_param: page_size}
+            response = await client.get(url, params=req_params)
+            items = items_extractor(response)
+            if not items:
+                break
+            for item in items:
+                yield item
+            page += 1
+            page_count += 1
+
+
+__all__ = ["paginate"]