@ngocsangairvds/vsaf 4.1.8 → 4.1.9

package/skills/vds-skill/vds-scripts/vds_agent_core/src/vds_agent_core/llm/provider.py

@@ -1,173 +0,0 @@
-"""LLM client abstraction — Protocol + LiteLLM-backed default.
-
-This is NOT a copy of audit_orchestrator/llm/provider.py. It defines a
-clean LLMClient Protocol with a LiteLLM-backed default implementation.
-The audit provider (Codex OAuth, native Anthropic SDK) stays in audit.
-"""
-
-from __future__ import annotations
-
-from typing import Any, Protocol, TypeVar
-
-from pydantic import BaseModel
-
-from vds_agent_core.config import LLMRuntimeConfig
-
-T = TypeVar("T", bound=BaseModel)
-
-
-class LLMClient(Protocol):
-    """Generic LLM client contract. All providers satisfy this."""
-
-    async def chat(self, messages: list[dict], *, model: str, **kwargs: Any) -> str: ...
-
-    async def chat_json(self, messages: list[dict], *, model: str, schema: type[T], **kwargs: Any) -> T: ...
-
-
-class LiteLLMClient:
-    """Default LLMClient backed by litellm (optional dep).
-
-    Install: pip install vds-agent-core[litellm]
-    """
-
-    def __init__(self, settings: Any = None) -> None:
-        self._settings = settings
-
-    async def chat(self, messages: list[dict], *, model: str, **kwargs: Any) -> str:
-        try:
-            import litellm
-        except ImportError as e:
-            raise ImportError("Install vds-agent-core[litellm] for LLM support") from e
-        response = await litellm.acompletion(model=model, messages=messages, **kwargs)
-        return response.choices[0].message.content or ""  # type: ignore[union-attr]
-
-    async def chat_json(self, messages: list[dict], *, model: str, schema: type[T], **kwargs: Any) -> T:
-        try:
-            import litellm
-        except ImportError as e:
-            raise ImportError("Install vds-agent-core[litellm] for LLM support") from e
-        response = await litellm.acompletion(model=model, messages=messages, response_format=schema, **kwargs)
-        content: str = response.choices[0].message.content or ""  # type: ignore[union-attr]
-        return schema.model_validate_json(content)
-
-
-class HttpxDirectClient:
-    """Minimal LLMClient using httpx directly — OpenAI-compatible endpoints only.
-
-    This is the base implementation when litellm is not installed.
-    """
-
-    def __init__(self, *, base_url: str = "https://api.openai.com/v1", api_key: str | None = None) -> None:
-        self._base_url = base_url.rstrip("/")
-        self._api_key = api_key
-
-    async def chat(self, messages: list[dict], *, model: str, **kwargs: Any) -> str:
-        import httpx
-
-        headers: dict[str, str] = {"Content-Type": "application/json"}
-        if self._api_key:
-            headers["Authorization"] = f"Bearer {self._api_key}"
-        payload = {"model": model, "messages": messages, "temperature": 0.0, **kwargs}
-        async with httpx.AsyncClient(timeout=60.0) as client:
-            resp = await client.post(f"{self._base_url}/chat/completions", json=payload, headers=headers)
-            resp.raise_for_status()
-            data = resp.json()
-        return data["choices"][0]["message"]["content"] or ""
-
-    async def chat_json(self, messages: list[dict], *, model: str, schema: type[T], **kwargs: Any) -> T:
-        content = await self.chat(messages, model=model, **kwargs)
-        # Strip markdown fences if present
-        if content.startswith("```"):
-            lines = content.splitlines()
-            content = "\n".join(lines[1:-1] if lines[-1].strip() == "```" else lines[1:])
-        return schema.model_validate_json(content)
-
-
-class LLMSettings(LLMRuntimeConfig):
-    """Config-backed factory for LLMClient instances.
-
-    Inherits BaseSettings — loads from env vars and ~/.vds/.env automatically.
-    """
-
-    @classmethod
-    def for_agent(
-        cls,
-        agent_id: str,
-        *,
-        profile: str | None = None,
-        api_settings: Any = None,
-        overrides: dict[str, Any] | None = None,
-    ) -> LLMSettings:
-        """Build per-agent LLM settings via the 3-layer cascade (FR-226.1).
-
-        Resolution precedence (highest -> lowest):
-            1. ``overrides`` -- programmatic kwargs (tests/replays).
-            2. ``api_settings.llm_settings`` -- Agent model from Redis/API.
-            3. ``VDS_AGENT_<AGENT_ID>_LLM__*`` env vars.
-            4. ``cls()`` defaults (env file + orchestrator subclass prefix).
-
-        Path B wins over Path A on conflict (FR-226.7); a structured WARN log
-        records the disagreement. Settings construction is side-effect-free
-        (FR-226.11) -- no I/O is performed here.
-
-        Args:
-            agent_id: Agent identifier (sanitized internally).
-            profile: Reserved for W3 profile YAML lookup; accepted now but
-                full enforcement (NFR-225.13 allowed_agents) lands in W3.
-            api_settings: Object with an optional ``llm_settings: dict`` field
-                (typically the Agent model loaded from Redis).
-            overrides: Programmatic field overrides.
-
-        Returns:
-            New ``LLMSettings`` instance with the resolved field values.
-        """
-        from vds_agent_core.llm._cascade import resolve_for_agent
-
-        return resolve_for_agent(
-            cls,
-            agent_id,
-            kind="llm",
-            env_prefix_template="VDS_AGENT_{sanitized}_LLM__",
-            api_attr="llm_settings",
-            profile=profile,
-            api_settings=api_settings,
-            overrides=overrides,
-        )
-
-    def build_client(self) -> LLMClient:
-        """Build an LLMClient. Uses LiteLLM if installed, falls back to httpx."""
-        try:
-            import litellm  # availability probe
-        except ImportError:
-            return HttpxDirectClient(
-                base_url=self.base_url or "https://api.openai.com/v1",
-                api_key=self.api_key,
-            )
-        return LiteLLMClient(settings=self)
-
-    def select_model(self, tier: str = "standard") -> str:
-        """Select model by tier name. Returns provider/model format for LiteLLM."""
-        models = {
-            "simple": self.model_simple,
-            "standard": self.model_standard,
-            "complex": self.model_complex,
-        }
-        return models.get(tier, self.model_standard)
-
-    def build_pydantic_ai_model(self, tier: str = "standard") -> Any:
-        """Build a pydantic-ai OpenAIChatModel backed by LiteLLMProvider.
-
-        Requires pydantic-ai with LiteLLM support. Model strings must use
-        LiteLLM's provider/model format: 'openai/gpt-4o', 'anthropic/claude-opus-4-5'.
-
-        Works with LiteLLM proxy endpoints too (point base_url at the proxy).
-        """
-        try:
-            from pydantic_ai.models.openai import OpenAIChatModel
-            from pydantic_ai.providers.litellm import LiteLLMProvider
-        except ImportError as e:
-            raise ImportError("Install vds-agent-core[litellm] and pydantic-ai for build_pydantic_ai_model()") from e
-        return OpenAIChatModel(
-            self.select_model(tier),  # must be "provider/model" format
-            provider=LiteLLMProvider(api_base=self.base_url, api_key=self.api_key),
-        )

package/skills/vds-skill/vds-scripts/vds_agent_core/src/vds_agent_core/llm/schemas.py

@@ -1,45 +0,0 @@
-"""Generic structured output schemas for LLM responses.
-
-Generic, audit-independent types extracted from audit_orchestrator.llm.schemas.
-Audit-specific schemas (GapFinding, RowEvaluationRequest/Response, etc.) live in
-audit_orchestrator.llm.audit_schemas.
-"""
-
-from __future__ import annotations
-
-import json
-
-from pydantic import BaseModel, Field
-
-
-class ProbeScore(BaseModel):
-    """Score from a single evaluation probe."""
-
-    name: str
-    score: float = Field(ge=0.0, le=10.0)
-    max_score: float = 10.0
-    reasoning: str
-    evidence_refs: list[str] = Field(default_factory=list)
-
-
-def parse_llm_response(response_text: str, schema_class: type[BaseModel]) -> BaseModel:
-    """Parse LLM response text into structured schema."""
-    text = response_text.strip()
-
-    # Handle markdown code blocks
-    if "```json" in text:
-        start = text.find("```json") + 7
-        end = text.find("```", start)
-        text = text[start:end].strip()
-    elif "```" in text:
-        start = text.find("```") + 3
-        end = text.find("```", start)
-        text = text[start:end].strip()
-
-    try:
-        data = json.loads(text)
-        return schema_class.model_validate(data)
-    except json.JSONDecodeError as e:
-        raise ValueError(f"Failed to parse LLM response as JSON: {e}") from e
-    except Exception as e:
-        raise ValueError(f"Failed to validate LLM response: {e}") from e

package/skills/vds-skill/vds-scripts/vds_agent_core/src/vds_agent_core/observability/__init__.py

@@ -1,77 +0,0 @@
-"""OpenTelemetry-based observability for VDS Agent Core.
-
-Provides distributed tracing with optional OpenTelemetry integration:
-- Agent loop span tracking
-- Tool call spans
-- LLM call spans with token accounting
-- Semantic convention attributes (gen_ai.agent.node.type)
-
-The tracer is a no-op when OpenTelemetry is not installed.
-Traces are opt-in only and activate when hooks return non-empty context.
-
-Phase 3 wiring (TSK-P3-Theta)
------------------------------
-``AgentTracer.__post_init__`` auto-registers a :class:`JsonlUsageExporter`
-observer whenever the env var ``VDS_LLM_USAGE_JSONL_ENABLED`` is unset or
-truthy. This closes the LLMUsageCollector ``[unavailable]`` gap by ensuring
-every LLM call traced through ``record_llm_call`` is persisted to
-``~/.vds/cache/llm-usage/<UTC-date>.jsonl``.
-
-For explicit / programmatic bootstrap (e.g. process startup hook), call
-:func:`bootstrap_observability` once with an explicit ``base_dir`` override.
-"""
-
-from collections.abc import Callable
-from pathlib import Path
-
-from vds_agent_core.observability.decorators import trace_audit, trace_llm_call, trace_span
-from vds_agent_core.observability.jsonl_exporter import JsonlUsageExporter
-from vds_agent_core.observability.tracer import (
-    ENV_JSONL_ENABLED,
-    AgentTracer,
-    BudgetEvent,
-    BudgetEventType,
-    TokenUsage,
-    TraceSpanType,
-)
-
-
-def bootstrap_observability(
-    tracer: AgentTracer,
-    *,
-    base_dir: Path | str | None = None,
-) -> Callable[[], None]:
-    """Explicitly wire a :class:`JsonlUsageExporter` onto ``tracer``.
-
-    This is the **Option C** entry point — a single, documented hook that
-    process bootstrap code can call to opt-in regardless of the env-var
-    default. Returns the unregister callable so callers can tear down
-    cleanly (e.g. tests, embedded use).
-
-    Parameters
-    ----------
-    tracer:
-        The :class:`AgentTracer` to receive the exporter.
-    base_dir:
-        Optional override for the JSONL output directory. ``None`` honors
-        the ``VDS_LLM_USAGE_DIR`` env var or falls back to
-        ``~/.vds/cache/llm-usage``.
-    """
-
-    exporter = JsonlUsageExporter(base_dir=base_dir)
-    return tracer.register_usage_observer(exporter)
-
-
-__all__ = [
-    "ENV_JSONL_ENABLED",
-    "AgentTracer",
-    "BudgetEvent",
-    "BudgetEventType",
-    "JsonlUsageExporter",
-    "TokenUsage",
-    "TraceSpanType",
-    "bootstrap_observability",
-    "trace_audit",
-    "trace_llm_call",
-    "trace_span",
-]

package/skills/vds-skill/vds-scripts/vds_agent_core/src/vds_agent_core/observability/decorators.py

@@ -1,258 +0,0 @@
-"""Tracing decorators for agent execution phases.
-
-Provides @trace_span and @trace_llm_call decorators that integrate
-with AgentTracer for OpenTelemetry span creation.
-"""
-
-from __future__ import annotations
-
-import functools
-from collections.abc import Callable
-from typing import Any, TypeVar
-
-F = TypeVar("F", bound=Callable[..., Any])
-
-
-# Bug W3 (Wave 8 audit): each `@trace_span` / `@trace_llm_call` decoration
-# was constructing a new AgentTracer instance whose __post_init__ calls
-# `trace.set_tracer_provider()` globally. Applying decorators to N functions
-# would clobber the global provider N times. Reuse a single module-level
-# instance so OTel provider initialization happens exactly once.
-_TRACER: Any = None
-
-
-def _module_tracer() -> Any:
-    """Return the singleton AgentTracer used by all decorators."""
-    global _TRACER
-    if _TRACER is None:
-        from vds_agent_core.observability.tracer import AgentTracer
-
-        _TRACER = AgentTracer()
-    return _TRACER
-
-
-def trace_span(
-    name: str,
-    attributes: dict[str, Any] | None = None,
-) -> Callable[[F], F]:
-    """Decorator to create a span around a function call.
-
-    The span is only created when AgentTracer tracing is enabled.
-    Otherwise the wrapped function executes normally.
-
-    Args:
-        name: Span name (e.g., "agent.decision", "search.vector")
-        attributes: Optional span attributes
-
-    Returns:
-        Decorated function that creates a span when called
-
-    Example:
-        @trace_span("agent.decision", {"decision.type": "route_selection"})
-        async def make_decision(context):
-            ...
-    """
-    # Bug W3 (Wave 8 audit): each decoration was constructing a new
-    # AgentTracer, whose __post_init__ calls trace.set_tracer_provider()
-    # globally — clobbering any previously-installed exporter / sampler
-    # config. Reuse the module-level singleton so OTel provider is set
-    # exactly once per process.
-    tracer = _module_tracer()
-
-    def decorator(func: F) -> F:
-        @functools.wraps(func)
-        def sync_wrapper(*args: Any, **kwargs: Any) -> Any:
-            if not tracer.is_enabled:
-                return func(*args, **kwargs)
-
-            with tracer._tracer.start_as_current_span(name, attributes=attributes or {}) as span:
-                try:
-                    result = func(*args, **kwargs)
-                    if span:
-                        span.set_attribute("success", True)
-                    return result
-                except Exception as e:
-                    if span:
-                        span.set_attribute("success", False)
-                        span.set_attribute("error", str(e))
-                    raise
-
-        @functools.wraps(func)
-        async def async_wrapper(*args: Any, **kwargs: Any) -> Any:
-            if not tracer.is_enabled:
-                return await func(*args, **kwargs)
-
-            with tracer._tracer.start_as_current_span(name, attributes=attributes or {}) as span:
-                try:
-                    result = await func(*args, **kwargs)
-                    if span:
-                        span.set_attribute("success", True)
-                    return result
-                except Exception as e:
-                    if span:
-                        span.set_attribute("success", False)
-                        span.set_attribute("error", str(e))
-                    raise
-
-        import inspect
-
-        if inspect.iscoroutinefunction(func):
-            return async_wrapper  # type: ignore[return-value]
-        return sync_wrapper  # type: ignore[return-value]
-
-    return decorator
-
-
-def trace_llm_call(model: str) -> Callable[[F], F]:
-    """Decorator to create a span around an LLM API call.
-
-    Extracts token usage from the response if available and records
-    it as span attributes.
-
-    Args:
-        model: Model identifier (e.g., "claude-3-5-sonnet")
-
-    Returns:
-        Decorated function that creates an LLM call span
-
-    Example:
-        @trace_llm_call("claude-3-5-sonnet")
-        async def call_llm(messages):
-            response = await client.messages.create(...)
-            return response
-    """
-    # Bug W3 (Wave 8 audit): reuse module-level singleton — see trace_span.
-    tracer = _module_tracer()
-
-    def decorator(func: F) -> F:
-        @functools.wraps(func)
-        async def async_wrapper(*args: Any, **kwargs: Any) -> Any:
-            if not tracer.is_enabled:
-                return await func(*args, **kwargs)
-
-            attributes = {
-                "gen_ai.agent.node.type": "llm_call",
-                "llm.model": model,
-            }
-            with tracer._tracer.start_as_current_span(f"llm.{model}", attributes=attributes) as span:
-                try:
-                    result = await func(*args, **kwargs)
-                    if span:
-                        span.set_attribute("success", True)
-                        _extract_and_record_usage(span, result)
-                    return result
-                except Exception as e:
-                    if span:
-                        span.set_attribute("success", False)
-                        span.set_attribute("llm.error", str(e))
-                    raise
-
-        @functools.wraps(func)
-        def sync_wrapper(*args: Any, **kwargs: Any) -> Any:
-            if not tracer.is_enabled:
-                return func(*args, **kwargs)
-
-            attributes = {
-                "gen_ai.agent.node.type": "llm_call",
-                "llm.model": model,
-            }
-            with tracer._tracer.start_as_current_span(f"llm.{model}", attributes=attributes) as span:
-                try:
-                    result = func(*args, **kwargs)
-                    if span:
-                        span.set_attribute("success", True)
-                        _extract_and_record_usage(span, result)
-                    return result
-                except Exception as e:
-                    if span:
-                        span.set_attribute("success", False)
-                        span.set_attribute("llm.error", str(e))
-                    raise
-
-        import inspect
-
-        if inspect.iscoroutinefunction(func):
-            return async_wrapper  # type: ignore[return-value]
-        return sync_wrapper  # type: ignore[return-value]
-
-    return decorator
-
-
-def _extract_and_record_usage(span: Any, result: Any) -> None:
-    """Extract token usage from LLM response and record on span."""
-    try:
-        usage = None
-        usage_attr = getattr(result, "usage", None)
-        if usage_attr is not None:
-            usage = usage_attr() if callable(usage_attr) else usage_attr
-        if usage is not None:
-            prompt_tokens = getattr(usage, "input_tokens", 0) or getattr(usage, "prompt_tokens", 0)
-            completion_tokens = getattr(usage, "output_tokens", 0) or getattr(usage, "completion_tokens", 0)
-            span.set_attribute("llm.prompt_tokens", int(prompt_tokens))
-            span.set_attribute("llm.completion_tokens", int(completion_tokens))
-            span.set_attribute("llm.total_tokens", int(prompt_tokens) + int(completion_tokens))
-    except Exception:
-        pass
-
-
-def trace_audit(audit_type: str) -> Callable[[F], F]:
-    """Decorator to create a span around an audit-specific operation.
-
-    Args:
-        audit_type: Audit operation identifier (e.g., "evidence.check",
-            "report.generate", "context.query")
-
-    Returns:
-        Decorated function that creates an audit span when called
-
-    Example:
-        @trace_audit("evidence.check")
-        async def verify_evidence(row):
-            ...
-    """
-    from vds_agent_core.observability.tracer import AgentTracer
-
-    tracer = AgentTracer()
-
-    def decorator(func: F) -> F:
-        @functools.wraps(func)
-        def sync_wrapper(*args: Any, **kwargs: Any) -> Any:
-            if not tracer.is_enabled:
-                return func(*args, **kwargs)
-            attributes = {"gen_ai.agent.node.type": "audit", "audit.type": audit_type}
-            with tracer._tracer.start_as_current_span(f"audit.{audit_type}", attributes=attributes) as span:
-                try:
-                    result = func(*args, **kwargs)
-                    if span:
-                        span.set_attribute("success", True)
-                    return result
-                except Exception as e:
-                    if span:
-                        span.set_attribute("success", False)
-                        span.set_attribute("error", str(e))
-                    raise
-
-        @functools.wraps(func)
-        async def async_wrapper(*args: Any, **kwargs: Any) -> Any:
-            if not tracer.is_enabled:
-                return await func(*args, **kwargs)
-            attributes = {"gen_ai.agent.node.type": "audit", "audit.type": audit_type}
-            with tracer._tracer.start_as_current_span(f"audit.{audit_type}", attributes=attributes) as span:
-                try:
-                    result = await func(*args, **kwargs)
-                    if span:
-                        span.set_attribute("success", True)
-                    return result
-                except Exception as e:
-                    if span:
-                        span.set_attribute("success", False)
-                        span.set_attribute("error", str(e))
-                    raise
-
-        import inspect
-
-        if inspect.iscoroutinefunction(func):
-            return async_wrapper  # type: ignore[return-value]
-        return sync_wrapper  # type: ignore[return-value]
-
-    return decorator