claude-smart 0.2.41 → 0.2.43

package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_manager.py

@@ -0,0 +1,280 @@
+"""
+Prompt management using file system prompt bank with markdown frontmatter files.
+"""
+
+import logging
+import re
+from pathlib import Path
+from typing import Any
+
+from .prompt_schema import Prompt
+
+logger = logging.getLogger(__name__)
+
+_FRONTMATTER_RE = re.compile(r"\A---\n(.*?\n)---\n(.*)", re.DOTALL)
+
+
+def _parse_frontmatter(raw: str) -> tuple[dict[str, Any], str]:
+    """
+    Parse YAML frontmatter from a markdown file using stdlib only.
+
+    Args:
+        raw (str): Raw file content with optional ``---`` delimited frontmatter.
+
+    Returns:
+        tuple[dict[str, Any], str]: Parsed metadata dict and the body content.
+
+    Raises:
+        ValueError: If frontmatter is missing or malformed.
+    """
+    if not (m := _FRONTMATTER_RE.match(raw)):
+        raise ValueError("Missing or malformed YAML frontmatter")
+
+    meta: dict[str, Any] = {}
+    for line in m.group(1).splitlines():
+        if ":" not in line:
+            continue
+        key, _, value = line.partition(":")
+        key = key.strip()
+        value = value.strip()
+
+        if not key:
+            continue
+
+        if value.startswith("[") and value.endswith("]"):
+            # Simple list: [a, b, c]
+            meta[key] = [
+                v.strip().strip("'\"") for v in value[1:-1].split(",") if v.strip()
+            ]
+        elif value.startswith("- "):
+            # First item of a block list on the same line — shouldn't happen in our format
+            meta[key] = [value[2:].strip()]
+        elif value.lower() in ("true", "false"):
+            meta[key] = value.lower() == "true"
+        elif value == "" or value.lower() == "null":
+            meta[key] = None
+        else:
+            meta[key] = value.strip("'\"")
+
+    # Handle block-style lists (- item per line)
+    current_key: str | None = None
+    for line in m.group(1).splitlines():
+        stripped = line.strip()
+        if ":" in line and not stripped.startswith("- "):
+            current_key = line.partition(":")[0].strip()
+        elif stripped.startswith("- ") and current_key:
+            if not isinstance(meta.get(current_key), list):
+                meta[current_key] = []
+            meta[current_key].append(stripped[2:].strip().strip("'\""))
+
+    return meta, m.group(2)
+
+
+class PromptManager:
+    """Prompt management using file system prompt bank."""
+
+    def __init__(
+        self,
+        prompt_bank_path: str | None = None,
+        version_override: dict[str, str] | None = None,
+    ):
+        """
+        Initialize the PromptManager.
+
+        Args:
+            prompt_bank_path (str, optional): Path to the prompt bank directory.
+            version_override (dict[str, str], optional): Map of prompt_id → version string to override the active version.
+        """
+        if prompt_bank_path is None:
+            current_dir = Path(__file__).parent
+            self.prompt_bank_path = current_dir / "prompt_bank"
+        else:
+            self.prompt_bank_path = Path(prompt_bank_path)
+
+        self.version_override = version_override
+
+        if not self.prompt_bank_path.exists():
+            logger.warning("Prompt bank path does not exist: %s", self.prompt_bank_path)
+
+        self._cache: dict[str, Prompt] = {}
+
+    # ==============================
+    # Public methods
+    # ==============================
+
+    def render_prompt(self, prompt_id: str, variables: dict[str, Any]) -> str:
+        """
+        Render prompt template with variables.
+
+        Args:
+            prompt_id (str): ID of the prompt.
+            variables (dict[str, Any]): Variables to substitute in template.
+
+        Returns:
+            str: Rendered prompt content.
+
+        Raises:
+            ValueError: If prompt not found or template rendering fails.
+        """
+        version = (
+            self.version_override.get(prompt_id) if self.version_override else None
+        )
+        prompt = self._get_prompt(prompt_id, version)
+        if not prompt:
+            raise ValueError(f"Prompt {prompt_id} not found")
+
+        missing_vars = set(prompt.variables) - set(variables.keys())
+        if missing_vars:
+            raise ValueError(
+                f"Missing required variables {missing_vars} for prompt {prompt_id}"
+            )
+
+        try:
+            return prompt.content.format(**variables)
+        except KeyError as e:
+            raise ValueError(
+                f"Missing required variable {e} for prompt {prompt_id}"
+            ) from e
+        except Exception as e:
+            raise ValueError(f"Error rendering prompt {prompt_id}: {e}") from e
+
+    def list_versions(self, prompt_id: str) -> list[str]:
+        """
+        List all versions of a prompt.
+
+        Args:
+            prompt_id (str): ID of the prompt.
+
+        Returns:
+            list[str]: List of version strings.
+        """
+        prompt_dir = self.prompt_bank_path / prompt_id
+        if not prompt_dir.is_dir():
+            return []
+        return [
+            p.name.removeprefix("v").removesuffix(".prompt.md")
+            for p in sorted(prompt_dir.glob("v*.prompt.md"))
+        ]
+
+    def get_active_version(self, prompt_id: str) -> str | None:
+        """
+        Get the active version for a prompt (considering overrides).
+
+        Args:
+            prompt_id (str): ID of the prompt.
+
+        Returns:
+            str | None: The active version string, or None if prompt not found.
+        """
+        if self.version_override and prompt_id in self.version_override:
+            return self.version_override[prompt_id]
+        return self._find_active_version(prompt_id)
+
+    def get_all_prompt_ids(self) -> list[str]:
+        """
+        Get list of all available prompt IDs.
+
+        Returns:
+            list[str]: List of prompt IDs.
+        """
+        if not self.prompt_bank_path.exists():
+            return []
+        return [
+            item.name
+            for item in self.prompt_bank_path.iterdir()
+            if item.is_dir() and any(item.glob("v*.prompt.md"))
+        ]
+
+    # ==============================
+    # Private methods
+    # ==============================
+
+    def _load_prompt(self, prompt_id: str, version: str) -> Prompt | None:
+        """Load a single prompt file by prompt_id and version string."""
+        path = self.prompt_bank_path / prompt_id / f"v{version}.prompt.md"
+        try:
+            raw = path.read_text(encoding="utf-8")
+        except FileNotFoundError:
+            logger.warning("Prompt file not found: %s", path)
+            return None
+        except Exception as e:
+            logger.error("Error reading prompt file %s: %s", path, e)
+            return None
+
+        try:
+            meta, content = _parse_frontmatter(raw)
+        except ValueError as e:
+            logger.error("Error parsing frontmatter in %s: %s", path, e)
+            return None
+
+        return Prompt(
+            active=meta.get("active", False),
+            description=meta.get("description"),
+            changelog=meta.get("changelog"),
+            variables=meta.get("variables", []),
+            content=content,
+        )
+
+    def _find_active_version(self, prompt_id: str) -> str | None:
+        """Scan .prompt.md files to find the latest one with active: true."""
+        prompt_dir = self.prompt_bank_path / prompt_id
+        if not prompt_dir.is_dir():
+            return None
+
+        def _semver_key(p: Path) -> tuple[int, ...]:
+            """Parse 'vX.Y.Z.prompt.md' into a comparable tuple.
+
+            Strips the leading 'v' before int parsing so versions sort by
+            their numeric semver tuple rather than collapsing to a single
+            fallback key. Without the prefix strip every version evaluates
+            to the (0,) fallback (because ``int('v1')`` raises) and the
+            "latest active" search becomes order-of-glob-dependent.
+            """
+            try:
+                return tuple(
+                    int(x)
+                    for x in p.stem.removeprefix("v").removesuffix(".prompt").split(".")
+                )
+            except ValueError:
+                return (0,)
+
+        latest: str | None = None
+        for path in sorted(
+            prompt_dir.glob("v*.prompt.md"), key=_semver_key, reverse=True
+        ):
+            try:
+                raw = path.read_text(encoding="utf-8")
+                meta, _ = _parse_frontmatter(raw)
+                if meta.get("active"):
+                    latest = path.name.removeprefix("v").removesuffix(".prompt.md")
+                    break
+            except (ValueError, OSError):
+                continue
+        return latest
+
+    def _get_prompt(self, prompt_id: str, version: str | None = None) -> Prompt | None:
+        """
+        Get prompt, using cache for active prompts.
+
+        Args:
+            prompt_id (str): ID of the prompt.
+            version (str, optional): Specific version to load.
+
+        Returns:
+            Prompt | None: The prompt, or None if not found.
+        """
+        if version:
+            return self._load_prompt(prompt_id, version)
+
+        if prompt_id in self._cache:
+            return self._cache[prompt_id]
+
+        active_version = self._find_active_version(prompt_id)
+        if not active_version:
+            logger.warning("No active version found for prompt %s", prompt_id)
+            return None
+
+        prompt = self._load_prompt(prompt_id, active_version)
+        if prompt:
+            self._cache[prompt_id] = prompt
+        return prompt

package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_schema.py

@@ -0,0 +1,11 @@
+from pydantic import BaseModel
+
+
+class Prompt(BaseModel):
+    """Self-contained prompt loaded from a .prompt.md file with YAML frontmatter."""
+
+    active: bool = False
+    description: str | None = None
+    changelog: str | None = None
+    variables: list[str]
+    content: str

package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/_eval_health.py

@@ -0,0 +1,131 @@
+"""Process-wide health singleton for the agent-success evaluation pipeline.
+
+Tracks three signals that the operator-facing healthcheck surfaces:
+
+1. Scheduler liveness — last tick timestamp (monotonic).
+2. Per-reason skip counts — why a session was skipped during evaluation.
+3. Producer failure counts in a rolling 24h window.
+
+This is intentionally a global singleton, not a per-request object — the
+healthcheck endpoint needs to read counts that accumulate across the
+process lifetime, and the scheduler / runner / service all need to write
+to the same store.
+"""
+
+from __future__ import annotations
+
+import enum
+import threading
+import time
+from collections import Counter, deque
+from typing import Any
+
+
+class SkipReason(enum.StrEnum):
+    """Why a session evaluation was skipped.
+
+    Members are the exact string the healthcheck exposes; do not rename
+    casually.
+    """
+
+    ALREADY_EVALUATED = "already_evaluated"
+    NO_REQUESTS = "no_requests"
+    NOT_YET_COMPLETE = "not_yet_complete"
+    NO_INTERACTIONS = "no_interactions"
+    NO_DATA_MODELS = "no_data_models"
+
+
+_FAILURE_WINDOW_SECONDS = 24 * 60 * 60
+
+
+class EvalHealth:
+    """Thread-safe counter store for evaluation-pipeline diagnostics.
+
+    A module-level singleton is exposed as `_HEALTH`; the public helpers
+    `record_skip`, `record_producer_failure`, `record_tick`, and
+    `get_status` proxy to it.
+    """
+
+    def __init__(self) -> None:
+        self._lock = threading.Lock()
+        self._skip_counts: Counter[SkipReason] = Counter()
+        self._failures: deque[float] = deque()
+        self._last_tick_monotonic: float | None = None
+
+    def record_skip(self, reason: SkipReason) -> None:
+        """Bump the counter for `reason` by one.
+
+        Args:
+            reason (SkipReason): The reason the runner skipped a session.
+        """
+        with self._lock:
+            self._skip_counts[reason] += 1
+
+    def record_producer_failure(self, at_ts: float | None = None) -> None:
+        """Record a producer failure (LLM error or persistent save failure).
+
+        Args:
+            at_ts (float | None): Wall-clock unix timestamp; defaults to now.
+        """
+        ts = time.time() if at_ts is None else at_ts
+        with self._lock:
+            self._failures.append(ts)
+            self._trim_locked(ts)
+
+    def record_tick(self, monotonic_ts: float | None = None) -> None:
+        """Record that the scheduler loop ticked.
+
+        Args:
+            monotonic_ts (float | None): Monotonic clock reading; defaults to now.
+        """
+        ts = time.monotonic() if monotonic_ts is None else monotonic_ts
+        with self._lock:
+            self._last_tick_monotonic = ts
+
+    def get_status(self, now_ts: float | None = None) -> dict[str, Any]:
+        """Return a snapshot of all health counters.
+
+        Args:
+            now_ts (float | None): Unix wall-clock used for the failure window;
+                defaults to now.
+
+        Returns:
+            dict[str, Any]: Snapshot suitable for the /healthz/eval payload.
+        """
+        ts = time.time() if now_ts is None else now_ts
+        with self._lock:
+            self._trim_locked(ts)
+            return {
+                "skip_counts": {r.value: self._skip_counts[r] for r in SkipReason},
+                "producer_failures_24h": len(self._failures),
+                "last_tick_monotonic": self._last_tick_monotonic,
+            }
+
+    def _trim_locked(self, now_ts: float) -> None:
+        """Drop failure timestamps older than the 24h window (caller holds lock)."""
+        cutoff = now_ts - _FAILURE_WINDOW_SECONDS
+        while self._failures and self._failures[0] < cutoff:
+            self._failures.popleft()
+
+
+_HEALTH = EvalHealth()
+
+
+def record_skip(reason: SkipReason) -> None:
+    """Module-level proxy to the singleton."""
+    _HEALTH.record_skip(reason)
+
+
+def record_producer_failure(at_ts: float | None = None) -> None:
+    """Module-level proxy to the singleton."""
+    _HEALTH.record_producer_failure(at_ts=at_ts)
+
+
+def record_tick(monotonic_ts: float | None = None) -> None:
+    """Module-level proxy to the singleton."""
+    _HEALTH.record_tick(monotonic_ts=monotonic_ts)
+
+
+def get_status(now_ts: float | None = None) -> dict[str, Any]:
+    """Module-level proxy to the singleton."""
+    return _HEALTH.get_status(now_ts=now_ts)

package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/agent_success_evaluation_constants.py

@@ -0,0 +1,60 @@
+"""Constants for agent success evaluation service"""
+
+from dataclasses import dataclass
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+@dataclass(frozen=True)
+class AgentSuccessEvaluationConstants:
+    """Constants for agent success evaluation prompts and configurations"""
+
+    # Prompt IDs
+    AGENT_SUCCESS_EVALUATION_PROMPT_ID = "agent_success_evaluation"
+
+
+class AgentSuccessEvaluationOutput(BaseModel):
+    """
+    Unified output schema for agent success evaluation.
+
+    For successful evaluations, only is_success=True is required.
+    For failed evaluations, all fields are required to provide failure details.
+
+    Attributes:
+        is_success (bool): Indicates whether the agent successfully responded to the user
+        failure_type (Optional[str]): Type of failure - 'missing_tool', 'wrong_tool', 'insufficient_info_from_tool', or 'wrong_answer'. Required when is_success=False
+        failure_reason (Optional[str]): Explanation for the failure and what the agent needs to do differently. Required when is_success=False
+    """
+
+    is_success: bool = Field(
+        description="Indicates whether the agent successfully responded to the user"
+    )
+    failure_type: (
+        Literal[
+            "missing_tool", "wrong_tool", "insufficient_info_from_tool", "wrong_answer"
+        ]
+        | None
+    ) = Field(
+        default=None,
+        description="Type of improvement the agent needs: 'missing_tool' (agent lacks necessary tools), 'wrong_tool' (agent used incorrect tool), 'insufficient_info_from_tool' (tool lacks necessary information), 'wrong_answer' (agent had info but answered incorrectly). Required when is_success=False",
+    )
+    failure_reason: str | None = Field(
+        default=None,
+        description="Explanation for the failure and what the agent needs to do differently. Required when is_success=False",
+    )
+    is_escalated: bool = Field(
+        default=False,
+        description="Whether the user was handed off to a human agent or another agent during the session.",
+    )
+    # OpenAI schema parsing requires explicitly forbidding additional properties
+    model_config = ConfigDict(
+        extra="allow",
+        json_schema_extra={"additionalProperties": False},
+    )
+
+
+# F1 cleanup: ``AgentSuccessEvaluationWithComparisonOutput`` (the combined
+# evaluation-plus-comparison schema) was retracted along with the session-level
+# shadow comparison branch in ``AgentSuccessEvaluator``. Per-turn shadow
+# comparison uses dedicated schemas under ``services/shadow_comparison/``.