claude-smart 0.2.41 → 0.2.43

package/plugin/vendor/reflexio/reflexio/benchmarks/retrieval_latency/embed_cache.py

@@ -0,0 +1,330 @@
+"""
+Embedding cache + deterministic fake embedder for retrieval latency benchmarks.
+
+This module controls the cost of embedding generation during a benchmark run.
+Retrieval latency measurement should not be dominated by embedder API calls,
+but query-side vectors must still be realistic so the vector-search path
+computes meaningful cosine similarities.
+
+Two helpers are provided:
+
+1. :class:`QueryEmbedCache` — on-disk cache of real embeddings for the fixed
+   query set. Populated once via :class:`LiteLLMClient`, then reused on every
+   subsequent run. First-time population requires a configured embedding API
+   key (typically ``OPENAI_API_KEY``).
+2. :func:`make_fake_document_embedder` — deterministic pseudo-random vector
+   generator used during corpus seeding. Document vector *content* does not
+   affect query-time latency (only row count does), so fake vectors are a
+   sound optimization for a latency benchmark.
+
+Both helpers expose a :func:`patch_storage_get_embedding` style context
+manager that swaps out the storage instance's bound ``_get_embedding`` method
+for the duration of a ``with`` block.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import struct
+from collections.abc import Callable, Iterable, Iterator
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Any, Literal
+
+from reflexio.models.config_schema import EMBEDDING_DIMENSIONS
+from reflexio.server.llm.litellm_client import LiteLLMClient, LiteLLMConfig
+
+logger = logging.getLogger(__name__)
+
+# Track the storage-level canonical dimension so fake document vectors
+# always fit the vec0 column width. Query cache entries are populated
+# through LiteLLMClient at this same dimension for consistency.
+DEFAULT_EMBEDDING_MODEL = "text-embedding-3-small"
+DEFAULT_EMBEDDING_DIMS = EMBEDDING_DIMENSIONS
+
+_CACHE_DIR = Path.home() / ".cache" / "reflexio-benchmarks"
+_DOCUMENT_PREFIX = "search_document: "
+_QUERY_PREFIX = "search_query: "
+
+
+EmbeddingPurpose = Literal["document", "query"]
+
+EmbeddingFn = Callable[[str, EmbeddingPurpose], list[float]]
+
+
+def _cache_path(model: str) -> Path:
+    """
+    Return the on-disk cache path for embeddings of a given model.
+
+    Args:
+        model (str): Embedding model identifier (e.g. text-embedding-3-small).
+
+    Returns:
+        Path: JSON cache file path. Parent directory is not created here.
+    """
+    safe = model.replace("/", "_")
+    return _CACHE_DIR / f"embeddings-{safe}.json"
+
+
+def _hash_key(text: str) -> str:
+    """
+    Return a stable short hash key for a full (prefixed) embedding input.
+
+    Args:
+        text (str): The full text that will be passed to the embedding model.
+
+    Returns:
+        str: Hex digest suitable for use as a JSON dict key.
+    """
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+class QueryEmbedCache:
+    """
+    On-disk cache of real embedding vectors for the benchmark query set.
+
+    The cache file is a flat JSON dict mapping SHA256(prefixed_text) → vector.
+    Population happens lazily via :meth:`ensure` and only touches the network
+    when the cache is missing an entry.
+
+    Attributes:
+        model (str): Embedding model used to populate the cache.
+        dimensions (int): Expected vector dimensionality.
+    """
+
+    def __init__(
+        self,
+        model: str = DEFAULT_EMBEDDING_MODEL,
+        dimensions: int = DEFAULT_EMBEDDING_DIMS,
+    ) -> None:
+        """
+        Initialize the cache, loading any existing on-disk entries.
+
+        Args:
+            model (str): Embedding model name. Must match whatever the live
+                storage backend is configured with.
+            dimensions (int): Expected embedding dimensionality. Must match
+                the storage vector column width.
+        """
+        self.model = model
+        self.dimensions = dimensions
+        self._path = _cache_path(model)
+        self._data: dict[str, list[float]] = {}
+        if self._path.exists():
+            try:
+                self._data = json.loads(self._path.read_text())
+            except (OSError, json.JSONDecodeError) as err:
+                logger.warning(
+                    "Failed to load embed cache at %s: %s — starting fresh",
+                    self._path,
+                    err,
+                )
+                self._data = {}
+
+    def ensure(self, queries: Iterable[str]) -> None:
+        """
+        Make sure every query has a real embedding in the cache.
+
+        Populates missing entries via the live embedding API in a single
+        batched call. Writes the cache to disk at the end of population so
+        subsequent runs skip the network entirely.
+
+        Args:
+            queries (Iterable[str]): Raw query strings (no prefix). The
+                ``search_query:`` prefix is applied internally.
+
+        Raises:
+            RuntimeError: If the embedder cannot be reached and the cache is
+                not already populated for every query.
+        """
+        missing: list[str] = []
+        missing_prefixed: list[str] = []
+        for q in queries:
+            prefixed = _QUERY_PREFIX + q
+            if _hash_key(prefixed) not in self._data:
+                missing.append(q)
+                missing_prefixed.append(prefixed)
+        if not missing:
+            return
+
+        logger.info(
+            "Embed cache miss for %d queries (model=%s) — fetching",
+            len(missing),
+            self.model,
+        )
+        client = LiteLLMClient(LiteLLMConfig(model=self.model))
+        try:
+            vectors = client.get_embeddings(
+                missing_prefixed, self.model, self.dimensions
+            )
+        except Exception as err:  # noqa: BLE001
+            raise RuntimeError(
+                f"Failed to populate query embed cache for model {self.model}: "
+                f"{err}. Set OPENAI_API_KEY (or the appropriate provider key) "
+                "and re-run."
+            ) from err
+
+        for prefixed, vec in zip(missing_prefixed, vectors, strict=True):
+            if len(vec) != self.dimensions:
+                raise RuntimeError(
+                    f"Embed cache got vector of length {len(vec)}, expected "
+                    f"{self.dimensions} for model {self.model}"
+                )
+            self._data[_hash_key(prefixed)] = list(vec)
+
+        self._path.parent.mkdir(parents=True, exist_ok=True)
+        self._path.write_text(json.dumps(self._data))
+
+    def lookup(self, text: str, purpose: EmbeddingPurpose) -> list[float]:
+        """
+        Return a cached embedding for the given raw text.
+
+        Args:
+            text (str): Raw text (no prefix).
+            purpose (EmbeddingPurpose): ``"document"`` or ``"query"``.
+
+        Returns:
+            list[float]: The cached embedding vector.
+
+        Raises:
+            KeyError: If the text is not in the cache.
+        """
+        prefix = _DOCUMENT_PREFIX if purpose == "document" else _QUERY_PREFIX
+        key = _hash_key(prefix + text)
+        try:
+            return self._data[key]
+        except KeyError as err:
+            raise KeyError(
+                f"Query embed cache miss for text={text!r} purpose={purpose}. "
+                "Did you forget to call QueryEmbedCache.ensure() for the query set?"
+            ) from err
+
+
+def _deterministic_vector(text: str, dimensions: int) -> list[float]:
+    """
+    Generate a deterministic pseudo-random unit vector from text.
+
+    Uses SHA256 as a cheap keyed PRNG, converts the digest stream into
+    little-endian float32 values, then normalizes to unit length. The
+    output is stable across runs and machines.
+
+    Args:
+        text (str): Input text (the hash seed).
+        dimensions (int): Desired vector length.
+
+    Returns:
+        list[float]: Unit-length vector of ``dimensions`` floats.
+    """
+    # Derive 4*dimensions pseudo-random bytes from chained SHA256 of
+    # (text, counter). This is not cryptographic — it just needs to be
+    # stable and unbiased enough to exercise cosine similarity paths.
+    need = dimensions * 4
+    buf = bytearray()
+    counter = 0
+    while len(buf) < need:
+        h = hashlib.sha256(f"{text}:{counter}".encode()).digest()
+        buf.extend(h)
+        counter += 1
+    floats = [
+        struct.unpack("<i", bytes(buf[i * 4 : (i + 1) * 4]))[0] / 2**31
+        for i in range(dimensions)
+    ]
+    norm = sum(x * x for x in floats) ** 0.5
+    if norm == 0.0:
+        return [0.0] * dimensions
+    return [x / norm for x in floats]
+
+
+def make_fake_document_embedder(
+    dimensions: int = DEFAULT_EMBEDDING_DIMS,
+) -> Callable[[str, EmbeddingPurpose], list[float]]:
+    """
+    Build a fake ``_get_embedding`` replacement for corpus seeding.
+
+    Returns deterministic hash-derived vectors so seeding is fast (no API
+    calls) while preserving vector column width and cosine-similarity
+    determinism across runs. Only valid for seeding — query-time retrieval
+    should use a real :class:`QueryEmbedCache` so BM25/vector ranking is
+    meaningful.
+
+    Args:
+        dimensions (int): Vector length to produce. Must match the storage
+            backend's vector column width.
+
+    Returns:
+        Callable: A bound-method-compatible ``(text, purpose) → vector``
+            function ready to patch onto a storage instance.
+    """
+
+    def fake(text: str, purpose: EmbeddingPurpose = "document") -> list[float]:
+        # Include purpose in the seed so doc and query versions of the same
+        # text get different vectors — mirrors how the real embedder treats
+        # the search_document/search_query prefixes as distinct inputs.
+        return _deterministic_vector(f"{purpose}:{text}", dimensions)
+
+    return fake
+
+
+def make_cached_query_embedder(
+    cache: QueryEmbedCache,
+    fallback_dims: int = DEFAULT_EMBEDDING_DIMS,
+) -> Callable[[str, EmbeddingPurpose], list[float]]:
+    """
+    Build a ``_get_embedding`` replacement backed by :class:`QueryEmbedCache`.
+
+    Document lookups fall back to fake deterministic vectors (seeding already
+    happened; any ``purpose='document'`` call during the timed loop would be
+    an oversight but must not crash). Query lookups raise if the cache is
+    empty — that indicates :meth:`QueryEmbedCache.ensure` was skipped.
+
+    Args:
+        cache (QueryEmbedCache): Populated cache for the benchmark query set.
+        fallback_dims (int): Vector width for any document-side lookups that
+            leak into the timed loop.
+
+    Returns:
+        Callable: A bound-method-compatible ``(text, purpose) → vector`` fn.
+    """
+
+    def cached(text: str, purpose: EmbeddingPurpose = "document") -> list[float]:
+        if purpose == "query":
+            return cache.lookup(text, "query")
+        return _deterministic_vector(f"document:{text}", fallback_dims)
+
+    return cached
+
+
+@contextmanager
+def patch_storage_get_embedding(
+    storage: Any,
+    replacement: EmbeddingFn,
+) -> Iterator[None]:
+    """
+    Temporarily replace a storage instance's ``_get_embedding`` bound method.
+
+    Used to swap in either the fake seeding embedder or the cached query
+    embedder for the duration of a benchmark phase.
+
+    ``storage`` is typed ``Any`` because ``_get_embedding`` is declared on the
+    sqlite storage mixins rather than on ``BaseStorage`` itself — a Protocol
+    that required the attribute would reject real storage instances.
+
+    Args:
+        storage (Any): A BaseStorage-like instance exposing a
+            ``_get_embedding`` method. Writing to ``storage._get_embedding``
+            goes into the instance dict, so subsequent lookups resolve to the
+            plain replacement function (no auto-binding).
+        replacement (EmbeddingFn): The ``(text, purpose) → vector`` fn to bind.
+
+    Yields:
+        None: During the ``with`` block, ``storage._get_embedding`` routes to
+        ``replacement``. On exit, the original bound method is restored.
+    """
+    original = storage._get_embedding
+    storage._get_embedding = replacement
+    try:
+        yield
+    finally:
+        storage._get_embedding = original

package/plugin/vendor/reflexio/reflexio/benchmarks/retrieval_latency/report.py

@@ -0,0 +1,317 @@
+"""
+Statistics and markdown rendering for retrieval latency benchmark results.
+
+Input: a flat list of trial rows produced by :mod:`bench`, each with
+``{backend, retrieval_type, layer, n, trial, elapsed_ms}``.
+
+Output: markdown tables grouped by retrieval type, optional diff column
+against a committed baseline file.
+
+All quantile math uses :mod:`statistics` to avoid pulling in numpy.
+"""
+
+from __future__ import annotations
+
+import json
+import statistics
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Any
+
+from reflexio.benchmarks.retrieval_latency.scenarios import QUERIES
+
+# Flag a cell when p95 has grown by more than this factor since the baseline.
+REGRESSION_THRESHOLD = 1.20
+
+
+@dataclass(frozen=True)
+class CellKey:
+    """
+    Immutable key identifying one (backend, retrieval_type, layer, n) cell.
+
+    Attributes:
+        backend (str): Storage backend name, e.g. ``"sqlite"``.
+        retrieval_type (str): One of ``profile``, ``user_playbook``,
+            ``agent_playbook``, ``unified``.
+        layer (str): ``service`` or ``http``.
+        n (int): Corpus size for this cell.
+    """
+
+    backend: str
+    retrieval_type: str
+    layer: str
+    n: int
+
+
+@dataclass
+class CellStats:
+    """
+    Aggregated latency stats for one benchmark cell.
+
+    Attributes:
+        trials (int): Number of timed samples summarized.
+        mean_ms (float): Arithmetic mean latency in milliseconds.
+        p50_ms (float): Median latency.
+        p95_ms (float): 95th percentile.
+        p99_ms (float): 99th percentile.
+        max_ms (float): Slowest sample observed.
+    """
+
+    trials: int
+    mean_ms: float
+    p50_ms: float
+    p95_ms: float
+    p99_ms: float
+    max_ms: float
+
+
+@dataclass
+class RunResults:
+    """
+    Full results of a benchmark run, aggregated and raw.
+
+    Attributes:
+        config (dict): Top-level metadata (CLI args, git sha, timestamp).
+        raw (list[dict]): All trial rows, unmodified.
+        stats (dict[str, CellStats]): Aggregated stats keyed by
+            ``backend|retrieval_type|layer|n``.
+    """
+
+    config: dict[str, Any]
+    raw: list[dict[str, Any]] = field(default_factory=list)
+    stats: dict[str, CellStats] = field(default_factory=dict)
+
+
+def _quantile(sorted_samples: list[float], q: float) -> float:
+    """
+    Return the ``q``-th quantile of an already-sorted sample list.
+
+    Uses the linear interpolation method so small N (e.g. 10 samples for
+    the smoke test) still gives a stable number.
+
+    Args:
+        sorted_samples (list[float]): Sorted list of samples.
+        q (float): Target quantile in ``[0, 1]``.
+
+    Returns:
+        float: Interpolated quantile value.
+    """
+    if not sorted_samples:
+        return 0.0
+    if len(sorted_samples) == 1:
+        return sorted_samples[0]
+    pos = q * (len(sorted_samples) - 1)
+    lo = int(pos)
+    hi = min(lo + 1, len(sorted_samples) - 1)
+    frac = pos - lo
+    return sorted_samples[lo] * (1 - frac) + sorted_samples[hi] * frac
+
+
+def compute_stats(samples: list[float]) -> CellStats:
+    """
+    Compute mean / p50 / p95 / p99 / max for one cell's samples.
+
+    Args:
+        samples (list[float]): Latency samples in milliseconds.
+
+    Returns:
+        CellStats: Aggregated stats. Empty input returns zeros.
+    """
+    if not samples:
+        return CellStats(
+            trials=0, mean_ms=0.0, p50_ms=0.0, p95_ms=0.0, p99_ms=0.0, max_ms=0.0
+        )
+    sorted_s = sorted(samples)
+    return CellStats(
+        trials=len(samples),
+        mean_ms=statistics.fmean(sorted_s),
+        p50_ms=_quantile(sorted_s, 0.50),
+        p95_ms=_quantile(sorted_s, 0.95),
+        p99_ms=_quantile(sorted_s, 0.99),
+        max_ms=sorted_s[-1],
+    )
+
+
+def cell_key_str(k: CellKey) -> str:
+    """
+    Serialize a :class:`CellKey` into a flat string for JSON keys.
+
+    Args:
+        k (CellKey): The cell identifier.
+
+    Returns:
+        str: ``backend|retrieval_type|layer|n`` form.
+    """
+    return f"{k.backend}|{k.retrieval_type}|{k.layer}|{k.n}"
+
+
+def aggregate(raw: list[dict[str, Any]]) -> dict[str, CellStats]:
+    """
+    Bucket raw trial rows by cell and compute aggregated stats per cell.
+
+    Args:
+        raw (list[dict]): Trial rows as emitted by :mod:`bench`.
+
+    Returns:
+        dict[str, CellStats]: Mapping from flat cell key to stats.
+    """
+    buckets: dict[str, list[float]] = {}
+    for row in raw:
+        k = CellKey(
+            backend=row["backend"],
+            retrieval_type=row["retrieval_type"],
+            layer=row["layer"],
+            n=row["n"],
+        )
+        buckets.setdefault(cell_key_str(k), []).append(row["elapsed_ms"])
+    return {key: compute_stats(samples) for key, samples in buckets.items()}
+
+
+def write_results_json(results: RunResults, path: Path) -> None:
+    """
+    Persist aggregated + raw results to disk as JSON.
+
+    Args:
+        results (RunResults): Fully populated run results.
+        path (Path): Output file path. Parent directory is created.
+    """
+    payload = {
+        "config": results.config,
+        "stats": {k: asdict(v) for k, v in results.stats.items()},
+        "raw": results.raw,
+    }
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(payload, indent=2))
+
+
+def load_baseline(path: Path) -> dict[str, CellStats]:
+    """
+    Load aggregated baseline stats from a previously-written results file.
+
+    Args:
+        path (Path): Baseline results.json file.
+
+    Returns:
+        dict[str, CellStats]: Stats indexed by flat cell key. Empty dict
+        if the file is missing or malformed.
+    """
+    if not path.exists():
+        return {}
+    data = json.loads(path.read_text())
+    return {
+        k: CellStats(**v)
+        for k, v in data.get("stats", {}).items()  # type: ignore[arg-type]
+    }
+
+
+def _fmt_cell(stats: CellStats) -> str:
+    """
+    Render a single cell's stats as ``p50 / p95 (mean)`` ms.
+
+    Args:
+        stats (CellStats): The cell's aggregated stats.
+
+    Returns:
+        str: Human-readable one-line summary, or ``"—"`` for empty cells.
+    """
+    if stats.trials == 0:
+        return "—"
+    return f"{stats.p50_ms:.1f} / {stats.p95_ms:.1f} ({stats.mean_ms:.1f})"
+
+
+def _diff_cell(current: CellStats, baseline: CellStats | None) -> str:
+    """
+    Render a baseline-diff cell showing ΔP95 and a regression marker.
+
+    Args:
+        current (CellStats): This run's stats for the cell.
+        baseline (CellStats | None): Prior run's stats, if any.
+
+    Returns:
+        str: ``+42%`` or ``-8%`` with a ``⚠`` marker if current p95 is
+        above the regression threshold.
+    """
+    if baseline is None or baseline.p95_ms == 0 or current.trials == 0:
+        return "—"
+    ratio = current.p95_ms / baseline.p95_ms
+    delta = (ratio - 1.0) * 100
+    marker = " ⚠" if ratio >= REGRESSION_THRESHOLD else ""
+    return f"{delta:+.0f}%{marker}"
+
+
+def render_markdown(
+    results: RunResults, baseline: dict[str, CellStats] | None = None
+) -> str:
+    """
+    Render the full run results as a markdown report.
+
+    One table per retrieval type, rows grouped by (backend, layer, N),
+    with an optional ΔP95 column against the committed baseline.
+
+    Args:
+        results (RunResults): Fully populated run results.
+        baseline (dict[str, CellStats] | None): Optional baseline stats.
+
+    Returns:
+        str: Markdown report suitable for writing to ``report.md``.
+    """
+    retrieval_types = sorted({row["retrieval_type"] for row in results.raw})
+    backends = sorted({row["backend"] for row in results.raw})
+    layers = sorted({row["layer"] for row in results.raw})
+    sizes = sorted({row["n"] for row in results.raw})
+
+    lines: list[str] = ["# Retrieval latency benchmark", ""]
+    lines.append(f"- Trials per cell: {results.config.get('trials', '?')}")
+    lines.append(f"- Warmup per cell: {results.config.get('warmup', '?')}")
+    lines.append(f"- Query set: {len(QUERIES)} fixed queries (see scenarios.py)")
+    lines.append("- Cell format: `p50 / p95 (mean)` in ms")
+    if baseline:
+        lines.append(f"- Regression threshold: p95 ≥ {REGRESSION_THRESHOLD:.0%}")
+    lines.append("")
+
+    for rt in retrieval_types:
+        lines.append(f"## {rt}")
+        lines.append("")
+        header = ["backend", "layer", *[f"N={n}" for n in sizes]]
+        if baseline:
+            header.append("ΔP95")
+        lines.append("| " + " | ".join(header) + " |")
+        lines.append("|" + "|".join(["---"] * len(header)) + "|")
+
+        for backend in backends:
+            for layer in layers:
+                row_cells: list[str] = [backend, layer]
+                latest_stats: CellStats | None = None
+                for n in sizes:
+                    key = cell_key_str(CellKey(backend, rt, layer, n))
+                    stats = results.stats.get(
+                        key,
+                        CellStats(0, 0.0, 0.0, 0.0, 0.0, 0.0),
+                    )
+                    row_cells.append(_fmt_cell(stats))
+                    if stats.trials > 0:
+                        latest_stats = stats
+                if baseline and latest_stats is not None:
+                    # Diff against the largest-N cell we have for this row.
+                    largest_n = max(
+                        (
+                            n
+                            for n in sizes
+                            if cell_key_str(CellKey(backend, rt, layer, n))
+                            in results.stats
+                        ),
+                        default=None,
+                    )
+                    base_key = (
+                        cell_key_str(CellKey(backend, rt, layer, largest_n))
+                        if largest_n is not None
+                        else None
+                    )
+                    base_stats = baseline.get(base_key) if base_key else None
+                    row_cells.append(_diff_cell(latest_stats, base_stats))
+                elif baseline:
+                    row_cells.append("—")
+                lines.append("| " + " | ".join(row_cells) + " |")
+        lines.append("")
+
+    return "\n".join(lines)

package/plugin/vendor/reflexio/reflexio/benchmarks/retrieval_latency/results/report.md

@@ -0,0 +1,43 @@
+# Retrieval latency benchmark
+
+- Trials per cell: 30
+- Warmup per cell: 3
+- Query set: 20 fixed queries (see scenarios.py)
+- Cell format: `p50 / p95 (mean)` in ms
+
+## agent_playbook
+
+
+| backend | layer   | N=100              | N=1000             |
+| ------- | ------- | ------------------ | ------------------ |
+| sqlite  | http    | 18.8 / 39.1 (21.4) | 19.2 / 21.3 (19.7) |
+| sqlite  | service | 15.2 / 16.9 (15.5) | 16.9 / 21.6 (17.6) |
+
+
+## profile
+
+
+| backend | layer   | N=100           | N=1000             |
+| ------- | ------- | --------------- | ------------------ |
+| sqlite  | http    | 4.1 / 5.6 (4.4) | 20.7 / 25.6 (21.2) |
+| sqlite  | service | 1.8 / 2.2 (2.0) | 17.4 / 21.2 (17.9) |
+
+
+## unified
+
+
+| backend | layer   | N=100              | N=1000             |
+| ------- | ------- | ------------------ | ------------------ |
+| sqlite  | http    | 24.7 / 30.3 (25.4) | 58.8 / 71.8 (59.9) |
+| sqlite  | service | 20.1 / 21.5 (20.1) | 57.5 / 72.7 (58.3) |
+
+
+## user_playbook
+
+
+| backend | layer   | N=100           | N=1000             |
+| ------- | ------- | --------------- | ------------------ |
+| sqlite  | http    | 3.9 / 4.8 (4.2) | 19.5 / 22.0 (19.8) |
+| sqlite  | service | 1.6 / 2.0 (1.7) | 17.1 / 17.8 (17.1) |
+
+