orizu 0.5.19 → 0.5.21

package/vendor/gepa-python/src/gepa/adapters/confidence_adapter/scoring.py

@@ -0,0 +1,206 @@
+# Copyright (c) 2025 Lakshya A Agrawal and the GEPA contributors
+# https://github.com/gepa-ai/gepa
+
+"""Scoring strategies that blend correctness with logprob confidence.
+
+Why logprobs?
+~~~~~~~~~~~~~
+When an LLM generates a structured JSON output constrained by an ``enum``,
+the model must choose one of the allowed values.  The token-level
+log-probabilities for those value tokens reveal **how certain** the model
+actually is about that choice -- even when the output always *looks*
+deterministic (temperature = 0, forced choice).
+
+The sum of the per-token logprobs for a value (``joint_logprob``) is the
+natural measure of confidence for the chosen classification:
+
+* **joint_logprob = 0** means the model considered every token virtually
+  certain (probability ≈ 1).  In practice values cluster between roughly
+  **-0.1** (very confident) and **-5.0** (very uncertain); values below
+  **-10** indicate near-random guessing.
+* Because logprobs are *negative*, **closer to 0 = more confident**.
+
+Using logprobs in GEPA
+~~~~~~~~~~~~~~~~~~~~~~~
+The :class:`ConfidenceAdapter` extracts ``joint_logprob`` from the
+``llm-structured-confidence`` library and passes it to a pluggable
+:class:`ScoringStrategy`.  Each strategy maps
+``(is_correct, logprob_score)`` to a single float in ``[0, 1]`` consumed
+by the GEPA engine (higher is better).
+
+Three built-in strategies are provided:
+
+* :class:`LinearBlendScoring` -- proportional penalty for low-confidence
+  correct answers.  **Recommended default** for most classification tasks.
+* :class:`ThresholdScoring` -- binary gate: ``1.0`` only if correct *and*
+  logprob exceeds a cutoff.
+* :class:`SigmoidScoring` -- smooth S-curve that maps logprobs to
+  ``[0, 1]`` when correct.
+
+All strategies degrade gracefully when logprobs are unavailable
+(``logprob_score=None``): correct answers receive ``1.0``, incorrect
+answers receive ``0.0`` -- identical to plain accuracy scoring.
+"""
+
+from __future__ import annotations
+
+import math
+from typing import Protocol, runtime_checkable
+
+
+@runtime_checkable
+class ScoringStrategy(Protocol):
+    """Protocol for combining correctness and logprob confidence into a score.
+
+    Every implementation must return a value in ``[0, 1]`` following GEPA's
+    convention: **higher is better**.
+
+    The ``logprob_score`` parameter is the **joint logprob** (sum of
+    per-token logprobs) for the target field, as returned by
+    ``llm-structured-confidence``.  It is always **<= 0** (closer to 0 =
+    more confident).
+    """
+
+    def score(self, is_correct: bool, logprob_score: float | None) -> float:
+        """Return a blended score in ``[0, 1]``.
+
+        Parameters
+        ----------
+        is_correct:
+            Whether the model's answer matches the expected answer.
+        logprob_score:
+            Joint logprob (sum of per-token logprobs) for the target field.
+            Always ``<= 0``.  ``None`` when logprobs are unavailable.
+        """
+        ...
+
+    def describe(self) -> str:
+        """Human-readable description for logs and reflective feedback.
+
+        .. note:: Not currently called by the adapter, but available for
+           custom logging or user-facing diagnostics.
+        """
+        ...
+
+
+class LinearBlendScoring:
+    """Proportional scoring that penalises low-confidence correct answers.
+
+    The joint logprob is converted to a probability via ``exp(logprob)``
+    and then compared against a threshold:
+
+    * **Correct + confident** (probability >= *low_confidence_threshold*)
+      -> ``1.0``
+    * **Correct + uncertain** (probability < threshold) -> linearly
+      interpolated between *min_score_on_correct* and ``1.0``
+    * **Incorrect** -> ``0.0``
+
+    When *logprob_score* is ``None`` (logprobs unavailable), correct answers
+    receive ``1.0`` so behaviour degrades gracefully to binary scoring.
+
+    Parameters
+    ----------
+    low_confidence_threshold:
+        Probability threshold (in ``(0, 1]``) above which a correct answer
+        receives a full score of ``1.0``.  Default ``0.5``.
+    min_score_on_correct:
+        Minimum score assigned to a correct answer whose probability falls
+        below the threshold.  Default ``0.3``.
+    """
+
+    def __init__(
+        self,
+        low_confidence_threshold: float = 0.5,
+        min_score_on_correct: float = 0.3,
+    ) -> None:
+        if not 0.0 < low_confidence_threshold <= 1.0:
+            raise ValueError("low_confidence_threshold must be in (0, 1]")
+        if not 0.0 <= min_score_on_correct < 1.0:
+            raise ValueError("min_score_on_correct must be in [0, 1)")
+        self.low_confidence_threshold = low_confidence_threshold
+        self.min_score_on_correct = min_score_on_correct
+
+    def score(self, is_correct: bool, logprob_score: float | None) -> float:
+        if not is_correct:
+            return 0.0
+        if logprob_score is None:
+            return 1.0
+        probability = math.exp(logprob_score)
+        if probability >= self.low_confidence_threshold:
+            return 1.0
+        t = probability / self.low_confidence_threshold
+        return self.min_score_on_correct + (1.0 - self.min_score_on_correct) * t
+
+    def describe(self) -> str:
+        return f"LinearBlendScoring(threshold={self.low_confidence_threshold}, min_score={self.min_score_on_correct})"
+
+
+class ThresholdScoring:
+    """Binary scoring gated on a logprob threshold.
+
+    * ``1.0`` only when correct **and** ``exp(logprob)`` >= *threshold*
+    * ``0.0`` otherwise
+
+    When *logprob_score* is ``None``, correct answers receive ``1.0``.
+
+    Parameters
+    ----------
+    threshold:
+        Probability threshold in ``(0, 1]``.  Default ``0.7``.
+    """
+
+    def __init__(self, threshold: float = 0.7) -> None:
+        if not 0.0 < threshold <= 1.0:
+            raise ValueError("threshold must be in (0, 1]")
+        self.threshold = threshold
+
+    def score(self, is_correct: bool, logprob_score: float | None) -> float:
+        if not is_correct:
+            return 0.0
+        if logprob_score is None:
+            return 1.0
+        probability = math.exp(logprob_score)
+        return 1.0 if probability >= self.threshold else 0.0
+
+    def describe(self) -> str:
+        return f"ThresholdScoring(threshold={self.threshold})"
+
+
+class SigmoidScoring:
+    """Smooth sigmoid curve mapping logprob confidence to ``[0, 1]``.
+
+    The logprob is converted to probability via ``exp(logprob)`` and then
+    passed through a sigmoid:
+
+        ``score = sigmoid(steepness * (probability - midpoint))``
+
+    when correct, ``0.0`` when incorrect.
+
+    When *logprob_score* is ``None``, correct answers receive ``1.0``.
+
+    Parameters
+    ----------
+    midpoint:
+        Probability value where the sigmoid outputs ``0.5``.  Default ``0.5``.
+    steepness:
+        Controls how sharply the curve transitions.  Default ``10.0``.
+    """
+
+    def __init__(self, midpoint: float = 0.5, steepness: float = 10.0) -> None:
+        if not 0.0 < midpoint < 1.0:
+            raise ValueError("midpoint must be in (0, 1)")
+        if steepness <= 0:
+            raise ValueError("steepness must be positive")
+        self.midpoint = midpoint
+        self.steepness = steepness
+
+    def score(self, is_correct: bool, logprob_score: float | None) -> float:
+        if not is_correct:
+            return 0.0
+        if logprob_score is None:
+            return 1.0
+        probability = math.exp(logprob_score)
+        return 1.0 / (1.0 + math.exp(-self.steepness * (probability - self.midpoint)))
+
+    def describe(self) -> str:
+        return f"SigmoidScoring(midpoint={self.midpoint}, steepness={self.steepness})"

package/vendor/gepa-python/src/gepa/adapters/default_adapter/default_adapter.py

@@ -0,0 +1,206 @@
+# Copyright (c) 2025 Lakshya A Agrawal and the GEPA contributors
+# https://github.com/gepa-ai/gepa
+
+from collections.abc import Mapping, Sequence
+from typing import Any, NamedTuple, Protocol, TypedDict, cast
+
+from gepa.core.adapter import EvaluationBatch, GEPAAdapter
+
+
+# DataInst, Trajectory, RolloutOutput
+class DefaultDataInst(TypedDict):
+    input: str
+    additional_context: dict[str, str]
+    answer: str
+
+
+class EvaluationResult(NamedTuple):
+    score: float
+    feedback: str
+    objective_scores: dict[str, float] | None = None
+
+
+class DefaultTrajectory(TypedDict):
+    data: DefaultDataInst
+    full_assistant_response: str
+    feedback: str
+
+
+class DefaultRolloutOutput(TypedDict):
+    full_assistant_response: str
+
+
+DefaultReflectiveRecord = TypedDict(
+    "DefaultReflectiveRecord",
+    {
+        "Inputs": str,
+        "Generated Outputs": str,
+        "Feedback": str,
+    },
+)
+
+
+class ChatMessage(TypedDict):
+    role: str
+    content: str
+
+
+class ChatCompletionCallable(Protocol):
+    """Protocol for chat completion callables (duck typing for custom model wrappers)."""
+
+    def __call__(self, messages: Sequence[ChatMessage]) -> str: ...
+
+
+# Callable that evaluates a response and returns (score, feedback, optional objective_scores)
+class Evaluator(Protocol):
+    def __call__(self, data: DefaultDataInst, response: str) -> EvaluationResult:
+        """
+        Evaluates a response and returns a score, feedback, and optional objective scores.
+        """
+        ...
+
+
+class ContainsAnswerEvaluator:
+    """Default evaluator that checks if the expected answer is contained in the response."""
+
+    def __init__(self, failure_score: float = 0.0):
+        self.failure_score = failure_score
+
+    def __call__(self, data: DefaultDataInst, response: str) -> EvaluationResult:
+        is_correct = data["answer"] in response
+        score = 1.0 if is_correct else self.failure_score
+
+        if is_correct:
+            feedback = f"The generated response is correct. The response include the correct answer '{data['answer']}'"
+        else:
+            additional_context_str = "\n".join(f"{k}: {v}" for k, v in data["additional_context"].items())
+            feedback = (
+                f"The generated response is incorrect. The correct answer is '{data['answer']}'. "
+                "Ensure that the correct answer is included in the response exactly as it is."
+            )
+            if additional_context_str:
+                feedback += f" Here is some additional context that might be helpful:\n{additional_context_str}"
+
+        return EvaluationResult(score=score, feedback=feedback, objective_scores=None)
+
+
+class DefaultAdapter(GEPAAdapter[DefaultDataInst, DefaultTrajectory, DefaultRolloutOutput]):
+    def __init__(
+        self,
+        model: str | ChatCompletionCallable,
+        evaluator: Evaluator | None = None,
+        max_litellm_workers: int = 10,
+        litellm_batch_completion_kwargs: dict[str, Any] | None = None,
+    ):
+        if isinstance(model, str):
+            from gepa.lm import LM
+
+            self._lm = LM(model)
+        else:
+            self._lm = None
+        self.model = model
+        self.evaluator = evaluator or ContainsAnswerEvaluator()
+        self.max_litellm_workers = max_litellm_workers
+        self.litellm_batch_completion_kwargs = litellm_batch_completion_kwargs or {}
+
+    def evaluate(
+        self,
+        batch: list[DefaultDataInst],
+        candidate: dict[str, str],
+        capture_traces: bool = False,
+    ) -> EvaluationBatch[DefaultTrajectory, DefaultRolloutOutput]:
+        outputs: list[DefaultRolloutOutput] = []
+        scores: list[float] = []
+        objective_scores: list[dict[str, float] | None] = []
+        trajectories: list[DefaultTrajectory] | None = [] if capture_traces else None
+
+        system_content = next(iter(candidate.values()))
+
+        litellm_requests = []
+
+        for data in batch:
+            user_content = f"{data['input']}"
+
+            messages: list[ChatMessage] = [
+                {"role": "system", "content": system_content},
+                {"role": "user", "content": user_content},
+            ]
+
+            litellm_requests.append(messages)
+
+        if self._lm is not None:
+            responses = self._lm.batch_complete(
+                litellm_requests, max_workers=self.max_litellm_workers, **self.litellm_batch_completion_kwargs
+            )
+        else:
+            model_fn = cast(ChatCompletionCallable, self.model)
+            responses = [model_fn(messages) for messages in litellm_requests]
+
+        for data, assistant_response in zip(batch, responses, strict=True):
+            eval_result = self.evaluator(data, assistant_response)
+            score = eval_result.score
+            feedback = eval_result.feedback
+            obj_scores = eval_result.objective_scores
+
+            output: DefaultRolloutOutput = {"full_assistant_response": assistant_response}
+
+            outputs.append(output)
+            scores.append(score)
+            objective_scores.append(obj_scores)
+
+            if trajectories is not None:
+                trajectories.append(
+                    {
+                        "data": data,
+                        "full_assistant_response": assistant_response,
+                        "feedback": feedback,
+                    }
+                )
+
+        objective_scores_arg: list[dict[str, float]] | None = None
+        if objective_scores:
+            all_none = all(x is None for x in objective_scores)
+            all_not_none = all(x is not None for x in objective_scores)
+            if not (all_none or all_not_none):
+                raise ValueError("Objective scores must either be all None or all not None.")
+            if all_not_none:
+                objective_scores_arg = cast(list[dict[str, float]], objective_scores)
+
+        return EvaluationBatch(
+            outputs=outputs,
+            scores=scores,
+            trajectories=trajectories,
+            objective_scores=objective_scores_arg,
+        )
+
+    def make_reflective_dataset(
+        self,
+        candidate: dict[str, str],
+        eval_batch: EvaluationBatch[DefaultTrajectory, DefaultRolloutOutput],
+        components_to_update: list[str],
+    ) -> Mapping[str, Sequence[Mapping[str, Any]]]:
+        ret_d: dict[str, list[DefaultReflectiveRecord]] = {}
+
+        assert len(components_to_update) == 1
+        comp = components_to_update[0]
+
+        trajectories = eval_batch.trajectories
+        assert trajectories is not None, "Trajectories are required to build a reflective dataset."
+
+        items: list[DefaultReflectiveRecord] = []
+
+        for traj in trajectories:
+            d: DefaultReflectiveRecord = {
+                "Inputs": traj["data"]["input"],
+                "Generated Outputs": traj["full_assistant_response"],
+                "Feedback": traj["feedback"],
+            }
+
+            items.append(d)
+
+        ret_d[comp] = items
+
+        if len(items) == 0:
+            raise Exception("No valid predictions found for any module.")
+
+        return ret_d