orizu 0.5.19 → 0.5.21

package/vendor/gepa-python/src/gepa/adapters/dspy_full_program_adapter/full_program_adapter.py

@@ -0,0 +1,269 @@
+import random
+from typing import Any, Callable
+
+import dspy
+from dspy.adapters.types import History
+from dspy.evaluate import Evaluate
+from dspy.primitives import Example, Prediction
+from dspy.teleprompt.bootstrap_trace import TraceData
+
+from gepa import EvaluationBatch, GEPAAdapter
+from gepa.proposer.reflective_mutation.base import LanguageModel
+
+
+class DspyAdapter(GEPAAdapter[Example, TraceData, Prediction]):
+    def __init__(
+        self,
+        task_lm: dspy.LM,
+        metric_fn: Callable,
+        reflection_lm: LanguageModel,
+        failure_score=0.0,
+        num_threads: int | None = None,
+        add_format_failure_as_feedback: bool = False,
+        rng: random.Random | None = None,
+    ):
+        self.task_lm = task_lm
+        self.metric_fn = metric_fn
+        assert reflection_lm is not None, (
+            "DspyAdapter for full-program evolution requires a reflection_lm to be provided"
+        )
+        self.reflection_lm = reflection_lm
+        self.failure_score = failure_score
+        self.num_threads = num_threads
+        self.add_format_failure_as_feedback = add_format_failure_as_feedback
+        self.rng = rng or random.Random(0)
+
+    def build_program(self, candidate: dict[str, str]) -> tuple[dspy.Module, None] | tuple[None, str]:
+        candidate_src = candidate["program"]
+        context = {}
+        o = self.load_dspy_program_from_code(candidate_src, context)
+        return o
+
+    def load_dspy_program_from_code(
+        self,
+        candidate_src: str,
+        context: dict,
+    ):
+        try:
+            compile(candidate_src, "<string>", "exec")
+        except SyntaxError as e:
+            # print(f"Syntax Error in original code {e}")
+            # return None
+            import traceback
+
+            tb = traceback.format_exc()
+            return None, f"Syntax Error in code: {e}\n{tb}"
+
+        try:
+            exec(candidate_src, context)  # expose to current namespace
+        except Exception as e:
+            import traceback
+
+            tb = traceback.format_exc()
+            return None, f"Error in executing code: {e}\n{tb}"
+
+        dspy_program = context.get("program")
+
+        if dspy_program is None:
+            return (
+                None,
+                "Your code did not define a `program` object. Please define a `program` object which is an instance of `dspy.Module`, either directly by dspy.Predict or dspy.ChainOfThought, or by instantiating a class that inherits from `dspy.Module`.",
+            )
+        else:
+            if not isinstance(dspy_program, dspy.Module):
+                return (
+                    None,
+                    f"Your code defined a `program` object, but it is an instance of {type(dspy_program)}, not `dspy.Module`. Please define a `program` object which is an instance of `dspy.Module`, either directly by dspy.Predict or dspy.ChainOfThought, or by instantiating a class that inherits from `dspy.Module`.",
+                )
+
+        dspy_program.set_lm(self.task_lm)
+
+        return dspy_program, None
+
+    def evaluate(self, batch, candidate, capture_traces=False):
+        program, feedback = self.build_program(candidate)
+
+        if program is None:
+            return EvaluationBatch(
+                outputs=[None for _ in batch], scores=[self.failure_score for _ in batch], trajectories=feedback
+            )
+
+        if capture_traces:
+            # bootstrap_trace_data-like flow with trace capture
+            from dspy.teleprompt.bootstrap_trace import bootstrap_trace_data
+
+            trajs = bootstrap_trace_data(
+                program=program,
+                dataset=batch,
+                metric=self.metric_fn,
+                num_threads=self.num_threads,
+                raise_on_error=False,
+                capture_failed_parses=True,
+                failure_score=self.failure_score,
+                format_failure_score=self.failure_score,
+            )
+            scores = []
+            outputs = []
+            for t in trajs:
+                outputs.append(t["prediction"])
+                if hasattr(t["prediction"], "__class__") and t.get("score") is None:
+                    scores.append(self.failure_score)
+                else:
+                    score = t["score"]
+                    if hasattr(score, "score"):
+                        score = score["score"]
+                    scores.append(score)
+            return EvaluationBatch(outputs=outputs, scores=scores, trajectories=trajs)
+        else:
+            evaluator = Evaluate(
+                devset=batch,
+                metric=self.metric_fn,
+                num_threads=self.num_threads,
+                return_all_scores=True,
+                failure_score=self.failure_score,
+                provide_traceback=True,
+                max_errors=len(batch) * 100,
+            )
+            res = evaluator(program)
+            outputs = [r[1] for r in res.results]
+            scores = [r[2] for r in res.results]
+            scores = [s["score"] if hasattr(s, "score") else s for s in scores]
+            return EvaluationBatch(outputs=outputs, scores=scores, trajectories=None)
+
+    def make_reflective_dataset(self, candidate, eval_batch, components_to_update):
+        proposed_program, _ = self.build_program(candidate)
+
+        assert set(components_to_update) == {"program"}, f"set(components_to_update) = {set(components_to_update)}"
+        from dspy.teleprompt.bootstrap_trace import FailedPrediction
+
+        ret_d: dict[str, list[dict[str, Any]]] = {}
+
+        if isinstance(eval_batch.trajectories, str):
+            feedback = eval_batch.trajectories
+            return {"program": {"Feedback": feedback}}
+
+        ########
+        items: list[dict[str, Any]] = []
+        for data in eval_batch.trajectories or []:
+            example_data = {}
+            trace = data["trace"]
+            example = data["example"]
+            example_data["Program Inputs"] = {**example.inputs()}
+            prediction = data["prediction"]
+            example_data["Program Outputs"] = {**prediction}
+            module_score = data["score"]
+
+            if hasattr(module_score, "feedback"):
+                feedback_text = module_score["feedback"]
+            else:
+                feedback_text = None
+
+            if hasattr(module_score, "score"):
+                module_score = module_score["score"]
+
+            trace_instances = trace
+
+            if len(trace_instances) == 0:
+                continue
+
+            selected = None
+            for t in trace_instances:
+                if isinstance(t[2], FailedPrediction):
+                    selected = t
+                    break
+
+            if selected is not None:
+                trace_instances = [selected]
+
+            trace_d = []
+            example_data["Program Trace"] = trace_d
+            for selected in trace_instances:
+                inputs = selected[1]
+                outputs = selected[2]
+
+                pred_name = None
+                for name, predictor in proposed_program.named_predictors():
+                    if predictor.signature.equals(selected[0].signature):
+                        pred_name = name
+                        break
+                assert pred_name is not None, f"Could not find predictor for {selected[0].signature}"
+
+                new_inputs = {}
+                new_outputs = {}
+
+                contains_history = False
+                history_key_name = None
+                for input_key, input_val in inputs.items():
+                    if isinstance(input_val, History):
+                        contains_history = True
+                        assert history_key_name is None
+                        history_key_name = input_key
+
+                if contains_history:
+                    s = "```json\n"
+                    for i, message in enumerate(inputs[history_key_name].messages):
+                        s += f"  {i}: {message}\n"
+                    s += "```"
+                    new_inputs["Context"] = s
+
+                for input_key, input_val in inputs.items():
+                    if contains_history and input_key == history_key_name:
+                        continue
+                    new_inputs[input_key] = str(input_val)
+
+                if isinstance(outputs, FailedPrediction):
+                    s = "Couldn't parse the output as per the expected output format. The model's raw response was:\n"
+                    s += "```\n"
+                    s += outputs.completion_text + "\n"
+                    s += "```\n\n"
+                    new_outputs = s
+                else:
+                    for output_key, output_val in outputs.items():
+                        new_outputs[output_key] = str(output_val)
+
+                d = {"Called Module": pred_name, "Inputs": new_inputs, "Generated Outputs": new_outputs}
+                # if isinstance(outputs, FailedPrediction):
+                #     adapter = ChatAdapter()
+                #     structure_instruction = ""
+                #     for dd in adapter.format(module.signature, [], {}):
+                #         structure_instruction += dd["role"] + ": " + dd["content"] + "\n"
+                #     d["Feedback"] = "Your output failed to parse. Follow this structure:\n" + structure_instruction
+                #     # d['score'] = self.failure_score
+                # else:
+                # assert fb["score"] == module_score, f"Currently, GEPA only supports feedback functions that return the same score as the module's score. However, the module-level score is {module_score} and the feedback score is {fb.score}."
+                # d['score'] = fb.score
+                trace_d.append(d)
+
+            if feedback_text is not None:
+                example_data["Feedback"] = feedback_text
+
+            items.append(example_data)
+
+        if len(items) == 0:
+            raise Exception("No valid predictions found for program.")
+
+        ret_d["program"] = items
+
+        ########
+        if len(ret_d) == 0:
+            raise Exception("No valid predictions found for any module.")
+
+        return ret_d
+
+    def propose_new_texts(
+        self,
+        candidate: dict[str, str],
+        reflective_dataset: dict[str, list[dict[str, Any]]],
+        components_to_update: list[str],
+    ) -> dict[str, str]:
+        from gepa.adapters.dspy_full_program_adapter.dspy_program_proposal_signature import DSPyProgramProposalSignature
+
+        new_texts: dict[str, str] = {}
+        for name in components_to_update:
+            base_instruction = candidate[name]
+            dataset_with_feedback = reflective_dataset[name]
+            new_texts[name] = DSPyProgramProposalSignature.run(
+                lm=self.reflection_lm,
+                input_dict={"curr_program": base_instruction, "dataset_with_feedback": dataset_with_feedback},
+            )["new_program"]
+        return new_texts

package/vendor/gepa-python/src/gepa/adapters/generic_rag_adapter/__init__.py

@@ -0,0 +1,56 @@
+# Copyright (c) 2025 Lakshya A Agrawal and the GEPA contributors
+# https://github.com/gepa-ai/gepa
+
+from .evaluation_metrics import RAGEvaluationMetrics
+from .generic_rag_adapter import (
+    GenericRAGAdapter,
+    RAGDataInst,
+    RAGOutput,
+    RAGTrajectory,
+)
+from .rag_pipeline import RAGPipeline
+from .vector_store_interface import VectorStoreInterface
+from .vector_stores.chroma_store import ChromaVectorStore
+from .vector_stores.weaviate_store import WeaviateVectorStore
+
+# Optional vector stores - import only if dependencies are available
+try:
+    from .vector_stores.qdrant_store import QdrantVectorStore
+
+    _QDRANT_AVAILABLE = True
+except ImportError:
+    _QDRANT_AVAILABLE = False
+
+try:
+    from .vector_stores.milvus_store import MilvusVectorStore
+
+    _MILVUS_AVAILABLE = True
+except ImportError:
+    _MILVUS_AVAILABLE = False
+
+try:
+    from .vector_stores.lancedb_store import LanceDBVectorStore
+
+    _LANCEDB_AVAILABLE = True
+except ImportError:
+    _LANCEDB_AVAILABLE = False
+
+__all__ = [
+    "GenericRAGAdapter",
+    "RAGDataInst",
+    "RAGOutput",
+    "RAGTrajectory",
+    "VectorStoreInterface",
+    "ChromaVectorStore",
+    "WeaviateVectorStore",
+    "RAGPipeline",
+    "RAGEvaluationMetrics",
+]
+
+# Add optional vector stores to __all__ if available
+if _QDRANT_AVAILABLE:
+    __all__.append("QdrantVectorStore")
+if _MILVUS_AVAILABLE:
+    __all__.append("MilvusVectorStore")
+if _LANCEDB_AVAILABLE:
+    __all__.append("LanceDBVectorStore")

package/vendor/gepa-python/src/gepa/adapters/generic_rag_adapter/evaluation_metrics.py

@@ -0,0 +1,226 @@
+# Copyright (c) 2025 Lakshya A Agrawal and the GEPA contributors
+# https://github.com/gepa-ai/gepa
+
+import re
+from typing import Any
+
+
+class RAGEvaluationMetrics:
+    """
+    Evaluation metrics for RAG systems.
+
+    Provides both retrieval and generation quality metrics
+    for comprehensive RAG system evaluation.
+    """
+
+    def evaluate_retrieval(self, retrieved_docs: list[dict[str, Any]], relevant_doc_ids: list[str]) -> dict[str, float]:
+        """
+        Evaluate retrieval quality metrics.
+
+        Args:
+            retrieved_docs: List of retrieved documents with metadata
+            relevant_doc_ids: List of ground truth relevant document IDs
+
+        Returns:
+            Dictionary with retrieval metrics (precision, recall, f1, mrr)
+        """
+        if not retrieved_docs or not relevant_doc_ids:
+            return {"retrieval_precision": 0.0, "retrieval_recall": 0.0, "retrieval_f1": 0.0, "retrieval_mrr": 0.0}
+
+        # Extract document IDs from retrieved docs
+        retrieved_ids = []
+        for doc in retrieved_docs:
+            doc_id = doc.get("metadata", {}).get("doc_id") or doc.get("metadata", {}).get("id")
+            if doc_id:
+                retrieved_ids.append(str(doc_id))
+
+        relevant_set = set(relevant_doc_ids)
+        retrieved_set = set(retrieved_ids)
+
+        # Calculate precision and recall
+        if len(retrieved_set) == 0:
+            precision = 0.0
+        else:
+            precision = len(relevant_set.intersection(retrieved_set)) / len(retrieved_set)
+
+        if len(relevant_set) == 0:
+            recall = 0.0
+        else:
+            recall = len(relevant_set.intersection(retrieved_set)) / len(relevant_set)
+
+        # Calculate F1
+        if precision + recall == 0:
+            f1 = 0.0
+        else:
+            f1 = 2 * (precision * recall) / (precision + recall)
+
+        # Calculate Mean Reciprocal Rank (MRR)
+        mrr = 0.0
+        for i, retrieved_id in enumerate(retrieved_ids):
+            if retrieved_id in relevant_set:
+                mrr = 1.0 / (i + 1)
+                break
+
+        return {"retrieval_precision": precision, "retrieval_recall": recall, "retrieval_f1": f1, "retrieval_mrr": mrr}
+
+    def evaluate_generation(self, generated_answer: str, ground_truth: str, context: str) -> dict[str, float]:
+        """
+        Evaluate generation quality metrics.
+
+        Args:
+            generated_answer: Generated answer text
+            ground_truth: Ground truth answer
+            context: Retrieved context used for generation
+
+        Returns:
+            Dictionary with generation metrics
+        """
+        # Exact match (case-insensitive)
+        exact_match = self._exact_match(generated_answer, ground_truth)
+
+        # F1 score based on token overlap
+        f1_score = self._token_f1(generated_answer, ground_truth)
+
+        # BLEU-like score
+        bleu_score = self._simple_bleu(generated_answer, ground_truth)
+
+        # Answer relevance (simple keyword overlap with context)
+        relevance_score = self._answer_relevance(generated_answer, context)
+
+        # Faithfulness (how well the answer is supported by context)
+        faithfulness_score = self._faithfulness_score(generated_answer, context)
+
+        return {
+            "exact_match": float(exact_match),
+            "token_f1": f1_score,
+            "bleu_score": bleu_score,
+            "answer_relevance": relevance_score,
+            "faithfulness": faithfulness_score,
+            "answer_confidence": (f1_score + relevance_score + faithfulness_score) / 3.0,
+        }
+
+    def combined_rag_score(
+        self,
+        retrieval_metrics: dict[str, float],
+        generation_metrics: dict[str, float],
+        retrieval_weight: float = 0.3,
+        generation_weight: float = 0.7,
+    ) -> float:
+        """
+        Combine retrieval and generation metrics into a single score.
+
+        Args:
+            retrieval_metrics: Output from evaluate_retrieval
+            generation_metrics: Output from evaluate_generation
+            retrieval_weight: Weight for retrieval score
+            generation_weight: Weight for generation score
+
+        Returns:
+            Combined score between 0 and 1
+        """
+        # Primary retrieval metric: F1 score
+        retrieval_score = retrieval_metrics.get("retrieval_f1", 0.0)
+
+        # Primary generation metric: weighted combination
+        generation_score = (
+            generation_metrics.get("token_f1", 0.0) * 0.4
+            + generation_metrics.get("answer_relevance", 0.0) * 0.3
+            + generation_metrics.get("faithfulness", 0.0) * 0.3
+        )
+
+        return retrieval_weight * retrieval_score + generation_weight * generation_score
+
+    def _exact_match(self, prediction: str, ground_truth: str) -> bool:
+        """Check if prediction exactly matches ground truth (case-insensitive)."""
+        return prediction.strip().lower() == ground_truth.strip().lower()
+
+    def _token_f1(self, prediction: str, ground_truth: str) -> float:
+        """Calculate F1 score based on token overlap."""
+        pred_tokens = set(self._normalize_text(prediction).split())
+        truth_tokens = set(self._normalize_text(ground_truth).split())
+
+        if len(pred_tokens) == 0 and len(truth_tokens) == 0:
+            return 1.0
+        if len(pred_tokens) == 0 or len(truth_tokens) == 0:
+            return 0.0
+
+        intersection = pred_tokens.intersection(truth_tokens)
+        precision = len(intersection) / len(pred_tokens)
+        recall = len(intersection) / len(truth_tokens)
+
+        if precision + recall == 0:
+            return 0.0
+
+        return 2 * (precision * recall) / (precision + recall)
+
+    def _simple_bleu(self, prediction: str, ground_truth: str, n: int = 2) -> float:
+        """Simple BLEU-like score for n-gram overlap."""
+        pred_words = self._normalize_text(prediction).split()
+        truth_words = self._normalize_text(ground_truth).split()
+
+        if len(pred_words) < n or len(truth_words) < n:
+            return self._token_f1(prediction, ground_truth)
+
+        pred_ngrams = {tuple(pred_words[i : i + n]) for i in range(len(pred_words) - n + 1)}
+        truth_ngrams = {tuple(truth_words[i : i + n]) for i in range(len(truth_words) - n + 1)}
+
+        if len(pred_ngrams) == 0 or len(truth_ngrams) == 0:
+            return 0.0
+
+        intersection = pred_ngrams.intersection(truth_ngrams)
+        return len(intersection) / len(pred_ngrams)
+
+    def _answer_relevance(self, answer: str, context: str) -> float:
+        """Measure how well the answer relates to the provided context."""
+        answer_words = set(self._normalize_text(answer).split())
+        context_words = set(self._normalize_text(context).split())
+
+        if len(answer_words) == 0:
+            return 0.0
+
+        overlap = answer_words.intersection(context_words)
+        return len(overlap) / len(answer_words)
+
+    def _faithfulness_score(self, answer: str, context: str) -> float:
+        """
+        Measure how well the answer is supported by the context.
+        Simple implementation based on shared key phrases.
+        """
+        # Extract key phrases (sequences of 2+ words)
+        answer_phrases = self._extract_phrases(answer)
+        context_phrases = self._extract_phrases(context)
+
+        if len(answer_phrases) == 0:
+            return 1.0  # Empty answer is technically faithful
+
+        supported_phrases = answer_phrases.intersection(context_phrases)
+        return len(supported_phrases) / len(answer_phrases)
+
+    def _extract_phrases(self, text: str, min_length: int = 2) -> set[str]:
+        """Extract meaningful phrases from text."""
+        words = self._normalize_text(text).split()
+        phrases = set()
+
+        # Add individual significant words (length > 3)
+        for word in words:
+            if len(word) > 3:
+                phrases.add(word)
+
+        # Add bi-grams and tri-grams
+        for n in range(min_length, min(4, len(words) + 1)):
+            for i in range(len(words) - n + 1):
+                phrase = " ".join(words[i : i + n])
+                if len(phrase) > 5:  # Only meaningful phrases
+                    phrases.add(phrase)
+
+        return phrases
+
+    def _normalize_text(self, text: str) -> str:
+        """Normalize text for comparison."""
+        # Convert to lowercase and remove extra whitespace
+        text = text.lower().strip()
+        # Remove punctuation and special characters
+        text = re.sub(r"[^\w\s]", " ", text)
+        # Normalize whitespace
+        text = re.sub(r"\s+", " ", text)
+        return text