@sylix/coworker 2.0.11 → 2.0.14

package/dist/skills/defaults/ai-ml/rag.md

@@ -0,0 +1,276 @@
+---
+name: rag
+description: Build production retrieval-augmented generation systems with hybrid search, reranking, and vector databases.
+---
+
+# RAG Implementation — CoWorker Edition
+
+Build production-ready LLM applications that ground responses in your data.
+
+## When to Use This Skill
+
+- Building knowledge base chatbots
+- Creating document Q&A systems
+- Implementing enterprise search
+- Adding context to LLM responses
+
+## Core Concepts
+
+### 1. RAG Architecture
+
+```
+User Query → Embedding Model → Vector Search → Context → LLM → Response
+                    ↓
+              Vector Database
+              (Pinecone/Qdrant/Weaviate)
+```
+
+### 2. Document Processing Pipeline
+
+```python
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_community.document_loaders import (
+    PyPDFLoader, TextLoader, WebLoader
+)
+from langchain.schema import Document
+
+class DocumentProcessor:
+    def __init__(
+        self,
+        chunk_size: int = 1000,
+        chunk_overlap: int = 200
+    ):
+        self.splitter = RecursiveCharacterTextSplitter(
+            chunk_size=chunk_size,
+            chunk_overlap=chunk_overlap,
+            separators=["\n\n", "\n", " ", ""]
+        )
+    
+    def load_and_split(self, file_path: str) -> list[Document]:
+        # Choose loader based on file type
+        if file_path.endswith('.pdf'):
+            loader = PyPDFLoader(file_path)
+        elif file_path.endswith('.txt'):
+            loader = TextLoader(file_path)
+        else:
+            raise ValueError(f"Unsupported file type: {file_path}")
+        
+        documents = loader.load()
+        return self.splitter.split_documents(documents)
+    
+    def process_batch(self, file_paths: list[str]) -> list[Document]:
+        all_docs = []
+        for path in file_paths:
+            docs = self.load_and_split(path)
+            all_docs.extend(docs)
+        return all_docs
+```
+
+### 3. Embedding Strategy
+
+```python
+from langchain_openai import OpenAIEmbeddings
+from langchain_community.embeddings import VoyageEmbeddings
+
+# OpenAI embeddings
+openai_embeddings = OpenAIEmbeddings(
+    model="text-embedding-3-large",
+    dimensions=3072
+)
+
+# Voyage AI (recommended for Claude)
+voyage_embeddings = VoyageEmbeddings(
+    model="voyage-2",
+    truncation=True
+)
+
+# For local/self-hosted
+from langchain_community.embeddings import HuggingFaceEmbeddings
+
+local_embeddings = HuggingFaceEmbeddings(
+    model_name="BAAI/bge-large-en-v1.5",
+    model_kwargs={'device': 'cuda'}
+)
+```
+
+### 4. Vector Store Integration
+
+```python
+# Pinecone
+from langchain_pinecone import PineconeVectorStore
+
+vectorstore = PineconeVectorStore.from_documents(
+    documents=docs,
+    embedding=openai_embeddings,
+    index_name="my-knowledge-base",
+    namespace="v1"
+)
+
+# Qdrant
+from langchain_qdrant import QdrantVectorStore
+from qdrant_client import QdrantClient
+
+qdrant = QdrantClient(host="localhost", port=6333)
+vectorstore = QdrantVectorStore.from_documents(
+    documents=docs,
+    embedding=voyage_embeddings,
+    collection_name="documents",
+    client=qdrant
+)
+
+# Weaviate
+from langchain_weaviate import WeaviateVectorStore
+import weaviate
+
+client = weaviate.Client(url="http://localhost:8080")
+vectorstore = WeaviateVectorStore.from_documents(
+    documents=docs,
+    embedding=openai_embeddings,
+    client=client,
+    index_name="Documents"
+)
+```
+
+### 5. Hybrid Search
+
+```python
+from langchain.retrievers import ContextualCompressionRetriever
+from langchain.retrievers.document_compressors import CohereRerank
+from langchain_community.retrievers import BM25Retriever
+
+class HybridRetriever:
+    def __init__(self, vectorstore, documents: list[Document]):
+        self.vector_retriever = vectorstore.as_retriever(
+            search_kwargs={"k": 10}
+        )
+        
+        self.bm25_retriever = BM25Retriever.from_documents(
+            documents,
+            k=10
+        )
+    
+    def get_relevant_documents(self, query: str) -> list[Document]:
+        # Parallel retrieval
+        vector_results = self.vector_retriever.get_relevant_documents(query)
+        keyword_results = self.bm25_retriever.get_relevant_documents(query)
+        
+        # Merge with ranking
+        seen = set()
+        merged = []
+        for doc in vector_results + keyword_results:
+            if doc.page_content not in seen:
+                seen.add(doc.page_content)
+                merged.append(doc)
+        
+        return merged[:5]
+
+# With reranking
+class RerankingRetriever:
+    def __init__(self, base_retriever):
+        self.base = base_retriever
+        self.reranker = CohereRerank(
+            cohere_api_key=os.getenv("COHERE_API_KEY"),
+            top_n=5
+        )
+    
+    def get_relevant_documents(self, query: str) -> list[Document]:
+        return self.reranker.compress_documents(
+            self.base.get_relevant_documents(query),
+            query=query
+        )
+```
+
+### 6. RAG Chain (LangGraph)
+
+```python
+from langgraph.graph import StateGraph, END
+from typing import TypedDict
+
+class RAGState(TypedDict):
+    question: str
+    context: list[Document]
+    answer: str
+
+def retrieve(state: RAGState) -> RAGState:
+    """Retrieve relevant documents."""
+    docs = retriever.get_relevant_documents(state["question"])
+    return {"context": docs}
+
+def generate(state: RAGState) -> RAGState:
+    """Generate answer from context."""
+    context_text = "\n\n".join(doc.page_content for doc in state["context"])
+    
+    prompt = f"""Answer the question based on the context.
+
+Context:
+{context_text}
+
+Question: {state["question"]}
+
+Answer:"""
+    
+    response = llm.invoke(prompt)
+    return {"answer": response.content}
+
+# Build graph
+graph = StateGraph(RAGState)
+graph.add_node("retrieve", retrieve)
+graph.add_node("generate", generate)
+
+graph.set_entry_point("retrieve")
+graph.add_edge("retrieve", "generate")
+graph.add_edge("generate", END)
+
+rag_agent = graph.compile()
+```
+
+### 7. Evaluation
+
+```python
+from langchain.evaluation import load_evaluator
+
+# Correctness evaluation
+evaluator = load_evaluator("labeled_criteria", criteria="correctness")
+
+eval_result = evaluator.evaluate_strings(
+    prediction=generated_answer,
+    input=user_question,
+    reference=ground_truth_answer
+)
+
+print(f"Score: {eval_result.score}")
+print(f"Reasoning: {eval_result.reasoning}")
+
+# Retrieval evaluation
+def evaluate_retrieval(retriever, questions: list[str]):
+    results = []
+    for q in questions:
+        docs = retriever.get_relevant_documents(q)
+        # Calculate relevant docs precision
+        relevant = sum(1 for d in docs if is_relevant(q, d))
+        precision = relevant / len(docs) if docs else 0
+        results.append({
+            "question": q,
+            "num_docs": len(docs),
+            "precision": precision
+        })
+    return results
+```
+
+## Best Practices
+
+1. **Chunk sizing** - 500-1000 chars typically works well
+2. **Overlap** - 10-20% overlap preserves context
+3. **Hybrid search** - Vector + keyword catches more
+4. **Reranking** - Improves relevance significantly
+5. **Metadata** - Store source, page, timestamps
+6. **Evaluation** - Test with real queries
+7. **Caching** - Cache embeddings for repeated queries
+
+## Common Issues
+
+- Lost context (chunks too small)
+- Noisy retrieval (missing metadata filtering)
+- Slow retrieval (not using approximate search)
+- Poor answer quality (wrong chunking strategy)
+- Hallucinations (insufficient context)