@cubis/foundry 0.3.83 → 0.3.84

package/workflows/workflows/agent-environment-setup/platforms/codex/skills/huggingface-ml/examples/02-advanced-usage.md

@@ -1,312 +0,0 @@
-# Example: RAG System with Hugging Face Embeddings and Reranking
-
-## Scenario
-
-A team is building a Retrieval-Augmented Generation (RAG) system for an internal knowledge base. The system must embed documents using a Hugging Face sentence-transformers model, store embeddings in a vector database, retrieve relevant context at query time, rerank candidates for precision, and feed the top results to a language model for answer generation. The pipeline must handle document chunking, embedding batching for large corpora, and hybrid search combining semantic and keyword matching.
-
-## Prompt
-
-"Build a RAG pipeline using Hugging Face models for embeddings and reranking. Use sentence-transformers for document and query embeddings, a cross-encoder for reranking retrieved candidates, and the Inference API for answer generation. Support document chunking, batch embedding, and hybrid search with BM25 + semantic similarity."
-
-## Expected Output
-
-### Document Chunking
-
-```python
-# rag/chunker.py
-from dataclasses import dataclass
-
-@dataclass
-class Chunk:
-    doc_id: str
-    chunk_index: int
-    text: str
-    metadata: dict
-
-def chunk_document(
-    doc_id: str,
-    text: str,
-    chunk_size: int = 512,
-    chunk_overlap: int = 64,
-    metadata: dict | None = None,
-) -> list[Chunk]:
-    """Split document into overlapping chunks by token count approximation."""
-    words = text.split()
-    chunks = []
-    start = 0
-
-    while start < len(words):
-        end = start + chunk_size
-        chunk_text = " ".join(words[start:end])
-        chunks.append(
-            Chunk(
-                doc_id=doc_id,
-                chunk_index=len(chunks),
-                text=chunk_text,
-                metadata=metadata or {},
-            )
-        )
-        start += chunk_size - chunk_overlap
-
-    return chunks
-```
-
-### Embedding Service
-
-```python
-# rag/embedder.py
-import torch
-import numpy as np
-from sentence_transformers import SentenceTransformer
-
-class EmbeddingService:
-    def __init__(
-        self,
-        model_id: str = "sentence-transformers/all-MiniLM-L6-v2",
-        device: str | None = None,
-    ):
-        self.device = device or ("cuda" if torch.cuda.is_available() else "cpu")
-        self.model = SentenceTransformer(model_id, device=self.device)
-        self.dimension = self.model.get_sentence_embedding_dimension()
-
-    def embed_texts(
-        self, texts: list[str], batch_size: int = 64, show_progress: bool = False
-    ) -> np.ndarray:
-        """Embed a list of texts. Returns array of shape (n, dimension)."""
-        embeddings = self.model.encode(
-            texts,
-            batch_size=batch_size,
-            show_progress_bar=show_progress,
-            normalize_embeddings=True,  # Required for cosine similarity via dot product
-            convert_to_numpy=True,
-        )
-        return embeddings
-
-    def embed_query(self, query: str) -> np.ndarray:
-        """Embed a single query. Returns array of shape (dimension,)."""
-        return self.model.encode(
-            query,
-            normalize_embeddings=True,
-            convert_to_numpy=True,
-        )
-```
-
-### Vector Store Integration
-
-```python
-# rag/vector_store.py
-import chromadb
-import numpy as np
-from rag.chunker import Chunk
-from rag.embedder import EmbeddingService
-
-class VectorStore:
-    def __init__(self, collection_name: str, embedder: EmbeddingService):
-        self.client = chromadb.PersistentClient(path="./chroma_db")
-        self.collection = self.client.get_or_create_collection(
-            name=collection_name,
-            metadata={"hnsw:space": "cosine"},
-        )
-        self.embedder = embedder
-
-    def index_chunks(self, chunks: list[Chunk], batch_size: int = 100):
-        """Index chunks with their embeddings."""
-        for i in range(0, len(chunks), batch_size):
-            batch = chunks[i : i + batch_size]
-            texts = [c.text for c in batch]
-            embeddings = self.embedder.embed_texts(texts, batch_size=batch_size)
-
-            self.collection.add(
-                ids=[f"{c.doc_id}:{c.chunk_index}" for c in batch],
-                embeddings=embeddings.tolist(),
-                documents=texts,
-                metadatas=[{**c.metadata, "doc_id": c.doc_id} for c in batch],
-            )
-
-    def semantic_search(
-        self, query: str, top_k: int = 20
-    ) -> list[dict]:
-        """Retrieve top-k chunks by semantic similarity."""
-        query_embedding = self.embedder.embed_query(query)
-
-        results = self.collection.query(
-            query_embeddings=[query_embedding.tolist()],
-            n_results=top_k,
-            include=["documents", "metadatas", "distances"],
-        )
-
-        return [
-            {
-                "id": results["ids"][0][i],
-                "text": results["documents"][0][i],
-                "metadata": results["metadatas"][0][i],
-                "score": 1 - results["distances"][0][i],  # Convert distance to similarity
-            }
-            for i in range(len(results["ids"][0]))
-        ]
-```
-
-### Hybrid Search with BM25
-
-```python
-# rag/hybrid_search.py
-from rank_bm25 import BM25Okapi
-from rag.vector_store import VectorStore
-
-class HybridSearcher:
-    def __init__(self, vector_store: VectorStore, corpus: list[dict]):
-        self.vector_store = vector_store
-        self.corpus = corpus
-        # Build BM25 index
-        tokenized = [doc["text"].lower().split() for doc in corpus]
-        self.bm25 = BM25Okapi(tokenized)
-        self.doc_lookup = {doc["id"]: doc for doc in corpus}
-
-    def search(
-        self,
-        query: str,
-        top_k: int = 20,
-        semantic_weight: float = 0.7,
-        bm25_weight: float = 0.3,
-    ) -> list[dict]:
-        # Semantic search
-        semantic_results = self.vector_store.semantic_search(query, top_k=top_k)
-        semantic_scores = {r["id"]: r["score"] for r in semantic_results}
-
-        # BM25 keyword search
-        tokenized_query = query.lower().split()
-        bm25_scores_raw = self.bm25.get_scores(tokenized_query)
-        max_bm25 = max(bm25_scores_raw) if max(bm25_scores_raw) > 0 else 1
-        bm25_scores = {
-            self.corpus[i]["id"]: bm25_scores_raw[i] / max_bm25
-            for i in range(len(self.corpus))
-        }
-
-        # Combine scores with reciprocal rank fusion
-        all_ids = set(semantic_scores.keys()) | set(
-            k for k, v in bm25_scores.items() if v > 0
-        )
-        combined = []
-        for doc_id in all_ids:
-            score = (
-                semantic_weight * semantic_scores.get(doc_id, 0)
-                + bm25_weight * bm25_scores.get(doc_id, 0)
-            )
-            combined.append({"id": doc_id, "score": score})
-
-        combined.sort(key=lambda x: x["score"], reverse=True)
-        return combined[:top_k]
-```
-
-### Cross-Encoder Reranker
-
-```python
-# rag/reranker.py
-from sentence_transformers import CrossEncoder
-
-class Reranker:
-    def __init__(self, model_id: str = "cross-encoder/ms-marco-MiniLM-L-6-v2"):
-        self.model = CrossEncoder(model_id)
-
-    def rerank(
-        self, query: str, candidates: list[dict], top_k: int = 5
-    ) -> list[dict]:
-        """Rerank candidates using a cross-encoder for higher precision."""
-        if not candidates:
-            return []
-
-        pairs = [(query, c["text"]) for c in candidates]
-        scores = self.model.predict(pairs)
-
-        for candidate, score in zip(candidates, scores):
-            candidate["rerank_score"] = float(score)
-
-        reranked = sorted(candidates, key=lambda x: x["rerank_score"], reverse=True)
-        return reranked[:top_k]
-```
-
-### RAG Pipeline with Answer Generation
-
-```python
-# rag/pipeline.py
-from huggingface_hub import InferenceClient
-from rag.embedder import EmbeddingService
-from rag.vector_store import VectorStore
-from rag.reranker import Reranker
-
-class RAGPipeline:
-    def __init__(self):
-        self.embedder = EmbeddingService()
-        self.vector_store = VectorStore("knowledge_base", self.embedder)
-        self.reranker = Reranker()
-        self.llm = InferenceClient(
-            model="meta-llama/Meta-Llama-3-8B-Instruct",
-            token="hf_YOUR_TOKEN",
-            timeout=30,
-        )
-
-    def answer(self, question: str) -> dict:
-        # Step 1: Retrieve candidates
-        candidates = self.vector_store.semantic_search(question, top_k=20)
-
-        # Step 2: Rerank for precision
-        reranked = self.reranker.rerank(question, candidates, top_k=5)
-
-        # Step 3: Build context from top results
-        context = "\n\n---\n\n".join(
-            f"[Source: {r['metadata'].get('doc_id', 'unknown')}]\n{r['text']}"
-            for r in reranked
-        )
-
-        # Step 4: Generate answer with LLM
-        prompt = f"""Based on the following context, answer the question.
-If the context does not contain enough information, say so.
-
-Context:
-{context}
-
-Question: {question}
-
-Answer:"""
-
-        response = self.llm.text_generation(
-            prompt, max_new_tokens=500, temperature=0.1
-        )
-
-        return {
-            "answer": response,
-            "sources": [
-                {"doc_id": r["metadata"].get("doc_id"), "score": r["rerank_score"]}
-                for r in reranked
-            ],
-        }
-```
-
-### Usage
-
-```python
-from rag.pipeline import RAGPipeline
-from rag.chunker import chunk_document
-
-pipeline = RAGPipeline()
-
-# Index documents
-documents = load_knowledge_base()  # your document loader
-for doc in documents:
-    chunks = chunk_document(doc["id"], doc["content"], metadata={"title": doc["title"]})
-    pipeline.vector_store.index_chunks(chunks)
-
-# Query
-result = pipeline.answer("How do I configure SSO for our enterprise plan?")
-print(result["answer"])
-print("Sources:", result["sources"])
-```
-
-## Key Decisions
-
-- **sentence-transformers for bi-encoder embeddings** -- bi-encoders embed queries and documents independently, enabling pre-computation of document embeddings for fast retrieval at scale.
-- **Cross-encoder reranker as a second stage** -- cross-encoders are more accurate than bi-encoders but too slow for full-corpus search. Using them to rerank the top 20 candidates balances accuracy with latency.
-- **Normalized embeddings with cosine similarity** -- `normalize_embeddings=True` allows using dot product as cosine similarity, which is faster and supported natively by most vector databases.
-- **Hybrid search with BM25** -- semantic search misses exact keyword matches (product names, error codes). BM25 complements semantic search for higher recall on technical queries.
-- **Chunking with overlap** -- 64-word overlap ensures that relevant context split across chunk boundaries is captured in at least one chunk.
-- **Inference API for generation** -- avoids hosting a large language model locally. The API handles scaling and GPU allocation.

package/workflows/workflows/agent-environment-setup/platforms/codex/skills/huggingface-ml/references/fine-tuning-guide.md

@@ -1,260 +0,0 @@
-# Fine-Tuning Guide
-
-Load this when training with Trainer, using PEFT/LoRA, preparing datasets, or evaluating fine-tuned models.
-
-## Dataset Preparation
-
-### Loading from the Hub
-
-```python
-from datasets import load_dataset
-
-# Load a public dataset
-dataset = load_dataset("imdb", split="train")
-
-# Load a specific configuration
-dataset = load_dataset("glue", "mrpc", split="train")
-
-# Stream large datasets without downloading fully
-dataset = load_dataset("c4", "en", streaming=True)
-```
-
-### Loading Custom Data
-
-```python
-from datasets import Dataset, DatasetDict
-
-# From a list of dictionaries
-data = [
-    {"text": "Great product!", "label": 1},
-    {"text": "Terrible experience.", "label": 0},
-]
-dataset = Dataset.from_list(data)
-
-# From CSV/JSON files
-dataset = load_dataset("csv", data_files="train.csv")
-dataset = load_dataset("json", data_files={"train": "train.jsonl", "test": "test.jsonl"})
-
-# Create train/test split
-split = dataset.train_test_split(test_size=0.2, seed=42)
-# split["train"], split["test"]
-```
-
-### Preprocessing
-
-```python
-from transformers import AutoTokenizer
-
-tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
-
-def preprocess(examples):
-    return tokenizer(
-        examples["text"],
-        truncation=True,
-        max_length=512,
-        padding="max_length",
-    )
-
-tokenized = dataset.map(preprocess, batched=True, remove_columns=["text"])
-```
-
-- Use `batched=True` for 10-100x faster preprocessing.
-- `remove_columns` drops raw text columns that the model does not need.
-- For large datasets, use `num_proc=4` for multi-process preprocessing.
-
-## Full Fine-Tuning with Trainer
-
-```python
-from transformers import (
-    AutoModelForSequenceClassification,
-    AutoTokenizer,
-    Trainer,
-    TrainingArguments,
-)
-
-model = AutoModelForSequenceClassification.from_pretrained(
-    "bert-base-uncased",
-    num_labels=4,  # Number of classification labels
-)
-tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
-
-training_args = TrainingArguments(
-    output_dir="./results",
-    num_train_epochs=3,
-    per_device_train_batch_size=16,
-    per_device_eval_batch_size=32,
-    learning_rate=2e-5,
-    weight_decay=0.01,
-    eval_strategy="steps",
-    eval_steps=500,
-    save_strategy="steps",
-    save_steps=500,
-    save_total_limit=3,           # Keep only the 3 best checkpoints
-    load_best_model_at_end=True,
-    metric_for_best_model="f1",
-    logging_steps=100,
-    fp16=True,                    # Mixed precision training
-    dataloader_num_workers=4,
-    report_to="wandb",            # or "tensorboard", "none"
-)
-
-trainer = Trainer(
-    model=model,
-    args=training_args,
-    train_dataset=train_dataset,
-    eval_dataset=eval_dataset,
-    tokenizer=tokenizer,
-    compute_metrics=compute_metrics,
-)
-
-trainer.train()
-```
-
-### Custom Metrics
-
-```python
-import evaluate
-import numpy as np
-
-accuracy = evaluate.load("accuracy")
-f1 = evaluate.load("f1")
-precision = evaluate.load("precision")
-recall = evaluate.load("recall")
-
-def compute_metrics(eval_pred):
-    logits, labels = eval_pred
-    predictions = np.argmax(logits, axis=-1)
-
-    return {
-        "accuracy": accuracy.compute(predictions=predictions, references=labels)["accuracy"],
-        "f1": f1.compute(predictions=predictions, references=labels, average="weighted")["f1"],
-        "precision": precision.compute(predictions=predictions, references=labels, average="weighted")["precision"],
-        "recall": recall.compute(predictions=predictions, references=labels, average="weighted")["recall"],
-    }
-```
-
-## PEFT: LoRA Fine-Tuning
-
-LoRA trains a small set of adapter weights (< 1% of model parameters) instead of the full model.
-
-```python
-from peft import LoraConfig, get_peft_model, TaskType
-
-# Configure LoRA
-lora_config = LoraConfig(
-    task_type=TaskType.SEQ_CLS,       # or CAUSAL_LM, SEQ_2_SEQ_LM
-    r=16,                              # Rank of the adapter matrices
-    lora_alpha=32,                     # Scaling factor
-    lora_dropout=0.1,
-    target_modules=["query", "value"], # Which layers to adapt
-    bias="none",
-)
-
-# Wrap the model
-model = AutoModelForSequenceClassification.from_pretrained("bert-base-uncased", num_labels=4)
-model = get_peft_model(model, lora_config)
-
-# Print trainable parameters
-model.print_trainable_parameters()
-# trainable params: 294,912 || all params: 109,482,240 || trainable%: 0.27%
-```
-
-### QLoRA (Quantized LoRA)
-
-Combines 4-bit quantization with LoRA for training large models on consumer GPUs.
-
-```python
-from transformers import AutoModelForCausalLM, BitsAndBytesConfig
-from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training
-
-# Load model in 4-bit
-bnb_config = BitsAndBytesConfig(
-    load_in_4bit=True,
-    bnb_4bit_quant_type="nf4",
-    bnb_4bit_compute_dtype=torch.float16,
-    bnb_4bit_use_double_quant=True,
-)
-
-model = AutoModelForCausalLM.from_pretrained(
-    "meta-llama/Meta-Llama-3-8B",
-    quantization_config=bnb_config,
-    device_map="auto",
-)
-
-# Prepare for training
-model = prepare_model_for_kbit_training(model)
-
-# Add LoRA
-lora_config = LoraConfig(
-    r=64,
-    lora_alpha=16,
-    target_modules=["q_proj", "k_proj", "v_proj", "o_proj"],
-    lora_dropout=0.05,
-    bias="none",
-    task_type="CAUSAL_LM",
-)
-model = get_peft_model(model, lora_config)
-```
-
-- QLoRA enables fine-tuning a 7B model on a single 24GB GPU.
-- `prepare_model_for_kbit_training` handles gradient checkpointing and layer freezing.
-
-## Pushing to the Hub
-
-```python
-# After training
-trainer.push_to_hub("your-org/ticket-classifier-v2")
-
-# Or manually
-model.push_to_hub("your-org/ticket-classifier-v2")
-tokenizer.push_to_hub("your-org/ticket-classifier-v2")
-
-# Push LoRA adapter only (much smaller)
-model.push_to_hub("your-org/ticket-classifier-lora-v2")
-```
-
-## Training Tips
-
-### Learning Rate
-
-- **Full fine-tuning**: 1e-5 to 5e-5 for BERT-size models.
-- **LoRA**: 1e-4 to 3e-4 (adapters can tolerate higher learning rates).
-- **Large models (7B+)**: 1e-5 to 2e-5 for full fine-tuning, 1e-4 for LoRA.
-
-### Batch Size and Gradient Accumulation
-
-```python
-TrainingArguments(
-    per_device_train_batch_size=4,
-    gradient_accumulation_steps=8,
-    # Effective batch size = 4 * 8 = 32
-)
-```
-
-- If GPU memory limits batch size, use gradient accumulation to simulate larger batches.
-- Larger effective batch sizes generally lead to more stable training.
-
-### Early Stopping
-
-```python
-from transformers import EarlyStoppingCallback
-
-trainer = Trainer(
-    ...,
-    callbacks=[EarlyStoppingCallback(early_stopping_patience=3)],
-)
-```
-
-- Stops training when the evaluation metric does not improve for N evaluations.
-- Requires `load_best_model_at_end=True` and `eval_strategy` set to `"steps"` or `"epoch"`.
-
-### Reproducibility
-
-```python
-from transformers import set_seed
-
-set_seed(42)  # Sets seed for torch, numpy, and random
-```
-
-- Set seed before training for reproducible results.
-- Note: some operations (attention dropout, data shuffling) may still vary across hardware.