@synsci/cli-darwin-x64 1.1.49

package/bin/skills/blip-2/references/advanced-usage.md

@@ -0,0 +1,680 @@
+# BLIP-2 Advanced Usage Guide
+
+## Fine-tuning BLIP-2
+
+### LoRA fine-tuning (recommended)
+
+```python
+import torch
+from transformers import Blip2ForConditionalGeneration, Blip2Processor
+from peft import LoraConfig, get_peft_model
+
+# Load base model
+model = Blip2ForConditionalGeneration.from_pretrained(
+    "Salesforce/blip2-opt-2.7b",
+    torch_dtype=torch.float16,
+    device_map="auto"
+)
+
+# Configure LoRA for the language model
+lora_config = LoraConfig(
+    r=16,
+    lora_alpha=32,
+    target_modules=["q_proj", "v_proj", "k_proj", "out_proj"],
+    lora_dropout=0.05,
+    bias="none",
+    task_type="CAUSAL_LM"
+)
+
+# Apply LoRA
+model = get_peft_model(model, lora_config)
+model.print_trainable_parameters()
+# trainable params: ~4M, all params: ~3.8B (0.1%)
+```
+
+### Fine-tuning Q-Former only
+
+```python
+# Freeze everything except Q-Former
+for name, param in model.named_parameters():
+    if "qformer" not in name.lower():
+        param.requires_grad = False
+    else:
+        param.requires_grad = True
+
+# Check trainable parameters
+trainable = sum(p.numel() for p in model.parameters() if p.requires_grad)
+total = sum(p.numel() for p in model.parameters())
+print(f"Trainable: {trainable:,} / {total:,} ({100*trainable/total:.2f}%)")
+```
+
+### Custom dataset for fine-tuning
+
+```python
+import torch
+from torch.utils.data import Dataset, DataLoader
+from PIL import Image
+
+class CaptionDataset(Dataset):
+    def __init__(self, data, processor, max_length=128):
+        self.data = data  # List of {"image_path": str, "caption": str}
+        self.processor = processor
+        self.max_length = max_length
+
+    def __len__(self):
+        return len(self.data)
+
+    def __getitem__(self, idx):
+        item = self.data[idx]
+        image = Image.open(item["image_path"]).convert("RGB")
+
+        # Process inputs
+        encoding = self.processor(
+            images=image,
+            text=item["caption"],
+            padding="max_length",
+            truncation=True,
+            max_length=self.max_length,
+            return_tensors="pt"
+        )
+
+        # Remove batch dimension
+        encoding = {k: v.squeeze(0) for k, v in encoding.items()}
+
+        # Labels for language modeling
+        encoding["labels"] = encoding["input_ids"].clone()
+
+        return encoding
+
+# Create dataloader
+dataset = CaptionDataset(train_data, processor)
+dataloader = DataLoader(dataset, batch_size=8, shuffle=True)
+```
+
+### Training loop
+
+```python
+from transformers import AdamW, get_linear_schedule_with_warmup
+from tqdm import tqdm
+
+# Optimizer
+optimizer = AdamW(model.parameters(), lr=1e-5, weight_decay=0.01)
+
+# Scheduler
+num_epochs = 3
+num_training_steps = len(dataloader) * num_epochs
+scheduler = get_linear_schedule_with_warmup(
+    optimizer,
+    num_warmup_steps=num_training_steps // 10,
+    num_training_steps=num_training_steps
+)
+
+# Training
+model.train()
+for epoch in range(num_epochs):
+    total_loss = 0
+
+    for batch in tqdm(dataloader, desc=f"Epoch {epoch+1}"):
+        batch = {k: v.to("cuda") for k, v in batch.items()}
+
+        outputs = model(**batch)
+        loss = outputs.loss
+
+        loss.backward()
+        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
+
+        optimizer.step()
+        scheduler.step()
+        optimizer.zero_grad()
+
+        total_loss += loss.item()
+
+    avg_loss = total_loss / len(dataloader)
+    print(f"Epoch {epoch+1} - Loss: {avg_loss:.4f}")
+
+# Save fine-tuned model
+model.save_pretrained("blip2-finetuned")
+processor.save_pretrained("blip2-finetuned")
+```
+
+### Fine-tuning with LAVIS
+
+```python
+from lavis.models import load_model_and_preprocess
+from lavis.common.registry import registry
+from lavis.datasets.builders import load_dataset
+
+# Load model
+model, vis_processors, txt_processors = load_model_and_preprocess(
+    name="blip2_opt",
+    model_type="pretrain_opt2.7b",
+    is_eval=False,  # Training mode
+    device="cuda"
+)
+
+# Load dataset
+dataset = load_dataset("coco_caption")
+
+# Get trainer class
+runner_cls = registry.get_runner_class("runner_base")
+runner = runner_cls(
+    cfg=cfg,
+    task=task,
+    model=model,
+    datasets=datasets
+)
+
+# Train
+runner.train()
+```
+
+## Multi-GPU Training
+
+### DataParallel
+
+```python
+import torch.nn as nn
+
+model = Blip2ForConditionalGeneration.from_pretrained(
+    "Salesforce/blip2-opt-2.7b",
+    torch_dtype=torch.float16
+)
+
+# Wrap with DataParallel
+if torch.cuda.device_count() > 1:
+    model = nn.DataParallel(model)
+
+model.to("cuda")
+```
+
+### DistributedDataParallel
+
+```python
+import torch.distributed as dist
+from torch.nn.parallel import DistributedDataParallel as DDP
+from torch.utils.data.distributed import DistributedSampler
+
+def setup(rank, world_size):
+    dist.init_process_group("nccl", rank=rank, world_size=world_size)
+    torch.cuda.set_device(rank)
+
+def train(rank, world_size):
+    setup(rank, world_size)
+
+    model = Blip2ForConditionalGeneration.from_pretrained(
+        "Salesforce/blip2-opt-2.7b",
+        torch_dtype=torch.float16
+    ).to(rank)
+
+    model = DDP(model, device_ids=[rank])
+
+    # Use DistributedSampler
+    sampler = DistributedSampler(dataset, num_replicas=world_size, rank=rank)
+    dataloader = DataLoader(dataset, sampler=sampler, batch_size=4)
+
+    # Training loop
+    for epoch in range(num_epochs):
+        sampler.set_epoch(epoch)
+        for batch in dataloader:
+            # ... training code
+            pass
+
+    dist.destroy_process_group()
+
+# Launch
+import torch.multiprocessing as mp
+world_size = torch.cuda.device_count()
+mp.spawn(train, args=(world_size,), nprocs=world_size)
+```
+
+### Accelerate integration
+
+```python
+from accelerate import Accelerator
+from transformers import Blip2ForConditionalGeneration, Blip2Processor
+
+accelerator = Accelerator(mixed_precision="fp16")
+
+model = Blip2ForConditionalGeneration.from_pretrained("Salesforce/blip2-opt-2.7b")
+optimizer = torch.optim.AdamW(model.parameters(), lr=1e-5)
+
+# Prepare for distributed training
+model, optimizer, dataloader = accelerator.prepare(
+    model, optimizer, dataloader
+)
+
+# Training loop
+for batch in dataloader:
+    outputs = model(**batch)
+    loss = outputs.loss
+
+    accelerator.backward(loss)
+    optimizer.step()
+    optimizer.zero_grad()
+```
+
+## Integration Patterns
+
+### Gradio interface
+
+```python
+import gradio as gr
+import torch
+from PIL import Image
+from transformers import Blip2Processor, Blip2ForConditionalGeneration
+
+# Load model
+processor = Blip2Processor.from_pretrained("Salesforce/blip2-opt-2.7b")
+model = Blip2ForConditionalGeneration.from_pretrained(
+    "Salesforce/blip2-opt-2.7b",
+    torch_dtype=torch.float16,
+    device_map="auto"
+)
+
+def caption_image(image, question=None):
+    if question:
+        inputs = processor(images=image, text=question, return_tensors="pt")
+    else:
+        inputs = processor(images=image, return_tensors="pt")
+
+    inputs = inputs.to("cuda", torch.float16)
+
+    generated_ids = model.generate(**inputs, max_new_tokens=100)
+    return processor.decode(generated_ids[0], skip_special_tokens=True)
+
+# Create interface
+demo = gr.Interface(
+    fn=caption_image,
+    inputs=[
+        gr.Image(type="pil", label="Upload Image"),
+        gr.Textbox(label="Question (optional)", placeholder="What is in this image?")
+    ],
+    outputs=gr.Textbox(label="Response"),
+    title="BLIP-2 Demo",
+    examples=[
+        ["example1.jpg", None],
+        ["example2.jpg", "What colors are in this image?"]
+    ]
+)
+
+demo.launch()
+```
+
+### FastAPI server
+
+```python
+from fastapi import FastAPI, UploadFile, File
+from PIL import Image
+import torch
+from transformers import Blip2Processor, Blip2ForConditionalGeneration
+import io
+
+app = FastAPI()
+
+# Load model at startup
+processor = None
+model = None
+
+@app.on_event("startup")
+async def load_model():
+    global processor, model
+    processor = Blip2Processor.from_pretrained("Salesforce/blip2-opt-2.7b")
+    model = Blip2ForConditionalGeneration.from_pretrained(
+        "Salesforce/blip2-opt-2.7b",
+        torch_dtype=torch.float16,
+        device_map="auto"
+    )
+
+@app.post("/caption")
+async def caption(file: UploadFile = File(...), question: str = None):
+    # Read image
+    contents = await file.read()
+    image = Image.open(io.BytesIO(contents)).convert("RGB")
+
+    # Process
+    if question:
+        inputs = processor(images=image, text=question, return_tensors="pt")
+    else:
+        inputs = processor(images=image, return_tensors="pt")
+
+    inputs = inputs.to("cuda", torch.float16)
+
+    # Generate
+    generated_ids = model.generate(**inputs, max_new_tokens=100)
+    caption = processor.decode(generated_ids[0], skip_special_tokens=True)
+
+    return {"caption": caption}
+
+@app.post("/batch_caption")
+async def batch_caption(files: list[UploadFile] = File(...)):
+    images = []
+    for file in files:
+        contents = await file.read()
+        images.append(Image.open(io.BytesIO(contents)).convert("RGB"))
+
+    inputs = processor(images=images, return_tensors="pt", padding=True)
+    inputs = inputs.to("cuda", torch.float16)
+
+    generated_ids = model.generate(**inputs, max_new_tokens=100)
+    captions = processor.batch_decode(generated_ids, skip_special_tokens=True)
+
+    return {"captions": captions}
+
+# Run: uvicorn server:app --host 0.0.0.0 --port 8000
+```
+
+### LangChain integration
+
+```python
+from langchain.tools import BaseTool
+from langchain.agents import initialize_agent, AgentType
+from langchain.llms import OpenAI
+import torch
+from PIL import Image
+from transformers import Blip2Processor, Blip2ForConditionalGeneration
+
+class ImageCaptionTool(BaseTool):
+    name = "image_caption"
+    description = "Generate a caption for an image. Input should be an image file path."
+
+    def __init__(self):
+        super().__init__()
+        self.processor = Blip2Processor.from_pretrained("Salesforce/blip2-opt-2.7b")
+        self.model = Blip2ForConditionalGeneration.from_pretrained(
+            "Salesforce/blip2-opt-2.7b",
+            torch_dtype=torch.float16,
+            device_map="auto"
+        )
+
+    def _run(self, image_path: str) -> str:
+        image = Image.open(image_path).convert("RGB")
+        inputs = self.processor(images=image, return_tensors="pt").to("cuda", torch.float16)
+        generated_ids = self.model.generate(**inputs, max_new_tokens=50)
+        return self.processor.decode(generated_ids[0], skip_special_tokens=True)
+
+class VisualQATool(BaseTool):
+    name = "visual_qa"
+    description = "Answer questions about an image. Input format: 'image_path|question'"
+
+    def __init__(self, processor, model):
+        super().__init__()
+        self.processor = processor
+        self.model = model
+
+    def _run(self, query: str) -> str:
+        image_path, question = query.split("|")
+        image = Image.open(image_path.strip()).convert("RGB")
+        inputs = self.processor(images=image, text=question.strip(), return_tensors="pt")
+        inputs = inputs.to("cuda", torch.float16)
+        generated_ids = self.model.generate(**inputs, max_new_tokens=50)
+        return self.processor.decode(generated_ids[0], skip_special_tokens=True)
+
+# Use with agent
+tools = [ImageCaptionTool(), VisualQATool(processor, model)]
+agent = initialize_agent(tools, llm, agent=AgentType.ZERO_SHOT_REACT_DESCRIPTION)
+```
+
+## ONNX Export and Deployment
+
+### Export to ONNX
+
+```python
+import torch
+from transformers import Blip2ForConditionalGeneration, Blip2Processor
+
+model = Blip2ForConditionalGeneration.from_pretrained("Salesforce/blip2-opt-2.7b")
+processor = Blip2Processor.from_pretrained("Salesforce/blip2-opt-2.7b")
+
+# Example inputs
+image = Image.open("example.jpg").convert("RGB")
+inputs = processor(images=image, return_tensors="pt")
+
+# Export vision encoder
+torch.onnx.export(
+    model.vision_model,
+    inputs["pixel_values"],
+    "blip2_vision.onnx",
+    input_names=["pixel_values"],
+    output_names=["image_embeds"],
+    dynamic_axes={
+        "pixel_values": {0: "batch_size"},
+        "image_embeds": {0: "batch_size"}
+    },
+    opset_version=14
+)
+```
+
+### TensorRT optimization
+
+```python
+import tensorrt as trt
+import pycuda.driver as cuda
+
+def build_engine(onnx_path, engine_path):
+    logger = trt.Logger(trt.Logger.WARNING)
+    builder = trt.Builder(logger)
+    network = builder.create_network(1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH))
+    parser = trt.OnnxParser(network, logger)
+
+    with open(onnx_path, 'rb') as f:
+        parser.parse(f.read())
+
+    config = builder.create_builder_config()
+    config.set_flag(trt.BuilderFlag.FP16)  # Enable FP16
+    config.max_workspace_size = 1 << 30  # 1GB
+
+    engine = builder.build_serialized_network(network, config)
+
+    with open(engine_path, 'wb') as f:
+        f.write(engine)
+
+build_engine("blip2_vision.onnx", "blip2_vision.trt")
+```
+
+## Specialized Use Cases
+
+### Video captioning (frame-by-frame)
+
+```python
+import cv2
+import torch
+from PIL import Image
+
+def caption_video(video_path, sample_rate=30):
+    """Caption video by sampling frames."""
+    cap = cv2.VideoCapture(video_path)
+    fps = cap.get(cv2.CAP_PROP_FPS)
+    frame_interval = int(fps * sample_rate / 30)  # Sample every N frames
+
+    captions = []
+    frame_count = 0
+
+    while cap.isOpened():
+        ret, frame = cap.read()
+        if not ret:
+            break
+
+        if frame_count % frame_interval == 0:
+            # Convert BGR to RGB
+            rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+            image = Image.fromarray(rgb_frame)
+
+            # Caption
+            inputs = processor(images=image, return_tensors="pt").to("cuda", torch.float16)
+            generated_ids = model.generate(**inputs, max_new_tokens=50)
+            caption = processor.decode(generated_ids[0], skip_special_tokens=True)
+
+            timestamp = frame_count / fps
+            captions.append({"timestamp": timestamp, "caption": caption})
+
+        frame_count += 1
+
+    cap.release()
+    return captions
+
+# Usage
+captions = caption_video("video.mp4", sample_rate=1)  # 1 frame per second
+for c in captions:
+    print(f"[{c['timestamp']:.1f}s] {c['caption']}")
+```
+
+### Document understanding
+
+```python
+def analyze_document(image_path):
+    """Extract information from document image."""
+    image = Image.open(image_path).convert("RGB")
+
+    questions = [
+        "What type of document is this?",
+        "What is the title of this document?",
+        "What are the main sections?",
+        "Summarize the key information."
+    ]
+
+    results = {}
+    for q in questions:
+        inputs = processor(images=image, text=q, return_tensors="pt").to("cuda", torch.float16)
+        generated_ids = model.generate(**inputs, max_new_tokens=100)
+        answer = processor.decode(generated_ids[0], skip_special_tokens=True)
+        results[q] = answer
+
+    return results
+
+# Usage
+doc_info = analyze_document("invoice.png")
+for q, a in doc_info.items():
+    print(f"Q: {q}\nA: {a}\n")
+```
+
+### Medical image analysis
+
+```python
+def analyze_medical_image(image_path, modality="xray"):
+    """Analyze medical images with specific prompts."""
+    image = Image.open(image_path).convert("RGB")
+
+    prompts = {
+        "xray": [
+            "Describe any abnormalities visible in this chest X-ray.",
+            "What anatomical structures are visible?",
+            "Is there any evidence of pathology?"
+        ],
+        "ct": [
+            "Describe the CT scan findings.",
+            "What organs are visible in this slice?",
+            "Are there any masses or lesions?"
+        ],
+        "mri": [
+            "Describe the MRI findings.",
+            "What tissues show abnormal signal intensity?",
+            "What is the most likely diagnosis?"
+        ]
+    }
+
+    results = []
+    for prompt in prompts.get(modality, prompts["xray"]):
+        inputs = processor(images=image, text=prompt, return_tensors="pt").to("cuda", torch.float16)
+        generated_ids = model.generate(**inputs, max_new_tokens=150)
+        answer = processor.decode(generated_ids[0], skip_special_tokens=True)
+        results.append({"question": prompt, "answer": answer})
+
+    return results
+
+# Note: BLIP-2 is not trained on medical data - use specialized models for clinical use
+```
+
+## Evaluation
+
+### Caption evaluation metrics
+
+```python
+from pycocoevalcap.bleu.bleu import Bleu
+from pycocoevalcap.meteor.meteor import Meteor
+from pycocoevalcap.rouge.rouge import Rouge
+from pycocoevalcap.cider.cider import Cider
+
+def evaluate_captions(predictions, references):
+    """
+    Evaluate generated captions against references.
+
+    Args:
+        predictions: dict {image_id: [caption]}
+        references: dict {image_id: [ref1, ref2, ...]}
+    """
+    scorers = [
+        (Bleu(4), ["Bleu_1", "Bleu_2", "Bleu_3", "Bleu_4"]),
+        (Meteor(), "METEOR"),
+        (Rouge(), "ROUGE_L"),
+        (Cider(), "CIDEr"),
+    ]
+
+    results = {}
+    for scorer, method in scorers:
+        score, _ = scorer.compute_score(references, predictions)
+        if isinstance(method, list):
+            for sc, m in zip(score, method):
+                results[m] = sc
+        else:
+            results[method] = score
+
+    return results
+
+# Usage
+preds = {0: ["a cat sitting on a mat"], 1: ["a dog running in the park"]}
+refs = {0: ["a cat on a mat", "cat sitting"], 1: ["dog in park", "running dog"]}
+scores = evaluate_captions(preds, refs)
+print(scores)
+```
+
+### VQA evaluation
+
+```python
+def vqa_accuracy(predictions, ground_truths):
+    """
+    VQA accuracy metric (soft accuracy from VQA challenge).
+
+    Args:
+        predictions: list of predicted answers
+        ground_truths: list of lists (multiple annotator answers)
+    """
+    def compute_accuracy(pred, gts):
+        pred = pred.lower().strip()
+        gts = [gt.lower().strip() for gt in gts]
+
+        # Count matches
+        matches = sum(1 for gt in gts if pred == gt)
+        return min(matches / 3, 1.0)  # Cap at 1.0
+
+    accuracies = []
+    for pred, gts in zip(predictions, ground_truths):
+        accuracies.append(compute_accuracy(pred, gts))
+
+    return sum(accuracies) / len(accuracies)
+
+# Usage
+preds = ["yes", "a dog", "blue"]
+gts = [["yes", "yes", "no"], ["dog", "a dog", "puppy"], ["blue", "light blue", "azure"]]
+acc = vqa_accuracy(preds, gts)
+print(f"VQA Accuracy: {acc:.2%}")
+```
+
+## Model Comparison
+
+### BLIP-2 variants benchmark
+
+| Model | COCO Caption (CIDEr) | VQAv2 (Acc) | GQA (Acc) | VRAM |
+|-------|---------------------|-------------|-----------|------|
+| blip2-opt-2.7b | 129.7 | 52.6 | 41.3 | 8GB |
+| blip2-opt-6.7b | 133.4 | 54.2 | 42.8 | 16GB |
+| blip2-flan-t5-xl | 138.1 | 62.9 | 44.1 | 10GB |
+| blip2-flan-t5-xxl | 145.8 | 65.0 | 45.9 | 26GB |
+
+### Comparison with other models
+
+| Model | Architecture | Zero-shot VQA | Training Cost |
+|-------|-------------|---------------|---------------|
+| BLIP-2 | Q-Former + LLM | Excellent | Low (Q-Former only) |
+| LLaVA | Linear + LLM | Good | Medium |
+| Flamingo | Perceiver + LLM | Excellent | High |
+| InstructBLIP | Q-Former + LLM | Best | Low |