@diottodev/runic 1.0.0 → 1.1.0

package/engineering-team/skills/senior-computer-vision/scripts/dataset_pipeline_builder.py

@@ -0,0 +1,1699 @@
+#!/usr/bin/env python3
+"""
+Dataset Pipeline Builder for Computer Vision
+
+Production-grade tool for building and managing CV dataset pipelines.
+Supports format conversion, splitting, augmentation config, and validation.
+
+Supported formats:
+- COCO (JSON annotations)
+- YOLO (txt per image)
+- Pascal VOC (XML annotations)
+- CVAT (XML export)
+
+Usage:
+    python dataset_pipeline_builder.py analyze --input /path/to/dataset
+    python dataset_pipeline_builder.py convert --input /path/to/coco --output /path/to/yolo --format yolo
+    python dataset_pipeline_builder.py split --input /path/to/dataset --train 0.8 --val 0.1 --test 0.1
+    python dataset_pipeline_builder.py augment-config --task detection --output augmentations.yaml
+    python dataset_pipeline_builder.py validate --input /path/to/dataset --format coco
+"""
+
+import os
+import sys
+import json
+import random
+import shutil
+import logging
+import argparse
+import hashlib
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple, Set, Any
+from datetime import datetime
+from collections import defaultdict
+import xml.etree.ElementTree as ET
+
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+
+# ============================================================================
+# Dataset Format Definitions
+# ============================================================================
+
+SUPPORTED_IMAGE_EXTENSIONS = {'.jpg', '.jpeg', '.png', '.bmp', '.tiff', '.webp'}
+
+COCO_CATEGORIES_TEMPLATE = {
+    "info": {
+        "description": "Custom Dataset",
+        "version": "1.0",
+        "year": datetime.now().year,
+        "contributor": "Dataset Pipeline Builder",
+        "date_created": datetime.now().isoformat()
+    },
+    "licenses": [{"id": 1, "name": "Unknown", "url": ""}],
+    "images": [],
+    "annotations": [],
+    "categories": []
+}
+
+YOLO_DATA_YAML_TEMPLATE = """# YOLO Dataset Configuration
+# Generated by Dataset Pipeline Builder
+
+path: {dataset_path}
+train: {train_path}
+val: {val_path}
+test: {test_path}
+
+# Classes
+nc: {num_classes}
+names: {class_names}
+
+# Optional: Download script
+# download:
+"""
+
+AUGMENTATION_PRESETS = {
+    'detection': {
+        'light': {
+            'horizontal_flip': 0.5,
+            'vertical_flip': 0.0,
+            'rotate': {'limit': 10, 'p': 0.3},
+            'brightness_contrast': {'brightness_limit': 0.1, 'contrast_limit': 0.1, 'p': 0.3},
+            'blur': {'blur_limit': 3, 'p': 0.1}
+        },
+        'medium': {
+            'horizontal_flip': 0.5,
+            'vertical_flip': 0.1,
+            'rotate': {'limit': 15, 'p': 0.5},
+            'scale': {'scale_limit': 0.2, 'p': 0.5},
+            'brightness_contrast': {'brightness_limit': 0.2, 'contrast_limit': 0.2, 'p': 0.5},
+            'hue_saturation': {'hue_shift_limit': 10, 'sat_shift_limit': 20, 'p': 0.3},
+            'blur': {'blur_limit': 5, 'p': 0.2},
+            'noise': {'var_limit': (10, 50), 'p': 0.2}
+        },
+        'heavy': {
+            'horizontal_flip': 0.5,
+            'vertical_flip': 0.2,
+            'rotate': {'limit': 30, 'p': 0.7},
+            'scale': {'scale_limit': 0.3, 'p': 0.6},
+            'brightness_contrast': {'brightness_limit': 0.3, 'contrast_limit': 0.3, 'p': 0.6},
+            'hue_saturation': {'hue_shift_limit': 20, 'sat_shift_limit': 30, 'p': 0.5},
+            'blur': {'blur_limit': 7, 'p': 0.3},
+            'noise': {'var_limit': (10, 80), 'p': 0.3},
+            'mosaic': {'p': 0.5},
+            'mixup': {'p': 0.3},
+            'cutout': {'num_holes': 8, 'max_h_size': 32, 'max_w_size': 32, 'p': 0.3}
+        }
+    },
+    'segmentation': {
+        'light': {
+            'horizontal_flip': 0.5,
+            'rotate': {'limit': 10, 'p': 0.3},
+            'elastic_transform': {'alpha': 50, 'sigma': 5, 'p': 0.1}
+        },
+        'medium': {
+            'horizontal_flip': 0.5,
+            'vertical_flip': 0.2,
+            'rotate': {'limit': 20, 'p': 0.5},
+            'scale': {'scale_limit': 0.2, 'p': 0.4},
+            'elastic_transform': {'alpha': 100, 'sigma': 10, 'p': 0.3},
+            'grid_distortion': {'num_steps': 5, 'distort_limit': 0.3, 'p': 0.3}
+        },
+        'heavy': {
+            'horizontal_flip': 0.5,
+            'vertical_flip': 0.3,
+            'rotate': {'limit': 45, 'p': 0.7},
+            'scale': {'scale_limit': 0.4, 'p': 0.6},
+            'elastic_transform': {'alpha': 200, 'sigma': 20, 'p': 0.5},
+            'grid_distortion': {'num_steps': 7, 'distort_limit': 0.5, 'p': 0.4},
+            'optical_distortion': {'distort_limit': 0.5, 'shift_limit': 0.5, 'p': 0.3}
+        }
+    },
+    'classification': {
+        'light': {
+            'horizontal_flip': 0.5,
+            'rotate': {'limit': 15, 'p': 0.3},
+            'brightness_contrast': {'p': 0.3}
+        },
+        'medium': {
+            'horizontal_flip': 0.5,
+            'rotate': {'limit': 30, 'p': 0.5},
+            'color_jitter': {'brightness': 0.2, 'contrast': 0.2, 'saturation': 0.2, 'hue': 0.1, 'p': 0.5},
+            'random_crop': {'height': 224, 'width': 224, 'p': 0.5},
+            'cutout': {'num_holes': 1, 'max_h_size': 40, 'max_w_size': 40, 'p': 0.3}
+        },
+        'heavy': {
+            'horizontal_flip': 0.5,
+            'vertical_flip': 0.2,
+            'rotate': {'limit': 45, 'p': 0.7},
+            'color_jitter': {'brightness': 0.4, 'contrast': 0.4, 'saturation': 0.4, 'hue': 0.2, 'p': 0.7},
+            'random_resized_crop': {'height': 224, 'width': 224, 'scale': (0.5, 1.0), 'p': 0.6},
+            'cutout': {'num_holes': 4, 'max_h_size': 60, 'max_w_size': 60, 'p': 0.5},
+            'auto_augment': {'policy': 'imagenet', 'p': 0.5},
+            'rand_augment': {'num_ops': 2, 'magnitude': 9, 'p': 0.5}
+        }
+    }
+}
+
+
+# ============================================================================
+# Dataset Analysis
+# ============================================================================
+
+class DatasetAnalyzer:
+    """Analyze dataset structure and statistics."""
+
+    def __init__(self, dataset_path: str):
+        self.dataset_path = Path(dataset_path)
+        self.stats = {}
+
+    def analyze(self) -> Dict[str, Any]:
+        """Run full dataset analysis."""
+        logger.info(f"Analyzing dataset at: {self.dataset_path}")
+
+        # Detect format
+        detected_format = self._detect_format()
+        self.stats['format'] = detected_format
+
+        # Count images
+        images = self._find_images()
+        self.stats['total_images'] = len(images)
+
+        # Analyze images
+        self.stats['image_stats'] = self._analyze_images(images)
+
+        # Analyze annotations based on format
+        if detected_format == 'coco':
+            self.stats['annotations'] = self._analyze_coco()
+        elif detected_format == 'yolo':
+            self.stats['annotations'] = self._analyze_yolo()
+        elif detected_format == 'voc':
+            self.stats['annotations'] = self._analyze_voc()
+        else:
+            self.stats['annotations'] = {'error': 'Unknown format'}
+
+        # Dataset quality checks
+        self.stats['quality'] = self._quality_checks()
+
+        return self.stats
+
+    def _detect_format(self) -> str:
+        """Auto-detect dataset format."""
+        # Check for COCO JSON
+        for json_file in self.dataset_path.rglob('*.json'):
+            try:
+                with open(json_file) as f:
+                    data = json.load(f)
+                if 'annotations' in data and 'images' in data:
+                    return 'coco'
+            except:
+                pass
+
+        # Check for YOLO txt files
+        txt_files = list(self.dataset_path.rglob('*.txt'))
+        if txt_files:
+            # Check if txt contains YOLO format (class x_center y_center width height)
+            for txt_file in txt_files[:5]:
+                if txt_file.name == 'classes.txt':
+                    continue
+                try:
+                    with open(txt_file) as f:
+                        line = f.readline().strip()
+                    if line:
+                        parts = line.split()
+                        if len(parts) == 5 and all(self._is_float(p) for p in parts):
+                            return 'yolo'
+                except:
+                    pass
+
+        # Check for VOC XML
+        xml_files = list(self.dataset_path.rglob('*.xml'))
+        for xml_file in xml_files[:5]:
+            try:
+                tree = ET.parse(xml_file)
+                root = tree.getroot()
+                if root.tag == 'annotation' and root.find('object') is not None:
+                    return 'voc'
+            except:
+                pass
+
+        return 'unknown'
+
+    def _is_float(self, s: str) -> bool:
+        """Check if string is a float."""
+        try:
+            float(s)
+            return True
+        except ValueError:
+            return False
+
+    def _find_images(self) -> List[Path]:
+        """Find all images in dataset."""
+        images = []
+        for ext in SUPPORTED_IMAGE_EXTENSIONS:
+            images.extend(self.dataset_path.rglob(f'*{ext}'))
+            images.extend(self.dataset_path.rglob(f'*{ext.upper()}'))
+        return images
+
+    def _analyze_images(self, images: List[Path]) -> Dict:
+        """Analyze image files without loading them."""
+        stats = {
+            'count': len(images),
+            'extensions': defaultdict(int),
+            'sizes': [],
+            'locations': defaultdict(int)
+        }
+
+        for img in images:
+            stats['extensions'][img.suffix.lower()] += 1
+            stats['sizes'].append(img.stat().st_size)
+            # Track which subdirectory
+            rel_path = img.relative_to(self.dataset_path)
+            if len(rel_path.parts) > 1:
+                stats['locations'][rel_path.parts[0]] += 1
+            else:
+                stats['locations']['root'] += 1
+
+        if stats['sizes']:
+            stats['total_size_mb'] = sum(stats['sizes']) / (1024 * 1024)
+            stats['avg_size_kb'] = (sum(stats['sizes']) / len(stats['sizes'])) / 1024
+            stats['min_size_kb'] = min(stats['sizes']) / 1024
+            stats['max_size_kb'] = max(stats['sizes']) / 1024
+
+        stats['extensions'] = dict(stats['extensions'])
+        stats['locations'] = dict(stats['locations'])
+        del stats['sizes']  # Don't include raw sizes
+
+        return stats
+
+    def _analyze_coco(self) -> Dict:
+        """Analyze COCO format annotations."""
+        stats = {
+            'total_annotations': 0,
+            'classes': {},
+            'images_with_annotations': 0,
+            'annotations_per_image': {},
+            'bbox_stats': {}
+        }
+
+        # Find COCO JSON files
+        for json_file in self.dataset_path.rglob('*.json'):
+            try:
+                with open(json_file) as f:
+                    data = json.load(f)
+
+                if 'annotations' not in data:
+                    continue
+
+                # Build category mapping
+                cat_map = {}
+                if 'categories' in data:
+                    for cat in data['categories']:
+                        cat_map[cat['id']] = cat['name']
+
+                # Count annotations per class
+                img_annotations = defaultdict(int)
+                bbox_widths = []
+                bbox_heights = []
+                bbox_areas = []
+
+                for ann in data['annotations']:
+                    stats['total_annotations'] += 1
+                    cat_id = ann.get('category_id')
+                    cat_name = cat_map.get(cat_id, f'class_{cat_id}')
+                    stats['classes'][cat_name] = stats['classes'].get(cat_name, 0) + 1
+                    img_annotations[ann.get('image_id')] += 1
+
+                    # Bbox stats
+                    if 'bbox' in ann:
+                        bbox = ann['bbox']  # [x, y, width, height]
+                        if len(bbox) == 4:
+                            bbox_widths.append(bbox[2])
+                            bbox_heights.append(bbox[3])
+                            bbox_areas.append(bbox[2] * bbox[3])
+
+                stats['images_with_annotations'] = len(img_annotations)
+                if img_annotations:
+                    counts = list(img_annotations.values())
+                    stats['annotations_per_image'] = {
+                        'min': min(counts),
+                        'max': max(counts),
+                        'avg': sum(counts) / len(counts)
+                    }
+
+                if bbox_areas:
+                    stats['bbox_stats'] = {
+                        'avg_width': sum(bbox_widths) / len(bbox_widths),
+                        'avg_height': sum(bbox_heights) / len(bbox_heights),
+                        'avg_area': sum(bbox_areas) / len(bbox_areas),
+                        'min_area': min(bbox_areas),
+                        'max_area': max(bbox_areas)
+                    }
+
+            except Exception as e:
+                logger.warning(f"Error parsing {json_file}: {e}")
+
+        return stats
+
+    def _analyze_yolo(self) -> Dict:
+        """Analyze YOLO format annotations."""
+        stats = {
+            'total_annotations': 0,
+            'classes': defaultdict(int),
+            'images_with_annotations': 0,
+            'bbox_stats': {}
+        }
+
+        # Find classes.txt if exists
+        class_names = {}
+        classes_file = self.dataset_path / 'classes.txt'
+        if classes_file.exists():
+            with open(classes_file) as f:
+                for i, line in enumerate(f):
+                    class_names[i] = line.strip()
+
+        bbox_widths = []
+        bbox_heights = []
+
+        for txt_file in self.dataset_path.rglob('*.txt'):
+            if txt_file.name == 'classes.txt':
+                continue
+
+            try:
+                with open(txt_file) as f:
+                    lines = f.readlines()
+
+                if lines:
+                    stats['images_with_annotations'] += 1
+
+                for line in lines:
+                    parts = line.strip().split()
+                    if len(parts) >= 5:
+                        stats['total_annotations'] += 1
+                        class_id = int(parts[0])
+                        class_name = class_names.get(class_id, f'class_{class_id}')
+                        stats['classes'][class_name] += 1
+
+                        # Bbox stats (normalized coords)
+                        w = float(parts[3])
+                        h = float(parts[4])
+                        bbox_widths.append(w)
+                        bbox_heights.append(h)
+
+            except Exception as e:
+                logger.warning(f"Error parsing {txt_file}: {e}")
+
+        stats['classes'] = dict(stats['classes'])
+
+        if bbox_widths:
+            stats['bbox_stats'] = {
+                'avg_width_normalized': sum(bbox_widths) / len(bbox_widths),
+                'avg_height_normalized': sum(bbox_heights) / len(bbox_heights),
+                'min_width_normalized': min(bbox_widths),
+                'max_width_normalized': max(bbox_widths)
+            }
+
+        return stats
+
+    def _analyze_voc(self) -> Dict:
+        """Analyze Pascal VOC format annotations."""
+        stats = {
+            'total_annotations': 0,
+            'classes': defaultdict(int),
+            'images_with_annotations': 0,
+            'difficulties': {'easy': 0, 'difficult': 0}
+        }
+
+        for xml_file in self.dataset_path.rglob('*.xml'):
+            try:
+                tree = ET.parse(xml_file)
+                root = tree.getroot()
+
+                if root.tag != 'annotation':
+                    continue
+
+                objects = root.findall('object')
+                if objects:
+                    stats['images_with_annotations'] += 1
+
+                for obj in objects:
+                    stats['total_annotations'] += 1
+                    name = obj.find('name')
+                    if name is not None:
+                        stats['classes'][name.text] += 1
+
+                    difficult = obj.find('difficult')
+                    if difficult is not None and difficult.text == '1':
+                        stats['difficulties']['difficult'] += 1
+                    else:
+                        stats['difficulties']['easy'] += 1
+
+            except Exception as e:
+                logger.warning(f"Error parsing {xml_file}: {e}")
+
+        stats['classes'] = dict(stats['classes'])
+        return stats
+
+    def _quality_checks(self) -> Dict:
+        """Run quality checks on dataset."""
+        checks = {
+            'issues': [],
+            'warnings': [],
+            'recommendations': []
+        }
+
+        # Check class imbalance
+        if 'annotations' in self.stats and 'classes' in self.stats['annotations']:
+            classes = self.stats['annotations']['classes']
+            if classes:
+                counts = list(classes.values())
+                max_count = max(counts)
+                min_count = min(counts)
+
+                if max_count > 0 and min_count / max_count < 0.1:
+                    checks['warnings'].append(
+                        f"Severe class imbalance detected: ratio {min_count/max_count:.2%}"
+                    )
+                    checks['recommendations'].append(
+                        "Consider oversampling minority classes or using focal loss"
+                    )
+                elif max_count > 0 and min_count / max_count < 0.3:
+                    checks['warnings'].append(
+                        f"Moderate class imbalance: ratio {min_count/max_count:.2%}"
+                    )
+
+        # Check image count
+        if self.stats.get('total_images', 0) < 100:
+            checks['warnings'].append(
+                f"Small dataset: only {self.stats.get('total_images', 0)} images"
+            )
+            checks['recommendations'].append(
+                "Consider data augmentation or transfer learning"
+            )
+
+        # Check for missing annotations
+        if 'annotations' in self.stats:
+            ann_stats = self.stats['annotations']
+            total_images = self.stats.get('total_images', 0)
+            images_with_ann = ann_stats.get('images_with_annotations', 0)
+
+            if total_images > 0 and images_with_ann < total_images:
+                missing = total_images - images_with_ann
+                checks['warnings'].append(
+                    f"{missing} images have no annotations"
+                )
+
+        return checks
+
+
+# ============================================================================
+# Format Conversion
+# ============================================================================
+
+class FormatConverter:
+    """Convert between dataset formats."""
+
+    def __init__(self, input_path: str, output_path: str):
+        self.input_path = Path(input_path)
+        self.output_path = Path(output_path)
+
+    def convert(self, target_format: str, source_format: str = None) -> Dict:
+        """Convert dataset to target format."""
+        # Auto-detect source format if not specified
+        if source_format is None:
+            analyzer = DatasetAnalyzer(str(self.input_path))
+            analyzer.analyze()
+            source_format = analyzer.stats.get('format', 'unknown')
+
+        logger.info(f"Converting from {source_format} to {target_format}")
+
+        conversion_key = f"{source_format}_to_{target_format}"
+
+        converters = {
+            'coco_to_yolo': self._coco_to_yolo,
+            'yolo_to_coco': self._yolo_to_coco,
+            'voc_to_coco': self._voc_to_coco,
+            'voc_to_yolo': self._voc_to_yolo,
+            'coco_to_voc': self._coco_to_voc,
+        }
+
+        if conversion_key not in converters:
+            return {'error': f"Unsupported conversion: {source_format} -> {target_format}"}
+
+        return converters[conversion_key]()
+
+    def _coco_to_yolo(self) -> Dict:
+        """Convert COCO format to YOLO format."""
+        results = {'converted_images': 0, 'converted_annotations': 0}
+
+        # Find COCO JSON
+        coco_files = list(self.input_path.rglob('*.json'))
+
+        for coco_file in coco_files:
+            try:
+                with open(coco_file) as f:
+                    coco_data = json.load(f)
+
+                if 'annotations' not in coco_data:
+                    continue
+
+                # Create output directories
+                self.output_path.mkdir(parents=True, exist_ok=True)
+                labels_dir = self.output_path / 'labels'
+                labels_dir.mkdir(exist_ok=True)
+
+                # Build category and image mappings
+                cat_map = {}
+                for i, cat in enumerate(coco_data.get('categories', [])):
+                    cat_map[cat['id']] = i
+
+                img_map = {}
+                for img in coco_data.get('images', []):
+                    img_map[img['id']] = {
+                        'file_name': img['file_name'],
+                        'width': img['width'],
+                        'height': img['height']
+                    }
+
+                # Group annotations by image
+                annotations_by_image = defaultdict(list)
+                for ann in coco_data['annotations']:
+                    annotations_by_image[ann['image_id']].append(ann)
+
+                # Write YOLO format labels
+                for img_id, annotations in annotations_by_image.items():
+                    if img_id not in img_map:
+                        continue
+
+                    img_info = img_map[img_id]
+                    label_name = Path(img_info['file_name']).stem + '.txt'
+                    label_path = labels_dir / label_name
+
+                    with open(label_path, 'w') as f:
+                        for ann in annotations:
+                            if 'bbox' not in ann:
+                                continue
+
+                            bbox = ann['bbox']  # [x, y, width, height]
+                            cat_id = cat_map.get(ann['category_id'], 0)
+
+                            # Convert to YOLO format (normalized x_center, y_center, width, height)
+                            x_center = (bbox[0] + bbox[2] / 2) / img_info['width']
+                            y_center = (bbox[1] + bbox[3] / 2) / img_info['height']
+                            w = bbox[2] / img_info['width']
+                            h = bbox[3] / img_info['height']
+
+                            f.write(f"{cat_id} {x_center:.6f} {y_center:.6f} {w:.6f} {h:.6f}\n")
+                            results['converted_annotations'] += 1
+
+                    results['converted_images'] += 1
+
+                # Write classes.txt
+                classes = [None] * len(cat_map)
+                for cat in coco_data.get('categories', []):
+                    idx = cat_map[cat['id']]
+                    classes[idx] = cat['name']
+
+                with open(self.output_path / 'classes.txt', 'w') as f:
+                    for class_name in classes:
+                        f.write(f"{class_name}\n")
+
+                # Write data.yaml for YOLO training
+                yaml_content = YOLO_DATA_YAML_TEMPLATE.format(
+                    dataset_path=str(self.output_path.absolute()),
+                    train_path='images/train',
+                    val_path='images/val',
+                    test_path='images/test',
+                    num_classes=len(classes),
+                    class_names=classes
+                )
+                with open(self.output_path / 'data.yaml', 'w') as f:
+                    f.write(yaml_content)
+
+            except Exception as e:
+                logger.error(f"Error converting {coco_file}: {e}")
+
+        return results
+
+    def _yolo_to_coco(self) -> Dict:
+        """Convert YOLO format to COCO format."""
+        results = {'converted_images': 0, 'converted_annotations': 0}
+
+        coco_data = COCO_CATEGORIES_TEMPLATE.copy()
+        coco_data['images'] = []
+        coco_data['annotations'] = []
+        coco_data['categories'] = []
+
+        # Read classes
+        classes_file = self.input_path / 'classes.txt'
+        class_names = []
+        if classes_file.exists():
+            with open(classes_file) as f:
+                class_names = [line.strip() for line in f.readlines()]
+
+        for i, name in enumerate(class_names):
+            coco_data['categories'].append({
+                'id': i,
+                'name': name,
+                'supercategory': 'object'
+            })
+
+        # Find images and labels
+        images = []
+        for ext in SUPPORTED_IMAGE_EXTENSIONS:
+            images.extend(self.input_path.rglob(f'*{ext}'))
+
+        annotation_id = 1
+        for img_id, img_path in enumerate(images, 1):
+            # Try to get image dimensions (without PIL)
+            # Assume 640x640 if can't determine
+            width, height = 640, 640
+
+            coco_data['images'].append({
+                'id': img_id,
+                'file_name': img_path.name,
+                'width': width,
+                'height': height
+            })
+            results['converted_images'] += 1
+
+            # Find corresponding label
+            label_path = img_path.with_suffix('.txt')
+            if not label_path.exists():
+                # Try labels subdirectory
+                label_path = img_path.parent.parent / 'labels' / (img_path.stem + '.txt')
+
+            if label_path.exists():
+                with open(label_path) as f:
+                    for line in f:
+                        parts = line.strip().split()
+                        if len(parts) >= 5:
+                            class_id = int(parts[0])
+                            x_center = float(parts[1]) * width
+                            y_center = float(parts[2]) * height
+                            w = float(parts[3]) * width
+                            h = float(parts[4]) * height
+
+                            # Convert to COCO format [x, y, width, height]
+                            x = x_center - w / 2
+                            y = y_center - h / 2
+
+                            coco_data['annotations'].append({
+                                'id': annotation_id,
+                                'image_id': img_id,
+                                'category_id': class_id,
+                                'bbox': [x, y, w, h],
+                                'area': w * h,
+                                'iscrowd': 0
+                            })
+                            annotation_id += 1
+                            results['converted_annotations'] += 1
+
+        # Write COCO JSON
+        self.output_path.mkdir(parents=True, exist_ok=True)
+        with open(self.output_path / 'annotations.json', 'w') as f:
+            json.dump(coco_data, f, indent=2)
+
+        return results
+
+    def _voc_to_coco(self) -> Dict:
+        """Convert Pascal VOC format to COCO format."""
+        results = {'converted_images': 0, 'converted_annotations': 0}
+
+        coco_data = COCO_CATEGORIES_TEMPLATE.copy()
+        coco_data['images'] = []
+        coco_data['annotations'] = []
+        coco_data['categories'] = []
+
+        class_to_id = {}
+        annotation_id = 1
+
+        for img_id, xml_file in enumerate(self.input_path.rglob('*.xml'), 1):
+            try:
+                tree = ET.parse(xml_file)
+                root = tree.getroot()
+
+                if root.tag != 'annotation':
+                    continue
+
+                # Get image info
+                filename = root.find('filename')
+                size = root.find('size')
+
+                if filename is None or size is None:
+                    continue
+
+                width = int(size.find('width').text)
+                height = int(size.find('height').text)
+
+                coco_data['images'].append({
+                    'id': img_id,
+                    'file_name': filename.text,
+                    'width': width,
+                    'height': height
+                })
+                results['converted_images'] += 1
+
+                # Convert objects
+                for obj in root.findall('object'):
+                    name = obj.find('name').text
+
+                    if name not in class_to_id:
+                        class_to_id[name] = len(class_to_id)
+                        coco_data['categories'].append({
+                            'id': class_to_id[name],
+                            'name': name,
+                            'supercategory': 'object'
+                        })
+
+                    bndbox = obj.find('bndbox')
+                    xmin = float(bndbox.find('xmin').text)
+                    ymin = float(bndbox.find('ymin').text)
+                    xmax = float(bndbox.find('xmax').text)
+                    ymax = float(bndbox.find('ymax').text)
+
+                    coco_data['annotations'].append({
+                        'id': annotation_id,
+                        'image_id': img_id,
+                        'category_id': class_to_id[name],
+                        'bbox': [xmin, ymin, xmax - xmin, ymax - ymin],
+                        'area': (xmax - xmin) * (ymax - ymin),
+                        'iscrowd': 0
+                    })
+                    annotation_id += 1
+                    results['converted_annotations'] += 1
+
+            except Exception as e:
+                logger.warning(f"Error parsing {xml_file}: {e}")
+
+        # Write output
+        self.output_path.mkdir(parents=True, exist_ok=True)
+        with open(self.output_path / 'annotations.json', 'w') as f:
+            json.dump(coco_data, f, indent=2)
+
+        return results
+
+    def _voc_to_yolo(self) -> Dict:
+        """Convert Pascal VOC format to YOLO format."""
+        # First convert to COCO, then to YOLO
+        temp_coco = self.output_path / '_temp_coco'
+
+        converter1 = FormatConverter(str(self.input_path), str(temp_coco))
+        converter1._voc_to_coco()
+
+        converter2 = FormatConverter(str(temp_coco), str(self.output_path))
+        results = converter2._coco_to_yolo()
+
+        # Clean up temp
+        shutil.rmtree(temp_coco, ignore_errors=True)
+
+        return results
+
+    def _coco_to_voc(self) -> Dict:
+        """Convert COCO format to Pascal VOC format."""
+        results = {'converted_images': 0, 'converted_annotations': 0}
+
+        self.output_path.mkdir(parents=True, exist_ok=True)
+        annotations_dir = self.output_path / 'Annotations'
+        annotations_dir.mkdir(exist_ok=True)
+
+        for coco_file in self.input_path.rglob('*.json'):
+            try:
+                with open(coco_file) as f:
+                    coco_data = json.load(f)
+
+                if 'annotations' not in coco_data:
+                    continue
+
+                # Build mappings
+                cat_map = {cat['id']: cat['name'] for cat in coco_data.get('categories', [])}
+                img_map = {img['id']: img for img in coco_data.get('images', [])}
+
+                # Group by image
+                ann_by_image = defaultdict(list)
+                for ann in coco_data['annotations']:
+                    ann_by_image[ann['image_id']].append(ann)
+
+                for img_id, annotations in ann_by_image.items():
+                    if img_id not in img_map:
+                        continue
+
+                    img_info = img_map[img_id]
+
+                    # Create VOC XML
+                    annotation = ET.Element('annotation')
+
+                    ET.SubElement(annotation, 'folder').text = 'images'
+                    ET.SubElement(annotation, 'filename').text = img_info['file_name']
+
+                    size = ET.SubElement(annotation, 'size')
+                    ET.SubElement(size, 'width').text = str(img_info['width'])
+                    ET.SubElement(size, 'height').text = str(img_info['height'])
+                    ET.SubElement(size, 'depth').text = '3'
+
+                    for ann in annotations:
+                        obj = ET.SubElement(annotation, 'object')
+                        ET.SubElement(obj, 'name').text = cat_map.get(ann['category_id'], 'unknown')
+                        ET.SubElement(obj, 'difficult').text = '0'
+
+                        bbox = ann['bbox']
+                        bndbox = ET.SubElement(obj, 'bndbox')
+                        ET.SubElement(bndbox, 'xmin').text = str(int(bbox[0]))
+                        ET.SubElement(bndbox, 'ymin').text = str(int(bbox[1]))
+                        ET.SubElement(bndbox, 'xmax').text = str(int(bbox[0] + bbox[2]))
+                        ET.SubElement(bndbox, 'ymax').text = str(int(bbox[1] + bbox[3]))
+
+                        results['converted_annotations'] += 1
+
+                    # Write XML
+                    xml_name = Path(img_info['file_name']).stem + '.xml'
+                    tree = ET.ElementTree(annotation)
+                    tree.write(annotations_dir / xml_name)
+                    results['converted_images'] += 1
+
+            except Exception as e:
+                logger.error(f"Error converting {coco_file}: {e}")
+
+        return results
+
+
+# ============================================================================
+# Dataset Splitting
+# ============================================================================
+
+class DatasetSplitter:
+    """Split dataset into train/val/test sets."""
+
+    def __init__(self, dataset_path: str, output_path: str = None):
+        self.dataset_path = Path(dataset_path)
+        self.output_path = Path(output_path) if output_path else self.dataset_path
+
+    def split(self, train: float = 0.8, val: float = 0.1, test: float = 0.1,
+              stratify: bool = True, seed: int = 42) -> Dict:
+        """Split dataset with optional stratification."""
+
+        if abs(train + val + test - 1.0) > 0.001:
+            raise ValueError(f"Split ratios must sum to 1.0, got {train + val + test}")
+
+        random.seed(seed)
+        logger.info(f"Splitting dataset: train={train}, val={val}, test={test}")
+
+        # Detect format and find images
+        analyzer = DatasetAnalyzer(str(self.dataset_path))
+        analyzer.analyze()
+        detected_format = analyzer.stats.get('format', 'unknown')
+
+        images = []
+        for ext in SUPPORTED_IMAGE_EXTENSIONS:
+            images.extend(self.dataset_path.rglob(f'*{ext}'))
+
+        if not images:
+            return {'error': 'No images found'}
+
+        # Stratify if requested and we have class info
+        if stratify and detected_format in ['coco', 'yolo']:
+            splits = self._stratified_split(images, detected_format, train, val, test)
+        else:
+            splits = self._random_split(images, train, val, test)
+
+        # Create output directories and copy/link files
+        results = self._create_split_directories(splits, detected_format)
+
+        return results
+
+    def _random_split(self, images: List[Path], train: float, val: float, test: float) -> Dict:
+        """Perform random split."""
+        images = list(images)
+        random.shuffle(images)
+
+        n = len(images)
+        train_end = int(n * train)
+        val_end = train_end + int(n * val)
+
+        return {
+            'train': images[:train_end],
+            'val': images[train_end:val_end],
+            'test': images[val_end:]
+        }
+
+    def _stratified_split(self, images: List[Path], format: str,
+                         train: float, val: float, test: float) -> Dict:
+        """Perform stratified split based on class distribution."""
+
+        # Group images by their primary class
+        image_classes = {}
+
+        for img in images:
+            if format == 'yolo':
+                label_path = img.with_suffix('.txt')
+                if not label_path.exists():
+                    label_path = img.parent.parent / 'labels' / (img.stem + '.txt')
+
+                if label_path.exists():
+                    with open(label_path) as f:
+                        line = f.readline()
+                    if line:
+                        class_id = int(line.split()[0])
+                        image_classes[img] = class_id
+                else:
+                    image_classes[img] = -1  # No annotation
+            else:
+                image_classes[img] = -1  # Default for other formats
+
+        # Group by class
+        class_images = defaultdict(list)
+        for img, class_id in image_classes.items():
+            class_images[class_id].append(img)
+
+        # Split each class proportionally
+        splits = {'train': [], 'val': [], 'test': []}
+
+        for class_id, class_imgs in class_images.items():
+            random.shuffle(class_imgs)
+            n = len(class_imgs)
+            train_end = int(n * train)
+            val_end = train_end + int(n * val)
+
+            splits['train'].extend(class_imgs[:train_end])
+            splits['val'].extend(class_imgs[train_end:val_end])
+            splits['test'].extend(class_imgs[val_end:])
+
+        # Shuffle final splits
+        for key in splits:
+            random.shuffle(splits[key])
+
+        return splits
+
+    def _create_split_directories(self, splits: Dict, format: str) -> Dict:
+        """Create split directories and organize files."""
+        results = {
+            'train_count': len(splits['train']),
+            'val_count': len(splits['val']),
+            'test_count': len(splits['test']),
+            'output_path': str(self.output_path)
+        }
+
+        # Create directory structure
+        for split_name in ['train', 'val', 'test']:
+            images_dir = self.output_path / 'images' / split_name
+            labels_dir = self.output_path / 'labels' / split_name
+            images_dir.mkdir(parents=True, exist_ok=True)
+            labels_dir.mkdir(parents=True, exist_ok=True)
+
+            for img_path in splits[split_name]:
+                # Create symlink for image
+                dst_img = images_dir / img_path.name
+                if not dst_img.exists():
+                    try:
+                        dst_img.symlink_to(img_path.absolute())
+                    except OSError:
+                        # Fall back to copy if symlink fails
+                        shutil.copy2(img_path, dst_img)
+
+                # Handle label file
+                if format == 'yolo':
+                    label_path = img_path.with_suffix('.txt')
+                    if not label_path.exists():
+                        label_path = img_path.parent.parent / 'labels' / (img_path.stem + '.txt')
+
+                    if label_path.exists():
+                        dst_label = labels_dir / (img_path.stem + '.txt')
+                        if not dst_label.exists():
+                            try:
+                                dst_label.symlink_to(label_path.absolute())
+                            except OSError:
+                                shutil.copy2(label_path, dst_label)
+
+        # Generate data.yaml for YOLO
+        if format == 'yolo':
+            # Read classes
+            classes_file = self.dataset_path / 'classes.txt'
+            class_names = []
+            if classes_file.exists():
+                with open(classes_file) as f:
+                    class_names = [line.strip() for line in f.readlines()]
+
+            yaml_content = YOLO_DATA_YAML_TEMPLATE.format(
+                dataset_path=str(self.output_path.absolute()),
+                train_path='images/train',
+                val_path='images/val',
+                test_path='images/test',
+                num_classes=len(class_names),
+                class_names=class_names
+            )
+            with open(self.output_path / 'data.yaml', 'w') as f:
+                f.write(yaml_content)
+
+        return results
+
+
+# ============================================================================
+# Augmentation Configuration
+# ============================================================================
+
+class AugmentationConfigGenerator:
+    """Generate augmentation configurations for different CV tasks."""
+
+    @staticmethod
+    def generate(task: str, intensity: str = 'medium',
+                 framework: str = 'albumentations') -> Dict:
+        """Generate augmentation config for task and intensity."""
+
+        if task not in AUGMENTATION_PRESETS:
+            return {'error': f"Unknown task: {task}. Use: detection, segmentation, classification"}
+
+        if intensity not in AUGMENTATION_PRESETS[task]:
+            return {'error': f"Unknown intensity: {intensity}. Use: light, medium, heavy"}
+
+        base_config = AUGMENTATION_PRESETS[task][intensity]
+
+        if framework == 'albumentations':
+            return AugmentationConfigGenerator._to_albumentations(base_config, task)
+        elif framework == 'torchvision':
+            return AugmentationConfigGenerator._to_torchvision(base_config, task)
+        elif framework == 'ultralytics':
+            return AugmentationConfigGenerator._to_ultralytics(base_config, task)
+        else:
+            return base_config
+
+    @staticmethod
+    def _to_albumentations(config: Dict, task: str) -> Dict:
+        """Convert to Albumentations format."""
+        transforms = []
+
+        for aug_name, params in config.items():
+            if aug_name == 'horizontal_flip':
+                transforms.append({
+                    'type': 'HorizontalFlip',
+                    'p': params
+                })
+            elif aug_name == 'vertical_flip':
+                transforms.append({
+                    'type': 'VerticalFlip',
+                    'p': params
+                })
+            elif aug_name == 'rotate':
+                transforms.append({
+                    'type': 'Rotate',
+                    'limit': params.get('limit', 15),
+                    'p': params.get('p', 0.5)
+                })
+            elif aug_name == 'scale':
+                transforms.append({
+                    'type': 'RandomScale',
+                    'scale_limit': params.get('scale_limit', 0.2),
+                    'p': params.get('p', 0.5)
+                })
+            elif aug_name == 'brightness_contrast':
+                transforms.append({
+                    'type': 'RandomBrightnessContrast',
+                    'brightness_limit': params.get('brightness_limit', 0.2),
+                    'contrast_limit': params.get('contrast_limit', 0.2),
+                    'p': params.get('p', 0.5)
+                })
+            elif aug_name == 'hue_saturation':
+                transforms.append({
+                    'type': 'HueSaturationValue',
+                    'hue_shift_limit': params.get('hue_shift_limit', 20),
+                    'sat_shift_limit': params.get('sat_shift_limit', 30),
+                    'p': params.get('p', 0.5)
+                })
+            elif aug_name == 'blur':
+                transforms.append({
+                    'type': 'Blur',
+                    'blur_limit': params.get('blur_limit', 5),
+                    'p': params.get('p', 0.3)
+                })
+            elif aug_name == 'noise':
+                transforms.append({
+                    'type': 'GaussNoise',
+                    'var_limit': params.get('var_limit', (10, 50)),
+                    'p': params.get('p', 0.3)
+                })
+            elif aug_name == 'elastic_transform':
+                transforms.append({
+                    'type': 'ElasticTransform',
+                    'alpha': params.get('alpha', 100),
+                    'sigma': params.get('sigma', 10),
+                    'p': params.get('p', 0.3)
+                })
+            elif aug_name == 'cutout':
+                transforms.append({
+                    'type': 'CoarseDropout',
+                    'max_holes': params.get('num_holes', 8),
+                    'max_height': params.get('max_h_size', 32),
+                    'max_width': params.get('max_w_size', 32),
+                    'p': params.get('p', 0.3)
+                })
+
+        # Add bbox format for detection
+        bbox_params = None
+        if task == 'detection':
+            bbox_params = {
+                'format': 'pascal_voc',
+                'label_fields': ['class_labels'],
+                'min_visibility': 0.3
+            }
+
+        return {
+            'framework': 'albumentations',
+            'task': task,
+            'transforms': transforms,
+            'bbox_params': bbox_params,
+            'code_example': AugmentationConfigGenerator._albumentations_code(transforms, task)
+        }
+
+    @staticmethod
+    def _albumentations_code(transforms: List, task: str) -> str:
+        """Generate Albumentations code example."""
+        code = """import albumentations as A
+from albumentations.pytorch import ToTensorV2
+
+transform = A.Compose([
+"""
+        for t in transforms:
+            params = ', '.join(f"{k}={v}" for k, v in t.items() if k != 'type')
+            code += f"    A.{t['type']}({params}),\n"
+
+        code += "    A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n"
+        code += "    ToTensorV2(),\n"
+        code += "]"
+
+        if task == 'detection':
+            code += ", bbox_params=A.BboxParams(format='pascal_voc', label_fields=['class_labels']))"
+        else:
+            code += ")"
+
+        return code
+
+    @staticmethod
+    def _to_torchvision(config: Dict, task: str) -> Dict:
+        """Convert to torchvision transforms format."""
+        transforms = []
+
+        for aug_name, params in config.items():
+            if aug_name == 'horizontal_flip':
+                transforms.append({
+                    'type': 'RandomHorizontalFlip',
+                    'p': params
+                })
+            elif aug_name == 'vertical_flip':
+                transforms.append({
+                    'type': 'RandomVerticalFlip',
+                    'p': params
+                })
+            elif aug_name == 'rotate':
+                transforms.append({
+                    'type': 'RandomRotation',
+                    'degrees': params.get('limit', 15)
+                })
+            elif aug_name == 'color_jitter':
+                transforms.append({
+                    'type': 'ColorJitter',
+                    'brightness': params.get('brightness', 0.2),
+                    'contrast': params.get('contrast', 0.2),
+                    'saturation': params.get('saturation', 0.2),
+                    'hue': params.get('hue', 0.1)
+                })
+
+        return {
+            'framework': 'torchvision',
+            'task': task,
+            'transforms': transforms
+        }
+
+    @staticmethod
+    def _to_ultralytics(config: Dict, task: str) -> Dict:
+        """Convert to Ultralytics YOLO format."""
+        yolo_config = {
+            'hsv_h': 0.015,
+            'hsv_s': 0.7,
+            'hsv_v': 0.4,
+            'degrees': config.get('rotate', {}).get('limit', 0.0),
+            'translate': 0.1,
+            'scale': config.get('scale', {}).get('scale_limit', 0.5),
+            'shear': 0.0,
+            'perspective': 0.0,
+            'flipud': config.get('vertical_flip', 0.0),
+            'fliplr': config.get('horizontal_flip', 0.5),
+            'mosaic': config.get('mosaic', {}).get('p', 1.0) if 'mosaic' in config else 0.0,
+            'mixup': config.get('mixup', {}).get('p', 0.0) if 'mixup' in config else 0.0,
+            'copy_paste': 0.0
+        }
+
+        return {
+            'framework': 'ultralytics',
+            'task': task,
+            'config': yolo_config,
+            'usage': "# Add to data.yaml or pass to Trainer\nmodel.train(data='data.yaml', augment=True, **aug_config)"
+        }
+
+
+# ============================================================================
+# Dataset Validation
+# ============================================================================
+
+class DatasetValidator:
+    """Validate dataset integrity and quality."""
+
+    def __init__(self, dataset_path: str, format: str = None):
+        self.dataset_path = Path(dataset_path)
+        self.format = format
+
+    def validate(self) -> Dict:
+        """Run all validation checks."""
+        results = {
+            'valid': True,
+            'errors': [],
+            'warnings': [],
+            'stats': {}
+        }
+
+        # Auto-detect format if not specified
+        if self.format is None:
+            analyzer = DatasetAnalyzer(str(self.dataset_path))
+            analyzer.analyze()
+            self.format = analyzer.stats.get('format', 'unknown')
+
+        results['format'] = self.format
+
+        # Run format-specific validation
+        if self.format == 'coco':
+            self._validate_coco(results)
+        elif self.format == 'yolo':
+            self._validate_yolo(results)
+        elif self.format == 'voc':
+            self._validate_voc(results)
+        else:
+            results['warnings'].append(f"Unknown format: {self.format}")
+
+        # General checks
+        self._validate_images(results)
+        self._check_duplicates(results)
+
+        # Set overall validity
+        results['valid'] = len(results['errors']) == 0
+
+        return results
+
+    def _validate_coco(self, results: Dict):
+        """Validate COCO format dataset."""
+        for json_file in self.dataset_path.rglob('*.json'):
+            try:
+                with open(json_file) as f:
+                    data = json.load(f)
+
+                if 'annotations' not in data:
+                    continue
+
+                # Check required fields
+                if 'images' not in data:
+                    results['errors'].append(f"{json_file}: Missing 'images' field")
+                if 'categories' not in data:
+                    results['warnings'].append(f"{json_file}: Missing 'categories' field")
+
+                # Validate annotations
+                image_ids = {img['id'] for img in data.get('images', [])}
+                category_ids = {cat['id'] for cat in data.get('categories', [])}
+
+                for ann in data['annotations']:
+                    if ann.get('image_id') not in image_ids:
+                        results['errors'].append(
+                            f"Annotation {ann.get('id')} references non-existent image {ann.get('image_id')}"
+                        )
+                    if ann.get('category_id') not in category_ids:
+                        results['warnings'].append(
+                            f"Annotation {ann.get('id')} references unknown category {ann.get('category_id')}"
+                        )
+
+                    # Validate bbox
+                    if 'bbox' in ann:
+                        bbox = ann['bbox']
+                        if len(bbox) != 4:
+                            results['errors'].append(
+                                f"Annotation {ann.get('id')}: Invalid bbox format"
+                            )
+                        elif any(v < 0 for v in bbox[:2]) or any(v <= 0 for v in bbox[2:]):
+                            results['warnings'].append(
+                                f"Annotation {ann.get('id')}: Suspicious bbox values {bbox}"
+                            )
+
+                results['stats']['coco_images'] = len(data.get('images', []))
+                results['stats']['coco_annotations'] = len(data['annotations'])
+                results['stats']['coco_categories'] = len(data.get('categories', []))
+
+            except json.JSONDecodeError as e:
+                results['errors'].append(f"{json_file}: Invalid JSON - {e}")
+            except Exception as e:
+                results['errors'].append(f"{json_file}: Error - {e}")
+
+    def _validate_yolo(self, results: Dict):
+        """Validate YOLO format dataset."""
+        label_files = list(self.dataset_path.rglob('*.txt'))
+        valid_labels = 0
+        invalid_labels = 0
+
+        for txt_file in label_files:
+            if txt_file.name == 'classes.txt':
+                continue
+
+            try:
+                with open(txt_file) as f:
+                    lines = f.readlines()
+
+                for line_num, line in enumerate(lines, 1):
+                    parts = line.strip().split()
+                    if not parts:
+                        continue
+
+                    if len(parts) < 5:
+                        results['errors'].append(
+                            f"{txt_file}:{line_num}: Expected 5 values, got {len(parts)}"
+                        )
+                        invalid_labels += 1
+                        continue
+
+                    try:
+                        class_id = int(parts[0])
+                        x, y, w, h = map(float, parts[1:5])
+
+                        # Check normalized coordinates
+                        if not (0 <= x <= 1 and 0 <= y <= 1):
+                            results['warnings'].append(
+                                f"{txt_file}:{line_num}: Center coords outside [0,1]: ({x}, {y})"
+                            )
+                        if not (0 < w <= 1 and 0 < h <= 1):
+                            results['warnings'].append(
+                                f"{txt_file}:{line_num}: Size outside (0,1]: ({w}, {h})"
+                            )
+
+                        valid_labels += 1
+
+                    except ValueError as e:
+                        results['errors'].append(
+                            f"{txt_file}:{line_num}: Invalid values - {e}"
+                        )
+                        invalid_labels += 1
+
+            except Exception as e:
+                results['errors'].append(f"{txt_file}: Error - {e}")
+
+        results['stats']['yolo_valid_labels'] = valid_labels
+        results['stats']['yolo_invalid_labels'] = invalid_labels
+
+    def _validate_voc(self, results: Dict):
+        """Validate Pascal VOC format dataset."""
+        xml_files = list(self.dataset_path.rglob('*.xml'))
+        valid_annotations = 0
+
+        for xml_file in xml_files:
+            try:
+                tree = ET.parse(xml_file)
+                root = tree.getroot()
+
+                if root.tag != 'annotation':
+                    continue
+
+                # Check required fields
+                filename = root.find('filename')
+                if filename is None:
+                    results['warnings'].append(f"{xml_file}: Missing filename")
+
+                size = root.find('size')
+                if size is None:
+                    results['warnings'].append(f"{xml_file}: Missing size")
+                else:
+                    for dim in ['width', 'height']:
+                        if size.find(dim) is None:
+                            results['errors'].append(f"{xml_file}: Missing {dim}")
+
+                # Validate objects
+                for obj in root.findall('object'):
+                    name = obj.find('name')
+                    if name is None or not name.text:
+                        results['errors'].append(f"{xml_file}: Object missing name")
+
+                    bndbox = obj.find('bndbox')
+                    if bndbox is None:
+                        results['errors'].append(f"{xml_file}: Object missing bndbox")
+                    else:
+                        for coord in ['xmin', 'ymin', 'xmax', 'ymax']:
+                            elem = bndbox.find(coord)
+                            if elem is None:
+                                results['errors'].append(f"{xml_file}: Missing {coord}")
+
+                    valid_annotations += 1
+
+            except ET.ParseError as e:
+                results['errors'].append(f"{xml_file}: XML parse error - {e}")
+            except Exception as e:
+                results['errors'].append(f"{xml_file}: Error - {e}")
+
+        results['stats']['voc_annotations'] = valid_annotations
+
+    def _validate_images(self, results: Dict):
+        """Check for image file issues."""
+        images = []
+        for ext in SUPPORTED_IMAGE_EXTENSIONS:
+            images.extend(self.dataset_path.rglob(f'*{ext}'))
+
+        results['stats']['total_images'] = len(images)
+
+        # Check for empty images
+        empty_images = [img for img in images if img.stat().st_size == 0]
+        if empty_images:
+            results['errors'].append(f"Found {len(empty_images)} empty image files")
+
+        # Check for very small images
+        small_images = [img for img in images if img.stat().st_size < 1000]
+        if small_images:
+            results['warnings'].append(f"Found {len(small_images)} very small images (<1KB)")
+
+    def _check_duplicates(self, results: Dict):
+        """Check for duplicate images by hash."""
+        images = []
+        for ext in SUPPORTED_IMAGE_EXTENSIONS:
+            images.extend(self.dataset_path.rglob(f'*{ext}'))
+
+        hashes = {}
+        duplicates = []
+
+        for img in images:
+            try:
+                with open(img, 'rb') as f:
+                    file_hash = hashlib.md5(f.read()).hexdigest()
+
+                if file_hash in hashes:
+                    duplicates.append((img, hashes[file_hash]))
+                else:
+                    hashes[file_hash] = img
+            except:
+                pass
+
+        if duplicates:
+            results['warnings'].append(f"Found {len(duplicates)} duplicate images")
+            results['stats']['duplicate_images'] = len(duplicates)
+
+
+# ============================================================================
+# Main CLI
+# ============================================================================
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Dataset Pipeline Builder for Computer Vision",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  Analyze dataset:
+    python dataset_pipeline_builder.py analyze --input /path/to/dataset
+
+  Convert COCO to YOLO:
+    python dataset_pipeline_builder.py convert --input /path/to/coco --output /path/to/yolo --format yolo
+
+  Split dataset:
+    python dataset_pipeline_builder.py split --input /path/to/dataset --train 0.8 --val 0.1 --test 0.1
+
+  Generate augmentation config:
+    python dataset_pipeline_builder.py augment-config --task detection --intensity heavy
+
+  Validate dataset:
+    python dataset_pipeline_builder.py validate --input /path/to/dataset --format coco
+        """
+    )
+
+    subparsers = parser.add_subparsers(dest='command', help='Command to run')
+
+    # Analyze command
+    analyze_parser = subparsers.add_parser('analyze', help='Analyze dataset structure and statistics')
+    analyze_parser.add_argument('--input', '-i', required=True, help='Path to dataset')
+    analyze_parser.add_argument('--json', action='store_true', help='Output as JSON')
+
+    # Convert command
+    convert_parser = subparsers.add_parser('convert', help='Convert between annotation formats')
+    convert_parser.add_argument('--input', '-i', required=True, help='Input dataset path')
+    convert_parser.add_argument('--output', '-o', required=True, help='Output dataset path')
+    convert_parser.add_argument('--format', '-f', required=True,
+                               choices=['yolo', 'coco', 'voc'],
+                               help='Target format')
+    convert_parser.add_argument('--source-format', '-s',
+                               choices=['yolo', 'coco', 'voc'],
+                               help='Source format (auto-detected if not specified)')
+
+    # Split command
+    split_parser = subparsers.add_parser('split', help='Split dataset into train/val/test')
+    split_parser.add_argument('--input', '-i', required=True, help='Input dataset path')
+    split_parser.add_argument('--output', '-o', help='Output path (default: same as input)')
+    split_parser.add_argument('--train', type=float, default=0.8, help='Train split ratio')
+    split_parser.add_argument('--val', type=float, default=0.1, help='Validation split ratio')
+    split_parser.add_argument('--test', type=float, default=0.1, help='Test split ratio')
+    split_parser.add_argument('--stratify', action='store_true', help='Stratify by class')
+    split_parser.add_argument('--seed', type=int, default=42, help='Random seed')
+
+    # Augmentation config command
+    aug_parser = subparsers.add_parser('augment-config', help='Generate augmentation configuration')
+    aug_parser.add_argument('--task', '-t', required=True,
+                           choices=['detection', 'segmentation', 'classification'],
+                           help='CV task type')
+    aug_parser.add_argument('--intensity', '-n', default='medium',
+                           choices=['light', 'medium', 'heavy'],
+                           help='Augmentation intensity')
+    aug_parser.add_argument('--framework', '-f', default='albumentations',
+                           choices=['albumentations', 'torchvision', 'ultralytics'],
+                           help='Target framework')
+    aug_parser.add_argument('--output', '-o', help='Output file path')
+
+    # Validate command
+    validate_parser = subparsers.add_parser('validate', help='Validate dataset integrity')
+    validate_parser.add_argument('--input', '-i', required=True, help='Path to dataset')
+    validate_parser.add_argument('--format', '-f',
+                                choices=['yolo', 'coco', 'voc'],
+                                help='Dataset format (auto-detected if not specified)')
+    validate_parser.add_argument('--json', action='store_true', help='Output as JSON')
+
+    args = parser.parse_args()
+
+    if args.command is None:
+        parser.print_help()
+        sys.exit(1)
+
+    try:
+        if args.command == 'analyze':
+            analyzer = DatasetAnalyzer(args.input)
+            results = analyzer.analyze()
+
+            if args.json:
+                print(json.dumps(results, indent=2, default=str))
+            else:
+                print("\n" + "="*60)
+                print("DATASET ANALYSIS REPORT")
+                print("="*60)
+                print(f"\nFormat: {results.get('format', 'unknown')}")
+                print(f"Total Images: {results.get('total_images', 0)}")
+
+                if 'image_stats' in results:
+                    stats = results['image_stats']
+                    print(f"\nImage Statistics:")
+                    print(f"  Total Size: {stats.get('total_size_mb', 0):.2f} MB")
+                    print(f"  Extensions: {stats.get('extensions', {})}")
+                    print(f"  Locations: {stats.get('locations', {})}")
+
+                if 'annotations' in results:
+                    ann = results['annotations']
+                    print(f"\nAnnotations:")
+                    print(f"  Total: {ann.get('total_annotations', 0)}")
+                    print(f"  Images with annotations: {ann.get('images_with_annotations', 0)}")
+                    if 'classes' in ann:
+                        print(f"  Classes: {len(ann['classes'])}")
+                        for cls, count in sorted(ann['classes'].items(), key=lambda x: -x[1])[:10]:
+                            print(f"    - {cls}: {count}")
+
+                if 'quality' in results:
+                    q = results['quality']
+                    if q.get('warnings'):
+                        print(f"\nWarnings:")
+                        for w in q['warnings']:
+                            print(f"  ⚠ {w}")
+                    if q.get('recommendations'):
+                        print(f"\nRecommendations:")
+                        for r in q['recommendations']:
+                            print(f"  → {r}")
+
+        elif args.command == 'convert':
+            converter = FormatConverter(args.input, args.output)
+            results = converter.convert(args.format, args.source_format)
+            print(json.dumps(results, indent=2))
+
+        elif args.command == 'split':
+            output = args.output if args.output else args.input
+            splitter = DatasetSplitter(args.input, output)
+            results = splitter.split(
+                train=args.train,
+                val=args.val,
+                test=args.test,
+                stratify=args.stratify,
+                seed=args.seed
+            )
+            print(json.dumps(results, indent=2))
+
+        elif args.command == 'augment-config':
+            config = AugmentationConfigGenerator.generate(
+                args.task,
+                args.intensity,
+                args.framework
+            )
+
+            output = json.dumps(config, indent=2)
+
+            if args.output:
+                with open(args.output, 'w') as f:
+                    f.write(output)
+                print(f"Configuration saved to {args.output}")
+            else:
+                print(output)
+
+        elif args.command == 'validate':
+            validator = DatasetValidator(args.input, args.format)
+            results = validator.validate()
+
+            if args.json:
+                print(json.dumps(results, indent=2))
+            else:
+                print("\n" + "="*60)
+                print("DATASET VALIDATION REPORT")
+                print("="*60)
+                print(f"\nFormat: {results.get('format', 'unknown')}")
+                print(f"Valid: {'✓' if results['valid'] else '✗'}")
+
+                if results.get('errors'):
+                    print(f"\nErrors ({len(results['errors'])}):")
+                    for err in results['errors'][:10]:
+                        print(f"  ✗ {err}")
+                    if len(results['errors']) > 10:
+                        print(f"  ... and {len(results['errors']) - 10} more")
+
+                if results.get('warnings'):
+                    print(f"\nWarnings ({len(results['warnings'])}):")
+                    for warn in results['warnings'][:10]:
+                        print(f"  ⚠ {warn}")
+                    if len(results['warnings']) > 10:
+                        print(f"  ... and {len(results['warnings']) - 10} more")
+
+                if results.get('stats'):
+                    print(f"\nStatistics:")
+                    for key, value in results['stats'].items():
+                        print(f"  {key}: {value}")
+
+        sys.exit(0)
+
+    except Exception as e:
+        logger.error(f"Error: {e}")
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()