aigroup-workflow 1.0.0 → 1.1.0

package/skills/jarvis/engineering-team/engineering-team/senior-computer-vision/references/production_vision_systems.md

@@ -1,1226 +0,0 @@
-# Production Vision Systems
-
-Comprehensive guide to deploying computer vision models in production environments.
-
-## Table of Contents
-
-- [Model Export and Optimization](#model-export-and-optimization)
-- [TensorRT Deployment](#tensorrt-deployment)
-- [ONNX Runtime Deployment](#onnx-runtime-deployment)
-- [Edge Device Deployment](#edge-device-deployment)
-- [Model Serving](#model-serving)
-- [Video Processing Pipelines](#video-processing-pipelines)
-- [Monitoring and Observability](#monitoring-and-observability)
-- [Scaling and Performance](#scaling-and-performance)
-
----
-
-## Model Export and Optimization
-
-### PyTorch to ONNX Export
-
-Basic export:
-```python
-import torch
-import torch.onnx
-
-def export_to_onnx(model, input_shape, output_path, dynamic_batch=True):
-    """
-    Export PyTorch model to ONNX format.
-
-    Args:
-        model: PyTorch model
-        input_shape: (C, H, W) input dimensions
-        output_path: Path to save .onnx file
-        dynamic_batch: Allow variable batch sizes
-    """
-    model.set_mode('inference')
-
-    # Create dummy input
-    dummy_input = torch.randn(1, *input_shape)
-
-    # Dynamic axes for variable batch size
-    dynamic_axes = None
-    if dynamic_batch:
-        dynamic_axes = {
-            'input': {0: 'batch_size'},
-            'output': {0: 'batch_size'}
-        }
-
-    # Export
-    torch.onnx.export(
-        model,
-        dummy_input,
-        output_path,
-        export_params=True,
-        opset_version=17,
-        do_constant_folding=True,
-        input_names=['input'],
-        output_names=['output'],
-        dynamic_axes=dynamic_axes
-    )
-
-    print(f"Exported to {output_path}")
-    return output_path
-```
-
-### ONNX Model Optimization
-
-Simplify and optimize ONNX graph:
-```python
-import onnx
-from onnxsim import simplify
-
-def optimize_onnx(input_path, output_path):
-    """
-    Simplify ONNX model for faster inference.
-    """
-    # Load model
-    model = onnx.load(input_path)
-
-    # Check validity
-    onnx.checker.check_model(model)
-
-    # Simplify
-    model_simplified, check = simplify(model)
-
-    if check:
-        onnx.save(model_simplified, output_path)
-        print(f"Simplified model saved to {output_path}")
-
-        # Print size reduction
-        import os
-        original_size = os.path.getsize(input_path) / 1024 / 1024
-        simplified_size = os.path.getsize(output_path) / 1024 / 1024
-        print(f"Size: {original_size:.2f}MB -> {simplified_size:.2f}MB")
-    else:
-        print("Simplification failed, saving original")
-        onnx.save(model, output_path)
-
-    return output_path
-```
-
-### Model Size Analysis
-
-```python
-def analyze_model(model_path):
-    """
-    Analyze ONNX model structure and size.
-    """
-    model = onnx.load(model_path)
-
-    # Count parameters
-    total_params = 0
-    param_sizes = {}
-
-    for initializer in model.graph.initializer:
-        param_count = 1
-        for dim in initializer.dims:
-            param_count *= dim
-        total_params += param_count
-        param_sizes[initializer.name] = param_count
-
-    # Print summary
-    print(f"Total parameters: {total_params:,}")
-    print(f"Model size: {total_params * 4 / 1024 / 1024:.2f} MB (FP32)")
-    print(f"Model size: {total_params * 2 / 1024 / 1024:.2f} MB (FP16)")
-    print(f"Model size: {total_params / 1024 / 1024:.2f} MB (INT8)")
-
-    # Top 10 largest layers
-    print("\nLargest layers:")
-    sorted_params = sorted(param_sizes.items(), key=lambda x: x[1], reverse=True)
-    for name, size in sorted_params[:10]:
-        print(f"  {name}: {size:,} params")
-
-    return total_params
-```
-
----
-
-## TensorRT Deployment
-
-### TensorRT Engine Build
-
-```python
-import tensorrt as trt
-
-def build_tensorrt_engine(onnx_path, engine_path, precision='fp16',
-                          max_batch_size=8, workspace_gb=4):
-    """
-    Build TensorRT engine from ONNX model.
-
-    Args:
-        onnx_path: Path to ONNX model
-        engine_path: Path to save TensorRT engine
-        precision: 'fp32', 'fp16', or 'int8'
-        max_batch_size: Maximum batch size
-        workspace_gb: GPU memory workspace in GB
-    """
-    logger = trt.Logger(trt.Logger.WARNING)
-    builder = trt.Builder(logger)
-    network = builder.create_network(
-        1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH)
-    )
-    parser = trt.OnnxParser(network, logger)
-
-    # Parse ONNX
-    with open(onnx_path, 'rb') as f:
-        if not parser.parse(f.read()):
-            for error in range(parser.num_errors):
-                print(parser.get_error(error))
-            raise RuntimeError("ONNX parsing failed")
-
-    # Configure builder
-    config = builder.create_builder_config()
-    config.set_memory_pool_limit(trt.MemoryPoolType.WORKSPACE,
-                                  workspace_gb * 1024 * 1024 * 1024)
-
-    # Set precision
-    if precision == 'fp16':
-        config.set_flag(trt.BuilderFlag.FP16)
-    elif precision == 'int8':
-        config.set_flag(trt.BuilderFlag.INT8)
-        # Requires calibrator for INT8
-
-    # Set optimization profile for dynamic shapes
-    profile = builder.create_optimization_profile()
-    input_name = network.get_input(0).name
-    input_shape = network.get_input(0).shape
-
-    # Min, optimal, max batch sizes
-    min_shape = (1,) + tuple(input_shape[1:])
-    opt_shape = (max_batch_size // 2,) + tuple(input_shape[1:])
-    max_shape = (max_batch_size,) + tuple(input_shape[1:])
-
-    profile.set_shape(input_name, min_shape, opt_shape, max_shape)
-    config.add_optimization_profile(profile)
-
-    # Build engine
-    serialized_engine = builder.build_serialized_network(network, config)
-
-    # Save engine
-    with open(engine_path, 'wb') as f:
-        f.write(serialized_engine)
-
-    print(f"TensorRT engine saved to {engine_path}")
-    return engine_path
-```
-
-### TensorRT Inference
-
-```python
-import numpy as np
-import pycuda.driver as cuda
-import pycuda.autoinit
-
-class TensorRTInference:
-    def __init__(self, engine_path):
-        """
-        Load TensorRT engine and prepare for inference.
-        """
-        self.logger = trt.Logger(trt.Logger.WARNING)
-
-        # Load engine
-        with open(engine_path, 'rb') as f:
-            engine_data = f.read()
-
-        runtime = trt.Runtime(self.logger)
-        self.engine = runtime.deserialize_cuda_engine(engine_data)
-        self.context = self.engine.create_execution_context()
-
-        # Allocate buffers
-        self.inputs = []
-        self.outputs = []
-        self.bindings = []
-        self.stream = cuda.Stream()
-
-        for i in range(self.engine.num_io_tensors):
-            name = self.engine.get_tensor_name(i)
-            dtype = trt.nptype(self.engine.get_tensor_dtype(name))
-            shape = self.engine.get_tensor_shape(name)
-            size = trt.volume(shape)
-
-            # Allocate host and device buffers
-            host_mem = cuda.pagelocked_empty(size, dtype)
-            device_mem = cuda.mem_alloc(host_mem.nbytes)
-
-            self.bindings.append(int(device_mem))
-
-            if self.engine.get_tensor_mode(name) == trt.TensorIOMode.INPUT:
-                self.inputs.append({'host': host_mem, 'device': device_mem,
-                                   'shape': shape, 'name': name})
-            else:
-                self.outputs.append({'host': host_mem, 'device': device_mem,
-                                    'shape': shape, 'name': name})
-
-    def infer(self, input_data):
-        """
-        Run inference on input data.
-
-        Args:
-            input_data: numpy array (batch, C, H, W)
-
-        Returns:
-            Output numpy array
-        """
-        # Copy input to host buffer
-        np.copyto(self.inputs[0]['host'], input_data.ravel())
-
-        # Transfer input to device
-        cuda.memcpy_htod_async(
-            self.inputs[0]['device'],
-            self.inputs[0]['host'],
-            self.stream
-        )
-
-        # Run inference
-        self.context.execute_async_v2(
-            bindings=self.bindings,
-            stream_handle=self.stream.handle
-        )
-
-        # Transfer output from device
-        cuda.memcpy_dtoh_async(
-            self.outputs[0]['host'],
-            self.outputs[0]['device'],
-            self.stream
-        )
-
-        # Synchronize
-        self.stream.synchronize()
-
-        # Reshape output
-        output = self.outputs[0]['host'].reshape(self.outputs[0]['shape'])
-        return output
-```
-
-### INT8 Calibration
-
-```python
-class Int8Calibrator(trt.IInt8EntropyCalibrator2):
-    def __init__(self, calibration_data, cache_file, batch_size=8):
-        """
-        INT8 calibrator for TensorRT.
-
-        Args:
-            calibration_data: List of numpy arrays
-            cache_file: Path to save calibration cache
-            batch_size: Calibration batch size
-        """
-        super().__init__()
-        self.calibration_data = calibration_data
-        self.cache_file = cache_file
-        self.batch_size = batch_size
-        self.current_index = 0
-
-        # Allocate device buffer
-        self.device_input = cuda.mem_alloc(
-            calibration_data[0].nbytes * batch_size
-        )
-
-    def get_batch_size(self):
-        return self.batch_size
-
-    def get_batch(self, names):
-        if self.current_index + self.batch_size > len(self.calibration_data):
-            return None
-
-        # Get batch
-        batch = self.calibration_data[
-            self.current_index:self.current_index + self.batch_size
-        ]
-        batch = np.stack(batch, axis=0)
-
-        # Copy to device
-        cuda.memcpy_htod(self.device_input, batch)
-        self.current_index += self.batch_size
-
-        return [int(self.device_input)]
-
-    def read_calibration_cache(self):
-        if os.path.exists(self.cache_file):
-            with open(self.cache_file, 'rb') as f:
-                return f.read()
-        return None
-
-    def write_calibration_cache(self, cache):
-        with open(self.cache_file, 'wb') as f:
-            f.write(cache)
-```
-
----
-
-## ONNX Runtime Deployment
-
-### Basic ONNX Runtime Inference
-
-```python
-import onnxruntime as ort
-
-class ONNXInference:
-    def __init__(self, model_path, device='cuda'):
-        """
-        Initialize ONNX Runtime session.
-
-        Args:
-            model_path: Path to ONNX model
-            device: 'cuda' or 'cpu'
-        """
-        # Set execution providers
-        if device == 'cuda':
-            providers = [
-                ('CUDAExecutionProvider', {
-                    'device_id': 0,
-                    'arena_extend_strategy': 'kNextPowerOfTwo',
-                    'gpu_mem_limit': 4 * 1024 * 1024 * 1024,  # 4GB
-                    'cudnn_conv_algo_search': 'EXHAUSTIVE',
-                }),
-                'CPUExecutionProvider'
-            ]
-        else:
-            providers = ['CPUExecutionProvider']
-
-        # Session options
-        sess_options = ort.SessionOptions()
-        sess_options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
-        sess_options.intra_op_num_threads = 4
-
-        # Create session
-        self.session = ort.InferenceSession(
-            model_path,
-            sess_options=sess_options,
-            providers=providers
-        )
-
-        # Get input/output info
-        self.input_name = self.session.get_inputs()[0].name
-        self.input_shape = self.session.get_inputs()[0].shape
-        self.output_name = self.session.get_outputs()[0].name
-
-        print(f"Loaded model: {model_path}")
-        print(f"Input: {self.input_name} {self.input_shape}")
-        print(f"Provider: {self.session.get_providers()[0]}")
-
-    def infer(self, input_data):
-        """
-        Run inference.
-
-        Args:
-            input_data: numpy array (batch, C, H, W)
-
-        Returns:
-            Model output
-        """
-        outputs = self.session.run(
-            [self.output_name],
-            {self.input_name: input_data.astype(np.float32)}
-        )
-        return outputs[0]
-
-    def benchmark(self, input_shape, num_iterations=100, warmup=10):
-        """
-        Benchmark inference speed.
-        """
-        import time
-
-        dummy_input = np.random.randn(*input_shape).astype(np.float32)
-
-        # Warmup
-        for _ in range(warmup):
-            self.infer(dummy_input)
-
-        # Benchmark
-        start = time.perf_counter()
-        for _ in range(num_iterations):
-            self.infer(dummy_input)
-        end = time.perf_counter()
-
-        avg_time = (end - start) / num_iterations * 1000
-        fps = 1000 / avg_time * input_shape[0]
-
-        print(f"Average latency: {avg_time:.2f}ms")
-        print(f"Throughput: {fps:.1f} images/sec")
-
-        return avg_time, fps
-```
-
----
-
-## Edge Device Deployment
-
-### NVIDIA Jetson Optimization
-
-```python
-def optimize_for_jetson(model_path, output_path, jetson_model='orin'):
-    """
-    Optimize model for NVIDIA Jetson deployment.
-
-    Args:
-        model_path: Path to ONNX model
-        output_path: Path to save optimized engine
-        jetson_model: 'nano', 'xavier', 'orin'
-    """
-    # Jetson-specific configurations
-    configs = {
-        'nano': {'precision': 'fp16', 'workspace': 1, 'dla': False},
-        'xavier': {'precision': 'fp16', 'workspace': 2, 'dla': True},
-        'orin': {'precision': 'int8', 'workspace': 4, 'dla': True},
-    }
-
-    config = configs[jetson_model]
-
-    # Build engine with Jetson-optimized settings
-    logger = trt.Logger(trt.Logger.WARNING)
-    builder = trt.Builder(logger)
-    network = builder.create_network(
-        1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH)
-    )
-    parser = trt.OnnxParser(network, logger)
-
-    with open(model_path, 'rb') as f:
-        parser.parse(f.read())
-
-    builder_config = builder.create_builder_config()
-    builder_config.set_memory_pool_limit(
-        trt.MemoryPoolType.WORKSPACE,
-        config['workspace'] * 1024 * 1024 * 1024
-    )
-
-    if config['precision'] == 'fp16':
-        builder_config.set_flag(trt.BuilderFlag.FP16)
-    elif config['precision'] == 'int8':
-        builder_config.set_flag(trt.BuilderFlag.INT8)
-
-    # Enable DLA if supported
-    if config['dla'] and builder.num_DLA_cores > 0:
-        builder_config.default_device_type = trt.DeviceType.DLA
-        builder_config.DLA_core = 0
-        builder_config.set_flag(trt.BuilderFlag.GPU_FALLBACK)
-
-    # Build and save
-    serialized = builder.build_serialized_network(network, builder_config)
-    with open(output_path, 'wb') as f:
-        f.write(serialized)
-
-    print(f"Jetson-optimized engine saved to {output_path}")
-```
-
-### OpenVINO for Intel Devices
-
-```python
-from openvino.runtime import Core
-
-class OpenVINOInference:
-    def __init__(self, model_path, device='CPU'):
-        """
-        Initialize OpenVINO inference.
-
-        Args:
-            model_path: Path to ONNX or OpenVINO IR model
-            device: 'CPU', 'GPU', 'MYRIAD' (Intel NCS)
-        """
-        self.core = Core()
-
-        # Load and compile model
-        self.model = self.core.read_model(model_path)
-        self.compiled = self.core.compile_model(self.model, device)
-
-        # Get input/output info
-        self.input_layer = self.compiled.input(0)
-        self.output_layer = self.compiled.output(0)
-
-        print(f"Loaded model on {device}")
-        print(f"Input shape: {self.input_layer.shape}")
-
-    def infer(self, input_data):
-        """
-        Run inference.
-        """
-        result = self.compiled([input_data])
-        return result[self.output_layer]
-
-    def benchmark(self, input_shape, num_iterations=100):
-        """
-        Benchmark inference speed.
-        """
-        import time
-
-        dummy = np.random.randn(*input_shape).astype(np.float32)
-
-        # Warmup
-        for _ in range(10):
-            self.infer(dummy)
-
-        # Benchmark
-        start = time.perf_counter()
-        for _ in range(num_iterations):
-            self.infer(dummy)
-        elapsed = time.perf_counter() - start
-
-        latency = elapsed / num_iterations * 1000
-        print(f"Latency: {latency:.2f}ms")
-        return latency
-
-
-def convert_to_openvino(onnx_path, output_dir, precision='FP16'):
-    """
-    Convert ONNX to OpenVINO IR format.
-    """
-    from openvino.tools import mo
-
-    mo.convert_model(
-        onnx_path,
-        output_model=f"{output_dir}/model.xml",
-        compress_to_fp16=(precision == 'FP16')
-    )
-    print(f"Converted to OpenVINO IR at {output_dir}")
-```
-
-### CoreML for Apple Silicon
-
-```python
-import coremltools as ct
-
-def convert_to_coreml(model_or_path, output_path, compute_units='ALL'):
-    """
-    Convert to CoreML for Apple devices.
-
-    Args:
-        model_or_path: PyTorch model or ONNX path
-        output_path: Path to save .mlpackage
-        compute_units: 'ALL', 'CPU_AND_GPU', 'CPU_AND_NE'
-    """
-    # Map compute units
-    units_map = {
-        'ALL': ct.ComputeUnit.ALL,
-        'CPU_AND_GPU': ct.ComputeUnit.CPU_AND_GPU,
-        'CPU_AND_NE': ct.ComputeUnit.CPU_AND_NE,  # Neural Engine
-    }
-
-    # Convert from ONNX
-    if isinstance(model_or_path, str) and model_or_path.endswith('.onnx'):
-        mlmodel = ct.convert(
-            model_or_path,
-            compute_units=units_map[compute_units],
-            minimum_deployment_target=ct.target.macOS13  # or iOS16
-        )
-    else:
-        # Convert from PyTorch
-        traced = torch.jit.trace(model_or_path, torch.randn(1, 3, 640, 640))
-        mlmodel = ct.convert(
-            traced,
-            inputs=[ct.TensorType(shape=(1, 3, 640, 640))],
-            compute_units=units_map[compute_units],
-        )
-
-    mlmodel.save(output_path)
-    print(f"CoreML model saved to {output_path}")
-```
-
----
-
-## Model Serving
-
-### Triton Inference Server
-
-Configuration file (`config.pbtxt`):
-```protobuf
-name: "yolov8"
-platform: "onnxruntime_onnx"
-max_batch_size: 8
-
-input [
-  {
-    name: "images"
-    data_type: TYPE_FP32
-    dims: [ 3, 640, 640 ]
-  }
-]
-
-output [
-  {
-    name: "output0"
-    data_type: TYPE_FP32
-    dims: [ 84, 8400 ]
-  }
-]
-
-instance_group [
-  {
-    count: 2
-    kind: KIND_GPU
-  }
-]
-
-dynamic_batching {
-  preferred_batch_size: [ 4, 8 ]
-  max_queue_delay_microseconds: 100
-}
-```
-
-Triton client:
-```python
-import tritonclient.http as httpclient
-
-class TritonClient:
-    def __init__(self, url='localhost:8000', model_name='yolov8'):
-        self.client = httpclient.InferenceServerClient(url=url)
-        self.model_name = model_name
-
-        # Check model is ready
-        if not self.client.is_model_ready(model_name):
-            raise RuntimeError(f"Model {model_name} is not ready")
-
-    def infer(self, images):
-        """
-        Send inference request to Triton.
-
-        Args:
-            images: numpy array (batch, C, H, W)
-        """
-        # Create input
-        inputs = [
-            httpclient.InferInput("images", images.shape, "FP32")
-        ]
-        inputs[0].set_data_from_numpy(images)
-
-        # Create output request
-        outputs = [
-            httpclient.InferRequestedOutput("output0")
-        ]
-
-        # Send request
-        response = self.client.infer(
-            model_name=self.model_name,
-            inputs=inputs,
-            outputs=outputs
-        )
-
-        return response.as_numpy("output0")
-```
-
-### TorchServe Deployment
-
-Model handler (`handler.py`):
-```python
-from ts.torch_handler.base_handler import BaseHandler
-import torch
-import cv2
-import numpy as np
-
-class YOLOHandler(BaseHandler):
-    def __init__(self):
-        super().__init__()
-        self.input_size = 640
-        self.conf_threshold = 0.25
-        self.iou_threshold = 0.45
-
-    def preprocess(self, data):
-        """Preprocess input images."""
-        images = []
-        for row in data:
-            image = row.get("data") or row.get("body")
-
-            if isinstance(image, (bytes, bytearray)):
-                image = np.frombuffer(image, dtype=np.uint8)
-                image = cv2.imdecode(image, cv2.IMREAD_COLOR)
-
-            # Resize and normalize
-            image = cv2.resize(image, (self.input_size, self.input_size))
-            image = image.astype(np.float32) / 255.0
-            image = np.transpose(image, (2, 0, 1))
-            images.append(image)
-
-        return torch.tensor(np.stack(images))
-
-    def inference(self, data):
-        """Run model inference."""
-        with torch.no_grad():
-            outputs = self.model(data)
-        return outputs
-
-    def postprocess(self, outputs):
-        """Postprocess model outputs."""
-        results = []
-        for output in outputs:
-            # Apply NMS and format results
-            detections = self._nms(output, self.conf_threshold, self.iou_threshold)
-            results.append(detections.tolist())
-        return results
-```
-
-TorchServe configuration (`config.properties`):
-```properties
-inference_address=http://0.0.0.0:8080
-management_address=http://0.0.0.0:8081
-metrics_address=http://0.0.0.0:8082
-number_of_netty_threads=4
-job_queue_size=100
-model_store=/opt/ml/model
-load_models=yolov8.mar
-```
-
-### FastAPI Serving
-
-```python
-from fastapi import FastAPI, File, UploadFile
-from fastapi.responses import JSONResponse
-import uvicorn
-import numpy as np
-import cv2
-
-app = FastAPI(title="YOLO Detection API")
-
-# Global model
-model = None
-
-@app.on_event("startup")
-async def load_model():
-    global model
-    model = ONNXInference("models/yolov8m.onnx", device='cuda')
-
-@app.post("/detect")
-async def detect(file: UploadFile = File(...), conf: float = 0.25):
-    """
-    Detect objects in uploaded image.
-    """
-    # Read image
-    contents = await file.read()
-    nparr = np.frombuffer(contents, np.uint8)
-    image = cv2.imdecode(nparr, cv2.IMREAD_COLOR)
-
-    # Preprocess
-    input_image = preprocess_image(image, 640)
-
-    # Inference
-    outputs = model.infer(input_image)
-
-    # Postprocess
-    detections = postprocess_detections(outputs, conf, 0.45)
-
-    return JSONResponse({
-        "detections": detections,
-        "image_size": list(image.shape[:2])
-    })
-
-@app.get("/health")
-async def health():
-    return {"status": "healthy", "model_loaded": model is not None}
-
-if __name__ == "__main__":
-    uvicorn.run(app, host="0.0.0.0", port=8000)
-```
-
----
-
-## Video Processing Pipelines
-
-### Real-Time Video Detection
-
-```python
-import cv2
-import time
-from collections import deque
-
-class VideoDetector:
-    def __init__(self, model, conf_threshold=0.25, track=True):
-        self.model = model
-        self.conf_threshold = conf_threshold
-        self.track = track
-        self.tracker = ByteTrack() if track else None
-        self.fps_buffer = deque(maxlen=30)
-
-    def process_video(self, source, output_path=None, show=True):
-        """
-        Process video stream with detection.
-
-        Args:
-            source: Video file path, camera index, or RTSP URL
-            output_path: Path to save output video
-            show: Display results in window
-        """
-        cap = cv2.VideoCapture(source)
-
-        if output_path:
-            fourcc = cv2.VideoWriter_fourcc(*'mp4v')
-            fps = cap.get(cv2.CAP_PROP_FPS)
-            width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
-            height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
-            writer = cv2.VideoWriter(output_path, fourcc, fps, (width, height))
-
-        frame_count = 0
-        start_time = time.time()
-
-        while cap.isOpened():
-            ret, frame = cap.read()
-            if not ret:
-                break
-
-            # Inference
-            t0 = time.perf_counter()
-            detections = self._detect(frame)
-
-            # Tracking
-            if self.track and len(detections) > 0:
-                detections = self.tracker.update(detections)
-
-            # Calculate FPS
-            inference_time = time.perf_counter() - t0
-            self.fps_buffer.append(1 / inference_time)
-            avg_fps = sum(self.fps_buffer) / len(self.fps_buffer)
-
-            # Draw results
-            frame = self._draw_detections(frame, detections, avg_fps)
-
-            # Output
-            if output_path:
-                writer.write(frame)
-
-            if show:
-                cv2.imshow('Detection', frame)
-                if cv2.waitKey(1) == ord('q'):
-                    break
-
-            frame_count += 1
-
-        # Cleanup
-        cap.release()
-        if output_path:
-            writer.release()
-        cv2.destroyAllWindows()
-
-        # Print statistics
-        total_time = time.time() - start_time
-        print(f"Processed {frame_count} frames in {total_time:.1f}s")
-        print(f"Average FPS: {frame_count / total_time:.1f}")
-
-    def _detect(self, frame):
-        """Run detection on single frame."""
-        # Preprocess
-        input_tensor = self._preprocess(frame)
-
-        # Inference
-        outputs = self.model.infer(input_tensor)
-
-        # Postprocess
-        detections = self._postprocess(outputs, frame.shape[:2])
-        return detections
-
-    def _preprocess(self, frame):
-        """Preprocess frame for model input."""
-        # Resize
-        input_size = 640
-        image = cv2.resize(frame, (input_size, input_size))
-
-        # Normalize and transpose
-        image = image.astype(np.float32) / 255.0
-        image = np.transpose(image, (2, 0, 1))
-        image = np.expand_dims(image, axis=0)
-
-        return image
-
-    def _draw_detections(self, frame, detections, fps):
-        """Draw detections on frame."""
-        for det in detections:
-            x1, y1, x2, y2 = det['bbox']
-            cls = det['class']
-            conf = det['confidence']
-            track_id = det.get('track_id', None)
-
-            # Draw box
-            color = self._get_color(cls)
-            cv2.rectangle(frame, (int(x1), int(y1)), (int(x2), int(y2)), color, 2)
-
-            # Draw label
-            label = f"{cls}: {conf:.2f}"
-            if track_id:
-                label = f"ID:{track_id} {label}"
-
-            cv2.putText(frame, label, (int(x1), int(y1) - 10),
-                       cv2.FONT_HERSHEY_SIMPLEX, 0.5, color, 2)
-
-        # Draw FPS
-        cv2.putText(frame, f"FPS: {fps:.1f}", (10, 30),
-                   cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2)
-
-        return frame
-```
-
-### Batch Video Processing
-
-```python
-import concurrent.futures
-from pathlib import Path
-
-def process_videos_batch(video_paths, model, output_dir, max_workers=4):
-    """
-    Process multiple videos in parallel.
-    """
-    output_dir = Path(output_dir)
-    output_dir.mkdir(parents=True, exist_ok=True)
-
-    def process_single(video_path):
-        detector = VideoDetector(model)
-        output_path = output_dir / f"{Path(video_path).stem}_detected.mp4"
-        detector.process_video(video_path, str(output_path), show=False)
-        return output_path
-
-    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
-        futures = {executor.submit(process_single, vp): vp for vp in video_paths}
-
-        for future in concurrent.futures.as_completed(futures):
-            video_path = futures[future]
-            try:
-                output_path = future.result()
-                print(f"Completed: {video_path} -> {output_path}")
-            except Exception as e:
-                print(f"Failed: {video_path} - {e}")
-```
-
----
-
-## Monitoring and Observability
-
-### Prometheus Metrics
-
-```python
-from prometheus_client import Counter, Histogram, Gauge, start_http_server
-
-# Define metrics
-INFERENCE_COUNT = Counter(
-    'model_inference_total',
-    'Total number of inferences',
-    ['model_name', 'status']
-)
-
-INFERENCE_LATENCY = Histogram(
-    'model_inference_latency_seconds',
-    'Inference latency in seconds',
-    ['model_name'],
-    buckets=[0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0]
-)
-
-GPU_MEMORY = Gauge(
-    'gpu_memory_used_bytes',
-    'GPU memory usage in bytes',
-    ['device']
-)
-
-DETECTIONS_COUNT = Counter(
-    'detections_total',
-    'Total detections by class',
-    ['model_name', 'class_name']
-)
-
-class MetricsWrapper:
-    def __init__(self, model, model_name='yolov8'):
-        self.model = model
-        self.model_name = model_name
-
-    def infer(self, input_data):
-        """Inference with metrics."""
-        start_time = time.perf_counter()
-
-        try:
-            result = self.model.infer(input_data)
-            INFERENCE_COUNT.labels(self.model_name, 'success').inc()
-
-            # Count detections by class
-            for det in result:
-                DETECTIONS_COUNT.labels(self.model_name, det['class']).inc()
-
-            return result
-
-        except Exception as e:
-            INFERENCE_COUNT.labels(self.model_name, 'error').inc()
-            raise
-
-        finally:
-            latency = time.perf_counter() - start_time
-            INFERENCE_LATENCY.labels(self.model_name).observe(latency)
-
-            # Update GPU memory
-            if torch.cuda.is_available():
-                memory = torch.cuda.memory_allocated()
-                GPU_MEMORY.labels('cuda:0').set(memory)
-
-# Start metrics server
-start_http_server(9090)
-```
-
-### Logging Configuration
-
-```python
-import logging
-import json
-from datetime import datetime
-
-class StructuredLogger:
-    def __init__(self, name, level=logging.INFO):
-        self.logger = logging.getLogger(name)
-        self.logger.setLevel(level)
-
-        # JSON formatter
-        handler = logging.StreamHandler()
-        handler.setFormatter(JsonFormatter())
-        self.logger.addHandler(handler)
-
-    def log_inference(self, model_name, latency, num_detections, input_shape):
-        self.logger.info(json.dumps({
-            'event': 'inference',
-            'timestamp': datetime.utcnow().isoformat(),
-            'model_name': model_name,
-            'latency_ms': latency * 1000,
-            'num_detections': num_detections,
-            'input_shape': list(input_shape)
-        }))
-
-    def log_error(self, model_name, error, input_shape):
-        self.logger.error(json.dumps({
-            'event': 'inference_error',
-            'timestamp': datetime.utcnow().isoformat(),
-            'model_name': model_name,
-            'error': str(error),
-            'error_type': type(error).__name__,
-            'input_shape': list(input_shape)
-        }))
-
-class JsonFormatter(logging.Formatter):
-    def format(self, record):
-        return record.getMessage()
-```
-
----
-
-## Scaling and Performance
-
-### Batch Processing Optimization
-
-```python
-class BatchProcessor:
-    def __init__(self, model, max_batch_size=8, max_wait_ms=100):
-        self.model = model
-        self.max_batch_size = max_batch_size
-        self.max_wait_ms = max_wait_ms
-        self.queue = []
-        self.lock = threading.Lock()
-        self.results = {}
-
-    async def process(self, image, request_id):
-        """Add image to batch and wait for result."""
-        future = asyncio.Future()
-
-        with self.lock:
-            self.queue.append((request_id, image, future))
-
-            if len(self.queue) >= self.max_batch_size:
-                self._process_batch()
-
-        # Wait for result with timeout
-        result = await asyncio.wait_for(future, timeout=5.0)
-        return result
-
-    def _process_batch(self):
-        """Process accumulated batch."""
-        batch_items = self.queue[:self.max_batch_size]
-        self.queue = self.queue[self.max_batch_size:]
-
-        # Stack images
-        images = np.stack([item[1] for item in batch_items])
-
-        # Inference
-        outputs = self.model.infer(images)
-
-        # Return results
-        for i, (request_id, image, future) in enumerate(batch_items):
-            future.set_result(outputs[i])
-```
-
-### Multi-GPU Inference
-
-```python
-import torch.nn as nn
-from torch.nn.parallel import DataParallel
-
-class MultiGPUInference:
-    def __init__(self, model, device_ids=None):
-        """
-        Wrap model for multi-GPU inference.
-
-        Args:
-            model: PyTorch model
-            device_ids: List of GPU IDs, e.g., [0, 1, 2, 3]
-        """
-        if device_ids is None:
-            device_ids = list(range(torch.cuda.device_count()))
-
-        self.device = torch.device('cuda:0')
-        self.model = DataParallel(model, device_ids=device_ids)
-        self.model.to(self.device)
-        self.model.set_mode('inference')
-
-    def infer(self, images):
-        """
-        Run inference across GPUs.
-        """
-        with torch.no_grad():
-            images = torch.from_numpy(images).to(self.device)
-            outputs = self.model(images)
-        return outputs.cpu().numpy()
-```
-
-### Performance Benchmarking
-
-```python
-def comprehensive_benchmark(model, input_sizes, batch_sizes, num_iterations=100):
-    """
-    Benchmark model across different configurations.
-    """
-    results = []
-
-    for input_size in input_sizes:
-        for batch_size in batch_sizes:
-            # Create input
-            dummy = np.random.randn(batch_size, 3, input_size, input_size).astype(np.float32)
-
-            # Warmup
-            for _ in range(10):
-                model.infer(dummy)
-
-            # Benchmark
-            latencies = []
-            for _ in range(num_iterations):
-                start = time.perf_counter()
-                model.infer(dummy)
-                latencies.append(time.perf_counter() - start)
-
-            # Calculate statistics
-            latencies = np.array(latencies) * 1000  # Convert to ms
-            result = {
-                'input_size': input_size,
-                'batch_size': batch_size,
-                'mean_latency_ms': np.mean(latencies),
-                'std_latency_ms': np.std(latencies),
-                'p50_latency_ms': np.percentile(latencies, 50),
-                'p95_latency_ms': np.percentile(latencies, 95),
-                'p99_latency_ms': np.percentile(latencies, 99),
-                'throughput_fps': batch_size * 1000 / np.mean(latencies)
-            }
-            results.append(result)
-
-            print(f"Size: {input_size}, Batch: {batch_size}")
-            print(f"  Latency: {result['mean_latency_ms']:.2f}ms (p99: {result['p99_latency_ms']:.2f}ms)")
-            print(f"  Throughput: {result['throughput_fps']:.1f} FPS")
-
-    return results
-```
-
----
-
-## Resources
-
-- [TensorRT Documentation](https://docs.nvidia.com/deeplearning/tensorrt/)
-- [ONNX Runtime Documentation](https://onnxruntime.ai/docs/)
-- [Triton Inference Server](https://github.com/triton-inference-server/server)
-- [OpenVINO Documentation](https://docs.openvino.ai/)
-- [CoreML Tools](https://coremltools.readme.io/)