@cloudstreamsoftware/claude-tools 1.0.0 → 1.2.0

package/skills/gcp-data-engineering/dataflow/error-handling.md

@@ -0,0 +1,496 @@
+# Dataflow Error Handling
+
+> Patterns for dead letter queues, retry logic, poison message handling, and monitoring in CloudStream Dataflow pipelines.
+
+## Dead Letter Queue (DLQ) Pattern
+
+The DLQ pattern routes unprocessable records to a separate destination for later investigation, preventing a single bad record from blocking the entire pipeline.
+
+```python
+# dlq_pipeline.py
+import apache_beam as beam
+from apache_beam.io.gcp.bigquery import WriteToBigQuery
+import json
+import traceback
+from datetime import datetime
+
+class ProcessWithDLQ(beam.DoFn):
+    """Process records, routing failures to dead letter queue."""
+
+    # Output tags
+    VALID_TAG = 'valid'
+    DLQ_TAG = 'dlq'
+
+    def process(self, element):
+        try:
+            record = json.loads(element.decode('utf-8'))
+
+            # Validation checks
+            if not record.get('record_id'):
+                raise ValueError("Missing required field: record_id")
+
+            if record.get('amount') and float(record['amount']) < 0:
+                raise ValueError(f"Negative amount: {record['amount']}")
+
+            # Type coercion
+            cleaned = {
+                'record_id': str(record['record_id']),
+                'amount': float(record.get('amount', 0)),
+                'stage': record.get('stage', 'Unknown'),
+                '_processed_at': datetime.utcnow().isoformat()
+            }
+
+            yield beam.pvalue.TaggedOutput(self.VALID_TAG, cleaned)
+
+        except Exception as e:
+            yield beam.pvalue.TaggedOutput(self.DLQ_TAG, {
+                'raw_message': element.decode('utf-8') if isinstance(element, bytes) else str(element),
+                'error_type': type(e).__name__,
+                'error_message': str(e),
+                'stack_trace': traceback.format_exc(),
+                'failed_at': datetime.utcnow().isoformat(),
+                'pipeline_name': 'zoho_deals_streaming'
+            })
+
+def run():
+    with beam.Pipeline(options=get_streaming_options()) as p:
+        messages = (
+            p | 'ReadPubSub' >> beam.io.ReadFromPubSub(
+                subscription='projects/cloudstream-prod/subscriptions/zoho-deals-sub'
+            )
+        )
+
+        results = messages | 'Process' >> beam.ParDo(
+            ProcessWithDLQ()
+        ).with_outputs(ProcessWithDLQ.VALID_TAG, ProcessWithDLQ.DLQ_TAG)
+
+        # Valid records to silver
+        results[ProcessWithDLQ.VALID_TAG] | 'WriteValid' >> WriteToBigQuery(
+            table='cloudstream-prod:silver.zoho_deals',
+            write_disposition='WRITE_APPEND'
+        )
+
+        # Failed records to DLQ table
+        results[ProcessWithDLQ.DLQ_TAG] | 'WriteDLQ' >> WriteToBigQuery(
+            table='cloudstream-prod:ops.dead_letter_queue',
+            write_disposition='WRITE_APPEND'
+        )
+```
+
+## Error Tagging and Routing
+
+```python
+# Multi-category error routing
+class ErrorRouter(beam.DoFn):
+    """Route errors to appropriate handlers based on type."""
+
+    RETRYABLE = 'retryable'      # Transient errors (API timeout, rate limit)
+    VALIDATION = 'validation'     # Data quality failures
+    POISON = 'poison'            # Unrecoverable (corrupt data, schema mismatch)
+
+    def process(self, element):
+        error_type = element.get('error_type', '')
+        error_msg = element.get('error_message', '')
+
+        if any(t in error_type for t in ['TimeoutError', 'ConnectionError', 'ServiceUnavailable']):
+            yield beam.pvalue.TaggedOutput(self.RETRYABLE, element)
+        elif any(t in error_type for t in ['ValueError', 'TypeError', 'ValidationError']):
+            yield beam.pvalue.TaggedOutput(self.VALIDATION, element)
+        else:
+            yield beam.pvalue.TaggedOutput(self.POISON, element)
+
+# Usage in pipeline
+routed = (
+    dlq_records
+    | 'RouteErrors' >> beam.ParDo(ErrorRouter()).with_outputs(
+        ErrorRouter.RETRYABLE, ErrorRouter.VALIDATION, ErrorRouter.POISON
+    )
+)
+
+# Retryable → back to input topic with delay
+routed[ErrorRouter.RETRYABLE] | 'RepublishRetryable' >> beam.ParDo(RepublishWithDelay())
+
+# Validation → quarantine table for data team
+routed[ErrorRouter.VALIDATION] | 'WriteValidation' >> WriteToBigQuery(
+    table='cloudstream-prod:ops.validation_failures'
+)
+
+# Poison → alert and archive
+routed[ErrorRouter.POISON] | 'AlertPoison' >> beam.ParDo(AlertAndArchive())
+```
+
+## Retry with Exponential Backoff
+
+```python
+import time
+import random
+from google.api_core import retry as api_retry
+from google.api_core.exceptions import ServiceUnavailable, TooManyRequests
+
+class RetryableTransform(beam.DoFn):
+    """Transform with built-in retry logic for external API calls."""
+
+    MAX_RETRIES = 5
+    BASE_DELAY = 1.0  # seconds
+    MAX_DELAY = 60.0  # seconds
+
+    def process(self, element):
+        for attempt in range(self.MAX_RETRIES):
+            try:
+                result = self._call_external_api(element)
+                yield result
+                return
+            except (ServiceUnavailable, TooManyRequests, ConnectionError) as e:
+                if attempt == self.MAX_RETRIES - 1:
+                    # Final attempt failed - route to DLQ
+                    yield beam.pvalue.TaggedOutput('dlq', {
+                        'record': element,
+                        'error': str(e),
+                        'attempts': self.MAX_RETRIES,
+                        'failed_at': datetime.utcnow().isoformat()
+                    })
+                    return
+
+                # Exponential backoff with jitter
+                delay = min(
+                    self.BASE_DELAY * (2 ** attempt) + random.uniform(0, 1),
+                    self.MAX_DELAY
+                )
+                time.sleep(delay)
+
+            except Exception as e:
+                # Non-retryable error - immediate DLQ
+                yield beam.pvalue.TaggedOutput('dlq', {
+                    'record': element,
+                    'error': str(e),
+                    'error_type': 'non_retryable',
+                    'failed_at': datetime.utcnow().isoformat()
+                })
+                return
+
+    def _call_external_api(self, element):
+        """Call Zoho API or other external service."""
+        # Implementation here
+        pass
+
+
+# For BigQuery write retries, use built-in retry strategy
+WriteToBigQuery(
+    table='...',
+    insert_retry_strategy='RETRY_ON_TRANSIENT_ERROR',
+    # Default: retries on 5xx errors and rate limiting
+)
+```
+
+## Poison Message Handling
+
+```python
+class PoisonMessageHandler(beam.DoFn):
+    """Handle messages that repeatedly fail processing."""
+
+    def __init__(self, max_attempts=3):
+        self.max_attempts = max_attempts
+
+    def process(self, element, window=beam.DoFn.WindowParam):
+        # Check retry count from PubSub attributes
+        attributes = element.attributes if hasattr(element, 'attributes') else {}
+        retry_count = int(attributes.get('retry_count', '0'))
+
+        if retry_count >= self.max_attempts:
+            # This is a poison message - archive and alert
+            yield beam.pvalue.TaggedOutput('poison', {
+                'message': element.data.decode('utf-8'),
+                'retry_count': retry_count,
+                'first_seen': attributes.get('first_seen', 'unknown'),
+                'last_attempt': datetime.utcnow().isoformat(),
+                'reason': 'max_retries_exceeded'
+            })
+        else:
+            try:
+                result = self._process_message(element)
+                yield result
+            except Exception as e:
+                # Increment retry count and republish
+                yield beam.pvalue.TaggedOutput('retry', {
+                    'data': element.data,
+                    'attributes': {
+                        **attributes,
+                        'retry_count': str(retry_count + 1),
+                        'first_seen': attributes.get('first_seen', datetime.utcnow().isoformat()),
+                        'last_error': str(e)
+                    }
+                })
+```
+
+```sql
+-- DLQ table schema for tracking poison messages
+CREATE TABLE `project.ops.dead_letter_queue` (
+  pipeline_name STRING,
+  source_topic STRING,
+  raw_message STRING,
+  error_type STRING,
+  error_message STRING,
+  stack_trace STRING,
+  retry_count INT64,
+  first_seen_at TIMESTAMP,
+  failed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP(),
+  resolved_at TIMESTAMP,
+  resolution STRING  -- 'reprocessed', 'discarded', 'manual_fix'
+)
+PARTITION BY DATE(failed_at)
+CLUSTER BY pipeline_name, error_type;
+```
+
+## Monitoring with Cloud Monitoring
+
+```python
+# Custom metrics for pipeline health
+from google.cloud import monitoring_v3
+from google.protobuf.timestamp_pb2 import Timestamp
+import time
+
+class MetricsReporter(beam.DoFn):
+    """Report custom metrics to Cloud Monitoring."""
+
+    def setup(self):
+        self.client = monitoring_v3.MetricServiceClient()
+        self.project_name = f"projects/cloudstream-prod"
+
+    def process(self, element):
+        # Report processing metrics
+        self._write_metric(
+            'custom.googleapis.com/dataflow/records_processed',
+            1,
+            labels={'pipeline': 'zoho_deals', 'status': 'success'}
+        )
+        yield element
+
+    def _write_metric(self, metric_type, value, labels=None):
+        series = monitoring_v3.TimeSeries()
+        series.metric.type = metric_type
+        if labels:
+            for k, v in labels.items():
+                series.metric.labels[k] = v
+
+        series.resource.type = 'global'
+        now = time.time()
+        interval = monitoring_v3.TimeInterval(
+            end_time=Timestamp(seconds=int(now))
+        )
+        point = monitoring_v3.Point(
+            interval=interval,
+            value=monitoring_v3.TypedValue(int64_value=value)
+        )
+        series.points = [point]
+
+        self.client.create_time_series(
+            request={"name": self.project_name, "time_series": [series]}
+        )
+```
+
+## Alerting Policies
+
+```yaml
+# terraform/dataflow_alerts.tf
+resource "google_monitoring_alert_policy" "dlq_rate" {
+  display_name = "Dataflow DLQ Rate Too High"
+  combiner     = "OR"
+
+  conditions {
+    display_name = "DLQ records > 100 in 5 minutes"
+    condition_threshold {
+      filter          = "metric.type=\"custom.googleapis.com/dataflow/dlq_count\" AND resource.type=\"global\""
+      comparison      = "COMPARISON_GT"
+      threshold_value = 100
+      duration        = "300s"
+      aggregations {
+        alignment_period   = "300s"
+        per_series_aligner = "ALIGN_SUM"
+      }
+    }
+  }
+
+  notification_channels = [
+    google_monitoring_notification_channel.slack_data_alerts.id
+  ]
+
+  alert_strategy {
+    auto_close = "1800s"  # Auto-close after 30 minutes of recovery
+  }
+}
+
+resource "google_monitoring_alert_policy" "pipeline_backlog" {
+  display_name = "Dataflow Backlog Growing"
+  combiner     = "OR"
+
+  conditions {
+    display_name = "PubSub unacked messages > 10000"
+    condition_threshold {
+      filter          = "metric.type=\"pubsub.googleapis.com/subscription/num_unacked_messages\""
+      comparison      = "COMPARISON_GT"
+      threshold_value = 10000
+      duration        = "600s"
+    }
+  }
+
+  notification_channels = [
+    google_monitoring_notification_channel.slack_data_alerts.id
+  ]
+}
+
+resource "google_monitoring_alert_policy" "pipeline_errors" {
+  display_name = "Dataflow System Errors"
+  combiner     = "OR"
+
+  conditions {
+    display_name = "Dataflow worker errors > 5/min"
+    condition_threshold {
+      filter          = "metric.type=\"dataflow.googleapis.com/job/system_error_count\""
+      comparison      = "COMPARISON_GT"
+      threshold_value = 5
+      duration        = "60s"
+    }
+  }
+
+  notification_channels = [
+    google_monitoring_notification_channel.pagerduty.id
+  ]
+}
+```
+
+## Graceful Shutdown
+
+```python
+import signal
+import apache_beam as beam
+
+class GracefulPipeline:
+    """Handle pipeline shutdown gracefully, flushing in-flight records."""
+
+    def __init__(self):
+        self.shutdown_requested = False
+        signal.signal(signal.SIGTERM, self._handle_shutdown)
+
+    def _handle_shutdown(self, signum, frame):
+        """Handle SIGTERM from Dataflow for drain operations."""
+        self.shutdown_requested = True
+        # Dataflow handles drain automatically when you call:
+        # gcloud dataflow jobs drain JOB_ID --region=us-central1
+
+    def run(self):
+        """
+        Dataflow drain vs cancel:
+        - drain: Finishes processing in-flight elements, stops reading new input
+        - cancel: Immediately stops all processing (may lose data)
+
+        Always prefer drain for graceful shutdown.
+        """
+        pass
+```
+
+```bash
+# Graceful shutdown commands
+# DRAIN: Process remaining elements, then stop (recommended)
+gcloud dataflow jobs drain JOB_ID --region=us-central1
+
+# CANCEL: Immediate stop (data loss possible)
+gcloud dataflow jobs cancel JOB_ID --region=us-central1
+
+# UPDATE: Replace running pipeline with new version (no downtime)
+gcloud dataflow jobs run NEW_JOB_NAME \
+  --gcs-location gs://cloudstream-dataflow-templates/template.json \
+  --region us-central1 \
+  --update
+```
+
+## Checkpoint/Restart Patterns
+
+```python
+# For batch pipelines: checkpoint progress in GCS
+class CheckpointedBatchPipeline:
+    """Resume batch processing from last successful checkpoint."""
+
+    CHECKPOINT_PATH = 'gs://cloudstream-dataflow-checkpoints/{pipeline}/{date}.json'
+
+    def get_last_checkpoint(self, pipeline_name):
+        """Read last successful offset from GCS."""
+        from google.cloud import storage
+        client = storage.Client()
+        bucket = client.bucket('cloudstream-dataflow-checkpoints')
+        blob = bucket.blob(f'{pipeline_name}/latest.json')
+
+        if blob.exists():
+            checkpoint = json.loads(blob.download_as_string())
+            return checkpoint.get('last_processed_id')
+        return None
+
+    def save_checkpoint(self, pipeline_name, last_id):
+        """Save checkpoint after successful batch completion."""
+        from google.cloud import storage
+        client = storage.Client()
+        bucket = client.bucket('cloudstream-dataflow-checkpoints')
+        blob = bucket.blob(f'{pipeline_name}/latest.json')
+        blob.upload_from_string(json.dumps({
+            'last_processed_id': last_id,
+            'checkpoint_time': datetime.utcnow().isoformat(),
+            'pipeline_name': pipeline_name
+        }))
+
+# For streaming pipelines: PubSub handles checkpointing automatically
+# via acknowledgment. Unacked messages will be redelivered.
+```
+
+## Error Rate Thresholds
+
+```python
+# Dynamic error rate monitoring within pipeline
+class ErrorRateMonitor(beam.DoFn):
+    """Track error rates and trigger alerts when thresholds exceeded."""
+
+    # Thresholds
+    WARN_THRESHOLD = 0.02   # 2% error rate = warning
+    ERROR_THRESHOLD = 0.05  # 5% error rate = alert
+    CRITICAL_THRESHOLD = 0.10  # 10% error rate = pause pipeline
+
+    def __init__(self):
+        self.total_counter = beam.metrics.Metrics.counter('pipeline', 'total_records')
+        self.error_counter = beam.metrics.Metrics.counter('pipeline', 'error_records')
+        self.error_rate_gauge = beam.metrics.Metrics.gauge('pipeline', 'error_rate')
+
+    def process(self, element):
+        self.total_counter.inc()
+
+        if element.get('_has_error'):
+            self.error_counter.inc()
+            # Beam metrics are reported to Cloud Monitoring automatically
+            # Alert policies trigger based on error_rate gauge
+
+        yield element
+```
+
+```sql
+-- Query DLQ for error rate analysis
+SELECT
+  DATE(failed_at) AS error_date,
+  pipeline_name,
+  error_type,
+  COUNT(*) AS error_count,
+  COUNT(*) * 100.0 / SUM(COUNT(*)) OVER (PARTITION BY DATE(failed_at), pipeline_name) AS pct_of_errors
+FROM `project.ops.dead_letter_queue`
+WHERE failed_at >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 7 DAY)
+  AND resolved_at IS NULL
+GROUP BY 1, 2, 3
+ORDER BY error_count DESC;
+```
+
+## Best Practices
+
+1. **Always implement DLQ** - Never drop records silently; route failures for investigation
+2. **Separate retryable from poison** - Transient errors should retry; corrupt data should not
+3. **Use exponential backoff** - Avoid thundering herd on service recovery
+4. **Prefer drain over cancel** - Graceful shutdown preserves data integrity
+5. **Monitor error rates, not just counts** - A spike from 0.1% to 2% is more significant than 100 errors
+6. **Set PubSub ack deadline appropriately** - Too short causes unnecessary retries; too long delays DLQ routing
+7. **Archive DLQ records with context** - Include stack trace, retry count, and raw message for debugging
+
+> **WARNING**: PubSub default ack deadline is 10 seconds. For pipelines with heavy processing, extend to 60-120 seconds to prevent duplicate processing.