claude-autopm 2.8.2 → 2.8.4

package/packages/plugin-data/scripts/examples/dbt-transform-example.sql

@@ -0,0 +1,238 @@
+/*
+dbt Transform Example - Context7 Best Practices
+
+Demonstrates dbt transformation patterns from Context7:
+- Jinja macros (log, debug, ref, source)
+- Materializations (table, view, incremental)
+- Tests and documentation
+- Configuration blocks
+
+Source: /dbt-labs/dbt-core (133 snippets, trust 10.0)
+
+To use this file:
+1. Place in models/ directory of your dbt project
+2. Create schema.yml for tests and documentation
+3. Run: dbt run --models fct_orders
+*/
+
+-- ===================================================================
+-- MODEL CONFIGURATION (Context7 Pattern)
+-- ===================================================================
+
+{{
+  config(
+    materialized='table',
+    tags=['finance', 'production'],
+    schema='analytics',
+    alias='fact_orders'
+  )
+}}
+
+-- ===================================================================
+-- JINJA MACROS (Context7 Pattern)
+-- ===================================================================
+
+-- ✅ CORRECT: Use {{ log() }} for debugging
+{{ log("Building fct_orders model", info=True) }}
+
+-- ===================================================================
+-- CTE STRUCTURE (Best Practice)
+-- ===================================================================
+
+WITH
+
+-- ✅ CORRECT: Use {{ ref() }} for model dependencies
+raw_orders AS (
+    SELECT *
+    FROM {{ ref('stg_orders') }}
+    {{ log("Loaded " ~ (select count(*) from ref('stg_orders')) ~ " orders", info=True) }}
+),
+
+-- ✅ CORRECT: Use {{ source() }} for raw data
+raw_customers AS (
+    SELECT *
+    FROM {{ source('raw', 'customers') }}
+),
+
+-- ✅ CORRECT: Use {{ ref() }} for other models
+raw_products AS (
+    SELECT *
+    FROM {{ ref('dim_products') }}
+),
+
+-- ===================================================================
+-- TRANSFORMATIONS
+-- ===================================================================
+
+enriched_orders AS (
+    SELECT
+        o.order_id,
+        o.order_date,
+        o.customer_id,
+        c.customer_name,
+        c.customer_email,
+        c.customer_segment,
+        o.product_id,
+        p.product_name,
+        p.product_category,
+        o.quantity,
+        o.unit_price,
+        -- ✅ CORRECT: Calculated columns
+        o.quantity * o.unit_price AS order_total,
+        CASE
+            WHEN o.quantity > 10 THEN 'bulk'
+            WHEN o.quantity > 5 THEN 'medium'
+            ELSE 'small'
+        END AS order_size,
+        -- Metadata
+        CURRENT_TIMESTAMP AS dbt_updated_at,
+        '{{ run_started_at }}' AS dbt_run_started_at
+
+    FROM raw_orders o
+    LEFT JOIN raw_customers c
+        ON o.customer_id = c.customer_id
+    LEFT JOIN raw_products p
+        ON o.product_id = p.product_id
+
+    -- ✅ CORRECT: Filter out invalid records
+    WHERE o.order_date IS NOT NULL
+        AND o.quantity > 0
+        AND o.unit_price > 0
+),
+
+-- ===================================================================
+-- AGGREGATIONS
+-- ===================================================================
+
+order_metrics AS (
+    SELECT
+        order_id,
+        order_date,
+        customer_id,
+        customer_name,
+        product_id,
+        product_name,
+        order_total,
+        order_size,
+
+        -- Customer-level aggregations
+        SUM(order_total) OVER (
+            PARTITION BY customer_id
+            ORDER BY order_date
+            ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
+        ) AS customer_lifetime_value,
+
+        -- Product-level aggregations
+        AVG(order_total) OVER (
+            PARTITION BY product_id
+        ) AS product_avg_order_value,
+
+        dbt_updated_at,
+        dbt_run_started_at
+
+    FROM enriched_orders
+)
+
+-- ===================================================================
+-- FINAL SELECT
+-- ===================================================================
+
+SELECT
+    order_id,
+    order_date,
+    customer_id,
+    customer_name,
+    product_id,
+    product_name,
+    order_total,
+    order_size,
+    customer_lifetime_value,
+    product_avg_order_value,
+    dbt_updated_at,
+    dbt_run_started_at
+
+FROM order_metrics
+
+-- ===================================================================
+-- TESTS AND DOCUMENTATION (schema.yml)
+-- ===================================================================
+
+/*
+Create a schema.yml file alongside this model:
+
+version: 2
+
+models:
+  - name: fct_orders
+    description: "Orders fact table with customer and product enrichment"
+
+    # ✅ CORRECT: Model-level tests
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - order_id
+
+    columns:
+      - name: order_id
+        description: "Primary key - unique order identifier"
+        tests:
+          - unique
+          - not_null
+
+      - name: customer_id
+        description: "Foreign key to customers dimension"
+        tests:
+          - not_null
+          - relationships:
+              to: ref('dim_customers')
+              field: customer_id
+
+      - name: order_date
+        description: "Order creation timestamp"
+        tests:
+          - not_null
+          - dbt_utils.expression_is_true:
+              expression: "order_date <= current_date"
+
+      - name: order_total
+        description: "Total order amount (quantity * unit_price)"
+        tests:
+          - not_null
+          - dbt_utils.accepted_range:
+              min_value: 0
+              inclusive: true
+
+      - name: customer_lifetime_value
+        description: "Cumulative customer spending"
+        tests:
+          - not_null
+          - dbt_utils.accepted_range:
+              min_value: 0
+
+sources:
+  - name: raw
+    description: "Raw data from source systems"
+    tables:
+      - name: customers
+        description: "Customer master data"
+        columns:
+          - name: customer_id
+            tests:
+              - unique
+              - not_null
+
+Context7 Patterns Demonstrated:
+================================
+1. ✅ {{ config() }} - Model configuration
+2. ✅ {{ ref() }} - Reference other models
+3. ✅ {{ source() }} - Reference raw data sources
+4. ✅ {{ log() }} - Debug logging
+5. ✅ CTE structure - Clean, readable SQL
+6. ✅ Calculated columns - Business logic
+7. ✅ Window functions - Advanced aggregations
+8. ✅ Tests in schema.yml - Data quality
+9. ✅ Documentation - Column descriptions
+10. ✅ Relationships - Foreign key tests
+
+Source: /dbt-labs/dbt-core (133 snippets, trust 10.0)
+*/

package/packages/plugin-data/scripts/examples/kafka-streaming-example.py

@@ -0,0 +1,257 @@
+#!/usr/bin/env python3
+"""
+Kafka Streaming Example - Context7 Best Practices
+
+Demonstrates Kafka producer/consumer patterns from Context7:
+- Producer with proper configuration and callbacks
+- Consumer with manual commits
+- Error handling and retry logic
+- Graceful shutdown
+
+Source: /apache/kafka (1,281 snippets, trust 9.1)
+"""
+
+from kafka import KafkaProducer, KafkaConsumer
+from kafka.errors import KafkaError
+import json
+import logging
+from datetime import datetime
+import time
+import signal
+import sys
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+# ===================================================================
+# PRODUCER EXAMPLE
+# ===================================================================
+
+class EventProducer:
+    """
+    Kafka producer with Context7 best practices.
+
+    Context7 Patterns:
+    - acks='all' for reliability
+    - Compression for performance
+    - Callbacks for delivery confirmation
+    """
+
+    def __init__(
+        self,
+        bootstrap_servers: str = "localhost:9092",
+        topic: str = "user_events",
+    ):
+        self.topic = topic
+
+        # ✅ CORRECT: Producer with proper configuration
+        self.producer = KafkaProducer(
+            bootstrap_servers=bootstrap_servers,
+            # Serialization
+            value_serializer=lambda v: json.dumps(v).encode('utf-8'),
+            key_serializer=lambda k: k.encode('utf-8') if k else None,
+            # Reliability
+            acks='all',  # Wait for all replicas
+            retries=3,
+            max_in_flight_requests_per_connection=1,
+            # Performance
+            compression_type='snappy',
+            batch_size=16384,
+            linger_ms=10,
+        )
+
+        logger.info(f"Producer initialized for topic: {self.topic}")
+
+    def send_event(self, key: str, value: dict) -> None:
+        """
+        Send event with Context7 callback pattern.
+
+        Context7 Pattern: Async send with delivery confirmation
+        """
+        value["timestamp"] = datetime.utcnow().isoformat()
+
+        future = self.producer.send(self.topic, key=key, value=value)
+        future.add_callback(self._on_success)
+        future.add_errback(self._on_error)
+
+    def _on_success(self, record_metadata):
+        """Callback for successful delivery."""
+        logger.info(
+            f"✓ Message delivered: topic={record_metadata.topic}, "
+            f"partition={record_metadata.partition}, offset={record_metadata.offset}"
+        )
+
+    def _on_error(self, excp):
+        """Callback for delivery failure."""
+        logger.error(f"✗ Message delivery failed: {excp}")
+
+    def close(self):
+        """Close producer after flushing."""
+        logger.info("Flushing and closing producer...")
+        self.producer.flush()
+        self.producer.close()
+
+
+# ===================================================================
+# CONSUMER EXAMPLE
+# ===================================================================
+
+class EventConsumer:
+    """
+    Kafka consumer with Context7 best practices.
+
+    Context7 Patterns:
+    - Manual commits for reliability
+    - Proper configuration (consumer.* prefix)
+    - Graceful shutdown
+    """
+
+    def __init__(
+        self,
+        bootstrap_servers: str = "localhost:9092",
+        topic: str = "user_events",
+        group_id: str = "event_processor_group",
+    ):
+        self.topic = topic
+        self.running = True
+
+        # ✅ CORRECT: Consumer with proper configuration
+        self.consumer = KafkaConsumer(
+            topic,
+            bootstrap_servers=bootstrap_servers,
+            group_id=group_id,
+            # Deserialization
+            value_deserializer=lambda m: json.loads(m.decode('utf-8')),
+            key_deserializer=lambda k: k.decode('utf-8') if k else None,
+            # Commit strategy
+            enable_auto_commit=False,  # Manual commits
+            auto_offset_reset='earliest',
+            # Performance
+            max_poll_records=100,
+            max_poll_interval_ms=300000,
+        )
+
+        # Setup graceful shutdown
+        signal.signal(signal.SIGINT, self._shutdown)
+        signal.signal(signal.SIGTERM, self._shutdown)
+
+        logger.info(f"Consumer initialized: topic={self.topic}, group={group_id}")
+
+    def consume(self) -> None:
+        """
+        Consume messages with Context7 manual commit pattern.
+
+        Context7 Pattern: Process then commit for exactly-once semantics
+        """
+        try:
+            while self.running:
+                # Context7 Pattern: Poll with timeout
+                messages = self.consumer.poll(timeout_ms=1000)
+
+                if not messages:
+                    continue
+
+                for topic_partition, records in messages.items():
+                    for record in records:
+                        try:
+                            # Process message
+                            self._process_message(record)
+
+                            # ✅ CORRECT: Commit after successful processing
+                            self.consumer.commit()
+
+                        except Exception as e:
+                            logger.error(f"Error processing message: {e}")
+                            # In production: decide to skip or retry
+
+        finally:
+            self._close()
+
+    def _process_message(self, record) -> None:
+        """Process a single message."""
+        logger.info(
+            f"Processing: partition={record.partition}, "
+            f"offset={record.offset}, key={record.key}"
+        )
+
+        # Your business logic here
+        logger.info(f"Message: {record.value}")
+
+    def _shutdown(self, signum, frame):
+        """Handle shutdown signal."""
+        logger.info("Shutdown signal received")
+        self.running = False
+
+    def _close(self):
+        """Close consumer."""
+        logger.info("Closing consumer...")
+        self.consumer.close()
+
+
+# ===================================================================
+# DEMO APPLICATION
+# ===================================================================
+
+def demo_producer():
+    """Demonstrate producer patterns."""
+    print("\n" + "=" * 60)
+    print("PRODUCER DEMO - Context7 Patterns")
+    print("=" * 60)
+
+    producer = EventProducer()
+
+    try:
+        # Send 10 events
+        for i in range(10):
+            event = {
+                "event_id": i,
+                "event_type": "page_view",
+                "user_id": f"user_{i % 3}",
+                "page": f"/page/{i}",
+            }
+
+            producer.send_event(key=f"user_{i % 3}", value=event)
+            time.sleep(0.1)
+
+        logger.info("All events sent")
+
+    finally:
+        producer.close()
+
+
+def demo_consumer():
+    """Demonstrate consumer patterns."""
+    print("\n" + "=" * 60)
+    print("CONSUMER DEMO - Context7 Patterns")
+    print("=" * 60)
+    print("Press Ctrl+C to stop consuming...")
+
+    consumer = EventConsumer()
+    consumer.consume()
+
+
+if __name__ == "__main__":
+    print("Kafka Streaming Example - Context7 Best Practices")
+    print("=" * 60)
+    print("")
+    print("Context7 Patterns Demonstrated:")
+    print("1. ✅ Producer: acks='all' for reliability")
+    print("2. ✅ Producer: Callbacks for delivery confirmation")
+    print("3. ✅ Producer: Compression and batching")
+    print("4. ✅ Consumer: Manual commits for exactly-once")
+    print("5. ✅ Consumer: Proper error handling")
+    print("6. ✅ Consumer: Graceful shutdown")
+    print("7. ✅ Serialization: JSON with proper encoding")
+    print("")
+    print("Source: /apache/kafka (1,281 snippets, trust 9.1)")
+    print("")
+
+    mode = input("Run (p)roducer or (c)onsumer? [p/c]: ").strip().lower()
+
+    if mode == 'p':
+        demo_producer()
+    elif mode == 'c':
+        demo_consumer()
+    else:
+        print("Invalid choice. Use 'p' or 'c'")