@cloudstreamsoftware/claude-tools 1.0.0 → 1.2.0

package/skills/gcp-data-engineering/dataflow/pipeline-patterns.md

@@ -0,0 +1,444 @@
+# Dataflow Pipeline Patterns
+
+> Apache Beam pipeline templates for batch and streaming workloads in CloudStream's GCP data platform.
+
+## Batch Pipeline Templates
+
+### TextIO: CSV/JSON File Processing
+
+```python
+# batch_csv_to_bigquery.py
+import apache_beam as beam
+from apache_beam.options.pipeline_options import PipelineOptions, GoogleCloudOptions
+from apache_beam.io.gcp.bigquery import WriteToBigQuery, BigQueryDisposition
+import csv
+import io
+from datetime import datetime
+
+class ParseCSV(beam.DoFn):
+    """Parse CSV rows into dictionaries."""
+
+    def __init__(self, headers):
+        self.headers = headers
+
+    def process(self, element):
+        reader = csv.reader(io.StringIO(element))
+        for row in reader:
+            if len(row) == len(self.headers):
+                record = dict(zip(self.headers, row))
+                record['_ingestion_timestamp'] = datetime.utcnow().isoformat()
+                record['_source_system'] = 'csv_import'
+                yield record
+
+def run_csv_pipeline(input_path, output_table, headers):
+    options = PipelineOptions(
+        runner='DataflowRunner',
+        project='cloudstream-prod',
+        region='us-central1',
+        temp_location='gs://cloudstream-dataflow-temp/tmp',
+        staging_location='gs://cloudstream-dataflow-temp/staging',
+        max_num_workers=10,
+        machine_type='n1-standard-4',
+    )
+
+    with beam.Pipeline(options=options) as p:
+        (
+            p
+            | 'ReadCSV' >> beam.io.ReadFromText(input_path, skip_header_lines=1)
+            | 'ParseRows' >> beam.ParDo(ParseCSV(headers))
+            | 'WriteBQ' >> WriteToBigQuery(
+                table=output_table,
+                write_disposition=BigQueryDisposition.WRITE_APPEND,
+                create_disposition=BigQueryDisposition.CREATE_IF_NEEDED,
+                custom_gcs_temp_location='gs://cloudstream-dataflow-temp/bq-tmp'
+            )
+        )
+```
+
+### BigQueryIO: Table-to-Table Transforms
+
+```python
+# bigquery_transform_pipeline.py
+import apache_beam as beam
+from apache_beam.io.gcp.bigquery import ReadFromBigQuery, WriteToBigQuery
+
+def enrich_record(record):
+    """Add computed fields to deal records."""
+    record['deal_age_days'] = (
+        datetime.now() - datetime.fromisoformat(record['created_time'])
+    ).days if record.get('created_time') else None
+    record['amount_bucket'] = (
+        'small' if (record.get('amount') or 0) < 10000
+        else 'medium' if (record.get('amount') or 0) < 100000
+        else 'large'
+    )
+    return record
+
+def run():
+    with beam.Pipeline(options=get_pipeline_options()) as p:
+        (
+            p
+            | 'ReadSilver' >> ReadFromBigQuery(
+                query="""
+                    SELECT record_id, deal_name, amount, stage, created_time, owner_id
+                    FROM `cloudstream-prod.silver.zoho_deals`
+                    WHERE _ingestion_date = CURRENT_DATE()
+                """,
+                use_standard_sql=True
+            )
+            | 'Enrich' >> beam.Map(enrich_record)
+            | 'WriteGold' >> WriteToBigQuery(
+                table='cloudstream-prod:gold.enriched_deals',
+                write_disposition='WRITE_TRUNCATE',
+                schema='SCHEMA_AUTODETECT'
+            )
+        )
+```
+
+### AvroIO: Schema-Aware Processing
+
+```python
+# avro_pipeline.py
+import apache_beam as beam
+from apache_beam.io.avroio import ReadFromAvro, WriteToAvro
+
+AVRO_SCHEMA = {
+    "type": "record",
+    "name": "ZohoDeal",
+    "fields": [
+        {"name": "record_id", "type": "string"},
+        {"name": "deal_name", "type": ["null", "string"]},
+        {"name": "amount", "type": ["null", "double"]},
+        {"name": "stage", "type": "string"},
+        {"name": "close_date", "type": ["null", "string"]},
+    ]
+}
+
+def run():
+    with beam.Pipeline(options=get_pipeline_options()) as p:
+        (
+            p
+            | 'ReadAvro' >> ReadFromAvro('gs://cloudstream-landing/avro/deals/*.avro')
+            | 'FilterActive' >> beam.Filter(lambda r: r['stage'] != 'Closed Lost')
+            | 'WriteAvro' >> WriteToAvro(
+                'gs://cloudstream-processed/deals/active',
+                schema=AVRO_SCHEMA,
+                file_name_suffix='.avro'
+            )
+        )
+```
+
+## Streaming Pipeline Templates
+
+### PubSub to BigQuery (Real-Time Ingestion)
+
+```python
+# streaming_pubsub_to_bq.py
+import apache_beam as beam
+from apache_beam.options.pipeline_options import PipelineOptions, StandardOptions
+from apache_beam.io.gcp.bigquery import WriteToBigQuery
+import json
+
+class ParsePubSubMessage(beam.DoFn):
+    def process(self, element):
+        message = json.loads(element.decode('utf-8'))
+        message['_ingestion_timestamp'] = datetime.utcnow().isoformat()
+        message['_ingestion_date'] = datetime.utcnow().strftime('%Y-%m-%d')
+        yield message
+
+def run():
+    options = PipelineOptions(
+        streaming=True,
+        project='cloudstream-prod',
+        region='us-central1',
+        temp_location='gs://cloudstream-dataflow-temp/tmp',
+        # Streaming-specific options
+        enable_streaming_engine=True,  # Reduces worker resource usage
+        experiments=['enable_streaming_engine'],
+    )
+
+    with beam.Pipeline(options=options) as p:
+        (
+            p
+            | 'ReadPubSub' >> beam.io.ReadFromPubSub(
+                subscription='projects/cloudstream-prod/subscriptions/zoho-events-sub'
+            )
+            | 'Parse' >> beam.ParDo(ParsePubSubMessage())
+            | 'WriteBQ' >> WriteToBigQuery(
+                table='cloudstream-prod:bronze.zoho_realtime_events',
+                write_disposition='WRITE_APPEND',
+                insert_retry_strategy='RETRY_ON_TRANSIENT_ERROR',
+                # Use streaming inserts for low-latency
+                method='STREAMING_INSERTS'
+            )
+        )
+```
+
+### PubSub to GCS (Raw Archive)
+
+```python
+# streaming_pubsub_to_gcs.py
+import apache_beam as beam
+from apache_beam.io.fileio import WriteToFiles, FileSink
+from apache_beam.transforms.window import FixedWindows
+
+class JsonSink(FileSink):
+    def write(self, record):
+        self._file_handle.write(json.dumps(record).encode('utf-8') + b'\n')
+
+def run():
+    with beam.Pipeline(options=streaming_options()) as p:
+        (
+            p
+            | 'ReadPubSub' >> beam.io.ReadFromPubSub(
+                topic='projects/cloudstream-prod/topics/zoho-webhooks'
+            )
+            | 'Parse' >> beam.Map(lambda x: json.loads(x.decode('utf-8')))
+            | 'Window5Min' >> beam.WindowInto(FixedWindows(300))  # 5-minute windows
+            | 'WriteGCS' >> WriteToFiles(
+                path='gs://cloudstream-landing/streaming/',
+                sink=JsonSink(),
+                file_naming=beam.io.fileio.destination_prefix_naming(
+                    suffix='.json'
+                )
+            )
+        )
+```
+
+## Windowing Strategies
+
+### Fixed Windows (Regular Intervals)
+
+```python
+# Aggregate every 5 minutes
+windowed = (
+    events
+    | 'FixedWindow' >> beam.WindowInto(
+        beam.window.FixedWindows(300),  # 5 minutes = 300 seconds
+        trigger=beam.trigger.AfterWatermark(
+            late=beam.trigger.AfterCount(100)  # Handle late data
+        ),
+        accumulation_mode=beam.trigger.AccumulationMode.ACCUMULATING
+    )
+    | 'CountPerWindow' >> beam.CombineGlobally(beam.combiners.CountCombineFn())
+)
+```
+
+### Sliding Windows (Overlapping)
+
+```python
+# 1-hour window, sliding every 5 minutes (for moving averages)
+sliding = (
+    events
+    | 'SlidingWindow' >> beam.WindowInto(
+        beam.window.SlidingWindows(
+            size=3600,    # 1 hour window
+            period=300    # Slide every 5 minutes
+        )
+    )
+    | 'AvgAmount' >> beam.CombineGlobally(
+        beam.combiners.MeanCombineFn()
+    ).without_defaults()
+)
+```
+
+### Session Windows (Activity-Based)
+
+```python
+# Group events by user session (10-minute gap = new session)
+sessions = (
+    events
+    | 'KeyByUser' >> beam.Map(lambda e: (e['user_id'], e))
+    | 'SessionWindow' >> beam.WindowInto(
+        beam.window.Sessions(600)  # 10-minute gap
+    )
+    | 'GroupSessions' >> beam.GroupByKey()
+    | 'AnalyzeSession' >> beam.ParDo(SessionAnalyzer())
+)
+```
+
+## Watermarks and Late Data
+
+```python
+# Handle late-arriving data with allowed lateness
+late_handling = (
+    events
+    | 'WindowWithLateness' >> beam.WindowInto(
+        beam.window.FixedWindows(300),
+        trigger=beam.trigger.AfterWatermark(
+            early=beam.trigger.AfterProcessingTime(60),  # Early results every 60s
+            late=beam.trigger.AfterCount(1)              # Fire on each late element
+        ),
+        allowed_lateness=beam.window.Duration(seconds=86400),  # 24h late tolerance
+        accumulation_mode=beam.trigger.AccumulationMode.ACCUMULATING
+    )
+)
+```
+
+> **WARNING**: Setting `allowed_lateness` too high increases memory usage. For Zoho webhook data, 1 hour is typically sufficient. Use 24 hours only for batch replay scenarios.
+
+## Side Inputs for Enrichment
+
+```python
+# Enrich streaming deals with customer dimension data
+class EnrichWithCustomer(beam.DoFn):
+    def process(self, deal, customers):
+        customer = customers.get(deal['customer_id'], {})
+        deal['customer_name'] = customer.get('account_name', 'Unknown')
+        deal['customer_industry'] = customer.get('industry', 'Unknown')
+        yield deal
+
+def run():
+    with beam.Pipeline(options=options) as p:
+        # Side input: customer dimension (refreshed periodically)
+        customers = (
+            p
+            | 'ReadCustomers' >> ReadFromBigQuery(
+                query='SELECT record_id, account_name, industry FROM silver.zoho_accounts'
+            )
+            | 'KeyById' >> beam.Map(lambda c: (c['record_id'], c))
+            | 'ToDict' >> beam.combiners.ToDict()
+        )
+
+        # Main input: streaming deals
+        deals = (
+            p
+            | 'ReadDeals' >> beam.io.ReadFromPubSub(topic=DEALS_TOPIC)
+            | 'ParseDeal' >> beam.Map(json.loads)
+        )
+
+        # Enrich with side input
+        enriched = (
+            deals
+            | 'Enrich' >> beam.ParDo(
+                EnrichWithCustomer(),
+                customers=beam.pvalue.AsSingleton(customers)
+            )
+        )
+```
+
+## Pipeline Options
+
+```python
+# Common pipeline options for CloudStream
+from apache_beam.options.pipeline_options import (
+    PipelineOptions, GoogleCloudOptions, WorkerOptions, SetupOptions
+)
+
+def get_pipeline_options(streaming=False, job_name='cloudstream-pipeline'):
+    options = PipelineOptions()
+
+    google_options = options.view_as(GoogleCloudOptions)
+    google_options.project = 'cloudstream-prod'
+    google_options.region = 'us-central1'
+    google_options.temp_location = 'gs://cloudstream-dataflow-temp/tmp'
+    google_options.staging_location = 'gs://cloudstream-dataflow-temp/staging'
+    google_options.job_name = f'{job_name}-{datetime.now().strftime("%Y%m%d-%H%M%S")}'
+    google_options.service_account_email = 'dataflow-sa@cloudstream-prod.iam.gserviceaccount.com'
+
+    worker_options = options.view_as(WorkerOptions)
+    worker_options.machine_type = 'n1-standard-4'
+    worker_options.max_num_workers = 10 if not streaming else 5
+    worker_options.disk_size_gb = 50
+    worker_options.network = 'cloudstream-vpc'
+    worker_options.subnetwork = 'regions/us-central1/subnetworks/dataflow-subnet'
+
+    if streaming:
+        options.view_as(StandardOptions).streaming = True
+
+    return options
+```
+
+## Autoscaling Configuration
+
+```python
+# Autoscaling options
+worker_options.autoscaling_algorithm = 'THROUGHPUT_BASED'
+worker_options.num_workers = 2        # Initial workers
+worker_options.max_num_workers = 20   # Max scale-out
+
+# For cost-sensitive batch jobs:
+worker_options.use_public_ips = False  # Reduces network cost
+worker_options.experiments = [
+    'shuffle_mode=service',            # Use Dataflow Shuffle (faster, less disk)
+    'use_runner_v2',                   # New Dataflow Runner (better autoscaling)
+]
+```
+
+## Flex Templates for Parameterized Pipelines
+
+```python
+# flex_template_pipeline.py
+import apache_beam as beam
+from apache_beam.options.pipeline_options import PipelineOptions
+
+class CloudStreamOptions(PipelineOptions):
+    @classmethod
+    def _add_argparse_args(cls, parser):
+        parser.add_argument('--source_table', required=True, help='Source BQ table')
+        parser.add_argument('--dest_table', required=True, help='Destination BQ table')
+        parser.add_argument('--processing_date', help='Date to process (YYYY-MM-DD)')
+        parser.add_argument('--mode', default='incremental', choices=['full', 'incremental'])
+
+def run():
+    options = PipelineOptions().view_as(CloudStreamOptions)
+
+    with beam.Pipeline(options=options) as p:
+        query = f"""
+            SELECT * FROM `{options.source_table}`
+            WHERE _ingestion_date = '{options.processing_date}'
+        """ if options.mode == 'incremental' else f"""
+            SELECT * FROM `{options.source_table}`
+        """
+
+        (
+            p
+            | 'Read' >> ReadFromBigQuery(query=query, use_standard_sql=True)
+            | 'Transform' >> beam.ParDo(TransformDoFn())
+            | 'Write' >> WriteToBigQuery(table=options.dest_table)
+        )
+```
+
+```yaml
+# flex_template_metadata.json
+{
+  "name": "CloudStream Bronze-to-Silver",
+  "description": "Parameterized pipeline for medallion layer transitions",
+  "parameters": [
+    {"name": "source_table", "label": "Source Table", "helpText": "Fully qualified BQ table"},
+    {"name": "dest_table", "label": "Destination Table", "helpText": "Fully qualified BQ table"},
+    {"name": "processing_date", "label": "Processing Date", "helpText": "YYYY-MM-DD format"},
+    {"name": "mode", "label": "Mode", "helpText": "full or incremental", "isOptional": true}
+  ]
+}
+```
+
+```bash
+# Build and deploy Flex Template
+gcloud dataflow flex-template build \
+  gs://cloudstream-dataflow-templates/bronze-to-silver.json \
+  --image-gcr-path gcr.io/cloudstream-prod/dataflow/bronze-to-silver:latest \
+  --sdk-language PYTHON \
+  --metadata-file flex_template_metadata.json
+
+# Launch from template
+gcloud dataflow flex-template run "bronze-to-silver-$(date +%Y%m%d)" \
+  --template-file-gcs-location gs://cloudstream-dataflow-templates/bronze-to-silver.json \
+  --region us-central1 \
+  --parameters source_table=cloudstream-prod:bronze.zoho_deals \
+  --parameters dest_table=cloudstream-prod:silver.zoho_deals \
+  --parameters processing_date=2024-01-15 \
+  --parameters mode=incremental
+```
+
+## Best Practices
+
+1. **Use Flex Templates** for reusable, parameterized pipelines
+2. **Enable Streaming Engine** for streaming jobs (reduces cost 30-50%)
+3. **Use Dataflow Shuffle** for batch jobs (`shuffle_mode=service`)
+4. **Set appropriate machine types** - n1-standard-4 for most, n1-highmem for memory-intensive
+5. **Monitor with Cloud Monitoring** - Track throughput, watermark lag, backlog
+6. **Use VPC-native networking** - No public IPs reduces cost and improves security
+7. **Test locally first** - Use `DirectRunner` before deploying to `DataflowRunner`
+
+> **COST WARNING**: Streaming pipelines run 24/7. A single n1-standard-4 worker costs ~$100/month. Design pipelines to scale down to 1 worker during low-traffic periods.

package/skills/gcp-data-engineering/dbt/model-organization.md

@@ -0,0 +1,63 @@
+# dbt Model Organization
+
+## Directory Structure
+```
+models/
+├── staging/          # Bronze → Silver (1:1 with source)
+│   ├── zoho/
+│   │   ├── stg_zoho__contacts.sql
+│   │   ├── stg_zoho__deals.sql
+│   │   └── _zoho__sources.yml
+│   └── gcs/
+│       └── stg_gcs__raw_events.sql
+├── intermediate/     # Silver transformations
+│   ├── int_contacts_enriched.sql
+│   └── int_deals_with_contacts.sql
+└── marts/            # Gold layer (business-ready)
+    ├── finance/
+    │   ├── fct_revenue.sql
+    │   └── dim_customers.sql
+    └── operations/
+        ├── fct_pipeline.sql
+        └── dim_sales_reps.sql
+```
+
+## Staging Model Pattern
+```sql
+-- models/staging/zoho/stg_zoho__contacts.sql
+WITH source AS (
+  SELECT * FROM {{ source('zoho', 'contacts_raw') }}
+),
+
+cleaned AS (
+  SELECT
+    id AS contact_id,
+    TRIM(LOWER(email)) AS email,
+    INITCAP(first_name) AS first_name,
+    INITCAP(last_name) AS last_name,
+    PARSE_TIMESTAMP('%Y-%m-%dT%H:%M:%S', created_time) AS created_at,
+    PARSE_TIMESTAMP('%Y-%m-%dT%H:%M:%S', modified_time) AS modified_at
+  FROM source
+  WHERE id IS NOT NULL
+)
+
+SELECT * FROM cleaned
+```
+
+## Testing Pattern
+```yaml
+# models/staging/zoho/_zoho__sources.yml
+version: 2
+sources:
+  - name: zoho
+    tables:
+      - name: contacts_raw
+        columns:
+          - name: id
+            tests:
+              - not_null
+              - unique
+          - name: email
+            tests:
+              - not_null
+```