@cloudstreamsoftware/claude-tools 1.0.0 → 1.2.0

package/skills/gcp-data-engineering/medallion-architecture/layer-transitions.md

@@ -0,0 +1,517 @@
+# Layer Transitions - Bronze to Silver to Gold
+
+> Orchestrating data movement between medallion layers using dbt, Dataflow, Cloud Functions, and Cloud Scheduler.
+
+## Transition Overview
+
+```
+[GCS Landing Zone] → Cloud Function trigger → [Bronze BQ Tables]
+                                                       │
+                                         dbt run (staging models)
+                                                       │
+                                              [Silver BQ Tables]
+                                                       │
+                                         dbt run (mart models)
+                                                       │
+                                              [Gold BQ Tables]
+                                                       │
+                                    ┌──────────────────┼──────────────────┐
+                              Looker Studio    Zoho Analytics    Custom Apps
+```
+
+## dbt Models: Bronze to Silver to Gold
+
+### Project Structure
+
+```yaml
+# dbt_project.yml
+models:
+  cloudstream:
+    staging:          # Bronze → Silver
+      +materialized: incremental
+      +schema: silver
+      +tags: ['silver']
+    intermediate:     # Silver internal transforms
+      +materialized: ephemeral
+      +schema: silver
+    marts:            # Silver → Gold
+      +materialized: table
+      +schema: gold
+      +tags: ['gold']
+```
+
+### Bronze to Silver (Staging Models)
+
+```sql
+-- models/staging/zoho_crm/stg_zoho__deals.sql
+{{
+  config(
+    materialized='incremental',
+    unique_key='record_id',
+    partition_by={'field': '_ingestion_date', 'data_type': 'date'},
+    cluster_by=['stage', 'owner_id'],
+    on_schema_change='append_new_columns'
+  )
+}}
+
+WITH source AS (
+  SELECT *
+  FROM {{ source('bronze', 'zoho_deals') }}
+  {% if is_incremental() %}
+  WHERE _ingestion_timestamp > (
+    SELECT COALESCE(MAX(_ingestion_timestamp), TIMESTAMP('2020-01-01'))
+    FROM {{ this }}
+  )
+  {% endif %}
+),
+
+deduplicated AS (
+  SELECT *, ROW_NUMBER() OVER (
+    PARTITION BY record_id ORDER BY modified_time DESC
+  ) AS _rn
+  FROM source
+),
+
+cleaned AS (
+  SELECT
+    record_id,
+    NULLIF(TRIM(deal_name), '') AS deal_name,
+    SAFE_CAST(amount AS FLOAT64) AS amount,
+    COALESCE(stage, 'Unknown') AS stage,
+    SAFE.PARSE_DATE('%Y-%m-%d', close_date) AS close_date,
+    owner_id,
+    account_id,
+    SAFE_CAST(probability AS INT64) AS probability,
+    _ingestion_date,
+    _ingestion_timestamp,
+    _source_system,
+    _batch_id
+  FROM deduplicated
+  WHERE _rn = 1
+    AND record_id IS NOT NULL
+)
+
+SELECT * FROM cleaned
+```
+
+### Silver to Gold (Mart Models)
+
+```sql
+-- models/marts/finance/fct_monthly_revenue.sql
+{{
+  config(
+    materialized='table',
+    partition_by={'field': 'revenue_month', 'data_type': 'date', 'granularity': 'month'},
+    cluster_by=['customer_id']
+  )
+}}
+
+WITH invoices AS (
+  SELECT * FROM {{ ref('stg_zoho__invoices') }}
+  WHERE status = 'paid'
+),
+
+customers AS (
+  SELECT * FROM {{ ref('stg_zoho__accounts') }}
+)
+
+SELECT
+  DATE_TRUNC(i.paid_date, MONTH) AS revenue_month,
+  i.customer_id,
+  c.account_name AS customer_name,
+  c.industry,
+  COUNT(*) AS invoice_count,
+  SUM(i.total) AS total_revenue,
+  SUM(i.tax_amount) AS total_tax,
+  AVG(DATE_DIFF(i.paid_date, i.invoice_date, DAY)) AS avg_days_to_pay
+FROM invoices i
+LEFT JOIN customers c ON i.customer_id = c.record_id
+GROUP BY 1, 2, 3, 4
+```
+
+## Dataflow Jobs for Streaming Transitions
+
+### PubSub to Silver (Streaming)
+
+```python
+# streaming_silver_pipeline.py
+import apache_beam as beam
+from apache_beam.options.pipeline_options import PipelineOptions
+from apache_beam.io.gcp.bigquery import WriteToBigQuery, BigQueryDisposition
+import json
+from datetime import datetime
+
+class ParseAndValidate(beam.DoFn):
+    """Bronze→Silver: parse, validate, enrich."""
+
+    def process(self, element, timestamp=beam.DoFn.TimestampParam):
+        try:
+            record = json.loads(element.decode('utf-8'))
+
+            # Validation
+            if not record.get('record_id'):
+                yield beam.pvalue.TaggedOutput('quarantine', {
+                    'raw': element.decode('utf-8'),
+                    'error': 'missing_record_id',
+                    'timestamp': datetime.utcnow().isoformat()
+                })
+                return
+
+            # Type coercion
+            cleaned = {
+                'record_id': str(record['record_id']),
+                'deal_name': (record.get('deal_name') or '').strip() or None,
+                'amount': float(record['amount']) if record.get('amount') else 0.0,
+                'stage': record.get('stage', 'Unknown'),
+                'owner_id': record.get('owner_id'),
+                'modified_time': record.get('modified_time'),
+                '_ingestion_timestamp': datetime.utcnow().isoformat(),
+                '_source_system': 'zoho_crm_webhook'
+            }
+
+            yield beam.pvalue.TaggedOutput('valid', cleaned)
+
+        except Exception as e:
+            yield beam.pvalue.TaggedOutput('quarantine', {
+                'raw': element.decode('utf-8'),
+                'error': str(e),
+                'timestamp': datetime.utcnow().isoformat()
+            })
+
+def run():
+    options = PipelineOptions(
+        streaming=True,
+        project='cloudstream-prod',
+        region='us-central1',
+        temp_location='gs://cloudstream-dataflow-temp/tmp',
+        autoscaling_algorithm='THROUGHPUT_BASED',
+        max_num_workers=5
+    )
+
+    with beam.Pipeline(options=options) as p:
+        messages = (
+            p
+            | 'ReadPubSub' >> beam.io.ReadFromPubSub(
+                topic='projects/cloudstream-prod/topics/zoho-webhooks'
+            )
+        )
+
+        results = messages | 'ParseValidate' >> beam.ParDo(
+            ParseAndValidate()
+        ).with_outputs('valid', 'quarantine')
+
+        # Valid records → Silver
+        results.valid | 'WriteToSilver' >> WriteToBigQuery(
+            table='cloudstream-prod:silver.zoho_deals_streaming',
+            write_disposition=BigQueryDisposition.WRITE_APPEND,
+            create_disposition=BigQueryDisposition.CREATE_IF_NEEDED
+        )
+
+        # Bad records → Quarantine
+        results.quarantine | 'WriteToQuarantine' >> WriteToBigQuery(
+            table='cloudstream-prod:silver._quarantine_streaming',
+            write_disposition=BigQueryDisposition.WRITE_APPEND
+        )
+
+if __name__ == '__main__':
+    run()
+```
+
+## Cloud Function Triggers on GCS File Arrival
+
+```python
+# cloud_function/bronze_loader/main.py
+"""Triggered when new file arrives in GCS landing zone.
+Loads raw file into Bronze BigQuery table."""
+
+from google.cloud import bigquery, storage
+import functions_framework
+import json
+
+PROJECT = 'cloudstream-prod'
+DATASET_BRONZE = 'bronze'
+
+# Source-to-table mapping
+SOURCE_MAP = {
+    'zoho-crm/deals': 'zoho_deals',
+    'zoho-crm/contacts': 'zoho_contacts',
+    'zoho-books/invoices': 'zoho_invoices',
+    'zoho-books/payments': 'zoho_payments',
+}
+
+@functions_framework.cloud_event
+def load_to_bronze(cloud_event):
+    """Triggered by GCS object finalize event."""
+    data = cloud_event.data
+    bucket_name = data['bucket']
+    file_path = data['name']
+
+    # Determine target table from file path
+    # Expected: landing/{source}/{entity}/YYYY/MM/DD/file.json
+    parts = file_path.split('/')
+    if len(parts) < 4:
+        print(f"Unexpected path format: {file_path}")
+        return
+
+    source_key = f"{parts[1]}/{parts[2]}"
+    table_name = SOURCE_MAP.get(source_key)
+    if not table_name:
+        print(f"Unknown source: {source_key}")
+        return
+
+    client = bigquery.Client(project=PROJECT)
+    table_ref = f"{PROJECT}.{DATASET_BRONZE}.{table_name}"
+
+    job_config = bigquery.LoadJobConfig(
+        source_format=bigquery.SourceFormat.NEWLINE_DELIMITED_JSON,
+        write_disposition=bigquery.WriteDisposition.WRITE_APPEND,
+        schema_update_options=[
+            bigquery.SchemaUpdateOption.ALLOW_FIELD_ADDITION
+        ],
+        # Add metadata columns
+        time_partitioning=bigquery.TimePartitioning(
+            type_=bigquery.TimePartitioningType.DAY,
+            field='_ingestion_date'
+        ),
+    )
+
+    uri = f"gs://{bucket_name}/{file_path}"
+    load_job = client.load_table_from_uri(uri, table_ref, job_config=job_config)
+    load_job.result()  # Wait for completion
+
+    print(f"Loaded {load_job.output_rows} rows from {uri} to {table_ref}")
+
+    # Trigger silver refresh if needed
+    trigger_silver_refresh(table_name)
+
+def trigger_silver_refresh(table_name):
+    """Publish message to trigger dbt staging model run."""
+    from google.cloud import pubsub_v1
+    publisher = pubsub_v1.PublisherClient()
+    topic = f"projects/{PROJECT}/topics/silver-refresh-trigger"
+
+    message = json.dumps({
+        'source_table': table_name,
+        'trigger_time': datetime.utcnow().isoformat()
+    }).encode('utf-8')
+
+    publisher.publish(topic, message)
+```
+
+## Orchestration with Cloud Composer/Scheduler
+
+### Cloud Composer DAG (Full Pipeline)
+
+```python
+# dags/medallion_pipeline.py
+from airflow import DAG
+from airflow.providers.google.cloud.operators.bigquery import BigQueryInsertJobOperator
+from airflow.providers.dbt.cloud.operators.dbt import DbtCloudRunJobOperator
+from airflow.operators.python import PythonOperator
+from airflow.utils.dates import days_ago
+from datetime import timedelta
+
+default_args = {
+    'owner': 'cloudstream',
+    'retries': 2,
+    'retry_delay': timedelta(minutes=5),
+    'email_on_failure': True,
+    'email': ['data-alerts@cloudstreamsoftware.com'],
+}
+
+with DAG(
+    'medallion_full_refresh',
+    default_args=default_args,
+    schedule_interval='0 6 * * *',  # 6 AM daily
+    start_date=days_ago(1),
+    catchup=False,
+    tags=['medallion', 'production'],
+) as dag:
+
+    # Step 1: Bronze freshness check
+    check_bronze = BigQueryInsertJobOperator(
+        task_id='check_bronze_freshness',
+        configuration={
+            'query': {
+                'query': """
+                    SELECT table_name, MAX(_ingestion_timestamp) AS latest
+                    FROM `bronze.INFORMATION_SCHEMA.COLUMNS`
+                    GROUP BY 1
+                    HAVING MAX(_ingestion_timestamp) < TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 25 HOUR)
+                """,
+                'useLegacySql': False,
+            }
+        },
+    )
+
+    # Step 2: dbt run staging (Bronze → Silver)
+    dbt_silver = DbtCloudRunJobOperator(
+        task_id='dbt_run_silver',
+        job_id=12345,  # dbt Cloud job ID for staging models
+        check_interval=30,
+        timeout=1800,
+    )
+
+    # Step 3: dbt test silver
+    dbt_test_silver = DbtCloudRunJobOperator(
+        task_id='dbt_test_silver',
+        job_id=12346,  # dbt Cloud job ID for staging tests
+        check_interval=30,
+        timeout=600,
+    )
+
+    # Step 4: dbt run marts (Silver → Gold)
+    dbt_gold = DbtCloudRunJobOperator(
+        task_id='dbt_run_gold',
+        job_id=12347,
+        check_interval=30,
+        timeout=1800,
+    )
+
+    # Step 5: dbt test gold
+    dbt_test_gold = DbtCloudRunJobOperator(
+        task_id='dbt_test_gold',
+        job_id=12348,
+        check_interval=30,
+        timeout=600,
+    )
+
+    check_bronze >> dbt_silver >> dbt_test_silver >> dbt_gold >> dbt_test_gold
+```
+
+### Cloud Scheduler (Lightweight Alternative)
+
+```yaml
+# terraform/scheduler.tf - For simpler orchestration without Composer
+resource "google_cloud_scheduler_job" "dbt_silver_refresh" {
+  name        = "dbt-silver-refresh"
+  schedule    = "0 */4 * * *"  # Every 4 hours
+  time_zone   = "America/Chicago"
+
+  http_target {
+    uri         = "https://cloud.getdbt.com/api/v2/accounts/12345/jobs/67890/run/"
+    http_method = "POST"
+    headers     = { "Authorization" = "Token ${var.dbt_cloud_token}" }
+    body        = base64encode(jsonencode({ "cause" = "Scheduled by Cloud Scheduler" }))
+  }
+}
+```
+
+## Dependency Management
+
+```yaml
+# models/staging/_staging__sources.yml
+sources:
+  - name: bronze
+    database: cloudstream-prod
+    schema: bronze
+    freshness:
+      warn_after: {count: 12, period: hour}
+      error_after: {count: 24, period: hour}
+    loaded_at_field: _ingestion_timestamp
+    tables:
+      - name: zoho_deals
+      - name: zoho_contacts
+      - name: zoho_invoices
+      - name: zoho_payments
+
+# models/marts/_marts__models.yml - explicit dependencies
+models:
+  - name: fct_revenue
+    config:
+      depends_on:
+        - ref('stg_zoho__invoices')
+        - ref('stg_zoho__accounts')
+```
+
+## Failure Handling and Replay
+
+```python
+# Replay failed loads from GCS archive
+def replay_failed_batch(batch_id: str, source_table: str):
+    """Re-process a specific failed batch from GCS landing zone."""
+    from google.cloud import bigquery, storage
+
+    client = bigquery.Client()
+    storage_client = storage.Client()
+
+    # Find files for the failed batch
+    bucket = storage_client.bucket('cloudstream-landing')
+    blobs = bucket.list_blobs(prefix=f"archive/{source_table}/{batch_id}/")
+
+    for blob in blobs:
+        uri = f"gs://cloudstream-landing/{blob.name}"
+        job_config = bigquery.LoadJobConfig(
+            source_format=bigquery.SourceFormat.NEWLINE_DELIMITED_JSON,
+            write_disposition=bigquery.WriteDisposition.WRITE_APPEND,
+        )
+
+        job = client.load_table_from_uri(
+            uri, f"cloudstream-prod.bronze.{source_table}", job_config=job_config
+        )
+        job.result()
+        print(f"Replayed: {uri} → {job.output_rows} rows")
+```
+
+## Monitoring and Alerts
+
+```yaml
+# Cloud Monitoring alert policies
+# Alert if silver layer is stale (no updates in 6+ hours)
+alertPolicies:
+  - displayName: "Silver Layer Staleness"
+    conditions:
+      - conditionThreshold:
+          filter: 'resource.type="bigquery_table" AND metric.type="bigquery.googleapis.com/storage/last_modified_time"'
+          comparison: COMPARISON_GT
+          thresholdValue: 21600  # 6 hours in seconds
+    notificationChannels:
+      - "projects/cloudstream-prod/notificationChannels/slack-data-alerts"
+
+  - displayName: "High Quarantine Rate"
+    conditions:
+      - conditionThreshold:
+          filter: 'metric.type="custom.googleapis.com/silver/quarantine_rate"'
+          comparison: COMPARISON_GT
+          thresholdValue: 0.05  # >5% quarantine rate
+```
+
+## SLA Tracking
+
+```sql
+-- Track layer transition SLAs
+CREATE TABLE `project.ops._sla_tracking` (
+  pipeline_name STRING,
+  layer_transition STRING,  -- 'bronze_to_silver', 'silver_to_gold'
+  started_at TIMESTAMP,
+  completed_at TIMESTAMP,
+  duration_seconds INT64,
+  sla_seconds INT64,       -- Target SLA
+  sla_met BOOL,
+  record_count INT64
+);
+
+-- SLA dashboard query
+SELECT
+  layer_transition,
+  DATE(started_at) AS run_date,
+  AVG(duration_seconds) AS avg_duration,
+  MAX(duration_seconds) AS max_duration,
+  COUNTIF(sla_met = FALSE) AS sla_violations,
+  COUNT(*) AS total_runs
+FROM `project.ops._sla_tracking`
+WHERE started_at >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 30 DAY)
+GROUP BY 1, 2
+ORDER BY 2 DESC;
+```
+
+## Best Practices
+
+1. **Never skip silver** - Even for "clean" sources, always apply deduplication and typing
+2. **Use dbt `--select` by tag** - Run `tag:silver` and `tag:gold` independently
+3. **Test between layers** - Run `dbt test` after silver before triggering gold
+4. **Archive raw files** - Move GCS files to archive/ after successful bronze load
+5. **Idempotent loads** - Use `unique_key` in incremental models for safe re-runs
+6. **Monitor freshness** - dbt source freshness checks catch upstream delays early
+
+> **WARNING**: Never run gold models if silver tests fail. Use `dbt build --select tag:silver` (runs + tests) before proceeding to gold.