@cloudstreamsoftware/claude-tools 1.0.0 → 1.1.0

package/skills/gcp-data-engineering/medallion-architecture/silver-layer.md

@@ -0,0 +1,305 @@
+# Silver Layer - Cleansed & Conformed Data
+
+> The silver layer transforms raw bronze data into cleansed, deduplicated, and validated records ready for business consumption.
+
+## Architecture Overview
+
+```
+Bronze (raw JSON/CSV) → Silver (cleansed, typed, deduplicated) → Gold (aggregated)
+                              │
+                              ├── Data quality rules applied
+                              ├── Deduplication by record_id + timestamp
+                              ├── Schema enforcement & evolution
+                              ├── SCD Type 2 tracking
+                              └── Error quarantine routing
+```
+
+## Deduplication Strategy
+
+CloudStream deduplicates using `record_id + modified_time` composite key from Zoho sources.
+
+```sql
+-- Deduplication: keep latest record per entity
+CREATE OR REPLACE TABLE `project.silver.zoho_contacts` AS
+WITH ranked AS (
+  SELECT
+    *,
+    ROW_NUMBER() OVER (
+      PARTITION BY record_id
+      ORDER BY modified_time DESC, _ingestion_timestamp DESC
+    ) AS row_num
+  FROM `project.bronze.zoho_contacts`
+  WHERE _ingestion_date >= DATE_SUB(CURRENT_DATE(), INTERVAL 3 DAY)
+)
+SELECT * EXCEPT(row_num)
+FROM ranked
+WHERE row_num = 1;
+```
+
+### dbt Deduplication Model
+
+```sql
+-- models/staging/stg_zoho__contacts.sql
+{{
+  config(
+    materialized='incremental',
+    unique_key='record_id',
+    partition_by={'field': '_ingestion_date', 'data_type': 'date'},
+    cluster_by=['owner_id', 'account_id']
+  )
+}}
+
+WITH source AS (
+  SELECT * FROM {{ source('bronze', 'zoho_contacts') }}
+  {% if is_incremental() %}
+  WHERE _ingestion_timestamp > (SELECT MAX(_ingestion_timestamp) FROM {{ this }})
+  {% endif %}
+),
+
+deduplicated AS (
+  SELECT
+    *,
+    ROW_NUMBER() OVER (
+      PARTITION BY record_id
+      ORDER BY modified_time DESC
+    ) AS _row_num
+  FROM source
+)
+
+SELECT * EXCEPT(_row_num)
+FROM deduplicated
+WHERE _row_num = 1
+```
+
+## Data Quality Rules
+
+### Null Handling
+
+| Column Type | Strategy | Example |
+|-------------|----------|---------|
+| Required ID | Reject to quarantine | `record_id IS NULL` → quarantine |
+| Email | Coalesce with fallback | `COALESCE(email, 'unknown@placeholder.com')` |
+| Numeric | Default to 0 or NULL | `COALESCE(SAFE_CAST(amount AS FLOAT64), 0.0)` |
+| Date | Flag as suspect | Keep NULL, add `_date_missing = TRUE` |
+| Name | Trim and validate | `NULLIF(TRIM(name), '')` |
+
+### Type Coercion
+
+```sql
+-- Safe type casting with error tracking
+SELECT
+  record_id,
+  SAFE_CAST(amount AS FLOAT64) AS amount,
+  SAFE.PARSE_DATE('%Y-%m-%d', date_field) AS parsed_date,
+  SAFE.PARSE_TIMESTAMP('%Y-%m-%dT%H:%M:%S%Ez', timestamp_field) AS parsed_ts,
+  -- Track coercion failures
+  CASE WHEN SAFE_CAST(amount AS FLOAT64) IS NULL AND amount IS NOT NULL
+       THEN TRUE ELSE FALSE END AS _amount_coercion_failed
+FROM `project.bronze.zoho_invoices`
+```
+
+### Range Validation
+
+```sql
+-- Business rule validations
+SELECT
+  *,
+  CASE
+    WHEN amount < 0 THEN 'NEGATIVE_AMOUNT'
+    WHEN amount > 10000000 THEN 'SUSPICIOUSLY_LARGE'
+    WHEN created_date > CURRENT_TIMESTAMP() THEN 'FUTURE_DATE'
+    WHEN created_date < TIMESTAMP('2015-01-01') THEN 'DATE_TOO_OLD'
+    ELSE NULL
+  END AS _validation_flag
+FROM deduplicated_source
+```
+
+## Schema Evolution Handling
+
+> **WARNING**: Schema changes from Zoho can break pipelines. Always use SAFE_CAST and handle missing columns gracefully.
+
+```sql
+-- Handle schema evolution with column existence checks (dbt macro)
+{% macro safe_column(column_name, default_value='NULL', cast_type=none) %}
+  {% if column_name in adapter.get_columns_in_relation(source('bronze', model_name)) | map(attribute='name') %}
+    {% if cast_type %}
+      SAFE_CAST({{ column_name }} AS {{ cast_type }})
+    {% else %}
+      {{ column_name }}
+    {% endif %}
+  {% else %}
+    {{ default_value }} AS {{ column_name }}
+  {% endif %}
+{% endmacro %}
+```
+
+## SCD Type 2 Implementation
+
+```sql
+-- models/intermediate/int_zoho__contacts_scd2.sql
+{{
+  config(
+    materialized='incremental',
+    unique_key='surrogate_key',
+    partition_by={'field': 'effective_from', 'data_type': 'timestamp', 'granularity': 'day'}
+  )
+}}
+
+WITH current_records AS (
+  SELECT * FROM {{ ref('stg_zoho__contacts') }}
+),
+
+{% if is_incremental() %}
+existing AS (
+  SELECT * FROM {{ this }}
+  WHERE is_current = TRUE
+),
+
+changes AS (
+  SELECT
+    c.*,
+    e.surrogate_key AS existing_key
+  FROM current_records c
+  LEFT JOIN existing e ON c.record_id = e.record_id
+  WHERE e.record_id IS NULL  -- new record
+     OR MD5(CONCAT(
+          COALESCE(c.email,''), '|',
+          COALESCE(c.phone,''), '|',
+          COALESCE(c.owner_id,''), '|',
+          COALESCE(CAST(c.account_id AS STRING),'')
+        )) != e.hash_diff  -- changed record
+)
+{% endif %}
+
+SELECT
+  {{ dbt_utils.generate_surrogate_key(['record_id', 'modified_time']) }} AS surrogate_key,
+  record_id,
+  email, phone, owner_id, account_id,
+  MD5(CONCAT(
+    COALESCE(email,''), '|', COALESCE(phone,''), '|',
+    COALESCE(owner_id,''), '|', COALESCE(CAST(account_id AS STRING),'')
+  )) AS hash_diff,
+  modified_time AS effective_from,
+  CAST(NULL AS TIMESTAMP) AS effective_to,
+  TRUE AS is_current,
+  CURRENT_TIMESTAMP() AS _loaded_at
+FROM {% if is_incremental() %} changes {% else %} current_records {% endif %}
+```
+
+## Incremental Processing
+
+```yaml
+# dbt_project.yml - Silver layer incremental config
+models:
+  cloudstream:
+    staging:
+      +materialized: incremental
+      +incremental_strategy: merge
+      +on_schema_change: append_new_columns
+      +partition_by:
+        field: _ingestion_date
+        data_type: date
+        granularity: day
+```
+
+## Partitioning Strategy
+
+All silver tables partition by `_ingestion_date` for efficient incremental loads:
+
+```sql
+CREATE TABLE `project.silver.zoho_deals` (
+  record_id STRING NOT NULL,
+  deal_name STRING,
+  amount FLOAT64,
+  stage STRING,
+  close_date DATE,
+  owner_id STRING,
+  _ingestion_date DATE NOT NULL,
+  _ingestion_timestamp TIMESTAMP NOT NULL,
+  _source_system STRING DEFAULT 'zoho_crm',
+  _batch_id STRING
+)
+PARTITION BY _ingestion_date
+CLUSTER BY owner_id, stage
+OPTIONS (
+  require_partition_filter = TRUE,
+  partition_expiration_days = 730  -- 2 year retention at silver
+);
+```
+
+## Error Quarantine Tables
+
+Records failing quality checks route to quarantine for investigation:
+
+```sql
+CREATE TABLE `project.silver._quarantine` (
+  source_table STRING,
+  record_id STRING,
+  error_type STRING,   -- 'NULL_KEY', 'TYPE_COERCION', 'RANGE_VIOLATION', 'DUPLICATE'
+  error_detail STRING,
+  raw_record STRING,   -- JSON serialized original
+  quarantined_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP(),
+  resolved_at TIMESTAMP,
+  resolution STRING    -- 'FIXED', 'IGNORED', 'SOURCE_CORRECTED'
+)
+PARTITION BY DATE(quarantined_at);
+
+-- Route bad records
+INSERT INTO `project.silver._quarantine`
+SELECT
+  'zoho_invoices' AS source_table,
+  record_id,
+  'NULL_KEY' AS error_type,
+  'record_id is NULL' AS error_detail,
+  TO_JSON_STRING(t) AS raw_record,
+  CURRENT_TIMESTAMP(),
+  NULL, NULL
+FROM `project.bronze.zoho_invoices` t
+WHERE record_id IS NULL;
+```
+
+## Quality Metrics Tracking
+
+```sql
+-- Track quality metrics per load
+CREATE TABLE `project.silver._quality_metrics` (
+  table_name STRING,
+  metric_date DATE,
+  batch_id STRING,
+  total_records INT64,
+  valid_records INT64,
+  quarantined_records INT64,
+  duplicate_records INT64,
+  null_key_count INT64,
+  type_coercion_failures INT64,
+  processing_time_seconds FLOAT64,
+  created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP()
+)
+PARTITION BY metric_date;
+
+-- Example quality check query
+SELECT
+  table_name,
+  metric_date,
+  ROUND(valid_records / NULLIF(total_records, 0) * 100, 2) AS quality_percentage,
+  quarantined_records,
+  CASE
+    WHEN valid_records / NULLIF(total_records, 0) < 0.95 THEN 'ALERT'
+    WHEN valid_records / NULLIF(total_records, 0) < 0.99 THEN 'WARNING'
+    ELSE 'HEALTHY'
+  END AS health_status
+FROM `project.silver._quality_metrics`
+WHERE metric_date >= DATE_SUB(CURRENT_DATE(), INTERVAL 7 DAY)
+ORDER BY metric_date DESC;
+```
+
+## Best Practices
+
+1. **Always partition silver tables by `_ingestion_date`** - enables efficient incremental loads
+2. **Use `SAFE_CAST` everywhere** - never let a type error crash a pipeline
+3. **Quarantine, do not drop** - bad records may contain fixable issues
+4. **Track quality over time** - degrading quality signals source system problems
+5. **Cluster by frequently-filtered columns** - owner_id, account_id, stage
+6. **Set `require_partition_filter = TRUE`** - prevents accidental full scans
+
+> **COST WARNING**: Silver tables with `require_partition_filter = FALSE` can cause expensive full-table scans. Always enforce partition filters in production.