@cloudstreamsoftware/claude-tools 1.0.0 → 1.2.0

package/skills/gcp-data-engineering/dbt/testing-patterns.md

@@ -0,0 +1,503 @@
+# dbt Testing Patterns
+
+> Comprehensive testing strategies for CloudStream's dbt project, covering schema tests, data tests, source freshness, CI/CD integration, and custom macros.
+
+## Schema Tests (Built-In)
+
+### Core Tests in YAML
+
+```yaml
+# models/staging/_stg_zoho__models.yml
+version: 2
+
+models:
+  - name: stg_zoho__deals
+    description: "Cleansed Zoho CRM deals, deduplicated by record_id"
+    columns:
+      - name: record_id
+        description: "Primary key from Zoho"
+        tests:
+          - not_null
+          - unique
+
+      - name: stage
+        description: "Deal pipeline stage"
+        tests:
+          - not_null
+          - accepted_values:
+              values: [
+                'Qualification', 'Needs Analysis', 'Proposal',
+                'Negotiation', 'Closed Won', 'Closed Lost', 'Unknown'
+              ]
+              config:
+                severity: warn  # New stages from Zoho should warn, not fail
+
+      - name: amount
+        description: "Deal value in USD"
+        tests:
+          - not_null:
+              config:
+                where: "stage = 'Closed Won'"  # Only required for won deals
+
+      - name: owner_id
+        description: "Sales rep who owns the deal"
+        tests:
+          - not_null
+          - relationships:
+              to: ref('stg_zoho__users')
+              field: user_id
+              config:
+                severity: warn  # Orphaned owners may exist during sync
+
+      - name: account_id
+        description: "Related account"
+        tests:
+          - relationships:
+              to: ref('stg_zoho__accounts')
+              field: record_id
+
+  - name: stg_zoho__invoices
+    columns:
+      - name: record_id
+        tests:
+          - not_null
+          - unique
+      - name: invoice_number
+        tests:
+          - not_null
+          - unique
+      - name: total
+        tests:
+          - not_null
+          - dbt_utils.accepted_range:
+              min_value: 0
+              max_value: 10000000
+              config:
+                severity: warn
+```
+
+## Data Tests (Custom SQL Assertions)
+
+```sql
+-- tests/assert_no_future_close_dates.sql
+-- Deals should not have close dates in the future for "Closed Won" stage
+SELECT
+  record_id,
+  deal_name,
+  close_date,
+  stage
+FROM {{ ref('stg_zoho__deals') }}
+WHERE stage = 'Closed Won'
+  AND close_date > CURRENT_DATE()
+
+-- If this returns any rows, the test FAILS
+```
+
+```sql
+-- tests/assert_invoice_total_matches_lines.sql
+-- Invoice total should match sum of line items
+SELECT
+  i.record_id AS invoice_id,
+  i.total AS header_total,
+  SUM(l.amount) AS line_total,
+  ABS(i.total - SUM(l.amount)) AS discrepancy
+FROM {{ ref('stg_zoho__invoices') }} i
+JOIN {{ ref('stg_zoho__invoice_lines') }} l ON i.record_id = l.invoice_id
+GROUP BY 1, 2
+HAVING ABS(i.total - SUM(l.amount)) > 0.01  -- Allow 1 cent rounding
+```
+
+```sql
+-- tests/assert_revenue_not_negative.sql
+-- Monthly revenue should never be negative
+SELECT
+  revenue_month,
+  customer_id,
+  total_revenue
+FROM {{ ref('fct_monthly_revenue') }}
+WHERE total_revenue < 0
+```
+
+## dbt Expectations Package
+
+```yaml
+# packages.yml
+packages:
+  - package: calogica/dbt_expectations
+    version: [">=0.10.0", "<0.11.0"]
+```
+
+```yaml
+# models/staging/_stg_zoho__models.yml
+models:
+  - name: stg_zoho__deals
+    tests:
+      # Table-level expectations
+      - dbt_expectations.expect_table_row_count_to_be_between:
+          min_value: 100  # We should always have at least 100 deals
+          config:
+            severity: error
+
+      - dbt_expectations.expect_table_row_count_to_equal_other_table:
+          compare_model: source('bronze', 'zoho_deals')
+          # Silver should have same or fewer rows than bronze (dedup removes some)
+          row_condition: "1=1"
+          compare_row_condition: "record_id IS NOT NULL"
+          config:
+            severity: warn
+
+    columns:
+      - name: amount
+        tests:
+          - dbt_expectations.expect_column_values_to_be_between:
+              min_value: 0
+              max_value: 50000000
+              row_condition: "stage != 'Closed Lost'"
+
+          - dbt_expectations.expect_column_mean_to_be_between:
+              min_value: 1000
+              max_value: 500000
+              config:
+                severity: warn
+
+      - name: email
+        tests:
+          - dbt_expectations.expect_column_values_to_match_regex:
+              regex: "^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$"
+              row_condition: "email IS NOT NULL"
+              config:
+                severity: warn
+
+      - name: close_date
+        tests:
+          - dbt_expectations.expect_column_values_to_be_between:
+              min_value: "'2015-01-01'"
+              max_value: "CAST(DATE_ADD(CURRENT_DATE(), INTERVAL 365 DAY) AS STRING)"
+```
+
+## Source Freshness Checks
+
+```yaml
+# models/staging/_sources.yml
+sources:
+  - name: bronze
+    database: cloudstream-prod
+    schema: bronze
+    freshness:
+      warn_after: {count: 6, period: hour}
+      error_after: {count: 24, period: hour}
+    loaded_at_field: _ingestion_timestamp
+    tables:
+      - name: zoho_deals
+        freshness:
+          warn_after: {count: 4, period: hour}   # Deals sync every 4 hours
+          error_after: {count: 8, period: hour}
+
+      - name: zoho_invoices
+        freshness:
+          warn_after: {count: 12, period: hour}  # Invoices sync twice daily
+          error_after: {count: 36, period: hour}
+
+      - name: zoho_contacts
+        freshness:
+          warn_after: {count: 24, period: hour}  # Contacts sync daily
+          error_after: {count: 48, period: hour}
+
+      - name: zoho_realtime_events
+        freshness:
+          warn_after: {count: 30, period: minute}  # Real-time should be fresh
+          error_after: {count: 2, period: hour}
+```
+
+```bash
+# Check source freshness
+dbt source freshness --select source:bronze
+
+# Output example:
+# 14:23:01 | WARN   | source bronze.zoho_deals is stale (last loaded 5 hours ago)
+# 14:23:01 | PASS   | source bronze.zoho_invoices is fresh (last loaded 2 hours ago)
+```
+
+## Test Severity Levels
+
+```yaml
+# Severity configuration matrix
+# ERROR: Pipeline stops, must fix before gold layer runs
+# WARN: Pipeline continues, alert sent to Slack
+
+models:
+  - name: stg_zoho__deals
+    columns:
+      - name: record_id
+        tests:
+          - not_null:
+              config:
+                severity: error    # HARD STOP: Cannot have null PKs
+          - unique:
+              config:
+                severity: error    # HARD STOP: Dedup failed
+
+      - name: stage
+        tests:
+          - accepted_values:
+              values: ['Qualification', 'Needs Analysis', 'Proposal',
+                       'Negotiation', 'Closed Won', 'Closed Lost', 'Unknown']
+              config:
+                severity: warn     # New values from Zoho are expected occasionally
+
+      - name: owner_id
+        tests:
+          - relationships:
+              to: ref('stg_zoho__users')
+              field: user_id
+              config:
+                severity: warn     # Orphaned refs possible during sync windows
+                error_if: ">100"   # But more than 100 orphans is suspicious
+                warn_if: ">10"
+```
+
+## Custom Test Macros
+
+```sql
+-- macros/tests/test_freshness_within_hours.sql
+{% test freshness_within_hours(model, column_name, max_hours=24) %}
+  SELECT
+    MAX({{ column_name }}) AS latest_record,
+    TIMESTAMP_DIFF(
+      CURRENT_TIMESTAMP(),
+      MAX({{ column_name }}),
+      HOUR
+    ) AS hours_since_latest
+  FROM {{ model }}
+  HAVING TIMESTAMP_DIFF(
+    CURRENT_TIMESTAMP(),
+    MAX({{ column_name }}),
+    HOUR
+  ) > {{ max_hours }}
+{% endtest %}
+```
+
+```sql
+-- macros/tests/test_no_orphaned_records.sql
+{% test no_orphaned_records(model, column_name, parent_model, parent_column) %}
+  SELECT
+    child.{{ column_name }}
+  FROM {{ model }} child
+  LEFT JOIN {{ parent_model }} parent
+    ON child.{{ column_name }} = parent.{{ parent_column }}
+  WHERE parent.{{ parent_column }} IS NULL
+    AND child.{{ column_name }} IS NOT NULL
+{% endtest %}
+```
+
+```sql
+-- macros/tests/test_row_count_anomaly.sql
+-- Alert if row count deviates more than 50% from 7-day average
+{% test row_count_anomaly(model, threshold=0.5) %}
+  WITH daily_counts AS (
+    SELECT
+      _ingestion_date,
+      COUNT(*) AS row_count
+    FROM {{ model }}
+    WHERE _ingestion_date >= DATE_SUB(CURRENT_DATE(), INTERVAL 8 DAY)
+    GROUP BY 1
+  ),
+  stats AS (
+    SELECT
+      AVG(row_count) AS avg_count,
+      STDDEV(row_count) AS stddev_count
+    FROM daily_counts
+    WHERE _ingestion_date < CURRENT_DATE()  -- Exclude today
+  )
+  SELECT
+    dc._ingestion_date,
+    dc.row_count,
+    s.avg_count,
+    ABS(dc.row_count - s.avg_count) / NULLIF(s.avg_count, 0) AS deviation_pct
+  FROM daily_counts dc
+  CROSS JOIN stats s
+  WHERE dc._ingestion_date = CURRENT_DATE()
+    AND ABS(dc.row_count - s.avg_count) / NULLIF(s.avg_count, 0) > {{ threshold }}
+{% endtest %}
+```
+
+Usage:
+
+```yaml
+models:
+  - name: stg_zoho__deals
+    tests:
+      - row_count_anomaly:
+          threshold: 0.5
+          config:
+            severity: warn
+    columns:
+      - name: _ingestion_timestamp
+        tests:
+          - freshness_within_hours:
+              max_hours: 8
+```
+
+## CI/CD Test Integration
+
+```yaml
+# .github/workflows/dbt-ci.yml
+name: dbt CI
+
+on:
+  pull_request:
+    paths:
+      - 'dbt/**'
+      - 'models/**'
+      - 'macros/**'
+
+jobs:
+  dbt-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install dbt
+        run: pip install dbt-bigquery==1.7.*
+
+      - name: dbt deps
+        run: dbt deps --target ci
+
+      - name: dbt compile
+        run: dbt compile --target ci
+
+      - name: dbt run (modified models only)
+        run: dbt run --select state:modified+ --target ci --defer --state ./prod-manifest
+        env:
+          DBT_CI_DATASET: ci_pr_${{ github.event.pull_request.number }}
+
+      - name: dbt test (modified models only)
+        run: dbt test --select state:modified+ --target ci --defer --state ./prod-manifest
+
+      - name: Cleanup CI dataset
+        if: always()
+        run: |
+          bq rm -r -f cloudstream-prod:ci_pr_${{ github.event.pull_request.number }}
+
+  dbt-production:
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/main'
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: dbt run + test (full)
+        run: |
+          dbt run --target prod
+          dbt test --target prod --store-failures
+
+      - name: Upload test results
+        run: |
+          dbt source freshness --target prod
+          # Store failures in dedicated schema for investigation
+```
+
+## Test Coverage Reporting
+
+```sql
+-- macros/test_coverage_report.sql
+-- Generate test coverage report
+{% macro test_coverage_report() %}
+  {% set models = graph.nodes.values() | selectattr("resource_type", "equalto", "model") | list %}
+  {% set tests = graph.nodes.values() | selectattr("resource_type", "equalto", "test") | list %}
+
+  {% for model in models %}
+    {% set model_tests = tests | selectattr("depends_on.nodes", "containing", model.unique_id) | list %}
+    {{ log(model.name ~ ": " ~ model_tests | length ~ " tests", info=True) }}
+  {% endfor %}
+{% endmacro %}
+```
+
+```bash
+# Quick coverage check
+dbt ls --resource-type test | wc -l    # Total tests
+dbt ls --resource-type model | wc -l   # Total models
+# Aim for at least 2 tests per model (not_null PK + unique PK minimum)
+```
+
+## Performance Testing for Slow Models
+
+```yaml
+# models/marts/_performance_thresholds.yml
+models:
+  - name: fct_monthly_revenue
+    config:
+      # Track model run time - alert if exceeds threshold
+      meta:
+        max_execution_seconds: 300  # 5 minutes max
+        owner: data-team
+
+  - name: kpi_daily_snapshot
+    config:
+      meta:
+        max_execution_seconds: 120  # 2 minutes max
+        owner: data-team
+```
+
+```python
+# scripts/check_model_performance.py
+"""Post-dbt-run check: alert if any model exceeded its time threshold."""
+import json
+import sys
+
+def check_performance(run_results_path='target/run_results.json'):
+    with open(run_results_path) as f:
+        results = json.load(f)
+
+    slow_models = []
+    for result in results['results']:
+        node = result['unique_id']
+        execution_time = result.get('execution_time', 0)
+        threshold = result.get('node', {}).get('config', {}).get('meta', {}).get(
+            'max_execution_seconds', 600  # Default 10 min
+        )
+
+        if execution_time > threshold:
+            slow_models.append({
+                'model': node,
+                'execution_time': round(execution_time, 1),
+                'threshold': threshold
+            })
+
+    if slow_models:
+        print("PERFORMANCE WARNING - Slow models detected:")
+        for m in slow_models:
+            print(f"  {m['model']}: {m['execution_time']}s (threshold: {m['threshold']}s)")
+        sys.exit(1)
+
+if __name__ == '__main__':
+    check_performance()
+```
+
+## Test Failure Playbook
+
+| Test Type | Severity | Action |
+|-----------|----------|--------|
+| `not_null` on PK | ERROR | Block pipeline. Check source for missing IDs. |
+| `unique` on PK | ERROR | Block pipeline. Deduplication logic is broken. |
+| `accepted_values` | WARN | Notify Slack. Zoho may have new picklist values. |
+| `relationships` | WARN | Notify. Sync timing issue or orphaned records. |
+| `freshness` | WARN/ERROR | Check CData Sync or Catalyst Cron job status. |
+| `row_count_anomaly` | WARN | Investigate source. May be legitimate (holiday, etc). |
+| Custom range test | WARN | Review outlier records. May need quarantine. |
+
+## Best Practices
+
+1. **Every model needs at minimum**: `not_null` + `unique` on primary key
+2. **Use `store_failures`** in production to persist failing rows for investigation
+3. **Run `dbt source freshness`** before model runs to catch upstream delays
+4. **Use `--select state:modified+`** in CI to only test affected models
+5. **Set severity wisely** - too many ERRORs causes alert fatigue; too many WARNs gets ignored
+6. **Custom tests for business rules** - SQL assertions catch domain-specific issues
+7. **Performance test regularly** - Models that slow down signal data growth problems
+
+> **WARNING**: `dbt test --store-failures` creates tables in your target schema for each failing test. Set up cleanup jobs or use a dedicated `dbt_test_failures` schema to avoid clutter.

package/skills/gcp-data-engineering/medallion-architecture/bronze-layer.md

@@ -0,0 +1,60 @@
+# Bronze Layer (Raw Ingestion)
+
+## Purpose
+Capture raw data from source systems with minimal transformation.
+Preserve original data for reprocessing if needed.
+
+## Zoho → GCS Ingestion Patterns
+
+### Scheduled Pull (Catalyst Cron)
+```javascript
+// Catalyst Cron function: Pull Zoho CRM data to GCS
+module.exports = async (cronDetails, context) => {
+  const { Storage } = require('@google-cloud/storage');
+  const storage = new Storage();
+  const bucket = storage.bucket('project-bronze');
+
+  const zohoToken = await refreshZohoToken();
+  const records = await fetchAllCRMRecords('Contacts', zohoToken);
+
+  const filename = `zoho-crm/contacts/${new Date().toISOString()}.json`;
+  const file = bucket.file(filename);
+  await file.save(JSON.stringify(records), {
+    contentType: 'application/json',
+    metadata: {
+      source: 'zoho-crm',
+      module: 'Contacts',
+      ingestion_time: new Date().toISOString(),
+      record_count: String(records.length)
+    }
+  });
+
+  console.log(`Ingested ${records.length} contacts to ${filename}`);
+  context.close();
+};
+```
+
+### Event-Driven (Webhook → Cloud Function)
+```javascript
+// Cloud Function: Receive Zoho webhook, write to GCS
+exports.zohoWebhook = async (req, res) => {
+  // Validate webhook signature
+  if (!validateZohoSignature(req)) {
+    return res.status(401).send('Invalid signature');
+  }
+
+  const event = req.body;
+  const bucket = storage.bucket('project-bronze');
+  const filename = `zoho-events/${event.module}/${Date.now()}.json`;
+
+  await bucket.file(filename).save(JSON.stringify(event));
+  res.status(200).send('OK');
+};
+```
+
+## Bronze Layer Rules
+- Never modify source data
+- Add metadata: source, timestamp, batch_id
+- Partition by ingestion date
+- Set retention policy (90 days typical for raw)
+- Schema-on-read (don't enforce schema at ingestion)