@cloudstreamsoftware/claude-tools 1.0.0 → 1.2.0

package/skills/bigquery-patterns/query-patterns.md

@@ -0,0 +1,372 @@
+# BigQuery Query Patterns
+
+## Common Table Expressions (CTEs)
+
+### Chaining Transformations
+
+```sql
+-- Multi-step transformation using CTEs
+WITH raw_orders AS (
+  SELECT *
+  FROM `project.bronze.zoho_orders`
+  WHERE DATE(ingestion_timestamp) = CURRENT_DATE()
+),
+cleaned_orders AS (
+  SELECT
+    CAST(JSON_VALUE(raw_payload, '$.id') AS STRING) AS order_id,
+    CAST(JSON_VALUE(raw_payload, '$.customer_id') AS STRING) AS customer_id,
+    CAST(JSON_VALUE(raw_payload, '$.amount') AS NUMERIC) AS amount,
+    PARSE_TIMESTAMP('%Y-%m-%dT%H:%M:%S', JSON_VALUE(raw_payload, '$.created_time')) AS created_at,
+    JSON_VALUE(raw_payload, '$.status') AS status
+  FROM raw_orders
+  WHERE JSON_VALUE(raw_payload, '$.id') IS NOT NULL
+),
+enriched_orders AS (
+  SELECT
+    o.*,
+    c.company_name,
+    c.segment,
+    c.account_manager
+  FROM cleaned_orders o
+  LEFT JOIN `project.silver.customers` c ON o.customer_id = c.customer_id
+)
+SELECT * FROM enriched_orders
+WHERE status = 'completed';
+```
+
+---
+
+## Window Functions
+
+### ROW_NUMBER (Deduplication)
+
+```sql
+-- Remove duplicates keeping latest version
+WITH ranked AS (
+  SELECT *,
+    ROW_NUMBER() OVER (
+      PARTITION BY record_id
+      ORDER BY ingestion_timestamp DESC
+    ) AS rn
+  FROM `project.bronze.raw_records`
+)
+SELECT * EXCEPT(rn)
+FROM ranked
+WHERE rn = 1;
+```
+
+### Running Totals and Moving Averages
+
+```sql
+SELECT
+  customer_id,
+  order_date,
+  order_total,
+  -- Running total
+  SUM(order_total) OVER (
+    PARTITION BY customer_id
+    ORDER BY order_date
+    ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
+  ) AS running_total,
+  -- 7-day moving average
+  AVG(order_total) OVER (
+    PARTITION BY customer_id
+    ORDER BY order_date
+    ROWS BETWEEN 6 PRECEDING AND CURRENT ROW
+  ) AS rolling_7day_avg,
+  -- Rank within customer
+  RANK() OVER (
+    PARTITION BY customer_id
+    ORDER BY order_total DESC
+  ) AS order_rank
+FROM `project.silver.orders`;
+```
+
+### LAG/LEAD (Period-over-Period)
+
+```sql
+-- Month-over-month revenue comparison
+WITH monthly_revenue AS (
+  SELECT
+    DATE_TRUNC(order_date, MONTH) AS month,
+    SUM(amount) AS revenue
+  FROM `project.silver.orders`
+  WHERE order_date >= DATE_SUB(CURRENT_DATE(), INTERVAL 12 MONTH)
+  GROUP BY 1
+)
+SELECT
+  month,
+  revenue,
+  LAG(revenue, 1) OVER (ORDER BY month) AS prev_month_revenue,
+  LAG(revenue, 12) OVER (ORDER BY month) AS same_month_last_year,
+  ROUND((revenue - LAG(revenue, 1) OVER (ORDER BY month))
+    / LAG(revenue, 1) OVER (ORDER BY month) * 100, 2) AS mom_growth_pct
+FROM monthly_revenue
+ORDER BY month DESC;
+```
+
+### NTILE (Segmentation)
+
+```sql
+-- Customer value segmentation into quartiles
+SELECT
+  customer_id,
+  total_revenue,
+  NTILE(4) OVER (ORDER BY total_revenue DESC) AS revenue_quartile,
+  CASE NTILE(4) OVER (ORDER BY total_revenue DESC)
+    WHEN 1 THEN 'Platinum'
+    WHEN 2 THEN 'Gold'
+    WHEN 3 THEN 'Silver'
+    WHEN 4 THEN 'Bronze'
+  END AS segment
+FROM `project.gold.customer_lifetime_value`;
+```
+
+---
+
+## UNNEST and Arrays
+
+### Working with Repeated Fields
+
+```sql
+-- Explode array of tags into rows
+SELECT
+  record_id,
+  tag
+FROM `project.silver.deals`,
+UNNEST(SPLIT(tags, ',')) AS tag;
+```
+
+### Aggregating into Arrays
+
+```sql
+-- Collect unique products per customer
+SELECT
+  customer_id,
+  ARRAY_AGG(DISTINCT product_name ORDER BY product_name) AS products_purchased,
+  ARRAY_LENGTH(ARRAY_AGG(DISTINCT product_name)) AS unique_products_count
+FROM `project.silver.order_items`
+GROUP BY customer_id;
+```
+
+---
+
+## STRUCT Handling
+
+### Accessing Nested Fields
+
+```sql
+-- Access struct fields
+SELECT
+  contact_id,
+  address.street,
+  address.city,
+  address.state,
+  address.zip,
+  _quality.is_valid_email,
+  _quality.quality_score
+FROM `project.silver.contacts`
+WHERE _quality.quality_score > 0.8;
+```
+
+### Creating Structs
+
+```sql
+-- Build quality metadata struct
+SELECT
+  customer_id,
+  STRUCT(
+    email IS NOT NULL AND REGEXP_CONTAINS(email, r'^[^@]+@[^@]+\.[^@]+$') AS is_valid_email,
+    phone IS NOT NULL AS has_phone,
+    CASE
+      WHEN email IS NOT NULL AND phone IS NOT NULL THEN 1.0
+      WHEN email IS NOT NULL OR phone IS NOT NULL THEN 0.5
+      ELSE 0.0
+    END AS quality_score
+  ) AS _quality
+FROM `project.bronze.raw_contacts`;
+```
+
+---
+
+## Approximate Aggregations (Cost Optimization)
+
+```sql
+-- Use approximate functions for large datasets
+SELECT
+  DATE(order_date) AS day,
+  APPROX_COUNT_DISTINCT(customer_id) AS approx_unique_customers,
+  APPROX_QUANTILES(order_total, 100)[OFFSET(50)] AS approx_median_order,
+  APPROX_QUANTILES(order_total, 100)[OFFSET(95)] AS approx_p95_order,
+  APPROX_TOP_COUNT(product_category, 5) AS top_5_categories
+FROM `project.silver.orders`
+WHERE order_date >= DATE_SUB(CURRENT_DATE(), INTERVAL 30 DAY)
+GROUP BY 1
+ORDER BY 1 DESC;
+```
+
+---
+
+## Data Quality Checks (Silver Layer)
+
+```sql
+-- Comprehensive quality validation
+WITH quality_results AS (
+  SELECT
+    'Null customer_id' AS check_name,
+    COUNTIF(customer_id IS NULL) AS failures,
+    COUNT(*) AS total
+  FROM `project.silver.orders`
+  UNION ALL
+  SELECT
+    'Negative amounts',
+    COUNTIF(amount < 0),
+    COUNT(*)
+  FROM `project.silver.orders`
+  UNION ALL
+  SELECT
+    'Future dates',
+    COUNTIF(order_date > CURRENT_DATE()),
+    COUNT(*)
+  FROM `project.silver.orders`
+  UNION ALL
+  SELECT
+    'Orphaned records',
+    COUNTIF(c.customer_id IS NULL),
+    COUNT(*)
+  FROM `project.silver.orders` o
+  LEFT JOIN `project.silver.customers` c ON o.customer_id = c.customer_id
+)
+SELECT
+  check_name,
+  failures,
+  total,
+  ROUND(failures / total * 100, 4) AS failure_rate_pct,
+  CASE
+    WHEN failures / total > 0.05 THEN 'FAIL'
+    WHEN failures / total > 0.01 THEN 'WARN'
+    ELSE 'PASS'
+  END AS status
+FROM quality_results;
+```
+
+---
+
+## Query Cost Estimation
+
+```sql
+-- Check bytes scanned BEFORE running expensive queries
+-- Use dry_run = true in BigQuery API or:
+SELECT
+  total_bytes_processed,
+  total_bytes_billed,
+  ROUND(total_bytes_billed / POW(1024, 4) * 5, 2) AS estimated_cost_usd
+FROM `region-us`.INFORMATION_SCHEMA.JOBS_BY_PROJECT
+WHERE creation_time > TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 1 HOUR)
+ORDER BY total_bytes_billed DESC
+LIMIT 10;
+```
+
+### Cost-Saving Patterns
+
+```sql
+-- GOOD: Partition pruning (scans only matching partitions)
+SELECT * FROM `project.silver.orders`
+WHERE order_date BETWEEN '2025-01-01' AND '2025-01-31';
+
+-- BAD: No partition filter (full table scan)
+SELECT * FROM `project.silver.orders`
+WHERE EXTRACT(MONTH FROM order_date) = 1;
+
+-- GOOD: Select specific columns
+SELECT customer_id, order_total, status
+FROM `project.silver.orders`;
+
+-- BAD: SELECT * (scans all columns)
+SELECT * FROM `project.silver.orders`;
+
+-- GOOD: Approximate distinct for dashboards
+SELECT APPROX_COUNT_DISTINCT(customer_id) FROM `project.silver.orders`;
+
+-- BAD: Exact distinct on massive tables
+SELECT COUNT(DISTINCT customer_id) FROM `project.silver.orders`;
+```
+
+---
+
+## Common Anti-Patterns
+
+| Anti-Pattern | Problem | Fix |
+|-------------|---------|-----|
+| `SELECT *` | Scans all columns (cost) | List needed columns |
+| No partition filter | Full table scan | Always filter partition column |
+| `ORDER BY` without `LIMIT` | Sorts entire result set | Add `LIMIT` or remove sort |
+| Correlated subquery | Executes per row (slow) | Use JOIN or window function |
+| `COUNT(DISTINCT x)` on >100M rows | Expensive exact count | Use `APPROX_COUNT_DISTINCT` |
+| Cross join without filter | Cartesian product explosion | Add join condition |
+| Regex on unpartitioned data | Full scan + CPU intensive | Filter first, then regex |
+| Repeated CTEs | Computed multiple times | Materialize to temp table |
+
+---
+
+## Medallion Layer Transitions
+
+### Bronze → Silver
+
+```sql
+-- Incremental silver update: only process new bronze records
+MERGE `project.silver.contacts` AS target
+USING (
+  WITH latest AS (
+    SELECT *,
+      ROW_NUMBER() OVER (PARTITION BY JSON_VALUE(raw_payload, '$.id') ORDER BY ingestion_timestamp DESC) AS rn
+    FROM `project.bronze.zoho_crm_contacts`
+    WHERE ingestion_timestamp > (SELECT MAX(_audit.processed_at) FROM `project.silver.contacts`)
+  )
+  SELECT
+    JSON_VALUE(raw_payload, '$.id') AS contact_id,
+    JSON_VALUE(raw_payload, '$.First_Name') AS first_name,
+    JSON_VALUE(raw_payload, '$.Last_Name') AS last_name,
+    JSON_VALUE(raw_payload, '$.Email') AS email,
+    JSON_VALUE(raw_payload, '$.Account_Name.name') AS company,
+    PARSE_DATE('%Y-%m-%d', JSON_VALUE(raw_payload, '$.Created_Time')) AS created_date,
+    PARSE_DATE('%Y-%m-%d', JSON_VALUE(raw_payload, '$.Modified_Time')) AS modified_date,
+    'zoho-crm' AS source_system
+  FROM latest
+  WHERE rn = 1
+) AS source
+ON target.contact_id = source.contact_id
+WHEN MATCHED THEN UPDATE SET
+  first_name = source.first_name,
+  last_name = source.last_name,
+  email = source.email,
+  company = source.company,
+  modified_date = source.modified_date,
+  _audit = STRUCT(CURRENT_TIMESTAMP() AS processed_at, GENERATE_UUID() AS processing_batch)
+WHEN NOT MATCHED THEN INSERT ROW;
+```
+
+### Silver → Gold
+
+```sql
+-- Customer 360 aggregation
+CREATE OR REPLACE TABLE `project.gold.customer_360` AS
+SELECT
+  c.customer_id,
+  c.first_name || ' ' || c.last_name AS customer_name,
+  COUNT(o.order_id) AS total_orders,
+  SUM(o.amount) AS total_revenue,
+  AVG(o.amount) AS avg_order_value,
+  MIN(o.order_date) AS first_order_date,
+  MAX(o.order_date) AS last_order_date,
+  DATE_DIFF(CURRENT_DATE(), MAX(o.order_date), DAY) AS days_since_last_order,
+  CASE
+    WHEN SUM(o.amount) > 50000 THEN 'high_value'
+    WHEN SUM(o.amount) > 10000 THEN 'medium'
+    ELSE 'low'
+  END AS segment,
+  STRUCT(CURRENT_TIMESTAMP() AS last_updated, 'daily' AS update_frequency) AS _refresh
+FROM `project.silver.customers` c
+LEFT JOIN `project.silver.orders` o ON c.customer_id = o.customer_id
+GROUP BY c.customer_id, c.first_name, c.last_name;
+```

package/skills/bigquery-patterns/schema-design.md

@@ -0,0 +1,78 @@
+# BigQuery Schema Design
+
+## Medallion Layer Schemas
+
+### Bronze Layer (Raw)
+```sql
+CREATE TABLE `project.bronze.zoho_crm_contacts` (
+  ingestion_id STRING NOT NULL,
+  ingestion_timestamp TIMESTAMP NOT NULL,
+  source_system STRING,
+  raw_payload JSON,
+  _metadata STRUCT<
+    source_file STRING,
+    batch_id STRING,
+    ingested_by STRING
+  >
+)
+PARTITION BY DATE(ingestion_timestamp)
+OPTIONS(
+  description='Raw CRM contacts from Zoho API',
+  labels=[('layer', 'bronze'), ('source', 'zoho-crm')]
+);
+```
+
+### Silver Layer (Cleansed)
+```sql
+CREATE TABLE `project.silver.contacts` (
+  contact_id STRING NOT NULL,
+  first_name STRING,
+  last_name STRING,
+  email STRING,
+  company STRING,
+  created_date DATE,
+  modified_date DATE,
+  source_system STRING,
+  _quality STRUCT<
+    is_valid_email BOOL,
+    is_duplicate BOOL,
+    quality_score FLOAT64
+  >,
+  _audit STRUCT<
+    processed_at TIMESTAMP,
+    processing_batch STRING
+  >
+)
+PARTITION BY modified_date
+CLUSTER BY company, source_system;
+```
+
+### Gold Layer (Business-Ready)
+```sql
+CREATE TABLE `project.gold.customer_360` (
+  customer_id STRING NOT NULL,
+  customer_name STRING,
+  total_orders INT64,
+  total_revenue NUMERIC,
+  avg_order_value NUMERIC,
+  first_order_date DATE,
+  last_order_date DATE,
+  segment STRING, -- 'high_value', 'medium', 'low'
+  churn_risk_score FLOAT64,
+  _refresh STRUCT<
+    last_updated TIMESTAMP,
+    update_frequency STRING
+  >
+)
+CLUSTER BY segment;
+```
+
+## Partitioning Strategy
+- Time-series data: Partition by DATE/TIMESTAMP
+- Large tables: Partition by ingestion date if no natural time column
+- Partition expiration: Set for bronze layer (90 days typical)
+
+## Clustering Strategy (max 4 columns)
+- First column: Most frequently filtered
+- Order: Most selective to least selective
+- Re-cluster after significant data changes

package/skills/cloudstream-project-template/SKILL.md

@@ -0,0 +1,20 @@
+---
+name: cloudstream-project-template
+description: CloudStream project structure template for new client projects. Covers Zoho + GCP project initialization, compliance mode selection, and standard directory layout.
+version: 1.0.0
+status: active
+introduced: 1.0.0
+lastUpdated: 2026-01-25
+activation: New project setup, client onboarding, project structure questions
+---
+
+# CloudStream Project Template
+
+Standard project structure for CloudStream client engagements.
+
+## Project Types
+1. **Zoho Creator App**: Forms, reports, workflows, widgets
+2. **Catalyst Full-Stack**: AppSail web apps + serverless functions
+3. **Widget Standalone**: React/Next.js app for Zoho or independent use
+4. **GCP Data Pipeline**: BigQuery + dbt medallion architecture
+5. **Integration Project**: Connecting multiple Zoho apps or external systems

package/skills/cloudstream-project-template/structure.md

@@ -0,0 +1,65 @@
+# Standard Project Structure
+
+## Zoho Creator + Catalyst Project
+```
+project-root/
+├── .claude/
+│   └── CLAUDE.md              # Client-specific config + compliance mode
+├── .env.example               # Required environment variables
+├── catalyst/
+│   ├── catalyst.json          # Catalyst project config
+│   ├── functions/
+│   │   ├── proxy/             # CORS proxy for widget API calls
+│   │   ├── webhooks/          # Webhook receivers
+│   │   └── batch/             # Cron functions for batch processing
+│   └── appsail/               # Full web app if needed
+│       ├── Dockerfile
+│       ├── package.json
+│       └── src/
+├── widgets/
+│   ├── dashboard/             # Each widget is a React app
+│   │   ├── package.json
+│   │   ├── src/
+│   │   └── dist/              # Built widget for upload
+│   └── data-entry/
+├── deluge/
+│   ├── workflows/             # Creator workflow scripts
+│   ├── functions/             # Standalone functions
+│   └── scheduled/             # Scheduled tasks
+├── docs/
+│   ├── architecture.md
+│   ├── api-reference.md
+│   └── runbook.md
+└── tests/
+    ├── widgets/               # Widget unit tests
+    └── catalyst/              # Catalyst function tests
+```
+
+## GCP Data Pipeline Project
+```
+project-root/
+├── .claude/
+│   └── CLAUDE.md
+├── dbt/
+│   ├── dbt_project.yml
+│   ├── models/
+│   │   ├── staging/           # Bronze → Silver
+│   │   ├── intermediate/      # Silver transformations
+│   │   └── marts/             # Gold layer
+│   ├── tests/
+│   └── macros/
+├── dataflow/
+│   ├── pipelines/
+│   └── templates/
+├── cloud-functions/
+│   ├── ingestion/             # Bronze layer loaders
+│   └── triggers/              # Event-driven processing
+├── bigquery/
+│   ├── schemas/               # Table definitions
+│   ├── views/                 # Materialized views
+│   └── procedures/            # Stored procedures
+└── terraform/                 # Infrastructure as code
+    ├── main.tf
+    ├── variables.tf
+    └── outputs.tf
+```