@cloudstreamsoftware/claude-tools 1.0.0 → 1.2.0

package/skills/zoho-patterns/analytics/zoho-to-bigquery-pipeline.md

@@ -0,0 +1,427 @@
+# Zoho to BigQuery Data Pipeline
+
+## Pipeline Architecture Options
+
+| Method | Complexity | Cost | Latency | Best For |
+|--------|-----------|------|---------|----------|
+| CData Sync | Low | $$$ | 15 min+ | Quick setup, many tables |
+| Catalyst Cron API Pull | Medium | $ | Configurable | Custom logic, transformations |
+| Zoho Analytics Export API | Medium | $ | Hours | Analytics-ready data |
+| Zoho Flow + Cloud Function | Low-Med | $$ | Near real-time | Event-driven sync |
+| Custom ETL (Dataflow/dbt) | High | $$ | Configurable | Complex transformations |
+
+## Method 1: CData Sync (Managed ETL)
+
+### Setup
+
+```
+1. CData Sync Console > Add Connection > Zoho CRM
+2. Enter OAuth credentials (Client ID, Client Secret, Refresh Token)
+3. Add Destination > Google BigQuery
+4. Enter GCP project ID, dataset name, service account key
+5. Select tables/modules to sync
+6. Configure schedule (every 15 min, hourly, daily)
+```
+
+### Pros/Cons
+
+```
++ No code required
++ Handles schema changes automatically
++ Built-in incremental sync
++ Handles pagination and rate limits
+- Expensive for large datasets ($$$)
+- Limited transformation during transit
+- Vendor lock-in
+- May not support all Zoho modules
+```
+
+## Method 2: Catalyst Cron API Pull (Recommended)
+
+### Architecture
+
+```
+Catalyst Cron (every 2 hours)
+    ↓ Fetch records from Zoho API (paginated)
+    ↓ Transform to BigQuery schema
+    ↓ Write to BigQuery via API
+    ↓ Update sync state (last_sync_timestamp)
+```
+
+### Implementation
+
+```javascript
+const catalyst = require("zcatalyst-sdk-node");
+const { BigQuery } = require("@google-cloud/bigquery");
+const axios = require("axios");
+
+module.exports = async (cronDetails, context) => {
+    const app = catalyst.initialize(context);
+    const bigquery = new BigQuery({
+        projectId: process.env.GCP_PROJECT_ID,
+        credentials: JSON.parse(process.env.GCP_SERVICE_ACCOUNT_KEY)
+    });
+
+    const startTime = Date.now();
+    const MAX_RUNTIME = 14 * 60 * 1000; // 14 minutes
+
+    try {
+        // Get last sync timestamp
+        const lastSync = await getLastSyncTime(app, "crm_deals");
+
+        // Fetch modified records from Zoho CRM
+        const records = await fetchModifiedRecords(
+            "Deals",
+            lastSync,
+            process.env.ZOHO_ACCESS_TOKEN
+        );
+
+        if (records.length === 0) {
+            console.log("No new records to sync");
+            context.close();
+            return;
+        }
+
+        // Transform to BigQuery schema
+        const bqRows = records.map(transformDealToBQ);
+
+        // Write to BigQuery
+        const dataset = bigquery.dataset(process.env.BQ_DATASET);
+        const table = dataset.table("crm_deals");
+
+        // Use streaming insert for incremental
+        await table.insert(bqRows, {
+            skipInvalidRows: true,
+            ignoreUnknownValues: true
+        });
+
+        // Update sync state
+        await updateSyncTime(app, "crm_deals", new Date().toISOString());
+
+        console.log(`Synced ${bqRows.length} deals to BigQuery`);
+        context.close();
+
+    } catch (error) {
+        console.error("Pipeline failed:", error);
+        await notifyPipelineFailure(app, error);
+        context.close();
+    }
+};
+
+// Fetch records modified since last sync
+async function fetchModifiedRecords(module, since, accessToken) {
+    let allRecords = [];
+    let page = 1;
+    let hasMore = true;
+
+    while (hasMore) {
+        const response = await axios.get(
+            `https://www.zohoapis.com/crm/v5/${module}`,
+            {
+                headers: { "Authorization": `Zoho-oauthtoken ${accessToken}` },
+                params: {
+                    modified_since: since,
+                    page: page,
+                    per_page: 200, // Max per page
+                    fields: "Deal_Name,Amount,Stage,Closing_Date,Created_Time,Modified_Time,Owner"
+                }
+            }
+        );
+
+        if (response.data && response.data.data) {
+            allRecords = allRecords.concat(response.data.data);
+            hasMore = response.data.info.more_records;
+            page++;
+        } else {
+            hasMore = false;
+        }
+    }
+
+    return allRecords;
+}
+```
+
+### Data Transformation
+
+```javascript
+// Transform Zoho CRM record to BigQuery schema
+function transformDealToBQ(crmDeal) {
+    return {
+        deal_id: crmDeal.id,
+        deal_name: crmDeal.Deal_Name || null,
+        amount: crmDeal.Amount ? parseFloat(crmDeal.Amount) : null,
+        stage: crmDeal.Stage || null,
+        closing_date: crmDeal.Closing_Date || null,
+        owner_name: crmDeal.Owner ? crmDeal.Owner.name : null,
+        owner_email: crmDeal.Owner ? crmDeal.Owner.email : null,
+        created_at: crmDeal.Created_Time || null,
+        modified_at: crmDeal.Modified_Time || null,
+        // Metadata
+        _synced_at: new Date().toISOString(),
+        _source: "zoho_crm"
+    };
+}
+```
+
+### BigQuery Schema
+
+```json
+[
+    {"name": "deal_id", "type": "STRING", "mode": "REQUIRED"},
+    {"name": "deal_name", "type": "STRING", "mode": "NULLABLE"},
+    {"name": "amount", "type": "FLOAT", "mode": "NULLABLE"},
+    {"name": "stage", "type": "STRING", "mode": "NULLABLE"},
+    {"name": "closing_date", "type": "DATE", "mode": "NULLABLE"},
+    {"name": "owner_name", "type": "STRING", "mode": "NULLABLE"},
+    {"name": "owner_email", "type": "STRING", "mode": "NULLABLE"},
+    {"name": "created_at", "type": "TIMESTAMP", "mode": "NULLABLE"},
+    {"name": "modified_at", "type": "TIMESTAMP", "mode": "NULLABLE"},
+    {"name": "_synced_at", "type": "TIMESTAMP", "mode": "REQUIRED"},
+    {"name": "_source", "type": "STRING", "mode": "REQUIRED"}
+]
+```
+
+## Method 3: Zoho Analytics Export API
+
+```javascript
+// Export data from Zoho Analytics to BigQuery
+async function exportFromAnalytics(workspaceId, viewId) {
+    // Step 1: Request export
+    const exportResponse = await axios.post(
+        `https://analyticsapi.zoho.com/restapi/v2/workspaces/${workspaceId}/views/${viewId}/data`,
+        null,
+        {
+            headers: { "Authorization": `Zoho-oauthtoken ${accessToken}` },
+            params: {
+                responseFormat: "json",
+                limit: 10000
+            }
+        }
+    );
+
+    const data = exportResponse.data.data;
+
+    // Step 2: Transform and load to BigQuery
+    const bqRows = data.map(row => transformAnalyticsRow(row));
+    await loadToBigQuery("analytics_export", bqRows);
+
+    return bqRows.length;
+}
+```
+
+## Incremental vs Full Loads
+
+### Decision Matrix
+
+| Factor | Incremental | Full Load |
+|--------|-------------|-----------|
+| Data volume | Large (>10K records) | Small (<10K records) |
+| Update frequency | Frequent | Infrequent |
+| Deleted records | Misses deletes | Catches deletes |
+| Data consistency | Eventually consistent | Fully consistent |
+| API quota usage | Low | High |
+| BigQuery cost | Low (streaming insert) | Higher (table replace) |
+| Complexity | Higher (state tracking) | Simple |
+
+### Incremental Load Pattern
+
+```javascript
+// Only fetch records modified since last sync
+async function incrementalSync(app, module, tableName) {
+    const lastSync = await getLastSyncTime(app, tableName);
+
+    const records = await fetchModifiedRecords(module, lastSync);
+    if (records.length === 0) return 0;
+
+    // Upsert pattern: Use MERGE in BigQuery
+    const tempTable = `${tableName}_staging`;
+    await loadToTempTable(records, tempTable);
+
+    // Merge staging into main table
+    const mergeQuery = `
+        MERGE \`${process.env.GCP_PROJECT_ID}.${process.env.BQ_DATASET}.${tableName}\` T
+        USING \`${process.env.GCP_PROJECT_ID}.${process.env.BQ_DATASET}.${tempTable}\` S
+        ON T.deal_id = S.deal_id
+        WHEN MATCHED THEN
+            UPDATE SET
+                deal_name = S.deal_name,
+                amount = S.amount,
+                stage = S.stage,
+                modified_at = S.modified_at,
+                _synced_at = S._synced_at
+        WHEN NOT MATCHED THEN
+            INSERT ROW
+    `;
+
+    await bigquery.query({ query: mergeQuery });
+    await updateSyncTime(app, tableName, new Date().toISOString());
+
+    return records.length;
+}
+```
+
+### Full Load Pattern
+
+```javascript
+// Replace entire table (for small datasets or periodic reconciliation)
+async function fullSync(module, tableName) {
+    let allRecords = [];
+    let page = 1;
+    let hasMore = true;
+
+    // Fetch ALL records (paginated)
+    while (hasMore) {
+        const batch = await fetchPage(module, page, 200);
+        allRecords = allRecords.concat(batch.data);
+        hasMore = batch.hasMore;
+        page++;
+    }
+
+    // Transform
+    const bqRows = allRecords.map(r => transformToBQ(r));
+
+    // Load to temp table, then swap
+    const tempTable = `${tableName}_full_load_${Date.now()}`;
+    await loadToTable(bqRows, tempTable);
+
+    // Atomic swap
+    await bigquery.query({
+        query: `
+            CREATE OR REPLACE TABLE \`${dataset}.${tableName}\`
+            AS SELECT * FROM \`${dataset}.${tempTable}\`
+        `
+    });
+
+    // Cleanup temp
+    await bigquery.dataset(dataset).table(tempTable).delete();
+
+    return allRecords.length;
+}
+```
+
+## Scheduling
+
+### Recommended Schedule
+
+| Data Type | Frequency | Method | Reason |
+|-----------|-----------|--------|--------|
+| CRM Deals | Every 2 hours | Incremental | Matches CRM-Books sync cycle |
+| Invoices | Every 4 hours | Incremental | Less volatile |
+| Contacts | Daily | Full load | Catch deletes |
+| Products | Daily | Full load | Small dataset |
+| Activities | Every 1 hour | Incremental | Time-sensitive |
+| Analytics aggregates | Daily | Full load | Pre-computed |
+
+### Catalyst Cron Schedule
+
+```json
+{
+    "crons": [
+        {
+            "function_name": "sync_crm_deals",
+            "cron_expression": "0 */2 * * *",
+            "description": "Sync CRM deals to BigQuery every 2 hours"
+        },
+        {
+            "function_name": "sync_invoices",
+            "cron_expression": "0 */4 * * *",
+            "description": "Sync invoices to BigQuery every 4 hours"
+        },
+        {
+            "function_name": "full_sync_contacts",
+            "cron_expression": "0 2 * * *",
+            "description": "Full sync contacts daily at 2 AM"
+        }
+    ]
+}
+```
+
+## Monitoring
+
+### Pipeline Health Dashboard (BigQuery SQL)
+
+```sql
+-- Monitor sync freshness
+SELECT
+    _source,
+    MAX(_synced_at) AS last_sync,
+    TIMESTAMP_DIFF(CURRENT_TIMESTAMP(), MAX(_synced_at), HOUR) AS hours_since_sync,
+    COUNT(*) AS total_records,
+    COUNTIF(_synced_at > TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 24 HOUR)) AS synced_today
+FROM `project.dataset.crm_deals`
+GROUP BY _source
+
+-- Alert if sync is stale (> 4 hours)
+-- Set up BigQuery scheduled query + email alert
+```
+
+### Error Notification
+
+```javascript
+async function notifyPipelineFailure(app, error) {
+    // Log to Catalyst Data Store
+    await app.datastore().table("PipelineErrors").insertRow({
+        pipeline_name: "zoho_to_bigquery",
+        error_message: error.message,
+        error_stack: error.stack,
+        timestamp: new Date().toISOString()
+    });
+
+    // Email alert
+    await app.email().sendMail({
+        from_email: "noreply@catalyst-project.com",
+        to_email: ["data-team@company.com"],
+        subject: "[ALERT] Zoho→BigQuery Pipeline Failure",
+        content: `Pipeline failed at ${new Date().toISOString()}\n\nError: ${error.message}\n\nStack: ${error.stack}`
+    });
+}
+```
+
+## Cost Optimization
+
+| Optimization | Savings | Trade-off |
+|-------------|---------|-----------|
+| Incremental sync | 80-90% less API calls | May miss deletes |
+| Batch insert (not streaming) | No streaming costs | Higher latency (minutes) |
+| Partitioned tables | Reduced query costs | Slightly complex schema |
+| Columnar selection | Less data transferred | Must specify fields |
+| Schedule during off-peak | Lower Zoho API contention | Delayed data |
+
+### BigQuery Table Partitioning
+
+```sql
+-- Create partitioned table for cost-effective queries
+CREATE TABLE `project.dataset.crm_deals` (
+    deal_id STRING NOT NULL,
+    deal_name STRING,
+    amount FLOAT64,
+    stage STRING,
+    closing_date DATE,
+    modified_at TIMESTAMP,
+    _synced_at TIMESTAMP NOT NULL
+)
+PARTITION BY DATE(_synced_at)
+CLUSTER BY stage, closing_date;
+
+-- Queries that filter by _synced_at only scan relevant partitions
+-- Queries that also filter by stage/closing_date benefit from clustering
+```
+
+## Medallion Architecture Mapping
+
+```
+Bronze (Raw):
+  zoho_raw.crm_deals        ← Direct API response, minimal transformation
+  zoho_raw.books_invoices   ← Raw JSON stored as-is
+
+Silver (Cleaned):
+  zoho_clean.deals          ← Deduplicated, typed, null-handled
+  zoho_clean.invoices       ← Standardized amounts, dates
+
+Gold (Business):
+  analytics.revenue_monthly ← Aggregated metrics
+  analytics.customer_ltv    ← Calculated business KPIs
+  analytics.pipeline_health ← Dashboard-ready views
+```
+
+> **NOTE:** Use dbt for Silver→Gold transformations. Store Bronze in BigQuery with the raw sync pipeline described above.