@cloudstreamsoftware/claude-tools 1.0.0 → 1.2.0

package/skills/zoho-patterns/catalyst/cron-batch-processing.md

@@ -0,0 +1,374 @@
+# Catalyst Cron Batch Processing
+
+## Critical Constraints
+
+> **TIMEOUT:** Cron functions have a 15-minute execution limit. Plan batch operations to complete within this window.
+
+> **MANDATORY:** Every Cron function MUST call `context.close()` in ALL exit paths. See context-close-patterns.md for details.
+
+## Cron Function Structure
+
+```javascript
+const catalyst = require("zcatalyst-sdk-node");
+
+module.exports = async (cronDetails, context) => {
+    const app = catalyst.initialize(context);
+    const startTime = Date.now();
+    const MAX_RUNTIME_MS = 14 * 60 * 1000; // 14 minutes (1 min buffer)
+
+    try {
+        const results = await processInBatches(app, startTime, MAX_RUNTIME_MS);
+        console.log("Batch complete:", JSON.stringify(results));
+        context.close();
+    } catch (error) {
+        console.error("Cron failed:", error.message);
+        await notifyAdminOnFailure(app, error);
+        context.close();
+    }
+};
+```
+
+## Chunking Patterns for Large Datasets
+
+### Pattern 1: Offset-Based Pagination
+
+```javascript
+async function processInBatches(app, startTime, maxRuntime) {
+    const BATCH_SIZE = 100;
+    let offset = await getLastOffset(app); // Resume from last run
+    let processedCount = 0;
+    let errors = [];
+
+    while (true) {
+        // Safety: Check remaining time
+        if (Date.now() - startTime > maxRuntime) {
+            console.warn("Approaching timeout, saving progress at offset:", offset);
+            await saveProgress(app, offset, processedCount, errors);
+            break;
+        }
+
+        // Fetch next batch
+        const records = await fetchBatch(app, offset, BATCH_SIZE);
+
+        if (records.length === 0) {
+            // All records processed - reset offset for next run
+            await saveProgress(app, 0, processedCount, errors);
+            break;
+        }
+
+        // Process each record
+        for (const record of records) {
+            try {
+                await processRecord(app, record);
+                processedCount++;
+            } catch (error) {
+                errors.push({
+                    recordId: record.ROWID,
+                    error: error.message
+                });
+            }
+        }
+
+        offset += BATCH_SIZE;
+    }
+
+    return { processedCount, errorCount: errors.length, errors };
+}
+```
+
+### Pattern 2: Cursor-Based (For APIs with Cursor Pagination)
+
+```javascript
+async function processWithCursor(app, startTime, maxRuntime) {
+    let cursor = await getSavedCursor(app); // null on first run
+    let processedCount = 0;
+
+    while (true) {
+        if (Date.now() - startTime > maxRuntime) {
+            await saveCursor(app, cursor);
+            break;
+        }
+
+        const response = await fetchWithCursor(cursor);
+        const { data, next_cursor, has_more } = response;
+
+        for (const item of data) {
+            await processItem(app, item);
+            processedCount++;
+        }
+
+        if (!has_more) {
+            await saveCursor(app, null); // Reset for next full run
+            break;
+        }
+
+        cursor = next_cursor;
+    }
+
+    return { processedCount, lastCursor: cursor };
+}
+```
+
+### Pattern 3: Date-Window Chunking
+
+```javascript
+async function processDateWindow(app) {
+    // Process records modified since last successful run
+    const lastRunDate = await getLastSuccessfulRun(app);
+    const now = new Date();
+
+    const criteria = `Modified_Date >= '${lastRunDate.toISOString()}' AND Modified_Date < '${now.toISOString()}'`;
+
+    const records = await fetchAllPages(app, criteria);
+
+    let processed = 0;
+    for (const record of records) {
+        await syncToExternalSystem(record);
+        processed++;
+    }
+
+    // Only update last-run timestamp on full success
+    await setLastSuccessfulRun(app, now);
+
+    return { processed, window: { from: lastRunDate, to: now } };
+}
+```
+
+## State Management Between Runs
+
+Use Catalyst Data Store (ZCQL) to persist state:
+
+```javascript
+// Save processing state
+async function saveProgress(app, offset, processedCount, errors) {
+    const dataStore = app.datastore();
+    const table = dataStore.table("BatchState");
+
+    const existingRows = await app.zcql().executeZCQLQuery(
+        "SELECT ROWID FROM BatchState WHERE job_name = 'sync_invoices'"
+    );
+
+    const stateData = {
+        job_name: "sync_invoices",
+        last_offset: offset,
+        last_run: new Date().toISOString(),
+        processed_count: processedCount,
+        error_count: errors.length,
+        errors_json: JSON.stringify(errors.slice(0, 50)), // Cap stored errors
+        status: errors.length > 0 ? "partial" : "complete"
+    };
+
+    if (existingRows.length > 0) {
+        stateData.ROWID = existingRows[0].BatchState.ROWID;
+        await table.updateRow(stateData);
+    } else {
+        await table.insertRow(stateData);
+    }
+}
+
+// Retrieve last state
+async function getLastOffset(app) {
+    const result = await app.zcql().executeZCQLQuery(
+        "SELECT last_offset FROM BatchState WHERE job_name = 'sync_invoices'"
+    );
+    return result.length > 0 ? parseInt(result[0].BatchState.last_offset) : 0;
+}
+```
+
+## Error Notification Setup
+
+```javascript
+async function notifyAdminOnFailure(app, error, jobName = "batch_job") {
+    // Option 1: Email via Catalyst Mail
+    const mailService = app.email();
+    await mailService.sendMail({
+        from_email: "noreply@your-catalyst-project.com",
+        to_email: ["admin@company.com"],
+        subject: `[ALERT] Cron Job Failed: ${jobName}`,
+        content: `
+            <h3>Batch Processing Failure</h3>
+            <p><strong>Job:</strong> ${jobName}</p>
+            <p><strong>Error:</strong> ${error.message}</p>
+            <p><strong>Stack:</strong> <pre>${error.stack}</pre></p>
+            <p><strong>Time:</strong> ${new Date().toISOString()}</p>
+        `,
+        content_type: "html"
+    });
+
+    // Option 2: Zoho Cliq notification
+    const axios = require("axios");
+    await axios.post(process.env.CLIQ_WEBHOOK_URL, {
+        text: `*Cron Job Failed:* ${jobName}\nError: ${error.message}\nTime: ${new Date().toISOString()}`
+    });
+}
+```
+
+## Retry Logic
+
+```javascript
+async function processWithRetry(fn, record, maxRetries = 3) {
+    for (let attempt = 1; attempt <= maxRetries; attempt++) {
+        try {
+            return await fn(record);
+        } catch (error) {
+            if (attempt === maxRetries) {
+                throw error; // Final attempt failed
+            }
+
+            // Exponential backoff
+            const delay = Math.pow(2, attempt) * 500;
+            console.warn(`Attempt ${attempt} failed for record ${record.ROWID}, retrying in ${delay}ms`);
+            await new Promise(resolve => setTimeout(resolve, delay));
+
+            // Don't retry non-transient errors
+            if (error.status === 400 || error.status === 404) {
+                throw error; // Bad data, retry won't help
+            }
+        }
+    }
+}
+```
+
+## Avoiding Duplicate Processing
+
+### Idempotency Pattern
+
+```javascript
+async function processRecordIdempotent(app, record) {
+    const idempotencyKey = `${record.ROWID}_${record.Modified_Date}`;
+
+    // Check if already processed
+    const existing = await app.zcql().executeZCQLQuery(
+        `SELECT ROWID FROM ProcessedLog WHERE idempotency_key = '${idempotencyKey}'`
+    );
+
+    if (existing.length > 0) {
+        console.log(`Skipping already-processed record: ${idempotencyKey}`);
+        return { skipped: true };
+    }
+
+    // Process the record
+    const result = await doActualProcessing(record);
+
+    // Mark as processed
+    await app.datastore().table("ProcessedLog").insertRow({
+        idempotency_key: idempotencyKey,
+        record_id: record.ROWID,
+        processed_at: new Date().toISOString(),
+        result: JSON.stringify(result)
+    });
+
+    return { skipped: false, result };
+}
+```
+
+### Locking Pattern (Prevent Concurrent Runs)
+
+```javascript
+async function acquireLock(app, jobName, ttlMinutes = 16) {
+    const lockExpiry = new Date(Date.now() + ttlMinutes * 60 * 1000).toISOString();
+
+    // Check for existing active lock
+    const existing = await app.zcql().executeZCQLQuery(
+        `SELECT ROWID, lock_expiry FROM CronLocks WHERE job_name = '${jobName}'`
+    );
+
+    if (existing.length > 0) {
+        const expiry = new Date(existing[0].CronLocks.lock_expiry);
+        if (expiry > new Date()) {
+            console.warn("Job already running, skipping this execution");
+            return false; // Lock held by another run
+        }
+        // Expired lock - update it
+        await app.datastore().table("CronLocks").updateRow({
+            ROWID: existing[0].CronLocks.ROWID,
+            lock_expiry: lockExpiry,
+            locked_at: new Date().toISOString()
+        });
+    } else {
+        await app.datastore().table("CronLocks").insertRow({
+            job_name: jobName,
+            lock_expiry: lockExpiry,
+            locked_at: new Date().toISOString()
+        });
+    }
+
+    return true; // Lock acquired
+}
+
+async function releaseLock(app, jobName) {
+    const existing = await app.zcql().executeZCQLQuery(
+        `SELECT ROWID FROM CronLocks WHERE job_name = '${jobName}'`
+    );
+    if (existing.length > 0) {
+        await app.datastore().table("CronLocks").deleteRow(existing[0].CronLocks.ROWID);
+    }
+}
+```
+
+## Complete Cron Function Template
+
+```javascript
+const catalyst = require("zcatalyst-sdk-node");
+
+module.exports = async (cronDetails, context) => {
+    const app = catalyst.initialize(context);
+    const JOB_NAME = "invoice_sync";
+    const startTime = Date.now();
+    const MAX_RUNTIME = 14 * 60 * 1000;
+
+    try {
+        // Prevent concurrent execution
+        const lockAcquired = await acquireLock(app, JOB_NAME);
+        if (!lockAcquired) {
+            console.log("Another instance running, exiting");
+            context.close();
+            return;
+        }
+
+        // Main processing
+        const results = await processInBatches(app, startTime, MAX_RUNTIME);
+
+        // Report results
+        console.log(`Completed: ${results.processedCount} processed, ${results.errorCount} errors`);
+
+        if (results.errorCount > 0) {
+            await notifyAdminOnFailure(app, new Error(
+                `${results.errorCount} records failed: ${JSON.stringify(results.errors.slice(0, 5))}`
+            ), JOB_NAME);
+        }
+
+        await releaseLock(app, JOB_NAME);
+        context.close();
+
+    } catch (error) {
+        console.error("Fatal error:", error);
+        await notifyAdminOnFailure(app, error, JOB_NAME);
+        await releaseLock(app, JOB_NAME);
+        context.close();
+    }
+};
+```
+
+## Cron Schedule Configuration
+
+Set in `catalyst-config.json`:
+
+```json
+{
+    "cron": {
+        "function_name": "invoice_sync_cron",
+        "cron_expression": "0 */2 * * *",
+        "description": "Sync invoices every 2 hours"
+    }
+}
+```
+
+| Expression | Schedule |
+|-----------|----------|
+| `0 * * * *` | Every hour |
+| `*/15 * * * *` | Every 15 minutes |
+| `0 */2 * * *` | Every 2 hours |
+| `0 6 * * *` | Daily at 6 AM |
+| `0 0 * * 0` | Weekly on Sunday midnight |
+| `0 0 1 * *` | First of every month |