bulltrackers-module 1.0.733 → 1.0.734

package/functions/computation-system-v2/framework/index.js

@@ -0,0 +1,45 @@
+/**
+ * @fileoverview Framework exports
+ * * This is the public API of the computation framework.
+ * Computations only need to import from here.
+ */
+
+// Core
+const { Computation } = require('./core/Computation');
+const { Orchestrator } = require('./execution/Orchestrator'); // <--- CHANGED
+const { ManifestBuilder } = require('./core/Manifest');
+const { RulesRegistry } = require('./core/Rules');
+
+// Scheduling
+const { ScheduleValidator } = require('./scheduling/ScheduleValidator');
+
+// Data Layer
+const { SchemaRegistry } = require('./data/SchemaRegistry');
+const { QueryBuilder } = require('./data/QueryBuilder');
+const { DataFetcher } = require('./data/DataFetcher');
+
+// Storage
+const { StorageManager } = require('./storage/StorageManager');
+
+module.exports = {
+    // Base class for computations
+    Computation,
+    
+    // Execution engine
+    Orchestrator, // <--- CHANGED
+    ManifestBuilder,
+    
+    // Business rules
+    RulesRegistry,
+    
+    // Scheduling
+    ScheduleValidator,
+    
+    // Data layer (for advanced use cases)
+    SchemaRegistry,
+    QueryBuilder,
+    DataFetcher,
+    
+    // Storage
+    StorageManager
+};

package/functions/computation-system-v2/framework/lineage/LineageTracker.js

@@ -0,0 +1,147 @@
+/**
+ * @fileoverview Data Lineage Tracker
+ * * Tracks the provenance of computation results.
+ * * Records which source data (tables, record counts) contributed to specific entity results.
+ * * Uses internal buffering to handle high-volume writes efficiently.
+ */
+
+const { BigQuery } = require('@google-cloud/bigquery');
+const crypto = require('crypto');
+
+class LineageTracker {
+    /**
+     * @param {Object} config - System configuration
+     * @param {Object} [logger] - Logger instance
+     */
+    constructor(config, logger = null) {
+        this.config = config;
+        this.logger = logger || console;
+        
+        this.bigquery = new BigQuery({
+            projectId: config.bigquery.projectId,
+            location: config.bigquery.location || 'US'
+        });
+        
+        this.datasetId = config.bigquery.dataset;
+        this.tableName = 'data_lineage';
+        
+        this.buffer = [];
+        this.BUFFER_SIZE = config.execution?.lineageBatchSize || 500;
+        this._tableChecked = false;
+    }
+
+    /**
+     * Track lineage for a specific computation result.
+     * @param {Object} params
+     * @param {string} params.computation - Computation name
+     * @param {string} params.date - Execution date
+     * @param {string} params.entityId - Entity ID
+     * @param {Object} params.sourceData - The actual input data object used
+     * @param {Object} params.result - The result object produced
+     */
+    async track({ computation, date, entityId, sourceData, result }) {
+        if (!sourceData) return;
+
+        const sourceSummary = this._summarizeSources(sourceData);
+        const resultHash = this._hash(result);
+
+        this.buffer.push({
+            date,
+            computation_name: computation,
+            entity_id: String(entityId),
+            sources_json: JSON.stringify(sourceSummary),
+            result_hash: resultHash,
+            timestamp: new Date().toISOString()
+        });
+
+        if (this.buffer.length >= this.BUFFER_SIZE) {
+            await this.flush();
+        }
+    }
+
+    /**
+     * Force write any buffered records to BigQuery.
+     */
+    async flush() {
+        if (this.buffer.length === 0) return;
+
+        const batch = [...this.buffer];
+        this.buffer = []; // Clear immediately
+
+        try {
+            await this._ensureTable();
+            
+            await this.bigquery
+                .dataset(this.datasetId)
+                .table(this.tableName)
+                .insert(batch);
+                
+            this._log('DEBUG', `Flushed ${batch.length} lineage records`);
+        } catch (e) {
+            this._log('ERROR', `Failed to flush lineage buffer: ${e.message}`);
+            // In a production system, we might want to retry or dump to a dead-letter file
+        }
+    }
+
+    /**
+     * Analyze source data to create a lightweight summary.
+     * @param {Object} data - Input data map { tableName: data }
+     * @returns {Array} Summary of sources [{ table, count, ... }]
+     */
+    _summarizeSources(data) {
+        return Object.entries(data).map(([table, content]) => {
+            let count = 0;
+            let meta = null;
+
+            if (Array.isArray(content)) {
+                count = content.length;
+            } else if (content && typeof content === 'object') {
+                count = Object.keys(content).length;
+            }
+
+            // Optional: If data has specific metadata fields (like 'version' or 'snapshot_id'), extract them
+            if (content && content._metadata) {
+                meta = content._metadata;
+            }
+
+            return { table, count, meta };
+        });
+    }
+
+    _hash(data) {
+        const str = typeof data === 'string' ? data : JSON.stringify(data);
+        return crypto.createHash('md5').update(str || '').digest('hex').substring(0, 16);
+    }
+
+    async _ensureTable() {
+        if (this._tableChecked) return;
+
+        const dataset = this.bigquery.dataset(this.datasetId);
+        const table = dataset.table(this.tableName);
+        const [exists] = await table.exists();
+        
+        if (!exists) {
+            this._log('INFO', `Creating lineage table: ${this.tableName}`);
+            await dataset.createTable(this.tableName, {
+                schema: [
+                    { name: 'date', type: 'DATE', mode: 'REQUIRED' },
+                    { name: 'computation_name', type: 'STRING', mode: 'REQUIRED' },
+                    { name: 'entity_id', type: 'STRING', mode: 'REQUIRED' },
+                    { name: 'sources_json', type: 'STRING', mode: 'REQUIRED' }, // JSON array of source summaries
+                    { name: 'result_hash', type: 'STRING', mode: 'NULLABLE' },
+                    { name: 'timestamp', type: 'TIMESTAMP', mode: 'REQUIRED' }
+                ],
+                timePartitioning: { type: 'DAY', field: 'date' },
+                clustering: { fields: ['computation_name', 'entity_id'] }
+            });
+        }
+        this._tableChecked = true;
+    }
+
+    _log(level, message) {
+        const prefix = '[LineageTracker]';
+        this.logger?.log ? this.logger.log(level, `${prefix} ${message}`) : console.log(`${level}: ${prefix} ${message}`);
+    }
+}
+
+module.exports = { LineageTracker };

package/functions/computation-system-v2/framework/monitoring/Profiler.js

@@ -0,0 +1,80 @@
+/**
+ * @fileoverview Computation Profiler
+ * Tracks execution metrics (duration, memory) for computations.
+ */
+
+class ComputationProfiler {
+    constructor() {
+        this.profiles = new Map();
+    }
+    
+    startProfile(computationName, entityId = null) {
+        const key = entityId ? `${computationName}:${entityId}` : computationName;
+        
+        this.profiles.set(key, {
+            startTime: Date.now(),
+            startMemory: process.memoryUsage().heapUsed,
+            queryCount: 0,
+            bytesProcessed: 0
+        });
+        
+        return key;
+    }
+    
+    endProfile(key, metadata = {}) {
+        const profile = this.profiles.get(key);
+        if (!profile) return;
+        
+        const endTime = Date.now();
+        const endMemory = process.memoryUsage().heapUsed;
+        
+        return {
+            duration: endTime - profile.startTime,
+            memoryDelta: endMemory - profile.startMemory,
+            queriesExecuted: profile.queryCount,
+            bytesProcessed: profile.bytesProcessed,
+            ...metadata
+        };
+    }
+    
+    async generateReport(dateStr, manifest) {
+        const report = {
+            date: dateStr,
+            totalDuration: 0,
+            totalQueries: 0,
+            totalBytes: 0,
+            computations: []
+        };
+        
+        for (const entry of manifest) {
+            const compProfiles = Array.from(this.profiles.entries())
+                .filter(([k]) => k.startsWith(entry.name));
+            
+            if (compProfiles.length === 0) continue;
+            
+            const durations = compProfiles.map(([, p]) => Date.now() - p.startTime);
+            const memories = compProfiles.map(([, p]) => process.memoryUsage().heapUsed - p.startMemory);
+            
+            report.computations.push({
+                name: entry.name,
+                entityCount: compProfiles.length,
+                avgDuration: durations.reduce((a, b) => a + b, 0) / durations.length,
+                maxDuration: Math.max(...durations),
+                p95Duration: this._percentile(durations, 0.95),
+                avgMemory: memories.reduce((a, b) => a + b, 0) / memories.length,
+                totalDuration: durations.reduce((a, b) => a + b, 0)
+            });
+        }
+        
+        report.computations.sort((a, b) => b.totalDuration - a.totalDuration);
+        return report;
+    }
+    
+    _percentile(arr, p) {
+        const sorted = [...arr].sort((a, b) => a - b);
+        const index = Math.ceil(sorted.length * p) - 1;
+        return sorted[index];
+    }
+}
+
+module.exports = { ComputationProfiler };

package/functions/computation-system-v2/framework/resilience/Checkpointer.js

@@ -0,0 +1,66 @@
+/**
+ * @fileoverview Computation Checkpointer
+ * Manages save/resume states for long-running computations using lightweight batch tracking.
+ */
+
+const crypto = require('crypto');
+
+class Checkpointer {
+    constructor(config, storage) {
+        this.config = config;
+        this.storage = storage;
+        this.enabled = config.checkpointing?.enabled !== false;
+    }
+    
+    async initCheckpoint(dateStr, computationName, totalEntities = 0) {
+        if (!this.enabled) return null;
+
+        // Check if there is an existing running checkpoint we can resume
+        const existing = await this.storage.getLatestCheckpoint(dateStr, computationName);
+        
+        if (existing && existing.status === 'running') {
+            return {
+                id: existing.checkpoint_id,
+                processedCount: existing.processed_count || 0,
+                completedBatches: new Set(existing.completed_batches || []),
+                lastEntityId: existing.last_entity_id,
+                isResumed: true
+            };
+        } else if (existing && existing.status === 'completed') {
+            // Already fully done
+            return { isCompleted: true };
+        }
+
+        // Start new
+        const checkpointId = crypto.randomUUID();
+        await this.storage.initCheckpoint(dateStr, computationName, checkpointId, totalEntities);
+        
+        return {
+            id: checkpointId,
+            processedCount: 0,
+            completedBatches: new Set(),
+            lastEntityId: null,
+            isResumed: false
+        };
+    }
+    
+    async markBatchComplete(dateStr, computationName, checkpointId, batchIndex, batchSize, lastEntityId) {
+        if (!this.enabled || !checkpointId) return;
+
+        // We update processed count approximately based on batch size
+        const processedCount = (batchIndex + 1) * batchSize;
+
+        await this.storage.updateCheckpoint(dateStr, computationName, checkpointId, {
+            processedCount,
+            lastEntityId,
+            batchIndex
+        });
+    }
+    
+    async complete(dateStr, computationName, checkpointId) {
+        if (!this.enabled || !checkpointId) return;
+        await this.storage.completeCheckpoint(dateStr, computationName, checkpointId);
+    }
+}
+
+module.exports = { Checkpointer };

package/functions/computation-system-v2/framework/scheduling/ScheduleValidator.js

@@ -0,0 +1,327 @@
+/**
+ * @fileoverview Schedule Validator
+ * 
+ * Validates computation schedules and enforces timing rules:
+ * 1. Parses schedule declarations
+ * 2. Validates schedule format
+ * 3. Checks 15-minute rule between dependent computations
+ * 4. Generates warnings/errors for problematic schedules
+ */
+
+/**
+ * @typedef {Object} Schedule
+ * @property {string} frequency - 'hourly' | 'daily' | 'weekly' | 'monthly'
+ * @property {string} [time] - Time in HH:MM format (UTC)
+ * @property {number} [dayOfWeek] - 0-6 (Sun-Sat) for weekly
+ * @property {number} [dayOfMonth] - 1-31 for monthly
+ * @property {string} [timezone] - Timezone (default: UTC)
+ */
+
+/**
+ * @typedef {Object} ValidationResult
+ * @property {boolean} valid - Whether all schedules are valid
+ * @property {Array} errors - Critical errors (must fix)
+ * @property {Array} warnings - Non-critical issues (should fix)
+ */
+
+class ScheduleValidator {
+    /**
+     * @param {Object} config - System configuration
+     * @param {Object} [logger] - Logger instance
+     */
+    constructor(config, logger = null) {
+        this.config = config;
+        this.logger = logger;
+        this.defaultSchedule = config.scheduling?.default || {
+            frequency: 'daily',
+            time: '02:00',
+            timezone: 'UTC'
+        };
+        this.dependencyGapMinutes = config.scheduling?.dependencyGapMinutes || 15;
+    }
+    
+    /**
+     * Validate all computation schedules in the manifest.
+     * @param {Array} manifest - Array of manifest entries
+     * @returns {ValidationResult}
+     */
+    validate(manifest) {
+        const errors = [];
+        const warnings = [];
+        
+        // Build lookup map
+        const entryMap = new Map();
+        for (const entry of manifest) {
+            entryMap.set(entry.name, entry);
+        }
+        
+        for (const entry of manifest) {
+            // 1. Validate schedule format
+            const scheduleErrors = this._validateScheduleFormat(entry);
+            errors.push(...scheduleErrors);
+            
+            // 2. Check dependency timing
+            const timingIssues = this._checkDependencyTiming(entry, entryMap);
+            for (const issue of timingIssues) {
+                if (issue.severity === 'error') {
+                    errors.push(issue);
+                } else {
+                    warnings.push(issue);
+                }
+            }
+        }
+        
+        return {
+            valid: errors.length === 0,
+            errors,
+            warnings
+        };
+    }
+    
+    /**
+     * Parse and normalize a schedule, applying defaults.
+     * @param {Object|undefined} schedule - Raw schedule from computation config
+     * @returns {Schedule} Normalized schedule
+     */
+    parseSchedule(schedule) {
+        if (!schedule) {
+            return { ...this.defaultSchedule };
+        }
+        
+        return {
+            frequency: schedule.frequency || this.defaultSchedule.frequency,
+            time: schedule.time || this.defaultSchedule.time,
+            dayOfWeek: schedule.dayOfWeek,
+            dayOfMonth: schedule.dayOfMonth,
+            timezone: schedule.timezone || 'UTC'
+        };
+    }
+    
+    /**
+     * Convert schedule to cron expression.
+     * @param {Schedule} schedule - Normalized schedule
+     * @returns {string} Cron expression
+     */
+    toCron(schedule) {
+        const [hour, minute] = (schedule.time || '00:00').split(':').map(Number);
+        
+        switch (schedule.frequency) {
+            case 'hourly':
+                return `${minute} * * * *`;
+            case 'daily':
+                return `${minute} ${hour} * * *`;
+            case 'weekly':
+                const dow = schedule.dayOfWeek ?? 0;
+                return `${minute} ${hour} * * ${dow}`;
+            case 'monthly':
+                const dom = schedule.dayOfMonth ?? 1;
+                return `${minute} ${hour} ${dom} * *`;
+            default:
+                return `${minute} ${hour} * * *`;  // Default to daily
+        }
+    }
+    
+    /**
+     * Calculate minutes between two schedules on a typical day.
+     * Returns null if schedules don't overlap (different frequencies).
+     * @param {Schedule} scheduleA - First schedule
+     * @param {Schedule} scheduleB - Second schedule
+     * @returns {number|null} Minutes between A and B (positive if B is after A)
+     */
+    calculateGap(scheduleA, scheduleB) {
+        // Only compare schedules with same or compatible frequencies
+        if (!this._frequenciesCompatible(scheduleA.frequency, scheduleB.frequency)) {
+            return null;
+        }
+        
+        const timeA = this._parseTime(scheduleA.time || '00:00');
+        const timeB = this._parseTime(scheduleB.time || '00:00');
+        
+        // For weekly, also check day of week
+        if (scheduleA.frequency === 'weekly' || scheduleB.frequency === 'weekly') {
+            const dowA = scheduleA.dayOfWeek ?? 0;
+            const dowB = scheduleB.dayOfWeek ?? 0;
+            if (dowA !== dowB) {
+                // Different days - calculate day gap
+                const dayDiff = ((dowB - dowA) + 7) % 7;
+                return dayDiff * 24 * 60 + (timeB - timeA);
+            }
+        }
+        
+        // For monthly, also check day of month
+        if (scheduleA.frequency === 'monthly' || scheduleB.frequency === 'monthly') {
+            const domA = scheduleA.dayOfMonth ?? 1;
+            const domB = scheduleB.dayOfMonth ?? 1;
+            if (domA !== domB) {
+                const dayDiff = domB - domA;
+                return dayDiff * 24 * 60 + (timeB - timeA);
+            }
+        }
+        
+        return timeB - timeA;
+    }
+    
+    /**
+     * Check if a computation should run on a given date.
+     * @param {Schedule} schedule - Computation schedule
+     * @param {Date|string} date - Date to check
+     * @returns {boolean}
+     */
+    shouldRunOnDate(schedule, date) {
+        const d = typeof date === 'string' ? new Date(date) : date;
+        
+        switch (schedule.frequency) {
+            case 'hourly':
+            case 'daily':
+                return true;
+            case 'weekly':
+                return d.getUTCDay() === (schedule.dayOfWeek ?? 0);
+            case 'monthly':
+                return d.getUTCDate() === (schedule.dayOfMonth ?? 1);
+            default:
+                return true;
+        }
+    }
+    
+    // =========================================================================
+    // PRIVATE METHODS
+    // =========================================================================
+    
+    _validateScheduleFormat(entry) {
+        const errors = [];
+        const schedule = entry.schedule;
+        
+        if (!schedule) return errors;  // Will use default
+        
+        // Validate frequency
+        const validFrequencies = ['hourly', 'daily', 'weekly', 'monthly'];
+        if (schedule.frequency && !validFrequencies.includes(schedule.frequency)) {
+            errors.push({
+                computation: entry.name,
+                field: 'schedule.frequency',
+                message: `Invalid frequency: ${schedule.frequency}. Must be one of: ${validFrequencies.join(', ')}`
+            });
+        }
+        
+        // Validate time format
+        if (schedule.time && !/^\d{2}:\d{2}$/.test(schedule.time)) {
+            errors.push({
+                computation: entry.name,
+                field: 'schedule.time',
+                message: `Invalid time format: ${schedule.time}. Must be HH:MM`
+            });
+        }
+        
+        // Validate time values
+        if (schedule.time) {
+            const [hour, minute] = schedule.time.split(':').map(Number);
+            if (hour < 0 || hour > 23 || minute < 0 || minute > 59) {
+                errors.push({
+                    computation: entry.name,
+                    field: 'schedule.time',
+                    message: `Invalid time values: ${schedule.time}`
+                });
+            }
+        }
+        
+        // Validate day of week
+        if (schedule.dayOfWeek !== undefined) {
+            if (schedule.dayOfWeek < 0 || schedule.dayOfWeek > 6) {
+                errors.push({
+                    computation: entry.name,
+                    field: 'schedule.dayOfWeek',
+                    message: `Invalid dayOfWeek: ${schedule.dayOfWeek}. Must be 0-6`
+                });
+            }
+        }
+        
+        // Validate day of month
+        if (schedule.dayOfMonth !== undefined) {
+            if (schedule.dayOfMonth < 1 || schedule.dayOfMonth > 31) {
+                errors.push({
+                    computation: entry.name,
+                    field: 'schedule.dayOfMonth',
+                    message: `Invalid dayOfMonth: ${schedule.dayOfMonth}. Must be 1-31`
+                });
+            }
+        }
+        
+        return errors;
+    }
+    
+    _checkDependencyTiming(entry, entryMap) {
+        const issues = [];
+        const entrySchedule = this.parseSchedule(entry.schedule);
+        
+        for (const depName of entry.dependencies) {
+            const dep = entryMap.get(depName);
+            if (!dep) continue;
+            
+            const depSchedule = this.parseSchedule(dep.schedule);
+            const gap = this.calculateGap(depSchedule, entrySchedule);
+            
+            if (gap === null) {
+                // Different frequencies - can't directly compare
+                issues.push({
+                    severity: 'warning',
+                    computation: entry.name,
+                    dependency: depName,
+                    message: `${entry.name} has different schedule frequency than dependency ${depName}. Manual verification recommended.`,
+                    suggestion: 'Ensure the dependency runs before this computation on all relevant dates'
+                });
+                continue;
+            }
+            
+            if (gap < 0) {
+                // Dependent runs BEFORE its dependency
+                issues.push({
+                    severity: 'error',
+                    computation: entry.name,
+                    dependency: depName,
+                    gap,
+                    message: `${entry.name} is scheduled BEFORE its dependency ${depName} (${Math.abs(gap)} minutes earlier)`,
+                    suggestion: `Move ${entry.name} to at least ${this.dependencyGapMinutes} minutes after ${depName}`
+                });
+            } else if (gap < this.dependencyGapMinutes) {
+                // Too close - race condition risk
+                issues.push({
+                    severity: 'warning',
+                    computation: entry.name,
+                    dependency: depName,
+                    gap,
+                    message: `${entry.name} scheduled only ${gap} minutes after dependency ${depName}`,
+                    suggestion: `Increase gap to at least ${this.dependencyGapMinutes} minutes`
+                });
+            }
+        }
+        
+        return issues;
+    }
+    
+    _frequenciesCompatible(freqA, freqB) {
+        // Hourly is compatible with everything (runs every hour anyway)
+        if (freqA === 'hourly' || freqB === 'hourly') return true;
+        
+        // Same frequency is always compatible
+        if (freqA === freqB) return true;
+        
+        // Daily is compatible with weekly/monthly (runs every day)
+        if (freqA === 'daily' || freqB === 'daily') return true;
+        
+        // Weekly and monthly are not directly comparable
+        return false;
+    }
+    
+    _parseTime(timeStr) {
+        const [hour, minute] = timeStr.split(':').map(Number);
+        return hour * 60 + minute;
+    }
+    
+    _log(level, message) {
+        if (this.logger && typeof this.logger.log === 'function') {
+            this.logger.log(level, `[ScheduleValidator] ${message}`);
+        }
+    }
+}
+
+module.exports = { ScheduleValidator };