bulltrackers-module 1.0.733 → 1.0.734

package/functions/computation-system-v2/framework/storage/StorageManager.js

@@ -0,0 +1,469 @@
+/**
+ * @fileoverview StorageManager - Unified storage for computation results
+ * Includes Optimized Checkpointing and Performance Monitoring.
+ */
+
+const { Firestore } = require('@google-cloud/firestore');
+const { BigQuery } = require('@google-cloud/bigquery');
+const pLimit = require('p-limit');
+
+class StorageManager {
+    constructor(config, logger = null) {
+        this.config = config;
+        this.logger = logger;
+        
+        this.bigquery = new BigQuery({
+            projectId: config.bigquery?.projectId,
+            location: config.bigquery?.location || 'EU'
+        });
+        
+        this._firestore = null;
+        this.tableExists = new Map();
+    }
+    
+    get firestore() {
+        if (!this._firestore) {
+            this._firestore = new Firestore({
+                projectId: this.config.bigquery?.projectId
+            });
+        }
+        return this._firestore;
+    }
+    
+    async commitResults(dateStr, entry, results, depResultHashes = {}) {
+        const storageConfig = this._resolveStorageConfig(entry);
+        const writeResults = { bigquery: null, firestore: null };
+        const startTime = Date.now();
+        
+        if (storageConfig.bigquery !== false) {
+            try {
+                writeResults.bigquery = await this._writeToBigQuery(
+                    dateStr, entry, results, depResultHashes
+                );
+            } catch (error) {
+                this._log('ERROR', `BigQuery write failed for ${entry.name}: ${error.message}`);
+                throw error;
+            }
+        }
+        
+        if (storageConfig.firestore?.enabled) {
+            try {
+                writeResults.firestore = await this._writeToFirestore(
+                    dateStr, entry, results, storageConfig.firestore
+                );
+            } catch (error) {
+                this._log('WARN', `Firestore write failed for ${entry.name}: ${error.message}`);
+                writeResults.firestore = { error: error.message };
+            }
+        }
+        
+        const duration = Date.now() - startTime;
+        this._log('INFO', `Committed ${entry.name} results in ${duration}ms`);
+        return writeResults;
+    }
+
+    // =========================================================================
+    // OPTIMIZED CHECKPOINTING (Lightweight State)
+    // =========================================================================
+
+    async initCheckpoint(dateStr, computationName, checkpointId, totalEntities) {
+        const table = 'computation_checkpoints';
+        const dataset = this.config.bigquery.dataset;
+        
+        await this._ensureCheckpointTable(table);
+
+        const row = {
+            date: dateStr,
+            computation_name: computationName,
+            checkpoint_id: checkpointId,
+            status: 'running',
+            processed_count: 0,
+            total_entities: totalEntities,
+            last_entity_id: null,
+            completed_batches: [],
+            // FIX: Use instance method, remove 'new', pass Date object
+            started_at: this.bigquery.timestamp(new Date()),
+            last_updated: this.bigquery.timestamp(new Date())
+        };
+
+        // Standard insert for initialization
+        await this.bigquery.dataset(dataset).table(table).insert([row]);
+    }
+
+    async updateCheckpoint(dateStr, computationName, checkpointId, state) {
+        const table = 'computation_checkpoints';
+        const fullTable = `\`${this.config.bigquery.projectId}.${this.config.bigquery.dataset}.${table}\``;
+
+        // Efficient UPDATE using structured fields
+        const query = `
+            UPDATE ${fullTable}
+            SET 
+                processed_count = @processedCount,
+                last_entity_id = @lastEntityId,
+                completed_batches = ARRAY_CONCAT(completed_batches, [@batchIndex]),
+                last_updated = CURRENT_TIMESTAMP()
+            WHERE date = @date 
+              AND computation_name = @computationName
+              AND checkpoint_id = @checkpointId
+        `;
+
+        await this.bigquery.query({
+            query,
+            params: {
+                date: dateStr,
+                computationName,
+                checkpointId,
+                processedCount: state.processedCount,
+                lastEntityId: state.lastEntityId,
+                batchIndex: state.batchIndex // Assuming we add one batch at a time
+            },
+            location: this.config.bigquery.location
+        });
+    }
+
+    async completeCheckpoint(dateStr, computationName, checkpointId) {
+        const table = 'computation_checkpoints';
+        const fullTable = `\`${this.config.bigquery.projectId}.${this.config.bigquery.dataset}.${table}\``;
+
+        const query = `
+            UPDATE ${fullTable}
+            SET status = 'completed', last_updated = CURRENT_TIMESTAMP()
+            WHERE date = @date AND computation_name = @computationName AND checkpoint_id = @checkpointId
+        `;
+
+        await this.bigquery.query({
+            query, 
+            params: { date: dateStr, computationName, checkpointId },
+            location: this.config.bigquery.location
+        });
+    }
+
+    async getLatestCheckpoint(dateStr, computationName) {
+        const table = 'computation_checkpoints';
+        const fullTable = `\`${this.config.bigquery.projectId}.${this.config.bigquery.dataset}.${table}\``;
+        
+        try {
+            // Get the most recent attempt
+            const query = `
+                SELECT checkpoint_id, status, processed_count, last_entity_id, completed_batches
+                FROM ${fullTable}
+                WHERE date = @date AND computation_name = @computationName
+                ORDER BY started_at DESC
+                LIMIT 1
+            `;
+
+            const [rows] = await this.bigquery.query({
+                query,
+                params: { date: dateStr, computationName },
+                location: this.config.bigquery.location
+            });
+
+            if (rows.length === 0) return null;
+            return rows[0];
+        } catch (e) {
+            return null; // Table might not exist yet
+        }
+    }
+
+    async _ensureCheckpointTable(tableName) {
+        if (this.tableExists.get(tableName)) return;
+        const dataset = this.bigquery.dataset(this.config.bigquery.dataset);
+        const table = dataset.table(tableName);
+        const [exists] = await table.exists();
+        
+        if (!exists) {
+            await dataset.createTable(tableName, {
+                schema: [
+                    { name: 'date', type: 'DATE', mode: 'REQUIRED' },
+                    { name: 'computation_name', type: 'STRING', mode: 'REQUIRED' },
+                    { name: 'checkpoint_id', type: 'STRING', mode: 'REQUIRED' },
+                    { name: 'status', type: 'STRING', mode: 'REQUIRED' }, // running, completed, failed
+                    { name: 'processed_count', type: 'INTEGER', mode: 'NULLABLE' },
+                    { name: 'total_entities', type: 'INTEGER', mode: 'NULLABLE' },
+                    { name: 'last_entity_id', type: 'STRING', mode: 'NULLABLE' },
+                    { name: 'completed_batches', type: 'INTEGER', mode: 'REPEATED' }, // ARRAY<INT64>
+                    { name: 'started_at', type: 'TIMESTAMP', mode: 'REQUIRED' },
+                    { name: 'last_updated', type: 'TIMESTAMP', mode: 'REQUIRED' }
+                ],
+                timePartitioning: { type: 'DAY', field: 'date' },
+                clustering: { fields: ['computation_name', 'status'] }
+            });
+        }
+        this.tableExists.set(tableName, true);
+    }
+
+    // =========================================================================
+    // PERFORMANCE REPORTING
+    // =========================================================================
+
+    async savePerformanceReport(dateStr, report) {
+        const table = 'computation_performance';
+        
+        await this._ensurePerformanceTable(table);
+        
+        const rows = report.computations.map(c => ({
+            date: dateStr,
+            computation_name: c.name,
+            entity_count: c.entityCount,
+            avg_duration_ms: c.avgDuration,
+            max_duration_ms: c.maxDuration,
+            p95_duration_ms: c.p95Duration,
+            avg_memory_delta: c.avgMemory,
+            total_duration_ms: c.totalDuration,
+            recorded_at: new Date().toISOString()
+        }));
+
+        if (rows.length === 0) return;
+        await this.bigquery.dataset(this.config.bigquery.dataset).table(table).insert(rows);
+    }
+
+    async _ensurePerformanceTable(tableName) {
+        if (this.tableExists.get(tableName)) return;
+        const dataset = this.bigquery.dataset(this.config.bigquery.dataset);
+        const table = dataset.table(tableName);
+        const [exists] = await table.exists();
+        
+        if (!exists) {
+            await dataset.createTable(tableName, {
+                schema: [
+                    { name: 'date', type: 'DATE', mode: 'REQUIRED' },
+                    { name: 'computation_name', type: 'STRING', mode: 'REQUIRED' },
+                    { name: 'entity_count', type: 'INTEGER', mode: 'NULLABLE' },
+                    { name: 'avg_duration_ms', type: 'FLOAT', mode: 'NULLABLE' },
+                    { name: 'max_duration_ms', type: 'FLOAT', mode: 'NULLABLE' },
+                    { name: 'p95_duration_ms', type: 'FLOAT', mode: 'NULLABLE' },
+                    { name: 'avg_memory_delta', type: 'FLOAT', mode: 'NULLABLE' },
+                    { name: 'total_duration_ms', type: 'FLOAT', mode: 'NULLABLE' },
+                    { name: 'recorded_at', type: 'TIMESTAMP', mode: 'REQUIRED' }
+                ],
+                timePartitioning: { type: 'DAY', field: 'date' }
+            });
+        }
+        this.tableExists.set(tableName, true);
+    }
+
+    // =========================================================================
+    // INTERNAL HELPERS
+    // =========================================================================
+
+    _resolveStorageConfig(entry) {
+        const entryStorage = entry.storage || {};
+        return {
+            bigquery: entryStorage.bigquery !== false,
+            firestore: {
+                enabled: entryStorage.firestore?.enabled === true,
+                path: entryStorage.firestore?.path || null,
+                merge: entryStorage.firestore?.merge || false,
+                includeMetadata: entryStorage.firestore?.includeMetadata !== false
+            }
+        };
+    }
+
+    async _writeToBigQuery(dateStr, entry, results, depResultHashes) {
+        const table = this.config.resultStore?.table || 'computation_results';
+        const fullTable = `\`${this.config.bigquery.projectId}.${this.config.bigquery.dataset}.${table}\``;
+        
+        const rows = this._buildBigQueryRows(dateStr, entry, results, depResultHashes);
+        if (rows.length === 0) return { rowCount: 0 };
+        
+        await this._ensureBigQueryTable(table);
+        
+        const batchSize = this.config.execution?.insertBatchSize || 500;
+        for (let i = 0; i < rows.length; i += batchSize) {
+            const batch = rows.slice(i, i + batchSize);
+            await this._upsertBatch(fullTable, batch);
+        }
+        return { rowCount: rows.length };
+    }
+
+    _buildBigQueryRows(dateStr, entry, results, depResultHashes) {
+        const rows = [];
+        const timestamp = new Date().toISOString();
+        const depResultHashesJson = JSON.stringify(depResultHashes);
+        
+        if (entry.type === 'per-entity' && typeof results === 'object') {
+            for (const [entityId, data] of Object.entries(results)) {
+                rows.push({
+                    date: dateStr,
+                    computation_name: entry.name,
+                    category: entry.category || 'uncategorized',
+                    entity_id: String(entityId),
+                    code_hash: entry.hash,
+                    result_hash: this._hashResult(data),
+                    dependency_result_hashes: depResultHashesJson,
+                    entity_count: 1,
+                    result_data: JSON.stringify(data),
+                    updated_at: timestamp
+                });
+            }
+        } else {
+            rows.push({
+                date: dateStr,
+                computation_name: entry.name,
+                category: entry.category || 'uncategorized',
+                entity_id: '_global',
+                code_hash: entry.hash,
+                result_hash: this._hashResult(results),
+                dependency_result_hashes: depResultHashesJson,
+                entity_count: typeof results === 'object' ? Object.keys(results).length : 1,
+                result_data: JSON.stringify(results),
+                updated_at: timestamp
+            });
+        }
+        return rows;
+    }
+
+    async _upsertBatch(fullTable, rows) {
+        const dates = rows.map(r => r.date);
+        const names = rows.map(r => r.computation_name);
+        const entityIds = rows.map(r => r.entity_id);
+        const codeHashes = rows.map(r => r.code_hash);
+        const resultHashes = rows.map(r => r.result_hash);
+        const depHashes = rows.map(r => r.dependency_result_hashes);
+        const entityCounts = rows.map(r => r.entity_count);
+        const resultData = rows.map(r => r.result_data);
+        const updatedAts = rows.map(r => r.updated_at);
+        
+        const mergeQuery = `
+            MERGE INTO ${fullTable} T
+            USING (
+                SELECT 
+                    PARSE_DATE('%Y-%m-%d', date) as date,
+                    computation_name, entity_id, code_hash, result_hash, 
+                    dependency_result_hashes, entity_count, result_data, 
+                    PARSE_TIMESTAMP('%Y-%m-%dT%H:%M:%E*SZ', updated_at) as updated_at
+                FROM UNNEST(@dates) AS date WITH OFFSET pos
+                JOIN UNNEST(@names) AS computation_name WITH OFFSET USING(pos)
+                JOIN UNNEST(@entity_ids) AS entity_id WITH OFFSET USING(pos)
+                JOIN UNNEST(@code_hashes) AS code_hash WITH OFFSET USING(pos)
+                JOIN UNNEST(@result_hashes) AS result_hash WITH OFFSET USING(pos)
+                JOIN UNNEST(@dep_hashes) AS dependency_result_hashes WITH OFFSET USING(pos)
+                JOIN UNNEST(@entity_counts) AS entity_count WITH OFFSET USING(pos)
+                JOIN UNNEST(@result_data) AS result_data WITH OFFSET USING(pos)
+                JOIN UNNEST(@updated_ats) AS updated_at WITH OFFSET USING(pos)
+            ) S
+            ON T.date = S.date AND T.computation_name = S.computation_name AND T.entity_id = S.entity_id
+            WHEN MATCHED THEN
+                UPDATE SET code_hash = S.code_hash, result_hash = S.result_hash, 
+                           dependency_result_hashes = S.dependency_result_hashes, 
+                           entity_count = S.entity_count, result_data = S.result_data, 
+                           updated_at = S.updated_at
+            WHEN NOT MATCHED THEN
+                INSERT (date, computation_name, category, entity_id, code_hash, result_hash, 
+                        dependency_result_hashes, entity_count, result_data, updated_at)
+                VALUES (S.date, S.computation_name, @category, S.entity_id, S.code_hash, S.result_hash, 
+                        S.dependency_result_hashes, S.entity_count, S.result_data, S.updated_at)
+        `;
+        
+        await this.bigquery.query({
+            query: mergeQuery,
+            params: {
+                dates, names, entity_ids: entityIds, code_hashes: codeHashes, 
+                result_hashes: resultHashes, dep_hashes: depHashes, 
+                entity_counts: entityCounts, result_data: resultData, 
+                updated_ats: updatedAts, category: rows[0].category
+            },
+            location: this.config.bigquery.location
+        });
+    }
+
+    async _ensureBigQueryTable(tableName) {
+        if (this.tableExists.get(tableName)) return;
+        const dataset = this.bigquery.dataset(this.config.bigquery.dataset);
+        const table = dataset.table(tableName);
+        const [exists] = await table.exists();
+        if (!exists) {
+            await dataset.createTable(tableName, {
+                schema: [
+                    { name: 'date', type: 'DATE', mode: 'REQUIRED' },
+                    { name: 'computation_name', type: 'STRING', mode: 'REQUIRED' },
+                    { name: 'category', type: 'STRING', mode: 'NULLABLE' },
+                    { name: 'entity_id', type: 'STRING', mode: 'REQUIRED' },
+                    { name: 'code_hash', type: 'STRING', mode: 'NULLABLE' },
+                    { name: 'result_hash', type: 'STRING', mode: 'NULLABLE' },
+                    { name: 'dependency_result_hashes', type: 'STRING', mode: 'NULLABLE' },
+                    { name: 'entity_count', type: 'INTEGER', mode: 'NULLABLE' },
+                    { name: 'result_data', type: 'STRING', mode: 'NULLABLE' },
+                    { name: 'updated_at', type: 'TIMESTAMP', mode: 'NULLABLE' }
+                ],
+                timePartitioning: { type: 'DAY', field: 'date' },
+                clustering: { fields: ['computation_name', 'category'] }
+            });
+        }
+        this.tableExists.set(tableName, true);
+    }
+
+    async _writeToFirestore(dateStr, entry, results, firestoreConfig) {
+        const { path, merge, includeMetadata } = firestoreConfig;
+        if (!path) throw new Error(`Firestore path not configured for ${entry.name}`);
+        
+        const timestamp = new Date();
+        const metadata = includeMetadata ? {
+            _computedAt: timestamp, _computationDate: dateStr, 
+            _computationName: entry.name, _codeHash: entry.hash
+        } : {};
+        
+        let docCount = 0;
+        
+        if (entry.type === 'per-entity' && typeof results === 'object') {
+            const batches = [];
+            let currentBatch = this.firestore.batch();
+            let batchCount = 0;
+            const MAX_BATCH = 500;
+            
+            for (const [entityId, data] of Object.entries(results)) {
+                const docPath = this._resolvePath(path, {
+                    entityId, date: dateStr, computationName: entry.name, category: entry.category || 'uncategorized'
+                });
+                
+                const docRef = this.firestore.doc(docPath);
+                const docData = { ...data, ...metadata };
+                
+                merge ? currentBatch.set(docRef, docData, { merge: true }) : currentBatch.set(docRef, docData);
+                
+                batchCount++; docCount++;
+                if (batchCount >= MAX_BATCH) {
+                    batches.push(currentBatch);
+                    currentBatch = this.firestore.batch();
+                    batchCount = 0;
+                }
+            }
+            if (batchCount > 0) batches.push(currentBatch);
+            
+            const limit = pLimit(10);
+            await Promise.all(batches.map(b => limit(() => b.commit())));
+            
+        } else {
+            const docPath = this._resolvePath(path, {
+                entityId: '_global', date: dateStr, computationName: entry.name, category: entry.category || 'uncategorized'
+            });
+            const docRef = this.firestore.doc(docPath);
+            const docData = { ...results, ...metadata };
+            merge ? await docRef.set(docData, { merge: true }) : await docRef.set(docData);
+            docCount = 1;
+        }
+        return { docCount };
+    }
+
+    _resolvePath(pathTemplate, values) {
+        let resolved = pathTemplate;
+        resolved = resolved.replace(/\{entityId\}/g, values.entityId || '_global');
+        resolved = resolved.replace(/\{date\}/g, values.date || '');
+        resolved = resolved.replace(/\{computationName\}/g, values.computationName || '');
+        resolved = resolved.replace(/\{category\}/g, values.category || '');
+        resolved = resolved.replace(/^\/+/, '');
+        return resolved;
+    }
+
+    _hashResult(data) {
+        const crypto = require('crypto');
+        const str = typeof data === 'string' ? data : JSON.stringify(data);
+        return crypto.createHash('md5').update(str).digest('hex').substring(0, 16);
+    }
+    
+    _log(level, message) {
+        this.logger?.log ? this.logger.log(level, `[StorageManager] ${message}`) : console.log(`[${level}] [StorageManager] ${message}`);
+    }
+}
+
+module.exports = { StorageManager };

package/functions/computation-system-v2/framework/storage/index.js

@@ -0,0 +1,9 @@
+/**
+ * @fileoverview Storage module exports
+ */
+
+const { StorageManager } = require('./StorageManager');
+
+module.exports = {
+    StorageManager
+};

package/functions/computation-system-v2/framework/testing/ComputationTester.js

@@ -0,0 +1,86 @@
+/**
+ * @fileoverview Unit Testing Utility for Computations
+ */
+
+class ComputationTester {
+    constructor(ComputationClass, config) {
+        this.ComputationClass = ComputationClass;
+        this.config = config;
+        this.mockData = {};
+        this.mockDependencies = {};
+    }
+    
+    withMockData(tableName, data) {
+        this.mockData[tableName] = data;
+        return this;
+    }
+    
+    withMockDependency(computationName, results) {
+        this.mockDependencies[computationName] = results;
+        return this;
+    }
+    
+    async run(entityId = 'test_entity') {
+        const instance = new this.ComputationClass();
+        
+        const context = {
+            date: '2026-01-24',
+            entityId,
+            data: this.mockData,
+            dependencies: this.mockDependencies,
+            rules: this.config.rules,
+            config: this.config,
+            logger: { log: () => {} }
+        };
+        
+        instance._meta = { logger: context.logger, config: this.config };
+        
+        await instance.process(context);
+        
+        const allResults = await instance.getResult();
+        return allResults[entityId] || allResults;
+    }
+    
+    async expect(entityId = 'test_entity') {
+        const result = await this.run(entityId);
+        
+        return {
+            result,
+            
+            toHaveProperty(prop, value) {
+                const actual = this._getNestedProperty(result, prop);
+                if (value !== undefined && actual !== value) {
+                    throw new Error(`Expected ${prop} to be ${value}, got ${actual}`);
+                }
+                if (actual === undefined) {
+                    throw new Error(`Expected ${prop} to exist`);
+                }
+                return this;
+            },
+            
+            toMatchSchema(schema) {
+                const errors = this._validateSchema(result, schema);
+                if (errors.length > 0) {
+                    throw new Error(`Schema validation failed: ${errors.join(', ')}`);
+                }
+                return this;
+            },
+            
+            _getNestedProperty(obj, path) {
+                return path.split('.').reduce((curr, key) => curr?.[key], obj);
+            },
+            
+            _validateSchema(data, schema) {
+                const errors = [];
+                for (const [key, type] of Object.entries(schema)) {
+                    if (typeof data[key] !== type) {
+                        errors.push(`${key}: expected ${type}, got ${typeof data[key]}`);
+                    }
+                }
+                return errors;
+            }
+        };
+    }
+}
+
+module.exports = { ComputationTester };