bulltrackers-module 1.0.733 → 1.0.734

package/functions/computation-system-v2/framework/storage/StateRepository.js

@@ -0,0 +1,286 @@
+/**
+ * @fileoverview State Repository
+ * * Manages the read-access to the system's state:
+ * 1. Loading daily execution status (hashes, timestamps)
+ * 2. Loading previous results (for dependencies and history)
+ * 3. Caching results for performance
+ */
+
+const { BigQuery } = require('@google-cloud/bigquery');
+
+class StateRepository {
+    /**
+     * @param {Object} config - System configuration
+     * @param {Object} [logger] - Logger instance
+     */
+    constructor(config, logger = null) {
+        this.config = config;
+        this.logger = logger || console;
+        
+        this.bigquery = new BigQuery({
+            projectId: config.bigquery.projectId,
+            location: config.bigquery.location
+        });
+
+        // Caches
+        this.statusCache = new Map();  // dateStr -> Map<compName, statusObj>
+        this.resultCache = new Map();  // key -> resultObj
+    }
+
+    /**
+     * Load status of all computations for a given date.
+     * @param {string} dateStr - Target date (YYYY-MM-DD)
+     * @returns {Promise<Map<string, Object>>} Map of computation name -> status
+     */
+    async getDailyStatus(dateStr) {
+        if (this.statusCache.has(dateStr)) {
+            return this.statusCache.get(dateStr);
+        }
+
+        const statusMap = new Map();
+        const table = this.config.resultStore?.table || 'computation_results';
+        const fullTable = `\`${this.config.bigquery.projectId}.${this.config.bigquery.dataset}.${table}\``;
+
+        try {
+            const query = `
+                SELECT 
+                    computation_name, 
+                    ANY_VALUE(code_hash) as code_hash, 
+                    ANY_VALUE(result_hash) as result_hash,
+                    ANY_VALUE(dependency_result_hashes) as dependency_result_hashes,
+                    COUNT(*) as entity_count, 
+                    MAX(updated_at) as updated_at
+                FROM ${fullTable}
+                WHERE date = @targetDate
+                GROUP BY computation_name
+            `;
+
+            const [rows] = await this.bigquery.query({
+                query,
+                params: { targetDate: dateStr },
+                location: this.config.bigquery.location
+            });
+
+            for (const row of rows) {
+                let depResultHashes = row.dependency_result_hashes;
+                if (typeof depResultHashes === 'string') {
+                    try { depResultHashes = JSON.parse(depResultHashes); } catch (e) { depResultHashes = {}; }
+                }
+
+                statusMap.set(row.computation_name.toLowerCase(), {
+                    hash: row.code_hash,
+                    resultHash: row.result_hash,
+                    dependencyResultHashes: depResultHashes || {},
+                    entityCount: parseInt(row.entity_count, 10),
+                    updatedAt: row.updated_at
+                });
+            }
+        } catch (e) {
+            if (e.message?.includes('Not found') || e.code === 404) {
+                this._log('DEBUG', `No status found for ${dateStr} (table may not exist)`);
+            } else {
+                this._log('WARN', `Failed to load status for ${dateStr}: ${e.message}`);
+            }
+        }
+
+        this.statusCache.set(dateStr, statusMap);
+        return statusMap;
+    }
+
+    /**
+     * Update the local status cache after a write.
+     * @param {string} dateStr 
+     * @param {string} name 
+     * @param {Object} statusObj 
+     */
+    updateStatusCache(dateStr, name, statusObj) {
+        const status = this.statusCache.get(dateStr) || new Map();
+        status.set(name.toLowerCase(), statusObj);
+        this.statusCache.set(dateStr, status);
+    }
+
+    /**
+     * Load a computation result (global or all entities).
+     */
+    async getResult(dateStr, computationName) {
+        const cacheKey = `${dateStr}:${computationName.toLowerCase()}`;
+        if (this.resultCache.has(cacheKey)) {
+            return this.resultCache.get(cacheKey);
+        }
+
+        try {
+            const table = this.config.resultStore?.table || 'computation_results';
+            const fullTable = `\`${this.config.bigquery.projectId}.${this.config.bigquery.dataset}.${table}\``;
+
+            const query = `
+                SELECT entity_id, result_data
+                FROM ${fullTable}
+                WHERE date = @targetDate AND computation_name = @compName
+            `;
+
+            const [rows] = await this.bigquery.query({
+                query,
+                params: { targetDate: dateStr, compName: computationName.toLowerCase() },
+                location: this.config.bigquery.location
+            });
+
+            if (rows.length === 0) {
+                this.resultCache.set(cacheKey, null);
+                return null;
+            }
+
+            const resultMap = {};
+            for (const row of rows) {
+                const entityId = row.entity_id || '_global';
+                let data = row.result_data;
+                if (typeof data === 'string') {
+                    try { data = JSON.parse(data); } catch (e) { /* keep as string */ }
+                }
+                resultMap[entityId] = data;
+            }
+
+            this.resultCache.set(cacheKey, resultMap);
+            return resultMap;
+        } catch (e) {
+            if (!e.message?.includes('Not found')) {
+                this._log('WARN', `Failed to load result for ${computationName}: ${e.message}`);
+            }
+            return null;
+        }
+    }
+
+    /**
+     * Load a specific entity's result (memory efficient).
+     */
+    async getEntityResult(dateStr, computationName, entityId) {
+        // Check full cache first
+        const fullCacheKey = `${dateStr}:${computationName.toLowerCase()}`;
+        if (this.resultCache.has(fullCacheKey)) {
+            const fullResult = this.resultCache.get(fullCacheKey);
+            return fullResult?.[entityId] || null;
+        }
+
+        // Check specific cache
+        const entityCacheKey = `${fullCacheKey}:${entityId}`;
+        if (this.resultCache.has(entityCacheKey)) {
+            return this.resultCache.get(entityCacheKey);
+        }
+
+        try {
+            const table = this.config.resultStore?.table || 'computation_results';
+            const fullTable = `\`${this.config.bigquery.projectId}.${this.config.bigquery.dataset}.${table}\``;
+
+            const query = `
+                SELECT result_data
+                FROM ${fullTable}
+                WHERE date = @targetDate 
+                  AND computation_name = @compName
+                  AND entity_id = @entityId
+                LIMIT 1
+            `;
+
+            const [rows] = await this.bigquery.query({
+                query,
+                params: { 
+                    targetDate: dateStr, 
+                    compName: computationName.toLowerCase(),
+                    entityId: String(entityId)
+                },
+                location: this.config.bigquery.location
+            });
+
+            if (rows.length === 0) {
+                this.resultCache.set(entityCacheKey, null);
+                return null;
+            }
+
+            let data = rows[0].result_data;
+            if (typeof data === 'string') {
+                try { data = JSON.parse(data); } catch (e) { /* keep */ }
+            }
+
+            this.resultCache.set(entityCacheKey, data);
+            return data;
+        } catch (e) {
+            return null;
+        }
+    }
+
+    /**
+     * Get multiple entity results in a single query (batch lazy load)
+     * FIXED: This solves the N+1 problem by allowing the Executor to fetch dependencies
+     * for an entire processing batch in one go.
+     */
+    async getBatchEntityResults(dateStr, computationName, entityIds) {
+        if (!entityIds || entityIds.length === 0) return {};
+        
+        const cacheKeyPrefix = `${dateStr}:${computationName.toLowerCase()}`;
+        const results = {};
+        const uncachedIds = [];
+        
+        // Check cache first
+        for (const entityId of entityIds) {
+            const key = `${cacheKeyPrefix}:${entityId}`;
+            if (this.resultCache.has(key)) {
+                results[entityId] = this.resultCache.get(key);
+            } else {
+                uncachedIds.push(entityId);
+            }
+        }
+        
+        if (uncachedIds.length === 0) return results;
+        
+        // Fetch uncached in batch
+        try {
+            const table = this.config.resultStore?.table || 'computation_results';
+            const fullTable = `\`${this.config.bigquery.projectId}.${this.config.bigquery.dataset}.${table}\``;
+
+            const query = `
+                SELECT entity_id, result_data
+                FROM ${fullTable}
+                WHERE date = @targetDate 
+                  AND computation_name = @compName
+                  AND entity_id IN UNNEST(@entityIds)
+            `;
+
+            const [rows] = await this.bigquery.query({
+                query,
+                params: { 
+                    targetDate: dateStr, 
+                    compName: computationName.toLowerCase(),
+                    entityIds: uncachedIds.map(String)
+                },
+                location: this.config.bigquery.location
+            });
+
+            // Parse and cache
+            for (const row of rows) {
+                const entityId = row.entity_id;
+                let data = row.result_data;
+                if (typeof data === 'string') {
+                    try { data = JSON.parse(data); } catch (e) { /* keep */ }
+                }
+                
+                results[entityId] = data;
+                this.resultCache.set(`${cacheKeyPrefix}:${entityId}`, data);
+            }
+            
+            return results;
+        } catch (e) {
+            this._log('ERROR', `Batch fetch failed: ${e.message}`);
+            return results;
+        }
+    }
+
+    cacheResult(dateStr, computationName, result) {
+        const cacheKey = `${dateStr}:${computationName.toLowerCase()}`;
+        this.resultCache.set(cacheKey, result);
+    }
+
+    _log(level, message) {
+        if (this.logger?.log) this.logger.log(level, `[StateRepo] ${message}`);
+        else console.log(`[${level}] [StateRepo] ${message}`);
+    }
+}
+
+module.exports = { StateRepository };