bulltrackers-module 1.0.721 → 1.0.722

package/functions/computation-system/persistence/ResultCommitter.js

@@ -1,697 +1,282 @@
 /**
- * @fileoverview Handles saving computation results with observability, Smart Cleanup, and GCS Support.
- * UPDATED: Added GCS Offloading logic (Hybrid Pointer System).
- * UPDATED: Preserved Legacy Sharding/Compression for backward compatibility.
- * UPDATED: Auto-cleanup of old Firestore shards when migrating a doc to GCS.
- * FIXED: Disabled "Single Doc Compression" strategy during intermediate flushes to ensure consistent sharding.
+ * @fileoverview Handles saving computation results.
+ * REFACTORED:
+ * 1. Writes ALL data to BigQuery (Source of Truth).
+ * 2. Writes to Firestore ONLY for 'Page' (Fan-out) and 'Alert' computations.
+ * 3. Removes GCS/Compression complexity for standard data (now BQ-only).
  */
 const { commitBatchInChunks, generateDataHash, FieldValue } = require('../utils/utils');
-const { updateComputationStatus }          = require('./StatusRepository');
-const { batchStoreSchemas }                = require('../utils/schema_capture');
+const { updateComputationStatus } = require('./StatusRepository');
+const { batchStoreSchemas } = require('../utils/schema_capture');
 const { generateProcessId, PROCESS_TYPES } = require('../logger/logger');
-const { HeuristicValidator }               = require('./ResultsValidator');
-const { PubSubUtils }                      = require('../../core/utils/pubsub_utils');
-const ContractValidator                    = require('./ContractValidator');
-const validationOverrides                  = require('../config/validation_overrides'); 
-const pLimit                               = require('p-limit'); 
-const zlib                                 = require('zlib'); 
-const { Storage }                          = require('@google-cloud/storage');
-const { ensureComputationResultsTable, insertRows } = require('../../core/utils/bigquery_utils');
-
-const storage = new Storage(); // Singleton GCS Client
-const NON_RETRYABLE_ERRORS = [ 'PERMISSION_DENIED', 'DATA_LOSS', 'FAILED_PRECONDITION' ];
+const { HeuristicValidator } = require('./ResultsValidator');
+const ContractValidator = require('./ContractValidator');
+const validationOverrides = require('../config/validation_overrides');
+const pLimit = require('p-limit');
+
+const DEFAULT_TTL_DAYS = 90;
 const SIMHASH_REGISTRY_COLLECTION = 'system_simhash_registry';
-const CONTRACTS_COLLECTION        = 'system_contracts';
-const DEFAULT_TTL_DAYS            = 90;
+const CONTRACTS_COLLECTION = 'system_contracts';
 
 async function commitResults(stateObj, dStr, passName, config, deps, skipStatusWrite = false, options = {}) {
     const successUpdates = {};
-    const failureReport  = [];
-    const schemas        = [];
-    const cleanupTasks   = []; 
-    const alertTriggers  = []; 
+    const failureReport = [];
+    const schemas = [];
+    const alertTriggers = [];
     const { logger, db, calculationUtils } = deps;
-    const withRetry = calculationUtils?.withRetry || (fn => fn());
 
     const pid = generateProcessId(PROCESS_TYPES.STORAGE, passName, dStr);
     const flushMode = options.flushMode || 'STANDARD';
-    const isInitialWrite = options.isInitialWrite === true; 
-    const shardIndexes = options.shardIndexes || {};
-    const nextShardIndexes = {};
-    const fanOutLimit = pLimit(10); 
+    const isInitialWrite = options.isInitialWrite === true;
 
+    // Pre-fetch contracts and hashes
     const calcNames = Object.keys(stateObj);
-    const hashKeys  = calcNames.map(n => stateObj[n].manifest?.hash).filter(Boolean);
-    
+    const hashKeys = calcNames.map(n => stateObj[n].manifest?.hash).filter(Boolean);
     const [contractMap, simHashMap] = await Promise.all([
         fetchContracts(db, calcNames),
         batchFetchSimHashes(db, hashKeys)
     ]);
 
     for (const name in stateObj) {
-        const calc      = stateObj[name];
+        const calc = stateObj[name];
         const execStats = calc._executionStats || { processedUsers: 0, skippedUsers: 0 };
-        const currentShardIndex = shardIndexes[name] || 0;
 
         const runMetrics = {
-            storage:    { sizeBytes: 0, isSharded: false, shardCount: 1, keys: 0, location: 'FIRESTORE' },
+            storage: { sizeBytes: 0, location: 'BIGQUERY', keys: 0 },
             validation: { isValid: true, anomalies: [] },
             execution: execStats,
             io: { writes: 0, deletes: 0 }
         };
 
-        const isAlertComputation = calc.manifest.isAlertComputation === true;
-        const isPageComputation = calc.manifest.isPage === true;
-        const ttlDays = calc.manifest.ttlDays !== undefined ? calc.manifest.ttlDays : DEFAULT_TTL_DAYS;
+        const manifest = calc.manifest;
+        const isAlert = manifest.isAlertComputation === true;
+        const isPage = manifest.isPage === true;
+        const ttlDays = manifest.ttlDays !== undefined ? manifest.ttlDays : DEFAULT_TTL_DAYS;
 
         try {
             const result = await calc.getResult();
-            const configOverrides = validationOverrides[calc.manifest.name] || {};
             
-            const dataDeps = calc.manifest.rootDataDependencies || [];
-            const isPriceOnly = (dataDeps.length === 1 && dataDeps[0] === 'price');
-            let effectiveOverrides = { ...configOverrides };
-
-            if (isPriceOnly) {
-                effectiveOverrides.maxZeroPct = 100;
-                effectiveOverrides.maxFlatlinePct = 100;
-                effectiveOverrides.maxNullPct = 100;
-                effectiveOverrides.maxNanPct = 100;
-                delete effectiveOverrides.weekend; 
+            // --- 1. VALIDATION ---
+            const configOverrides = validationOverrides[manifest.name] || {};
+            const dataDeps = manifest.rootDataDependencies || [];
+            // Relax validation for price-only computations
+            if (dataDeps.length === 1 && dataDeps[0] === 'price') {
+                Object.assign(configOverrides, { maxZeroPct: 100, maxFlatlinePct: 100, maxNullPct: 100, maxNanPct: 100 });
+                delete configOverrides.weekend;
             }
 
+            // Contract Validation
             const contract = contractMap[name];
             if (contract) {
                 const contractCheck = ContractValidator.validate(result, contract);
-                if (!contractCheck.valid) {
-                    runMetrics.validation.isValid = false;
-                    runMetrics.validation.anomalies.push(contractCheck.reason);
-                    const semanticError = new Error(contractCheck.reason);
-                    semanticError.stage = 'SEMANTIC_GATE';
-                    throw semanticError;
-                }
+                if (!contractCheck.valid) throw new Error(`[SEMANTIC_GATE] ${contractCheck.reason}`);
             }
 
+            // Heuristic Validation (Circuit Breaker)
             if (result && Object.keys(result).length > 0) {
-                const healthCheck = HeuristicValidator.analyze(calc.manifest.name, result, dStr, effectiveOverrides);
+                const healthCheck = HeuristicValidator.analyze(manifest.name, result, dStr, configOverrides);
                 if (!healthCheck.valid) {
                     runMetrics.validation.isValid = false;
                     runMetrics.validation.anomalies.push(healthCheck.reason);
-                    const validationError = new Error(healthCheck.reason);
-                    validationError.stage = 'QUALITY_CIRCUIT_BREAKER';
-                    throw validationError;
+                    throw new Error(`[QUALITY_CIRCUIT_BREAKER] ${healthCheck.reason}`);
                 }
             }
 
             const isEmpty = !result || (typeof result === 'object' && Object.keys(result).length === 0);
             const resultHash = isEmpty ? 'empty' : generateDataHash(result);
-            const simHash = (flushMode !== 'INTERMEDIATE') ? (simHashMap[calc.manifest.hash] || null) : null;
-
-            if (isEmpty) { 
-                if (flushMode === 'INTERMEDIATE') { 
-                    nextShardIndexes[name] = currentShardIndex; 
-                    continue; 
-                }
-                
-                if (isAlertComputation && flushMode === 'FINAL') {
-                    const docPath = `${config.resultsCollection}/${dStr}/${config.resultsSubcollection}/${calc.manifest.category}/${config.computationsSubcollection}/${name}`;
-                    alertTriggers.push({ date: dStr, computationName: name, documentPath: docPath });
+            const simHash = (flushMode !== 'INTERMEDIATE') ? (simHashMap[manifest.hash] || null) : null;
+
+            // --- 2. HANDLE EMPTY RESULTS ---
+            if (isEmpty) {
+                if (flushMode === 'INTERMEDIATE') continue;
+                if (manifest.hash) {
+                    successUpdates[name] = {
+                        hash: manifest.hash, simHash, resultHash,
+                        dependencyResultHashes: manifest.dependencyResultHashes || {},
+                        category: manifest.category, composition: manifest.composition, metrics: runMetrics
+                    };
                 }
+                continue;
+            }
 
-                if (calc.manifest.hash) { 
-                    successUpdates[name] = { 
-                        hash: calc.manifest.hash, simHash: simHash, resultHash: resultHash, 
-                        dependencyResultHashes: calc.manifest.dependencyResultHashes || {},
-                        category: calc.manifest.category, composition: calc.manifest.composition,
-                        metrics: runMetrics 
-                    }; 
-                } 
-                continue; 
+            // --- 3. WRITE TO BIGQUERY (UNIVERSAL) ---
+            // ALL data goes to BigQuery first. This is the primary storage.
+            // Using a fire-and-forget approach or await based on critical need. 
+            // We await here to ensure data safety before reporting success.
+            await writeToBigQuery(result, name, dStr, manifest.category, logger, isAlert).catch(err => {
+                logger.log('WARN', `[BigQuery] Write warning for ${name}: ${err.message}`);
+            });
+            
+            // If it's NOT Page or Alert, we are done (No Firestore write)
+            if (!isPage && !isAlert) {
+                if (manifest.hash) {
+                    successUpdates[name] = {
+                        hash: manifest.hash, simHash, resultHash,
+                        dependencyResultHashes: manifest.dependencyResultHashes || {},
+                        category: manifest.category, composition: manifest.composition, metrics: runMetrics
+                    };
+                }
+                continue; // Skip Firestore logic
             }
 
-            // [NEW] Page Computation Logic (Fan-Out) with TTL
-            if (isPageComputation && !isEmpty) {
-                const expireAt = calculateExpirationDate(dStr, ttlDays);
+            // --- 4. FIRESTORE WRITES (SELECTIVE) ---
+            const expireAt = calculateExpirationDate(dStr, ttlDays);
+
+            // A. PAGE COMPUTATIONS (Fan-Out)
+            if (isPage) {
                 const mainDocRef = db.collection(config.resultsCollection).doc(dStr)
-                    .collection(config.resultsSubcollection).doc(calc.manifest.category)
+                    .collection(config.resultsSubcollection).doc(manifest.category)
                     .collection(config.computationsSubcollection).doc(name);
 
-                // Optimization: Only attempt cleanup on the initial write to save reads
-                if (isInitialWrite) {
-                    await cleanupOldShards(mainDocRef, name, config, deps, runMetrics);
-                }
-
-                // 1. Fan-out writes for each user
+                // Fan-out writes: One document per User ID
                 const pageWrites = [];
                 for (const [cid, userData] of Object.entries(result)) {
+                    // Unique document for each user ID
                     const userDocRef = mainDocRef.collection('pages').doc(cid);
                     
                     const payload = (typeof userData === 'object' && userData !== null) 
                         ? { ...userData, _expireAt: expireAt }
                         : { value: userData, _expireAt: expireAt };
 
-                    pageWrites.push({
-                        ref: userDocRef,
-                        data: payload,
-                        options: { merge: false } // Overwrite specifically for this run
-                    });
+                    pageWrites.push({ ref: userDocRef, data: payload, options: { merge: false } });
                 }
 
-                // 2. Commit the fan-out writes
                 if (pageWrites.length > 0) {
                     await commitBatchInChunks(config, deps, pageWrites, `${name}::PageFanOut`);
                     runMetrics.io.writes += pageWrites.length;
-                    runMetrics.storage.keys = pageWrites.length;
-                    logger.log('INFO', `[PageMode] ${name}: Wrote ${pageWrites.length} user pages. TTL: ${ttlDays}d.`);
-                }
-
-                // 3. Write or Update the "Header" document
-                const isFinalFlush = (flushMode !== 'INTERMEDIATE');
-                
-                let pageCountValue = pageWrites.length;
-                if (!isInitialWrite) {
-                    pageCountValue = FieldValue.increment(pageWrites.length);
+                    runMetrics.storage.location = 'FIRESTORE_PAGES';
                 }
 
+                // Write Header Document (Metadata for frontend/indexing)
                 const headerData = {
-                    _isPageMode: true, 
-                    _pageCount: pageCountValue,
+                    _isPageMode: true,
+                    _pageCount: isInitialWrite ? pageWrites.length : FieldValue.increment(pageWrites.length),
                     _lastUpdated: new Date().toISOString(),
                     _expireAt: expireAt,
-                    _completed: isFinalFlush ? true : false // Always a boolean: true when final flush, false otherwise
+                    _completed: flushMode !== 'INTERMEDIATE'
                 };
-
                 await mainDocRef.set(headerData, { merge: !isInitialWrite });
                 
-                runMetrics.io.writes += 1;
-                
-                // 4. Write to BigQuery (for analytics) - same structure as other computations
-                // Page computations store the full result object { cid1: {...}, cid2: {...}, ... } in result_data
-                await writeToBigQuery(result, name, dStr, calc.manifest.category, logger, false).catch(err => {
-                    logger.log('WARN', `[BigQuery] Failed to write page computation ${name} for ${dStr}: ${err.message}`);
-                });
-                
-                if (isFinalFlush && calc.manifest.hash) { 
-                    successUpdates[name] = { 
-                        hash: calc.manifest.hash, simHash: simHash, resultHash: resultHash,
-                        category: calc.manifest.category, composition: calc.manifest.composition, 
-                        metrics: runMetrics 
-                    }; 
-                }
-                
-                continue; 
+                logger.log('INFO', `[ResultCommitter] ${name}: Wrote ${pageWrites.length} user pages to Firestore.`);
             }
-            
-            // Standard Computation Logic (GCS, Compression or Sharding) with TTL
-            if (typeof result === 'object') runMetrics.storage.keys = Object.keys(result).length;
-            const resultKeys = Object.keys(result || {});
-            const isMultiDate = resultKeys.length > 0 && resultKeys.every(k => /^\d{4}-\d{2}-\d{2}$/.test(k));
-
-            if (isMultiDate) {
-                const datePromises  = resultKeys.map((historicalDate) => fanOutLimit(async () => {
-                    const dailyData = result[historicalDate];
-                    if (!dailyData || Object.keys(dailyData).length === 0) return;
-                    
-                    const dailyExpireAt = calculateExpirationDate(historicalDate, ttlDays);
-                    
-                    const historicalDocRef = db.collection(config.resultsCollection).doc(historicalDate).collection(config.resultsSubcollection).doc(calc.manifest.category).collection(config.computationsSubcollection).doc(name);
-                    
-                    // Recursive call allows GCS logic to apply per-day
-                    const stats = await writeSingleResult(dailyData, historicalDocRef, name, historicalDate, calc.manifest.category, logger, config, deps, 0, 'STANDARD', false, dailyExpireAt, isAlertComputation, isPageComputation);
-                    runMetrics.io.writes += stats.opCounts.writes;
-                    runMetrics.io.deletes += stats.opCounts.deletes;
-
-                    if (isAlertComputation && flushMode !== 'INTERMEDIATE') {
-                        alertTriggers.push({ date: historicalDate, computationName: name, documentPath: historicalDocRef.path });
-                    }
-                }));
-                await Promise.all(datePromises);
-                
-                if (calc.manifest.hash) { successUpdates[name] = { hash: calc.manifest.hash, simHash, resultHash, dependencyResultHashes: calc.manifest.dependencyResultHashes || {}, category: calc.manifest.category, composition: calc.manifest.composition, metrics: runMetrics }; }
-            } else {
-                const runExpireAt = calculateExpirationDate(dStr, ttlDays);
 
-                const mainDocRef = db.collection(config.resultsCollection).doc(dStr).collection(config.resultsSubcollection).doc(calc.manifest.category).collection(config.computationsSubcollection).doc(name);
-                const writeStats = await writeSingleResult(result, mainDocRef, name, dStr, calc.manifest.category, logger, config, deps, currentShardIndex, flushMode, isInitialWrite, runExpireAt, isAlertComputation, isPageComputation);
-                
-                runMetrics.storage.sizeBytes  = writeStats.totalSize;
-                runMetrics.storage.isSharded  = writeStats.isSharded;
-                runMetrics.storage.shardCount = writeStats.shardCount;
-                runMetrics.storage.location   = writeStats.location;
-                runMetrics.io.writes += writeStats.opCounts.writes;
-                runMetrics.io.deletes += writeStats.opCounts.deletes;
-                
-                nextShardIndexes[name] = writeStats.nextShardIndex;
-                if (calc.manifest.hash) { successUpdates[name] = { hash: calc.manifest.hash, simHash, resultHash, dependencyResultHashes: calc.manifest.dependencyResultHashes || {}, category: calc.manifest.category, composition: calc.manifest.composition, metrics: runMetrics }; }
+            // B. ALERT COMPUTATIONS (Single Doc for Triggers)
+            if (isAlert) {
+                // Alerts are written to a single document to trigger the listener
+                const mainDocRef = db.collection(config.resultsCollection).doc(dStr)
+                    .collection(config.resultsSubcollection).doc(manifest.category)
+                    .collection(config.computationsSubcollection).doc(name);
+
+                const alertPayload = {
+                    ...result,
+                    _isAlert: true,
+                    _lastUpdated: new Date().toISOString(),
+                    _expireAt: expireAt
+                };
 
-                if (isAlertComputation && flushMode !== 'INTERMEDIATE') {
+                await mainDocRef.set(alertPayload);
+                runMetrics.io.writes += 1;
+                runMetrics.storage.location = 'FIRESTORE_ALERT';
+                
+                // Add to triggers list for logging
+                if (flushMode !== 'INTERMEDIATE') {
                     alertTriggers.push({ date: dStr, computationName: name, documentPath: mainDocRef.path });
                 }
             }
 
-            if (calc.manifest.class.getSchema && flushMode !== 'INTERMEDIATE') {
-                const { class: _cls, ...safeMetadata } = calc.manifest;
-                // Ensure ttlDays is set to the resolved value (defaults to 90 if undefined)
-                safeMetadata.ttlDays = ttlDays;
-                schemas.push({ name, category: calc.manifest.category, schema: calc.manifest.class.getSchema(), metadata: safeMetadata });
+            // --- 5. FINALIZE ---
+            if (manifest.hash) {
+                successUpdates[name] = {
+                    hash: manifest.hash, simHash, resultHash,
+                    dependencyResultHashes: manifest.dependencyResultHashes || {},
+                    category: manifest.category, composition: manifest.composition, metrics: runMetrics
+                };
             }
-            if (calc.manifest.previousCategory && calc.manifest.previousCategory !== calc.manifest.category && flushMode !== 'INTERMEDIATE') {
-                cleanupTasks.push(deleteOldCalculationData(dStr, calc.manifest.previousCategory, name, config, deps));
+
+            // Store Schema
+            if (manifest.class.getSchema && flushMode !== 'INTERMEDIATE') {
+                const { class: _cls, ...safeMetadata } = manifest;
+                safeMetadata.ttlDays = ttlDays;
+                schemas.push({ name, category: manifest.category, schema: manifest.class.getSchema(), metadata: safeMetadata });
             }
 
         } catch (e) {
-            const stage = e.stage || 'EXECUTION';
-            if (logger && logger.log) { logger.log('ERROR', `Commit failed for ${name} [${stage}]`, { processId: pid, error: e }); }
-            failureReport.push({ name, error: { message: e.message, stack: e.stack, stage }, metrics: runMetrics });
+            logger.log('ERROR', `Commit failed for ${name}`, { error: e });
+            failureReport.push({ name, error: { message: e.message, stack: e.stack }, metrics: runMetrics });
         }
     }
 
     if (schemas.length) batchStoreSchemas(deps, config, schemas).catch(() => {});
-    if (cleanupTasks.length > 0) { await Promise.allSettled(cleanupTasks); }
-    if (!skipStatusWrite && Object.keys(successUpdates).length > 0 && flushMode !== 'INTERMEDIATE') { 
-        await updateComputationStatus(dStr, successUpdates, config, deps); 
+    if (!skipStatusWrite && Object.keys(successUpdates).length > 0 && flushMode !== 'INTERMEDIATE') {
+        await updateComputationStatus(dStr, successUpdates, config, deps);
     }
-
-    if (alertTriggers.length > 0) {
-        logger.log('INFO', `[Alert System] ${alertTriggers.length} alert computations written to Firestore - triggers will fire automatically`);
-    }
-    
-    return { successUpdates, failureReport, shardIndexes: nextShardIndexes };
-}
-
-async function batchFetchSimHashes(db, hashes) {
-    if (!hashes || hashes.length === 0) return {};
-    const map = {};
-    const refs = hashes.map(h => db.collection(SIMHASH_REGISTRY_COLLECTION).doc(h));
-    try {
-        const snaps = await db.getAll(...refs);
-        snaps.forEach(snap => { if (snap.exists) map[snap.id] = snap.data().simHash; });
-    } catch (e) {}
-    return map;
-}
-
-async function fetchContracts(db, calcNames) {
-    if (!calcNames || calcNames.length === 0) return {};
-    const map = {};
-    const refs = calcNames.map(name => db.collection(CONTRACTS_COLLECTION).doc(name));
-    try {
-        const snaps = await db.getAll(...refs);
-        snaps.forEach(snap => { if (snap.exists) map[snap.id] = snap.data(); });
-    } catch (e) {}
-    return map;
-}
-
-async function writeSingleResult(result, docRef, name, dateContext, category, logger, config, deps, startShardIndex = 0, flushMode = 'STANDARD', isInitialWrite = false, expireAt = null, isAlertComputation = false) {
-    const opCounts = { writes: 0, deletes: 0 };
     
-    // Check if previously sharded (so we can clean up if moving to GCS or Compressed)
-    let wasSharded = false;
-    try {
-        const currentSnap = await docRef.get();
-        if (currentSnap.exists) {
-            wasSharded = (currentSnap.data()._sharded === true);
-        }
-    } catch (e) {}
-
-    const jsonString = JSON.stringify(result);
-    const rawBuffer = Buffer.from(jsonString);
-    const totalSize = rawBuffer.length;
-
-    // --- STRATEGY 1: GCS OFFLOAD ---
-    // Trigger if bucket defined AND (UseGCS config set OR size > 800KB)
-    // This keeps small files in Firestore (faster/cheaper reads) but offloads dangerous sizes
-    const GCS_THRESHOLD = 800 * 1024; // 800KB
-    const bucketName = config.gcsBucketName || 'bulltrackers';
-    const useGCS = config.forceGCS || totalSize > GCS_THRESHOLD;
-
-    if (useGCS) {
-        try {
-            const bucket = storage.bucket(bucketName);
-            const fileName = `${dateContext}/${category}/${name}.json.gz`;
-            const file = bucket.file(fileName);
-
-            // 1. Compress & Upload
-            const compressedBuffer = zlib.gzipSync(rawBuffer);
-            await file.save(compressedBuffer, {
-                contentType: 'application/json',
-                contentEncoding: 'gzip',
-                metadata: {
-                    created: new Date().toISOString(),
-                    originalSize: totalSize,
-                    computation: name
-                }
-            });
-
-            // 2. Clean up old Firestore shards (Crucial for cost/consistency)
-            if (wasSharded) {
-                await cleanupOldShards(docRef, name, config, deps, { io: opCounts });
-            }
-
-            // 3. Write Pointer Document
-            const pointerPayload = {
-                _completed: true,
-                _gcs: true, // Flag for the Reader
-                gcsUri: `gs://${bucketName}/${fileName}`,
-                gcsBucket: bucketName,
-                gcsPath: fileName,
-                _lastUpdated: new Date().toISOString(),
-                sizeBytes: totalSize
-            };
-            if (expireAt) pointerPayload._expireAt = expireAt;
-
-            // Overwrite existing doc (merge: false ensures we clear old schema/data fields)
-            await docRef.set(pointerPayload, { merge: false });
-            opCounts.writes += 1;
-
-            logger.log('INFO', `[GCS] ${name}: Offloaded ${(totalSize/1024).toFixed(0)}KB to ${fileName}`);
-            
-            // Write to BigQuery (await to ensure completion before function returns)
-            // Errors are caught and logged but don't fail the operation
-            // Pass isAlertComputation flag to use streaming for alerts, load jobs for others
-            await writeToBigQuery(result, name, dateContext, category, logger, isAlertComputation).catch(err => {
-                logger.log('WARN', `[BigQuery] Failed to write ${name} for ${dateContext}: ${err.message}`);
-            });
-            
-            return { totalSize, isSharded: false, shardCount: 1, nextShardIndex: startShardIndex, opCounts, location: 'GCS' };
-
-        } catch (gcsErr) {
-            logger.log('ERROR', `[GCS] Upload failed for ${name}, falling back to Firestore: ${gcsErr.message}`);
-            // Fallthrough to Standard Logic...
-        }
-    }
-
-    // --- STRATEGY 2: FIRESTORE COMPRESSION ---
-    // FIX: Only compress if this is a single, atomic write (not part of a stream).
-    // Streaming relies on Strategy 3 (Sharding) to create distinct files (shard_0, shard_1...).
-    // If flushMode is INTERMEDIATE or we are already at a high shard index, we MUST fall through to sharding.
-    if (totalSize > 50 * 1024 && startShardIndex === 0 && flushMode !== 'INTERMEDIATE') { 
-        try {
-            const compressedBuffer = zlib.gzipSync(rawBuffer);
-            if (compressedBuffer.length < 900 * 1024) {
-                const payloadBuffer = Buffer.from(compressedBuffer); 
-                const compressedPayload = { 
-                    _compressed: true, 
-                    _completed: true, 
-                    _lastUpdated: new Date().toISOString(), 
-                    payload: payloadBuffer 
-                };
-                if (expireAt) compressedPayload._expireAt = expireAt;
-                
-                if (wasSharded) {
-                    await cleanupOldShards(docRef, name, config, deps, { io: opCounts });
-                    // Use merge: false (overwrite)
-                    await docRef.set(compressedPayload, { merge: false });
-                } else {
-                    await docRef.set(compressedPayload, { merge: false });
-                }
-                
-                opCounts.writes += 1;
-                logger.log('INFO', `[Compression] ${name}: Compressed ${(totalSize/1024).toFixed(0)}KB -> ${(compressedBuffer.length/1024).toFixed(0)}KB.`);
-                
-                // Write to BigQuery (await to ensure completion before function returns)
-                // Errors are caught and logged but don't fail the operation
-                await writeToBigQuery(result, name, dateContext, category, logger).catch(err => {
-                    logger.log('WARN', `[BigQuery] Failed to write ${name} for ${dateContext}: ${err.message}`);
-                });
-                
-                return { totalSize: compressedBuffer.length, isSharded: false, shardCount: 1, nextShardIndex: startShardIndex, opCounts, location: 'FIRESTORE' };
-            }
-        } catch (compErr) {
-            logger.log('WARN', `[SelfHealing] Compression failed for ${name}, reverting to sharding. Error: ${compErr.message}`);
-        }
+    if (alertTriggers.length > 0) {
+        logger.log('INFO', `[ResultCommitter] ${alertTriggers.length} alert computations updated in Firestore.`);
     }
 
-    // --- STRATEGY 3: FIRESTORE SHARDING (Fallback) ---
-    const strategies = [ { bytes: 900 * 1024, keys: null }, { bytes: 450 * 1024, keys: 10000 }, { bytes: 200 * 1024, keys: 2000 }, { bytes: 100 * 1024, keys: 50 } ];
-    let committed = false; let lastError = null; 
-    let finalStats = { totalSize: 0, isSharded: false, shardCount: 1, nextShardIndex: startShardIndex, location: 'FIRESTORE' };
-    let rootMergeOption = !isInitialWrite; 
-
-    // Only wipe existing shards if this is the INITIAL write for this batch run.
-    let shouldWipeShards = wasSharded && isInitialWrite; 
-
-    for (let attempt = 0; attempt < strategies.length; attempt++) {
-        if (committed) break;
-        const constraints = strategies[attempt];
-        try {
-            const updates = await prepareAutoShardedWrites(result, docRef, logger, constraints.bytes, constraints.keys, startShardIndex, flushMode, expireAt);
-            
-            if (shouldWipeShards) {
-                 const shardCol = docRef.collection('_shards');
-                 const shardDocs = await shardCol.listDocuments();
-                 shardDocs.forEach(d => updates.unshift({ type: 'DELETE', ref: d }));
-                 shouldWipeShards = false; 
-            }
-            
-            const rootUpdate = updates.find(u => u.ref.path === docRef.path && u.type !== 'DELETE');
-            // FIX: Always use merge: false to ensure old fields (like _compressed/payload) are wiped
-            if (rootUpdate) { rootUpdate.options = { merge: false }; }
-
-            const writes = updates.filter(u => u.type !== 'DELETE').length;
-            const deletes = updates.filter(u => u.type === 'DELETE').length;
-
-            await commitBatchInChunks(config, deps, updates, `${name}::${dateContext}`);
-            
-            opCounts.writes += writes;
-            opCounts.deletes += deletes;
-            finalStats.totalSize = updates.reduce((acc, u) => acc + (u.data ? JSON.stringify(u.data).length : 0), 0);
-            
-            // Determine shard count from updates
-            let maxIndex = startShardIndex;
-            updates.forEach(u => {
-                if (u.type === 'DELETE') return;
-                const segs = u.ref.path.split('/');
-                const last = segs[segs.length - 1];
-                if (last.startsWith('shard_')) {
-                    const idx = parseInt(last.split('_')[1]);
-                    if (!isNaN(idx) && idx > maxIndex) maxIndex = idx;
-                    finalStats.isSharded = true;
-                }
-            });
-            const pointer = updates.find(u => u.data && u.data._shardCount !== undefined);
-            if (pointer) {
-                finalStats.shardCount = pointer.data._shardCount;
-                finalStats.nextShardIndex = finalStats.shardCount; 
-            } else if (updates.length > 0) {
-                finalStats.nextShardIndex = maxIndex + 1;
-            }
-            
-            committed = true;
-        } catch (commitErr) {
-             lastError = commitErr;
-             const msg = commitErr.message || '';
-             const code = commitErr.code || '';
-             
-             if (NON_RETRYABLE_ERRORS.includes(code)) { 
-                 logger.log('ERROR', `[SelfHealing] ${name} FATAL error: ${msg}.`); 
-                 throw commitErr; 
-             }
-             logger.log('WARN', `[SelfHealing] ${name} on ${dateContext} failed attempt ${attempt+1}. Error: ${msg}. Retrying...`); 
-             continue;
-        }
-    }
-    if (!committed) { 
-        const shardingError = new Error(`Exhausted sharding strategies for ${name}. Last error: ${lastError?.message}`);
-        shardingError.stage = 'SHARDING_LIMIT_EXCEEDED';
-        throw shardingError; 
-    }
-    
-    // Write to BigQuery (await to ensure completion before function returns)
-    // Errors are caught and logged but don't fail the operation
-    await writeToBigQuery(result, name, dateContext, category, logger).catch(err => {
-        logger.log('WARN', `[BigQuery] Failed to write ${name} for ${dateContext}: ${err.message}`);
-    });
-    
-    finalStats.opCounts = opCounts;
-    return finalStats;
+    return { successUpdates, failureReport };
 }
 
-// =============================================================================
-// HELPERS
-// =============================================================================
+// --- HELPERS ---
 
-/**
- * Write computation result to BigQuery (errors are logged but don't fail Firestore writes)
- * @param {object} result - Computation result data
- * @param {string} name - Computation name
- * @param {string} dateContext - Date string (YYYY-MM-DD)
- * @param {string} category - Category (e.g., 'popular-investor', 'alerts')
- * @param {object} logger - Logger instance
- * @param {boolean} isAlertComputation - If true, uses streaming inserts (immediate). If false, uses load jobs (batched, free).
- */
 async function writeToBigQuery(result, name, dateContext, category, logger, isAlertComputation = false) {
-    // Skip if BigQuery is disabled via environment variable
-    if (process.env.BIGQUERY_ENABLED === 'false') {
-        return;
-    }
-    
+    if (process.env.BIGQUERY_ENABLED === 'false') return;
+
     try {
-        // Size check: BigQuery streaming inserts have a 10MB limit per row
-        // Estimate size by stringifying the result
-        const estimatedSize = JSON.stringify(result).length;
-        const MAX_BIGQUERY_ROW_SIZE = 9 * 1024 * 1024; // 9MB safety limit (10MB is hard limit)
-        
-        if (estimatedSize > MAX_BIGQUERY_ROW_SIZE) {
-            if (logger) {
-                logger.log('WARN', `[BigQuery] Skipping ${name} (${dateContext}): Result too large for streaming (${(estimatedSize/1024/1024).toFixed(2)}MB). Data is in GCS/Firestore.`);
-            }
-            // Return early - don't attempt insert that will fail
-            // The data is still available in Firestore/GCS, so this is acceptable
-            return;
-        }
-        
-        // Ensure table exists
+        const { ensureComputationResultsTable, insertRowsWithMerge } = require('../../core/utils/bigquery_utils');
         await ensureComputationResultsTable(logger);
-        
-        // Extract metadata (cids if present)
-        const metadata = {};
-        if (result.cids && Array.isArray(result.cids)) {
-            metadata.cids = result.cids;
-        }
-        
-        // Prepare row for BigQuery
+
+        // Simple metadata extraction
+        const metadata = result.cids && Array.isArray(result.cids) ? { cids: result.cids } : null;
+
         const row = {
             date: dateContext,
             computation_name: name,
             category: category,
-            result_data: result, // Full result as JSON
-            metadata: Object.keys(metadata).length > 0 ? metadata : null,
+            result_data: result, // BigQuery handles JSON wrapping
+            metadata: metadata,
             created_at: new Date().toISOString()
         };
-        
+
         const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
         
-        // Use MERGE operation to overwrite existing results (by date + computation_name + category)
-        // This ensures re-running a computation overwrites the old result
-        // Key fields: date, computation_name, category (ignoring created_at)
-        const { insertRowsWithMerge } = require('../../core/utils/bigquery_utils');
-        const keyFields = ['date', 'computation_name', 'category'];
-        
-        // For alert computations, we still want to use MERGE but it will use load jobs (free)
-        // This ensures overwrites work correctly for both alert and non-alert computations
-        await insertRowsWithMerge(datasetId, 'computation_results', [row], keyFields, logger);
-        
-    } catch (error) {
-        // Log but don't throw - BigQuery write failure shouldn't break Firestore writes
-        if (logger) {
-            logger.log('WARN', `[BigQuery] Write failed for ${name} (${dateContext}): ${error.message}`);
-        }
-        // Don't re-throw - ensure Firestore writes always succeed
-    }
-}
-
-async function cleanupOldShards(docRef, name, config, deps, metrics) {
-    const shardCol = docRef.collection('_shards');
-    const shardDocs = await shardCol.listDocuments();
-    if (shardDocs.length > 0) {
-        const updates = shardDocs.map(d => ({ type: 'DELETE', ref: d }));
-        await commitBatchInChunks(config, deps, updates, `${name}::CleanupOldShards`);
-        if (metrics && metrics.io) metrics.io.deletes += updates.length;
-    }
-}
-
-async function prepareAutoShardedWrites(result, docRef, logger, maxBytes = 900 * 1024, maxKeys = null, startShardIndex = 0, flushMode = 'STANDARD', expireAt = null) {
-    const OVERHEAD_ALLOWANCE = 20 * 1024; const CHUNK_LIMIT = maxBytes - OVERHEAD_ALLOWANCE;
-    const totalSize = calculateFirestoreBytes(result); const docPathSize = Buffer.byteLength(docRef.path, 'utf8') + 16;
-    const writes = []; const shardCollection = docRef.collection('_shards');
-    let currentChunk = {}; let currentChunkSize = 0; let currentKeyCount = 0; 
-    let shardIndex = startShardIndex;
-
-    const injectTTL = (data) => expireAt ? { ...data, _expireAt: expireAt } : data;
+        // Always use merge to ensure idempotency (overwrite previous run for same date/calc)
+        await insertRowsWithMerge(datasetId, 'computation_results', [row], ['date', 'computation_name', 'category'], logger);
 
-    if (!maxKeys && (totalSize + docPathSize) < CHUNK_LIMIT && flushMode === 'STANDARD' && startShardIndex === 0) {  
-        const data = { ...result, _completed: true, _sharded: false, _lastUpdated: new Date().toISOString() };
-        return [{ ref: docRef, data: injectTTL(data), options: { merge: true } }];  
-    }
-
-    for (const [key, value] of Object.entries(result)) { 
-        if (key.startsWith('_')) continue;
-        const keySize = Buffer.byteLength(key, 'utf8') + 1; const valueSize = calculateFirestoreBytes(value); const itemSize = keySize + valueSize;
-        const byteLimitReached = (currentChunkSize + itemSize > CHUNK_LIMIT); const keyLimitReached = (maxKeys && currentKeyCount + 1 >= maxKeys);
-        
-        if (byteLimitReached || keyLimitReached) { 
-            const chunkData = injectTTL(currentChunk);
-            writes.push({ ref: shardCollection.doc(`shard_${shardIndex}`), data: chunkData, options: { merge: false } }); 
-            shardIndex++; currentChunk = {}; currentChunkSize = 0; currentKeyCount = 0;
-        }
-        currentChunk[key] = value; currentChunkSize += itemSize; currentKeyCount++;
-    }
-    
-    if (Object.keys(currentChunk).length > 0) { 
-        const chunkData = injectTTL(currentChunk);
-        writes.push({ ref: shardCollection.doc(`shard_${shardIndex}`), data: chunkData, options: { merge: false } });
-        shardIndex++; 
-    }
-    
-    if (flushMode !== 'INTERMEDIATE') {
-        const pointerData = { 
-            _completed: true, 
-            _sharded: true, 
-            _shardCount: shardIndex, 
-            _lastUpdated: new Date().toISOString() 
-        };
-        writes.push({ ref: docRef, data: injectTTL(pointerData), options: { merge: true } }); 
+    } catch (error) {
+        if (logger) logger.log('WARN', `[BigQuery] Write failed for ${name}: ${error.message}`);
+        // Do not throw; we don't want to crash the computation pipeline if metrics fail
     }
-
-    return writes;
 }
 
-async function deleteOldCalculationData(dateStr, oldCategory, calcName, config, deps) {
-    const { db, logger, calculationUtils } = deps;
-    const { withRetry } = calculationUtils || { withRetry: (fn) => fn() };
+async function batchFetchSimHashes(db, hashes) {
+    if (!hashes || hashes.length === 0) return {};
+    const map = {};
+    const refs = hashes.map(h => db.collection(SIMHASH_REGISTRY_COLLECTION).doc(h));
     try {
-        const oldDocRef = db.collection(config.resultsCollection).doc(dateStr).collection(config.resultsSubcollection).doc(oldCategory).collection(config.computationsSubcollection).doc(calcName);
-        
-        const batch = db.batch(); let ops = 0;
-
-        // Clean up 'pages' subcollection if it exists (for Page Mode)
-        const pagesCol = oldDocRef.collection('pages');
-        const pageDocs = await withRetry(() => pagesCol.listDocuments(), 'ListOldPages');
-        for (const pDoc of pageDocs) { batch.delete(pDoc); ops++; }
-
-        // Clean up '_shards' subcollection (for Standard Mode)
-        const shardsCol = oldDocRef.collection('_shards');
-        const shardsSnap = await withRetry(() => shardsCol.listDocuments(), 'ListOldShards');
-        
-        for (const shardDoc of shardsSnap) { batch.delete(shardDoc); ops++; }
-        
-        batch.delete(oldDocRef); ops++;
-        
-        await withRetry(() => batch.commit(), 'CleanupOldCategory');
-        logger.log('INFO', `[Migration] Cleaned up ${ops} docs for ${calcName} in '${oldCategory}'`);
-    } catch (e) { logger.log('WARN', `[Migration] Failed to clean up ${calcName}: ${e.message}`); }
+        const snaps = await db.getAll(...refs);
+        snaps.forEach(snap => { if (snap.exists) map[snap.id] = snap.data().simHash; });
+    } catch (e) {}
+    return map;
 }
 
-function calculateFirestoreBytes(value) {
-    if (value === null) return 1; if (value === undefined) return 0; if (typeof value === 'boolean') return 1; if (typeof value === 'number') return 8; if (typeof value === 'string') return Buffer.byteLength(value, 'utf8') + 1; if (value instanceof Date) return 8; if (value.constructor && value.constructor.name === 'DocumentReference') { return Buffer.byteLength(value.path, 'utf8') + 16; }
-    if (Array.isArray(value)) { let sum = 0; for (const item of value) sum += calculateFirestoreBytes(item); return sum; }
-    if (typeof value === 'object') { let sum = 0; for (const k in value) { if (Object.prototype.hasOwnProperty.call(value, k)) { sum += (Buffer.byteLength(k, 'utf8') + 1) + calculateFirestoreBytes(value[k]); } } return sum; } return 0;
+async function fetchContracts(db, calcNames) {
+    if (!calcNames || calcNames.length === 0) return {};
+    const map = {};
+    const refs = calcNames.map(name => db.collection(CONTRACTS_COLLECTION).doc(name));
+    try {
+        const snaps = await db.getAll(...refs);
+        snaps.forEach(snap => { if (snap.exists) map[snap.id] = snap.data(); });
+    } catch (e) {}
+    return map;
 }
 
 function calculateExpirationDate(dateStr, ttlDays) {
-    // Validate inputs
-    if (!dateStr || typeof dateStr !== 'string') {
-        return null; // Invalid date string
-    }
-    
-    if (ttlDays === undefined || ttlDays === null || isNaN(Number(ttlDays))) {
-        return null; // Invalid TTL days
-    }
-    
+    if (!dateStr || !ttlDays || isNaN(Number(ttlDays))) return null;
     const base = new Date(dateStr);
-    
-    // Check if date is valid (invalid dates have NaN getTime())
-    if (isNaN(base.getTime())) {
-        return null; // Invalid date
-    }
-    
+    if (isNaN(base.getTime())) return null;
     base.setDate(base.getDate() + Number(ttlDays));
-    
-    // Double-check the result is still valid
-    if (isNaN(base.getTime())) {
-        return null; // Resulting date is invalid
-    }
-    
     return base;
 }