bulltrackers-module 1.0.731 → 1.0.733

package/functions/computation-system/helpers/computation_worker.js

@@ -1,375 +0,0 @@
-/**
- * FILENAME: computation-system/helpers/computation_worker.js
- * UPDATED: Implements Google Cloud Monitoring Heartbeats.
- * UPDATED: Implements Structured Logging with Trace Context.
- * FIXED: 'manifest' scope issue (declared outside try/catch).
- */
-
-const { executeDispatchTask } = require('../WorkflowOrchestrator.js');
-const { getManifest }         = require('../topology/ManifestLoader');
-const { StructuredLogger }    = require('../logger/logger'); 
-const { recordRunAttempt }    = require('../persistence/RunRecorder'); 
-const { normalizeName }       = require('../utils/utils');
-const os                      = require('os');
-const monitoring              = require('@google-cloud/monitoring'); // [NEW]
-const monConfig               = require('../config/monitoring_config'); // [NEW]
-
-// Initialize the Google Monitoring Client once
-const metricClient = new monitoring.MetricServiceClient();
-
-let calculationPackage;
-try { calculationPackage = require('aiden-shared-calculations-unified'); } catch (e) { throw e; }
-const calculations = calculationPackage.calculations;
-
-const MAX_RETRIES = 3;
-const STALE_LOCK_THRESHOLD_MS = 1000 * 60 * 15; // 15 minutes
-
-// =============================================================================
-// HELPER: Firestore Timestamp Conversion
-// =============================================================================
-/**
- * Converts a Firestore Timestamp or Date to milliseconds.
- * Firestore stores Date objects as Timestamp objects, which have a .toDate() method.
- * This function handles both cases correctly.
- * @param {any} field - Firestore Timestamp, Date object, or string
- * @returns {number} Milliseconds since epoch, or 0 if invalid
- */
-function getMillis(field) {
-    if (!field) return 0;
-    // Handle Firestore Timestamp (has .toDate() method)
-    if (field.toDate && typeof field.toDate === 'function') {
-        return field.toDate().getTime();
-    }
-    // Handle standard Date object or string
-    const date = new Date(field);
-    return isNaN(date.getTime()) ? 0 : date.getTime();
-}
-
-// [NEW] Helper to push metric to Google Cloud
-async function pushMetric(type, value, labels) {
-    if (!monConfig.enabled) return;
-    try {
-        const dataPoint = {
-            interval: { endTime: { seconds: Date.now() / 1000 } },
-            value: { int64Value: value }, // Assumes memory in MB
-        };
-
-        const timeSeriesData = {
-            name: metricClient.projectPath(monConfig.project),
-            timeSeries: [{
-                metric: {
-                    type: type,
-                    labels: labels,
-                },
-                resource: {
-                    type: 'global',
-                    labels: { project_id: monConfig.project },
-                },
-                points: [dataPoint],
-            }],
-        };
-
-        // Fire and forget (don't await to avoid blocking compute)
-        metricClient.createTimeSeries(timeSeriesData).catch(err => console.error('[Monitoring] Push failed', err.message));
-    } catch (e) { /* Ignore setup errors */ }
-}
-
-function startMemoryHeartbeat(db, ledgerPath, workerId, computationName, traceId, intervalMs = 5000) {
-    let peakRss = 0;
-    
-    // Firestore Heartbeat (Legacy/State)
-    const firestoreTimer = setInterval(async () => {
-        const rssMB = Math.round(process.memoryUsage().rss / 1024 / 1024);
-        if (rssMB > peakRss) peakRss = rssMB;
-        await db.doc(ledgerPath).update({ 
-            'telemetry.lastMemoryMB': rssMB, 
-            'telemetry.lastHeartbeat': new Date() 
-        }).catch(() => {});
-    }, 2000); // 2s for DB
-
-    // Google Monitoring Heartbeat (New/Out-of-Band)
-    const monitoringTimer = setInterval(async () => {
-        const rssMB = Math.round(process.memoryUsage().rss / 1024 / 1024);
-        
-        // Push Memory Metric
-        pushMetric(monConfig.metrics.memory, rssMB, {
-            worker_id: workerId,
-            computation: computationName,
-            // Including trace_id in metric labels allows strict correlation 
-            // BUT can cause high cardinality. Use with caution or only for debugging.
-            // Per user request, we include it to "attach eyeball".
-            trace_id: traceId || 'unknown' 
-        });
-
-    }, intervalMs); // 5s for API
-
-    firestoreTimer.unref();
-    monitoringTimer.unref();
-
-    return { 
-        stop: () => { clearInterval(firestoreTimer); clearInterval(monitoringTimer); },
-        getPeak: () => peakRss 
-    };
-}
-
-/**
- * STRICT IDEMPOTENCY GATE
- */
-async function checkIdempotencyAndClaimLease(db, ledgerPath, dispatchId, workerId, onDemand = false) {
-    const docRef = db.doc(ledgerPath);
-
-    try {
-        return await db.runTransaction(async (t) => {
-            const doc = await t.get(docRef);
-            
-            if (doc.exists) {
-                const data = doc.data();
-                
-                // [ON-DEMAND OVERRIDE] If this is an on-demand request, force re-run even if completed
-                if (onDemand && ['COMPLETED', 'FAILED', 'CRASH'].includes(data.status)) {
-                    // Force overwrite - on-demand requests should always run
-                    // Log will be handled by caller
-                    // Continue to claim lease below
-                } else {
-                    // [FIX] Only block if it's the EXACT SAME Dispatch ID (Duplicate Delivery)
-                    if (['COMPLETED', 'FAILED', 'CRASH'].includes(data.status)) {
-                        if (data.dispatchId === dispatchId) {
-                            return { shouldRun: false, reason: `Task already in terminal state: ${data.status}` };
-                        }
-                        // If dispatchId differs, we allow the overwrite (Re-Run).
-                        // The Dispatcher is the authority; if it sent a message, we run it.
-                    }
-                }
-
-                if (data.status === 'IN_PROGRESS' && data.dispatchId === dispatchId) {
-                    return { shouldRun: false, reason: 'Duplicate delivery: Task already IN_PROGRESS with same ID.' };
-                }
-                if (data.status === 'IN_PROGRESS' && !onDemand) {
-                    // CRITICAL FIX: Check if the lock is stale before blocking
-                    // If the task has been IN_PROGRESS for more than 15 minutes, break the lock
-                    const lastActivityTime = getMillis(data.telemetry?.lastHeartbeat) || getMillis(data.startedAt);
-                    const lockAge = Date.now() - lastActivityTime;
-                    
-                    if (lockAge > STALE_LOCK_THRESHOLD_MS) {
-                        // Stale lock detected - break it and allow this worker to claim it
-                        // The transaction will overwrite the stale lock below
-                        // Note: We can't log here because we're in a transaction, but the caller will log
-                        // Continue to claim lease below
-                    } else {
-                        // Valid active lock - wait for it to complete
-                        return { shouldRun: false, reason: 'Collision: Task currently IN_PROGRESS by another worker.' };
-                    }
-                }
-            }
-
-            const lease = {
-                status: 'IN_PROGRESS',
-                workerId: workerId,
-                dispatchId: dispatchId || 'unknown',
-                startedAt: new Date(),
-                onDemand: onDemand || false
-            };
-            
-            t.set(docRef, lease, { merge: true });
-            return { shouldRun: true, leaseData: lease };
-        });
-    } catch (e) {
-        return { shouldRun: false, reason: `Transaction Error: ${e.message}` };
-    }
-}
-
-async function handleComputationTask(message, config, dependencies) {
-    // 1. Parse Message
-    let data;
-    try {
-        const raw = message.data?.message?.data || message.data || message.json;
-        data = (typeof raw === 'string') ? JSON.parse(Buffer.from(raw, 'base64').toString()) : raw;
-    } catch (e) { return; }
-
-    if (!data || data.action !== 'RUN_COMPUTATION_DATE') return;
-    
-    const { date, pass, computation, previousCategory, triggerReason, dispatchId, dependencyResultHashes, resources, traceContext, metadata } = data;
-    const resourceTier = resources || 'standard';
-    const onDemand = metadata?.onDemand === true || false; // Extract on-demand flag
-    const ledgerPath = `computation_audit_ledger/${date}/passes/${pass}/tasks/${computation}`;
-    const workerId = process.env.K_REVISION || os.hostname();
-
-    // 2. Initialize Trace-Aware Logger (The "Eyeball")
-    const globalMetadata = {};
-    if (traceContext && monConfig.enabled) {
-        globalMetadata['logging.googleapis.com/trace'] = `projects/${monConfig.project}/traces/${traceContext.traceId}`;
-        globalMetadata['logging.googleapis.com/spanId'] = traceContext.spanId;
-        globalMetadata['logging.googleapis.com/trace_sampled'] = traceContext.sampled;
-    }
-
-    const logger = new StructuredLogger({ 
-        minLevel: config.minLevel || 'INFO', 
-        enableStructured: true, 
-        globalMetadata, 
-        ...config 
-    });
-    
-    const runDeps = { ...dependencies, logger };
-    const db = dependencies.db;
-
-    // --- STEP 1: IDEMPOTENCY CHECK (with on-demand override) ---
-    // Check for stale locks before the transaction (for logging purposes)
-    const preCheckSnap = await db.doc(ledgerPath).get();
-    let wasStaleLock = false;
-    if (preCheckSnap.exists) {
-        const preCheckData = preCheckSnap.data();
-        if (preCheckData.status === 'IN_PROGRESS' && preCheckData.workerId !== workerId) {
-            const lastActivityTime = getMillis(preCheckData.telemetry?.lastHeartbeat) || getMillis(preCheckData.startedAt);
-            const lockAge = Date.now() - lastActivityTime;
-            if (lockAge > STALE_LOCK_THRESHOLD_MS) {
-                wasStaleLock = true;
-                logger.log('WARN', `[Worker] 🧟 Breaking stale lock for ${computation}. Previous worker ${preCheckData.workerId} was inactive for ${Math.round(lockAge / 1000 / 60)} minutes.`);
-            }
-        }
-    }
-    
-    const gate = await checkIdempotencyAndClaimLease(db, ledgerPath, dispatchId, workerId, onDemand);
-    if (!gate.shouldRun) {
-        logger.log('WARN', `[Worker] 🛑 Idempotency Gate: Skipping ${computation}. Reason: ${gate.reason}`);
-        return;
-    }
-    
-    if (wasStaleLock) {
-        logger.log('INFO', `[Worker] ✅ Successfully claimed lease for ${computation} after breaking stale lock.`);
-    }
-    
-    if (onDemand) {
-        logger.log('INFO', `[Worker] 🔄 On-demand request: Forcing re-run of ${computation} for ${date}`);
-    }
-
-    logger.log('INFO', `[Worker] 📥 Task: ${computation} (${date}) [Tier: ${resourceTier}] [ID: ${dispatchId}]`);
-
-    // --- STEP 2: START DUAL HEARTBEATS ---
-    const heartbeats = startMemoryHeartbeat(db, ledgerPath, workerId, computation, traceContext?.traceId);
-
-    // [FIX] Declare manifest here so it is accessible in both try and catch blocks
-    let manifest;
-    const taskStartTime = Date.now(); // Declare outside try/catch for access in error handler
-
-    try {
-        manifest = getManifest(config.activeProductLines || [], calculations, runDeps);
-        const startTime = taskStartTime;
-        
-        const result = await executeDispatchTask(
-            date, pass, computation, config, runDeps, 
-            manifest, previousCategory, dependencyResultHashes,
-            metadata // Pass metadata (including targetCid) to orchestrator
-        );
-
-        heartbeats.stop();
-        const failureReport = result?.updates?.failureReport || [];
-        const successUpdates = result?.updates?.successUpdates || {};
-
-        if (failureReport.length > 0) {
-            const reportedError = failureReport[0].error;
-            const errorObj = new Error(reportedError.message);
-            errorObj.stage = reportedError.stage; 
-            throw errorObj;
-        }
-
-        const calcUpdate = successUpdates[normalizeName(computation)] || {};
-        const metrics = {
-            durationMs:    Date.now() - startTime,
-            peakMemoryMB:  heartbeats.getPeak(),
-            io:            calcUpdate.metrics?.io,
-            storage:       calcUpdate.metrics?.storage,
-            execution:     calcUpdate.metrics?.execution,
-            validation:    calcUpdate.metrics?.validation,
-            composition:   calcUpdate.composition
-        };
-
-        await db.doc(ledgerPath).update({ status: 'COMPLETED', completedAt: new Date() });
-        await recordRunAttempt(db, { date, computation, pass }, 'SUCCESS', null, metrics, triggerReason, resourceTier);
-        
-        const { notifyComputationComplete, getComputationDisplayName } = require('../../api-v2/helpers/notification_helpers.js');
-        // Send notification if this was an on-demand computation
-        if (metadata?.onDemand && metadata?.requestId && metadata?.requestingUserCid) {
-            try {
-                await notifyComputationComplete(
-                    dependencies.db,
-                    dependencies.logger,
-                    metadata.requestingUserCid,
-                    metadata.requestId,
-                    computation,
-                    getComputationDisplayName(computation),
-                    true,
-                    null,
-                    {
-                        collectionRegistry: dependencies.collectionRegistry,
-                        config: config,
-                        notificationType: 'userActionCompletions'
-                    }
-                );
-            } catch (notifError) {
-                // Non-critical, log and continue
-                if (dependencies.logger) {
-                    dependencies.logger.log('WARN', `[Worker] Failed to send completion notification for ${computation}`, notifError);
-                }
-            }
-        }
-
-    } catch (err) { 
-        heartbeats.stop();
-        
-        const isDeterministic = ['SHARDING_LIMIT_EXCEEDED', 'QUALITY_CIRCUIT_BREAKER', 'SEMANTIC_GATE'].includes(err.stage);
-        
-        // [FIX] Now we can safely access 'manifest' to find the current hash
-        let currentHash = 'unknown';
-        if (typeof manifest !== 'undefined' && Array.isArray(manifest)) {
-             const item = manifest.find(c => normalizeName(c.name) === normalizeName(computation));
-             if (item) currentHash = item.hash;
-        }
-
-        if (isDeterministic || (message.deliveryAttempt || 1) >= MAX_RETRIES) {
-            const errorPayload = { message: err.message, stage: err.stage || 'FATAL' };
-
-            // [UPDATED] We now save 'hash' and 'resourceTier' to the ledger on failure
-            await db.doc(ledgerPath).set({ 
-                status: 'FAILED', 
-                error: errorPayload, 
-                failedAt: new Date(),
-                hash: currentHash, // <--- CRITICAL FOR DISPATCHER DEAD-LETTER CHECK
-                resourceTier: resourceTier 
-            }, { merge: true });
-
-            await recordRunAttempt(db, { date, computation, pass }, 'FAILURE', { message: err.message, stage: err.stage || 'FATAL' }, { durationMs: Date.now() - taskStartTime, peakMemoryMB: heartbeats.getPeak() }, triggerReason, resourceTier);
-            
-            // Send error notification if this was an on-demand computation
-            if (metadata?.onDemand && metadata?.requestId && metadata?.requestingUserCid) {
-                try {
-                    await notifyComputationComplete(
-                        db,
-                        dependencies.logger,
-                        metadata.requestingUserCid,
-                        metadata.requestId,
-                        computation,
-                        getComputationDisplayName(computation),
-                        false,
-                        err.message,
-                        {
-                            collectionRegistry: dependencies.collectionRegistry,
-                            config: config,
-                            notificationType: 'userActionCompletions'
-                        }
-                    );
-                } catch (notifError) {
-                    // Non-critical, log and continue
-                    if (dependencies.logger) {
-                        dependencies.logger.log('WARN', `[Worker] Failed to send error notification for ${computation}`, notifError);
-                    }
-                }
-            }
-            
-            return;
-        }
-        
-        throw err;
-    }
-}
-
-module.exports = { handleComputationTask };

package/functions/computation-system/helpers/monitor.js

@@ -1,64 +0,0 @@
-/**
- * @fileoverview Monitor helper for Cloud Workflows.
- * Checks the state of the Audit Ledger to determine if a pass is complete.
- * This function is stateless and receives dependencies via injection.
- * THIS FILE IS NOW REDUNDANT, LOGIC HAS MOVED TO THE DISPATCHER AND WORKFLOW
- */
-
-/**
- * Checks the status of a specific computation pass.
- * @param {object} req - Express request object (query: date, pass).
- * @param {object} res - Express response object.
- * @param {object} dependencies - Contains db (Firestore), logger.
- */
-async function checkPassStatus(req, res, dependencies) {
-    const { db, logger } = dependencies;
-    const { date, pass } = req.query;
-
-    if (!date || !pass) {
-        return res.status(400).json({ error: "Missing 'date' or 'pass' query parameters." });
-    }
-
-    const ledgerPath = `computation_audit_ledger/${date}/passes/${pass}/tasks`;
-    logger.log('INFO', `[Monitor] Checking status for ${date} Pass ${pass} at ${ledgerPath}`);
-
-    try {
-        const tasksRef = db.collection(ledgerPath);
-
-        // 1. Check for Active Tasks (Blocking)
-        // If anything is PENDING or IN_PROGRESS, the system is still working.
-        const runningSnap = await tasksRef.where('status', 'in', ['PENDING', 'IN_PROGRESS']).get();
-        
-        if (!runningSnap.empty) {
-            logger.log('INFO', `[Monitor] Pass ${pass} is RUNNING. Active tasks: ${runningSnap.size}`);
-            return res.status(200).json({ 
-                state: 'RUNNING', 
-                activeCount: runningSnap.size 
-            });
-        }
-
-        // 2. Check for Failures (Retry Condition)
-        // If nothing is running, we check if anything ended in FAILED state.
-        // We consider these "retryable" by re-triggering the dispatcher.
-        const failedSnap = await tasksRef.where('status', '==', 'FAILED').get();
-        
-        if (!failedSnap.empty) {
-            logger.log('WARN', `[Monitor] Pass ${pass} finished with FAILURES. Count: ${failedSnap.size}`);
-            return res.status(200).json({ 
-                state: 'HAS_FAILURES', 
-                failureCount: failedSnap.size 
-            });
-        }
-
-        // 3. Clean Success
-        // No running tasks, no failed tasks.
-        logger.log('INFO', `[Monitor] Pass ${pass} COMPLETED successfully.`);
-        return res.status(200).json({ state: 'SUCCESS' });
-
-    } catch (error) {
-        logger.log('ERROR', `[Monitor] Failed to check status: ${error.message}`);
-        return res.status(500).json({ error: error.message });
-    }
-}
-
-module.exports = { checkPassStatus };

package/functions/computation-system/helpers/on_demand_helpers.js

@@ -1,154 +0,0 @@
-/**
- * @fileoverview Helpers for on-demand computation requests
- * Handles dependency chain resolution and ordered triggering
- */
-
-const { getManifest } = require('../topology/ManifestLoader');
-const { normalizeName } = require('../utils/utils');
-
-// Import calculations package (matching computation_worker.js pattern)
-let calculationPackage;
-try { 
-    calculationPackage = require('aiden-shared-calculations-unified'); 
-} catch (e) { 
-    throw new Error(`Failed to load calculations package: ${e.message}`); 
-}
-const calculations = calculationPackage.calculations;
-
-/**
- * Resolves all dependencies for a given computation and returns them grouped by pass
- * @param {string} computationName - The target computation name
- * @param {Array} manifest - The computation manifest
- * @returns {Array<{pass: number, computations: Array<string>}>} Dependencies grouped by pass
- */
-function resolveDependencyChain(computationName, manifest) {
-    const manifestMap = new Map(manifest.map(c => [normalizeName(c.name), c]));
-    const normalizedTarget = normalizeName(computationName);
-    
-    const targetCalc = manifestMap.get(normalizedTarget);
-    if (!targetCalc) {
-        throw new Error(`Computation ${computationName} not found in manifest`);
-    }
-    
-    // Build adjacency list (dependencies map)
-    const adjacency = new Map();
-    for (const calc of manifest) {
-        const normName = normalizeName(calc.name);
-        adjacency.set(normName, (calc.dependencies || []).map(d => normalizeName(d)));
-    }
-    
-    // Get all dependencies recursively
-    const required = new Set([normalizedTarget]);
-    const queue = [normalizedTarget];
-    
-    while (queue.length > 0) {
-        const calcName = queue.shift();
-        const dependencies = adjacency.get(calcName) || [];
-        for (const dep of dependencies) {
-            if (!required.has(dep)) {
-                required.add(dep);
-                queue.push(dep);
-            }
-        }
-    }
-    
-    // Group by pass
-    const byPass = new Map();
-    for (const calcName of required) {
-        const calc = manifestMap.get(calcName);
-        if (calc) {
-            const pass = calc.pass || 1;
-            if (!byPass.has(pass)) {
-                byPass.set(pass, []);
-            }
-            byPass.get(pass).push(calc.name); // Use original name, not normalized
-        }
-    }
-    
-    // Convert to sorted array
-    const passes = Array.from(byPass.entries())
-        .sort((a, b) => a[0] - b[0])
-        .map(([pass, computations]) => ({ pass, computations }));
-    
-    return passes;
-}
-
-/**
- * Triggers computations for a given date, handling dependencies in order
- * @param {string} targetComputation - The target computation to run
- * @param {string} date - The date string (YYYY-MM-DD)
- * @param {object} dependencies - Contains pubsub, logger, etc.
- * @param {object} config - Computation system config
- * @param {object} metadata - Additional metadata for the computation request
- * @returns {Promise<Array>} Array of triggered computation messages
- */
-async function triggerComputationWithDependencies(targetComputation, date, dependencies, config, metadata = {}) {
-    const { pubsub, logger } = dependencies;
-    // Use on-demand topic for on-demand computation requests
-    const computationTopic = config.computationTopicOnDemand || 
-                            (config.getComputationTopic && config.getComputationTopic(true)) ||
-                            'computation-tasks-ondemand';
-    const topic = pubsub.topic(computationTopic);
-    const crypto = require('crypto');
-    
-    // Get manifest to resolve dependencies
-    const manifest = getManifest(config.activeProductLines || [], calculations);
-    
-    // Resolve dependency chain
-    const dependencyPasses = resolveDependencyChain(targetComputation, manifest);
-    
-    logger.log('INFO', `[On-Demand] Resolved dependency chain for ${targetComputation}:`, {
-        totalPasses: dependencyPasses.length,
-        passes: dependencyPasses.map(p => `Pass ${p.pass}: ${p.computations.length} computations`)
-    });
-    
-    const triggeredMessages = [];
-    
-    // Trigger each pass in order
-    for (const passGroup of dependencyPasses) {
-        const { pass, computations } = passGroup;
-        
-        // Trigger all computations in this pass
-        for (const computation of computations) {
-            const dispatchId = crypto.randomUUID();
-            const isTarget = normalizeName(computation) === normalizeName(targetComputation);
-            
-            const computationMessage = {
-                action: 'RUN_COMPUTATION_DATE',
-                computation: computation,
-                date: date,
-                pass: String(pass),
-                dispatchId: dispatchId,
-                triggerReason: metadata.triggerReason || 'on_demand',
-                resources: metadata.resources || 'standard',
-                metadata: {
-                    ...metadata,
-                    onDemand: true,
-                    isTargetComputation: isTarget,
-                    targetCid: metadata.targetCid || null // Pass through targetCid for optimization
-                },
-                traceContext: {
-                    traceId: crypto.randomBytes(16).toString('hex'),
-                    spanId: crypto.randomBytes(8).toString('hex'),
-                    sampled: true
-                }
-            };
-            
-            await topic.publishMessage({
-                data: Buffer.from(JSON.stringify(computationMessage))
-            });
-            
-            triggeredMessages.push(computationMessage);
-            
-            logger.log('INFO', `[On-Demand] Triggered ${computation} (Pass ${pass})${isTarget ? ' [TARGET]' : ' [DEPENDENCY]'}`);
-        }
-    }
-    
-    return triggeredMessages;
-}
-
-module.exports = {
-    resolveDependencyChain,
-    triggerComputationWithDependencies
-};
-