bulltrackers-module 1.0.295 → 1.0.296

package/functions/generic-api/admin-api/index.js

@@ -19,130 +19,129 @@ const createAdminRouter = (config, dependencies, unifiedCalculations) => {
     const router = express.Router();
     const { db, logger } = dependencies;
 
+    // Helper to get fresh manifest
+    const getFullManifest = () => getManifest([], unifiedCalculations, dependencies);
+
     // --- 1. TOPOLOGY VISUALIZER ---
-    // Returns nodes/edges for React Flow or Cytoscape
     router.get('/topology', async (req, res) => {
         try {
-            // Build manifest using the INJECTED calculations object
-            // Passing [] for productLines ensures we get the FULL graph
-            const manifest = getManifest([], unifiedCalculations, dependencies);
-            
+            const manifest = getFullManifest();
             const nodes = [];
             const edges = [];
 
             manifest.forEach(calc => {
-                // Nodes
                 nodes.push({
                     id: calc.name,
                     data: { 
                         label: calc.name,
                         layer: calc.category,
-                        pass: calc.pass, // Visualization can group columns by Pass
+                        pass: calc.pass,
                         isHistorical: calc.isHistorical,
                         type: calc.type
                     },
-                    position: { x: 0, y: 0 } // Frontend handles layout (e.g. Dagre)
+                    position: { x: 0, y: 0 }
                 });
 
-                // Dependency Edges (Calc -> Calc)
                 if (calc.dependencies) {
                     calc.dependencies.forEach(dep => {
                         edges.push({
                             id: `e-${dep}-${calc.name}`,
                             source: normalizeName(dep),
                             target: calc.name,
-                            type: 'default',
-                            animated: false
+                            type: 'smoothstep'
                         });
                     });
                 }
                 
-                // Root Data Edges (Data -> Calc)
                 if (calc.rootDataDependencies) {
                     calc.rootDataDependencies.forEach(root => {
-                        // Ensure a node exists for the root data type
                         const rootId = `ROOT_${root.toUpperCase()}`;
                         if (!nodes.find(n => n.id === rootId)) {
                             nodes.push({
                                 id: rootId,
-                                type: 'input', // Special React Flow type
+                                type: 'input',
                                 data: { label: `${root.toUpperCase()} DB` },
-                                position: { x: 0, y: 0 }
+                                position: { x: 0, y: 0 },
+                                style: { background: '#f0f0f0', border: '1px solid #777' }
                             });
                         }
-
                         edges.push({
                             id: `e-root-${root}-${calc.name}`,
                             source: rootId,
                             target: calc.name,
                             animated: true,
-                            style: { stroke: '#ff0072' } // Highlight data flow
+                            style: { stroke: '#ff0072' }
                         });
                     });
                 }
             });
 
-            res.json({ 
-                summary: { 
-                    totalNodes: nodes.length, 
-                    totalEdges: edges.length 
-                }, 
-                nodes, 
-                edges 
-            });
+            res.json({ summary: { totalNodes: nodes.length, totalEdges: edges.length }, nodes, edges });
         } catch (e) {
             logger.log('ERROR', '[AdminAPI] Topology build failed', e);
             res.status(500).json({ error: e.message });
         }
     });
 
-    // --- 2. STATUS MATRIX (Calendar View) ---
-    // ?start=2023-01-01&end=2023-01-30
+    // --- 2. STATUS MATRIX (Calendar / State UI) ---
+    // Returns status of ALL computations across a date range.
+    // ENHANCED: Cross-references Manifest to detect "PENDING" (Not run yet) vs "MISSING".
     router.get('/matrix', async (req, res) => {
         const { start, end } = req.query;
-        if (!start || !end) return res.status(400).json({ error: "Start (YYYY-MM-DD) and End dates required." });
+        if (!start || !end) return res.status(400).json({ error: "Start and End dates required." });
 
         try {
-            const startDate = new Date(start);
-            const endDate = new Date(end);
+            const startDate = new Date(String(start));
+            const endDate = new Date(String(end));
             const dates = [];
-            
-            // Generate date range
             for (let d = new Date(startDate); d <= endDate; d.setDate(d.getDate() + 1)) {
                 dates.push(d.toISOString().slice(0, 10));
             }
 
-            const limit = pLimit(20); // Concurrent Firestore reads
+            const manifest = getFullManifest();
+            const allCalcNames = new Set(manifest.map(c => c.name));
+
+            const limit = pLimit(20);
             const matrix = {};
 
             await Promise.all(dates.map(date => limit(async () => {
-                // Fetch Global Status and Root Data Availability
+                // Fetch Status and Root Data Availability
                 const [statusSnap, rootSnap] = await Promise.all([
                     db.collection('computation_status').doc(date).get(),
                     db.collection('system_root_data_index').doc(date).get()
                 ]);
 
-                // Flatten status for frontend (calcName -> { status: 'COMPLETED' | 'IMPOSSIBLE' })
                 const statusData = statusSnap.exists ? statusSnap.data() : {};
                 const rootData = rootSnap.exists ? rootSnap.data() : { status: { hasPortfolio: false } };
 
-                // Clean up status map
-                const cleanStatus = {};
-                Object.keys(statusData).forEach(key => {
-                    const entry = statusData[key];
-                    if (typeof entry === 'object') {
-                        if (entry.hash && entry.hash.startsWith('IMPOSSIBLE')) cleanStatus[key] = 'IMPOSSIBLE';
-                        else cleanStatus[key] = 'COMPLETED';
+                const dateStatus = {};
+
+                // Check every calculation in the Manifest
+                allCalcNames.forEach(calcName => {
+                    const entry = statusData[calcName];
+                    
+                    if (!entry) {
+                        // If root data exists but calc is missing -> PENDING
+                        // If no root data -> WAITING_DATA
+                        dateStatus[calcName] = rootData.status?.hasPortfolio ? 'PENDING' : 'WAITING_DATA';
+                    } else if (typeof entry === 'object') {
+                        if (entry.hash && typeof entry.hash === 'string' && entry.hash.startsWith('IMPOSSIBLE')) {
+                            dateStatus[calcName] = 'IMPOSSIBLE';
+                        } else if (entry.hash === false) {
+                            dateStatus[calcName] = 'BLOCKED';
+                        } else {
+                            dateStatus[calcName] = 'COMPLETED';
+                        }
                     } else if (entry === 'IMPOSSIBLE') {
-                        cleanStatus[key] = 'IMPOSSIBLE';
-                    } else if (entry === true || typeof entry === 'string') {
-                        cleanStatus[key] = 'COMPLETED';
+                        dateStatus[calcName] = 'IMPOSSIBLE';
+                    } else {
+                        dateStatus[calcName] = 'COMPLETED';
                     }
                 });
 
                 matrix[date] = {
-                    dataAvailable: rootData.status || {}, // e.g. { hasPortfolio: true }
-                    calculations: cleanStatus
+                    dataAvailable: rootData.status || {},
+                    calculations: dateStatus
                 };
             })));
 
@@ -153,74 +152,227 @@ const createAdminRouter = (config, dependencies, unifiedCalculations) => {
         }
     });
 
-    // --- 3. FLIGHT RECORDER (Inspection) ---
-    // Look up execution details for a specific Calc + Date
-    router.get('/inspect/:date/:calcName', async (req, res) => {
-        const { date, calcName } = req.params;
+    // --- 3. PIPELINE STATE (Progress Bar) ---
+    // Shows realtime status of the 5-pass system for a specific date
+    router.get('/pipeline/state', async (req, res) => {
+        const { date } = req.query;
+        if (!date) return res.status(400).json({ error: "Date required" });
+
         try {
-            // We search across all potential passes (1-5) because we might not know which one it belongs to
             const passes = ['1', '2', '3', '4', '5'];
-            let executionRecord = null;
+            const state = await Promise.all(passes.map(async (pass) => {
+                // We use the Audit Ledger which is the source of truth for execution state
+                const tasksSnap = await db.collection(`computation_audit_ledger/${date}/passes/${pass}/tasks`).get();
+                
+                const stats = {
+                    pending: 0,
+                    inProgress: 0,
+                    completed: 0,
+                    failed: 0,
+                    totalMemoryMB: 0,
+                    avgDurationMs: 0
+                };
+                
+                const durations = [];
+                
+                tasksSnap.forEach(doc => {
+                    const data = doc.data();
+                    const s = (data.status || 'UNKNOWN').toLowerCase();
+                    if (stats[s] !== undefined) stats[s]++;
+                    else stats[s] = 1;
 
-            // Run in parallel to find the record fast
-            await Promise.all(passes.map(async (pass) => {
-                if (executionRecord) return; // Optimization
-                const ref = db.doc(`computation_audit_ledger/${date}/passes/${pass}/tasks/${calcName}`);
-                const snap = await ref.get();
-                if (snap.exists) {
-                    executionRecord = { pass, ...snap.data() };
-                }
+                    if (data.telemetry?.lastMemory?.rssMB) {
+                        stats.totalMemoryMB += data.telemetry.lastMemory.rssMB;
+                    }
+                    if (data.completedAt && data.startedAt) {
+                        durations.push(new Date(data.completedAt).getTime() - new Date(data.startedAt).getTime());
+                    }
+                });
+
+                stats.avgDurationMs = durations.length ? 
+                    Math.round(durations.reduce((a, b) => a + b, 0) / durations.length) : 0;
+
+                return { pass, stats, totalTasks: tasksSnap.size };
             }));
 
-            if (!executionRecord) {
-                return res.status(404).json({ 
-                    status: 'NOT_FOUND', 
-                    message: `No execution record found in ledger for ${calcName} on ${date}` 
+            res.json({ date, passes: state });
+        } catch (e) {
+            res.status(500).json({ error: e.message });
+        }
+    });
+
+    // --- 4. DEPENDENCY TRACER (Blast Radius) ---
+    router.get('/trace/:calcName', async (req, res) => {
+        const { calcName } = req.params;
+        const mode = req.query.mode || 'downstream'; // 'upstream' or 'downstream'
+        
+        try {
+            const manifest = getFullManifest();
+            const manifestMap = new Map(manifest.map(c => [c.name, c]));
+            
+            if (!manifestMap.has(calcName)) return res.status(404).json({ error: 'Calculation not found' });
+
+            const trace = { root: calcName, chain: [] };
+
+            if (mode === 'upstream') {
+                // What does X depend on?
+                const visited = new Set();
+                const walk = (name, depth = 0) => {
+                    if (visited.has(name) || depth > 10) return;
+                    visited.add(name);
+                    const calc = manifestMap.get(name);
+                    if (!calc) return;
+                    
+                    trace.chain.push({ 
+                        name, depth, pass: calc.pass, type: calc.type 
+                    });
+                    
+                    calc.dependencies?.forEach(dep => walk(dep, depth + 1));
+                };
+                walk(calcName);
+            } else {
+                // What depends on X? (Downstream / Impact)
+                const reverseGraph = new Map();
+                manifest.forEach(c => {
+                    c.dependencies?.forEach(dep => {
+                        const normDep = normalizeName(dep);
+                        if (!reverseGraph.has(normDep)) reverseGraph.set(normDep, []);
+                        reverseGraph.get(normDep).push(c.name);
+                    });
                 });
+
+                const visited = new Set();
+                const walk = (name, depth = 0) => {
+                    if (visited.has(name) || depth > 10) return;
+                    visited.add(name);
+                    
+                    const calc = manifestMap.get(name);
+                    trace.chain.push({ 
+                        name, depth, pass: calc?.pass 
+                    });
+
+                    reverseGraph.get(name)?.forEach(child => walk(child, depth + 1));
+                };
+                walk(calcName);
             }
 
-            // Also fetch the "Contract" if it exists (for volatility analysis)
-            const contractSnap = await db.collection('system_contracts').doc(calcName).get();
-            
-            res.json({
-                execution: executionRecord,
-                contract: contractSnap.exists ? contractSnap.data() : null
+            res.json(trace);
+        } catch (e) {
+            res.status(500).json({ error: e.message });
+        }
+    });
+
+    // --- 5. CONTRACT VIOLATIONS (Quality Gate) ---
+    router.get('/violations', async (req, res) => {
+        const days = parseInt(String(req.query.days)) || 7;
+        const cutoff = new Date();
+        cutoff.setDate(cutoff.getDate() - days);
+
+        try {
+            // Check DLQ for Semantic Failures (Hard Violations)
+            const dlqSnap = await db.collection('computation_dead_letter_queue')
+                .where('finalAttemptAt', '>', cutoff)
+                .where('error.stage', '==', 'SEMANTIC_GATE')
+                .limit(50)
+                .get();
+
+            const violations = [];
+            dlqSnap.forEach(doc => {
+                const data = doc.data();
+                violations.push({
+                    id: doc.id,
+                    computation: data.originalData.computation,
+                    date: data.originalData.date,
+                    reason: data.error.message,
+                    type: 'HARD_VIOLATION',
+                    timestamp: data.finalAttemptAt
+                });
+            });
+
+            // Check Audit Logs for Soft Anomalies (Statistical warnings)
+            const anomalySnap = await db.collectionGroup('history')
+                .where('triggerTime', '>', cutoff.toISOString())
+                .where('anomalies', '!=', []) // Firestore != operator
+                .limit(50)
+                .get();
+
+            anomalySnap.forEach(doc => {
+                const data = doc.data();
+                data.anomalies?.forEach(anomaly => {
+                    violations.push({
+                        id: doc.id,
+                        computation: data.computationName,
+                        date: data.targetDate,
+                        reason: anomaly,
+                        type: 'SOFT_ANOMALY',
+                        timestamp: data.triggerTime
+                    });
+                });
             });
 
+            // Sort by time desc
+            violations.sort((a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime());
+
+            res.json({ count: violations.length, violations });
         } catch (e) {
-            logger.log('ERROR', `[AdminAPI] Inspect failed for ${calcName}`, e);
             res.status(500).json({ error: e.message });
         }
     });
 
-    // --- 4. ANOMALY DETECTOR ---
-    // Finds recent crashes and chronic failures
-    router.get('/anomalies', async (req, res) => {
+    // --- 6. MEMORY HOTSPOTS (Forensics) ---
+    router.get('/memory/hotspots', async (req, res) => {
+        const thresholdMB = parseInt(String(req.query.threshold)) || 1000; // 1GB default
+        
         try {
-            const [dlqSnap, statsSnap] = await Promise.all([
-                db.collection('computation_dead_letter_queue').orderBy('finalAttemptAt', 'desc').limit(50).get(),
-                db.collection('computation_audit_logs').orderBy('failureCount', 'desc').limit(20).get()
-            ]);
-
-            const recentCrashes = [];
-            dlqSnap.forEach(doc => recentCrashes.push({ id: doc.id, ...doc.data() }));
-
-            const chronicFailures = [];
-            statsSnap.forEach(doc => {
-                const d = doc.data();
-                if (d.failureCount > 0) {
-                    chronicFailures.push({
-                        computation: doc.id,
-                        failures: d.failureCount,
-                        successes: d.successCount || 0,
-                        lastError: d.lastRunStatus
-                    });
-                }
+            // Ledger tasks maintain 'telemetry.lastMemory'
+            // We use collectionGroup to search across all dates/passes
+            const snapshot = await db.collectionGroup('tasks')
+                .where('telemetry.lastMemory.rssMB', '>', thresholdMB)
+                .orderBy('telemetry.lastMemory.rssMB', 'desc')
+                .limit(20)
+                .get();
+
+            const hotspots = [];
+            snapshot.forEach(doc => {
+                const data = doc.data();
+                hotspots.push({
+                    computation: data.computation,
+                    rssMB: data.telemetry.lastMemory.rssMB,
+                    heapMB: data.telemetry.lastMemory.heapUsedMB,
+                    status: data.status,
+                    worker: data.workerId,
+                    date: doc.ref.parent.parent.parent.parent.id // traversing path to get date
+                });
             });
+            
+            res.json({ count: hotspots.length, hotspots });
+        } catch (e) {
+            res.status(500).json({ error: e.message });
+        }
+    });
+
+    // --- 7. FLIGHT RECORDER (Inspection) ---
+    // Existing inspection endpoint kept for drill-down
+    router.get('/inspect/:date/:calcName', async (req, res) => {
+        const { date, calcName } = req.params;
+        try {
+            const passes = ['1', '2', '3', '4', '5'];
+            let executionRecord = null;
 
+            await Promise.all(passes.map(async (pass) => {
+                if (executionRecord) return;
+                const ref = db.doc(`computation_audit_ledger/${date}/passes/${pass}/tasks/${calcName}`);
+                const snap = await ref.get();
+                if (snap.exists) executionRecord = { pass, ...snap.data() };
+            }));
+
+            if (!executionRecord) return res.status(404).json({ status: 'NOT_FOUND' });
+
+            const contractSnap = await db.collection('system_contracts').doc(calcName).get();
+            
             res.json({
-                recentCrashes,
-                chronicFailures
+                execution: executionRecord,
+                contract: contractSnap.exists ? contractSnap.data() : null
             });
         } catch (e) {
             res.status(500).json({ error: e.message });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.295",
+  "version": "1.0.296",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [