bulltrackers-module 1.0.296 → 1.0.298

package/functions/computation-system/utils/utils.js

@@ -6,12 +6,14 @@ const { FieldValue, FieldPath } = require('@google-cloud/firestore');
 const crypto                    = require('crypto');
 
 // [NEW] Single Source of Truth for Data Availability
+// These are defaults/fallbacks. The dynamic function getEarliestDataDates below is the primary source.
 const DEFINITIVE_EARLIEST_DATES = {
-    portfolio: new Date('2025-09-25T00:00:00Z'),
-    history:   new Date('2025-11-05T00:00:00Z'),
-    social:    new Date('2025-10-30T00:00:00Z'),
-    insights:  new Date('2025-08-26T00:00:00Z'),
-    price:     new Date('2025-08-01T00:00:00Z')
+    portfolio: new Date('2025-01-01T00:00:00Z'),
+    history:   new Date('2025-08-01T00:00:00Z'),
+    social:    new Date('2025-08-01T00:00:00Z'),
+    insights:  new Date('2025-08-01T00:00:00Z'),
+    price:     new Date('2025-08-01T00:00:00Z'),
+    absoluteEarliest: new Date('2023-08-01T00:00:00Z')
 };
 
 /** Stage 1: Normalize a calculation name to kebab-case */
@@ -162,95 +164,75 @@ function getExpectedDateStrings(startDate, endDate) {
     return dateStrings;
 }
 
-/** Stage 4: Get the earliest date in a *flat* collection where doc IDs are dates. */
-async function getFirstDateFromSimpleCollection(config, deps, collectionName) {
-    const { db, logger } = deps;
-    try {
-        if (!collectionName) { logger.log('WARN', `[Core Utils] Collection name not provided for simple date query.`); return null; }
-        const query = db.collection(collectionName).where(FieldPath.documentId(), '>=', '2000-01-01').orderBy(FieldPath.documentId(), 'asc').limit(1);
-        const snapshot = await withRetry(() => query.get(), `GetEarliestDoc(${collectionName})`);
-        if (!snapshot.empty && /^\d{4}-\d{2}-\d{2}$/.test(snapshot.docs[0].id)) { return new Date(snapshot.docs[0].id + 'T00:00:00Z'); }
-    } catch (e) { logger.log('ERROR', `GetFirstDate failed for ${collectionName}`, { errorMessage: e.message }); }
-    return null;
-}
-
-/** Stage 4: Get the earliest date in a sharded collection */
-async function getFirstDateFromCollection(config, deps, collectionName) {
+/** * Stage 4: Get the earliest date from the Centralized Root Data Index.
+ * This REPLACES the expensive/error-prone raw collection scanning.
+ * It queries the 'system_root_data_index' to find the first date where data flags are TRUE.
+ */
+async function getEarliestDataDates(config, deps) {
     const { db, logger } = deps;
-    let earliestDate = null;
-    try {
-        if (!collectionName) { logger.log('WARN', `[Core Utils] Collection name not provided for sharded date query.`); return null; }
-        const blockDocRefs = await withRetry(() => db.collection(collectionName).listDocuments(), `GetBlocks(${collectionName})`);
-        if (!blockDocRefs.length) { logger.log('WARN', `No block documents in collection: ${collectionName}`); return null; }
-        for (const blockDocRef of blockDocRefs) {
-            const snapshotQuery = blockDocRef.collection(config.snapshotsSubcollection).where(FieldPath.documentId(), '>=', '2000-01-01').orderBy(FieldPath.documentId(), 'asc').limit(1);
-            const snapshotSnap = await withRetry(() => snapshotQuery.get(), `GetEarliestSnapshot(${blockDocRef.path})`);
-            if (!snapshotSnap.empty && /^\d{4}-\d{2}-\d{2}$/.test(snapshotSnap.docs[0].id)) {
-                const foundDate = new Date(snapshotSnap.docs[0].id + 'T00:00:00Z');
-                if (!earliestDate || foundDate < earliestDate) earliestDate = foundDate;
+    // Default to 'system_root_data_index' if not configured
+    const indexCollection = process.env.ROOT_DATA_AVAILABILITY_COLLECTION || 
+                           config.rootDataAvailabilityCollection || 
+                           'system_root_data_index';
+    
+    // Helper to find earliest date where a specific flag is true
+    // Efficient: Uses the document ID (date string) index.
+    const getEarliestForType = async (flagName) => {
+        try {
+            const snapshot = await db.collection(indexCollection)
+                .where(flagName, '==', true)
+                .orderBy(FieldPath.documentId(), 'asc')
+                .limit(1)
+                .get();
+
+            if (!snapshot.empty) {
+                const dateStr = snapshot.docs[0].id; // YYYY-MM-DD
+                // Safety check on date format
+                if (/^\d{4}-\d{2}-\d{2}$/.test(dateStr)) {
+                    return new Date(dateStr + 'T00:00:00Z');
+                }
             }
+        } catch (e) {
+            logger.log('WARN', `[Utils] Failed to query index for ${flagName}: ${e.message}`);
         }
-    } catch (e) { logger.log('ERROR', `GetFirstDate failed for ${collectionName}`, { errorMessage: e.message }); }
-    return earliestDate;
-}
+        return null;
+    };
 
-/** Stage 5: Determine the earliest date from *all* source data. */
-async function getEarliestDataDates(config, deps) {
-    const { logger } = deps;
-    const [ investorDate, speculatorDate, investorHistoryDate, speculatorHistoryDate, insightsDate, socialDate, priceDate ] = await Promise.all([
-        getFirstDateFromCollection       (config, deps, config.normalUserPortfolioCollection),
-        getFirstDateFromCollection       (config, deps, config.speculatorPortfolioCollection),
-        getFirstDateFromCollection       (config, deps, config.normalUserHistoryCollection),
-        getFirstDateFromCollection       (config, deps, config.speculatorHistoryCollection),
-        getFirstDateFromSimpleCollection (config, deps, config.insightsCollectionName),
-        getFirstDateFromSimpleCollection (config, deps, config.socialInsightsCollectionName),
-        getFirstDateFromPriceCollection  (config, deps)
+    // Parallel query for all data types
+    const [portfolioDate, historyDate, socialDate, insightsDate, priceDate] = await Promise.all([
+        getEarliestForType('hasPortfolio'),
+        getEarliestForType('hasHistory'),
+        getEarliestForType('hasSocial'),
+        getEarliestForType('hasInsights'),
+        getEarliestForType('hasPrices')
     ]);
 
-    const getMinDate = (...dates) => {
-        const validDates = dates.filter(Boolean);
-        if (validDates.length === 0) return null;
-        return new Date(Math.min(...validDates));
-    };
-
-    const earliestPortfolioDate = getMinDate(investorDate, speculatorDate);
-    const earliestHistoryDate   = getMinDate(investorHistoryDate, speculatorHistoryDate);
-    const earliestInsightsDate  = getMinDate(insightsDate);
-    const earliestSocialDate    = getMinDate(socialDate);
-    const earliestPriceDate     = getMinDate(priceDate);
-    const absoluteEarliest      = getMinDate(earliestPortfolioDate, earliestHistoryDate, earliestInsightsDate, earliestSocialDate, earliestPriceDate);
+    // Calculate absolute earliest among found dates
+    const foundDates = [portfolioDate, historyDate, socialDate, insightsDate, priceDate].filter(d => d !== null);
+    
+    let absoluteEarliest = null;
+    if (foundDates.length > 0) {
+        absoluteEarliest = new Date(Math.min(...foundDates));
+    } else {
+        // Fallback if index is empty
+        const configStart = config.earliestComputationDate || '2023-01-01';
+        absoluteEarliest = new Date(configStart + 'T00:00:00Z');
+        logger.log('WARN', `[Utils] No data found in Root Data Index (${indexCollection}). Defaulting to ${configStart}`);
+    }
 
-    const fallbackDate = new Date(config.earliestComputationDate + 'T00:00:00Z' || '2023-01-01T00:00:00Z');
+    // Update the static export for consumers who use it (best effort synchronization)
+    DEFINITIVE_EARLIEST_DATES.absoluteEarliest = absoluteEarliest;
 
     return {
-        portfolio:        earliestPortfolioDate || new Date('2999-12-31T00:00:00Z'),
-        history:          earliestHistoryDate   || new Date('2999-12-31T00:00:00Z'),
-        insights:         earliestInsightsDate  || new Date('2999-12-31T00:00:00Z'),
-        social:           earliestSocialDate    || new Date('2999-12-31T00:00:00Z'),
-        price:            earliestPriceDate     || new Date('2999-12-31T00:00:00Z'),
-        absoluteEarliest: absoluteEarliest      || fallbackDate
+        portfolio:        portfolioDate || new Date('2999-12-31T00:00:00Z'),
+        history:          historyDate   || new Date('2999-12-31T00:00:00Z'),
+        insights:         insightsDate  || new Date('2999-12-31T00:00:00Z'),
+        social:           socialDate    || new Date('2999-12-31T00:00:00Z'),
+        price:            priceDate     || new Date('2999-12-31T00:00:00Z'),
+        absoluteEarliest: absoluteEarliest
     };
 }
 
-async function getFirstDateFromPriceCollection(config, deps) {
-    const { db, logger } = deps;
-    const collection = config.priceCollection || 'asset_prices';
-    try {
-        const snapshot = await withRetry(() => db.collection(collection).limit(10).get(), `GetPriceShards(${collection})`);
-        let earliestDate = null;
-        snapshot.forEach(doc => {
-            const shardData = doc.data();
-            for (const instrumentId in shardData) {
-                const instrumentData = shardData[instrumentId];
-                if (!instrumentData.prices) continue;
-                const dates = Object.keys(instrumentData.prices).filter(d => /^\d{4}-\d{2}-\d{2}$/.test(d)).sort();
-                if (dates.length > 0) { const firstDate = new Date(dates[0] + 'T00:00:00Z'); if (!earliestDate || firstDate < earliestDate) earliestDate = firstDate; }
-            }
-        });
-        return earliestDate;
-    } catch (e) { logger.log('ERROR', `Failed to get earliest price date from ${collection}`, { errorMessage: e.message }); return null; }
-}
-
 module.exports = { 
     FieldValue, 
     FieldPath, 
@@ -259,7 +241,7 @@ module.exports = {
     getExpectedDateStrings, 
     getEarliestDataDates, 
     generateCodeHash,
-    generateDataHash, // Exported
+    generateDataHash, 
     withRetry,
     DEFINITIVE_EARLIEST_DATES
-};
+};

package/functions/generic-api/admin-api/index.js

@@ -2,6 +2,7 @@
  * @fileoverview Admin API Router
  * Sub-module for system observability, debugging, and visualization.
  * Mounted at /admin within the Generic API.
+ * UPDATED: Added advanced cost, performance, and live monitoring endpoints.
  */
 
 const express = require('express');
@@ -84,8 +85,6 @@ const createAdminRouter = (config, dependencies, unifiedCalculations) => {
     });
 
     // --- 2. STATUS MATRIX (Calendar / State UI) ---
-    // Returns status of ALL computations across a date range.
-    // ENHANCED: Cross-references Manifest to detect "PENDING" (Not run yet) vs "MISSING".
     router.get('/matrix', async (req, res) => {
         const { start, end } = req.query;
         if (!start || !end) return res.status(400).json({ error: "Start and End dates required." });
@@ -153,7 +152,6 @@ const createAdminRouter = (config, dependencies, unifiedCalculations) => {
     });
 
     // --- 3. PIPELINE STATE (Progress Bar) ---
-    // Shows realtime status of the 5-pass system for a specific date
     router.get('/pipeline/state', async (req, res) => {
         const { date } = req.query;
         if (!date) return res.status(400).json({ error: "Date required" });
@@ -352,7 +350,6 @@ const createAdminRouter = (config, dependencies, unifiedCalculations) => {
     });
 
     // --- 7. FLIGHT RECORDER (Inspection) ---
-    // Existing inspection endpoint kept for drill-down
     router.get('/inspect/:date/:calcName', async (req, res) => {
         const { date, calcName } = req.params;
         try {
@@ -379,6 +376,156 @@ const createAdminRouter = (config, dependencies, unifiedCalculations) => {
         }
     });
 
+    // --- 8. COST & RESOURCE ANALYSIS ---
+    router.get('/analytics/costs', async (req, res) => {
+        const { date, days } = req.query;
+        // Default to today if no date, or range if days provided
+        const targetDate = date || new Date().toISOString().slice(0, 10);
+        
+        // Simple Cost Model (Estimates)
+        const COSTS = {
+            write: 0.18 / 100000,
+            read:  0.06 / 100000,
+            delete: 0.02 / 100000,
+            compute_std_sec:  0.000023, // 1vCPU 2GB (approx)
+            compute_high_sec: 0.000092  // 2vCPU 8GB (approx)
+        };
+
+        try {
+            const auditRef = db.collection('computation_audit_logs');
+            // We scan the 'history' subcollectionGroup for the given date(s)
+            // Note: This can be expensive. In prod, you'd want aggregate counters.
+            const query = db.collectionGroup('history').where('targetDate', '==', targetDate);
+            const snap = await query.get();
+
+            let totalCost = 0;
+            const byPass = {};
+            const byCalc = {};
+
+            snap.forEach(doc => {
+                const data = doc.data();
+                const ops = data.firestoreOps || { reads: 0, writes: 0, deletes: 0 };
+                const durationSec = (data.durationMs || 0) / 1000;
+                const tier = data.resourceTier || 'standard';
+
+                const ioCost = (ops.writes * COSTS.write) + (ops.reads * COSTS.read) + (ops.deletes * COSTS.delete);
+                const computeCost = durationSec * (tier === 'high-mem' ? COSTS.compute_high_sec : COSTS.compute_std_sec);
+                const itemCost = ioCost + computeCost;
+
+                totalCost += itemCost;
+
+                // Aggregations
+                const pass = data.pass || 'unknown';
+                if (!byPass[pass]) byPass[pass] = { cost: 0, runs: 0, duration: 0 };
+                byPass[pass].cost += itemCost;
+                byPass[pass].runs++;
+                byPass[pass].duration += durationSec;
+
+                const calc = data.computationName;
+                if (!byCalc[calc]) byCalc[calc] = { cost: 0, runs: 0, ops: { r:0, w:0 } };
+                byCalc[calc].cost += itemCost;
+                byCalc[calc].runs++;
+                byCalc[calc].ops.r += ops.reads;
+                byCalc[calc].ops.w += ops.writes;
+            });
+
+            // Top 10 Expensive Calcs
+            const topCalcs = Object.entries(byCalc)
+                .sort((a, b) => b[1].cost - a[1].cost)
+                .slice(0, 10)
+                .map(([name, stats]) => ({ name, ...stats }));
+
+            res.json({
+                date: targetDate,
+                totalCostUSD: totalCost,
+                breakdown: {
+                    byPass,
+                    topCalculations: topCalcs
+                },
+                meta: { model: COSTS }
+            });
+
+        } catch (e) { res.status(500).json({ error: e.message }); }
+    });
+
+    // --- 9. REROUTE (OOM) ANALYSIS ---
+    router.get('/analytics/reroutes', async (req, res) => {
+        const { date } = req.query;
+        if (!date) return res.status(400).json({ error: "Date required" });
+
+        try {
+            // Find all runs that used high-mem
+            const query = db.collectionGroup('history')
+                .where('targetDate', '==', date)
+                .where('resourceTier', '==', 'high-mem');
+            
+            const snap = await query.get();
+            const reroutes = [];
+
+            snap.forEach(doc => {
+                const data = doc.data();
+                reroutes.push({
+                    computation: data.computationName,
+                    pass: data.pass,
+                    trigger: data.trigger?.reason,
+                    peakMemoryMB: data.peakMemoryMB,
+                    durationMs: data.durationMs,
+                    runId: data.runId
+                });
+            });
+
+            res.json({ count: reroutes.length, reroutes });
+        } catch (e) { res.status(500).json({ error: e.message }); }
+    });
+
+    // --- 10. LIVE DASHBOARD (Snapshot) ---
+    // Poll this endpoint to simulate a WebSocket feed
+    router.get('/live/dashboard', async (req, res) => {
+        const today = new Date().toISOString().slice(0, 10);
+        try {
+            // Query the Ledger for Active Tasks
+            // We look at all passes for today
+            const passes = ['1', '2', '3', '4', '5'];
+            const activeTasks = [];
+            const recentFailures = [];
+
+            await Promise.all(passes.map(async (pass) => {
+                const colRef = db.collection(`computation_audit_ledger/${today}/passes/${pass}/tasks`);
+                
+                // Get Running
+                const runningSnap = await colRef.where('status', 'in', ['PENDING', 'IN_PROGRESS']).get();
+                runningSnap.forEach(doc => {
+                    activeTasks.push({ pass, ...doc.data() });
+                });
+
+                // Get Recent Failures (last 10 mins?? hard to query without index, just grab failures)
+                const failSnap = await colRef.where('status', '==', 'FAILED').get();
+                failSnap.forEach(doc => {
+                    recentFailures.push({ pass, ...doc.data() });
+                });
+            }));
+
+            // Get Pipeline Stage (which pass is active?)
+            // We infer this by seeing which pass has pending tasks
+            let currentStage = 'IDLE';
+            for (const p of passes) {
+                const hasActive = activeTasks.some(t => t.pass === p);
+                if (hasActive) { currentStage = `PASS_${p}`; break; }
+            }
+
+            res.json({
+                status: 'success',
+                timestamp: new Date(),
+                pipelineState: currentStage,
+                activeCount: activeTasks.length,
+                failureCount: recentFailures.length,
+                tasks: activeTasks,
+                failures: recentFailures
+            });
+
+        } catch (e) { res.status(500).json({ error: e.message }); }
+    });
+
     return router;
 };
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.296",
+  "version": "1.0.298",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [