bulltrackers-module 1.0.731 → 1.0.733

package/functions/computation-system/context/ManifestBuilder.js

@@ -1,379 +0,0 @@
-/**
- * @fileoverview Dynamic Manifest Builder - Handles Topological Sort and Auto-Discovery.
- * UPDATED: Removed Automatic Infra Hashing. Now relies strictly on SYSTEM_EPOCH.
- * UPDATED: Whitelisted 'rootDataSeries', 'dependencySeries', 'mandatoryRoots', and 'isTest'.
- * UPDATED: Whitelisted 'schedule' to allow calculations to define their own execution cadence (Daily/Weekly/Monthly).
- * UPDATED: Whitelisted 'ttlDays' to allow calculations to define custom retention policies.
- */
-const { generateCodeHash, LEGACY_MAPPING } = require('../topology/HashManager.js');
-const { normalizeName } = require('../utils/utils');
-
-const SYSTEM_EPOCH = require('../system_epoch');
-
-// Import Layers
-const MathematicsLayer = require('../layers/mathematics');
-const ExtractorsLayer  = require('../layers/extractors');
-const ProfilingLayer   = require('../layers/profiling');
-const ValidatorsLayer  = require('../layers/validators');
-
-const LAYER_GROUPS = {
-    'mathematics': MathematicsLayer,
-    'extractors':  ExtractorsLayer,
-    'profiling':   ProfilingLayer,
-    'validators':  ValidatorsLayer
-};
-
-/**
- * Heuristic to estimate the "weight" of a calculation based on its output structure.
- */
-function estimateComplexity(Class, metadata) {
-    let weight = 1.0; 
-
-    try {
-        const schema = typeof Class.getSchema === 'function' ? Class.getSchema() : {};
-        if (schema.patternProperties || (schema.type === 'object' && !schema.properties)) {
-            weight *= 3.0; 
-        }
-        const name = Class.name.toLowerCase();
-        if (name.includes('perstock') || name.includes('perticker')) weight *= 2.0;
-        if (name.includes('peruser')) weight *= 10.0; 
-
-        if (metadata.rootDataDependencies && metadata.rootDataDependencies.includes('portfolio')) {
-            weight *= 1.5;
-        }
-
-    } catch (e) { }
-
-    return weight;
-}
-
-function generateLayerHashes(layerExports, layerName) {
-    const hashes = {};
-    const keys = Object.keys(layerExports).sort(); 
-    for (const key of keys) {
-        const item = layerExports[key];
-        let source = `LAYER:${layerName}:EXPORT:${key}`;
-        if (typeof item === 'function') source += item.toString();
-        else if (typeof item === 'object' && item !== null) source += JSON.stringify(item);
-        else source += String(item);
-        hashes[key] = generateCodeHash(source);
-    }
-    return hashes;
-}
-
-function buildDynamicTriggers() {
-    const triggers = {};
-    for (const [layerName, layerExports] of Object.entries(LAYER_GROUPS)) {
-        triggers[layerName] = {};
-        for (const exportName of Object.keys(layerExports)) {
-            const patterns = [];
-            patterns.push(exportName);
-            const alias = LEGACY_MAPPING[exportName];
-            if (alias) {
-                patterns.push(`math.${alias}`);
-                patterns.push(`${alias}.`);
-            }
-            triggers[layerName][exportName] = patterns;
-        }
-    }
-    return triggers;
-}
-
-const LAYER_HASHES = {};
-for (const [name, exports] of Object.entries(LAYER_GROUPS)) { LAYER_HASHES[name] = generateLayerHashes(exports, name); }
-
-const LAYER_TRIGGERS = buildDynamicTriggers();
-
-const log = {
-    info:    (msg) => console.log('ℹ︎ ' + msg),
-    step:    (msg) => console.log('› ' + msg),
-    warn:    (msg) => console.warn('⚠︎ ' + msg),
-    success: (msg) => console.log('✔︎ ' + msg),
-    error:   (msg) => console.error('✖ ' + msg),
-    fatal:   (msg) => { console.error('✖ FATAL ✖ ' + msg); console.error('✖ FATAL ✖ Manifest build FAILED.'); },
-    divider: (label) => { const line = ''.padEnd(60, '─'); console.log(`\n${line}\n${label}\n${line}\n`); },
-};
-
-function getDependencySet(endpoints, adjacencyList) {
-    const required = new Set(endpoints);
-    const queue = [...endpoints];
-    while (queue.length > 0) { 
-        const calcName = queue.shift(); 
-        const dependencies = adjacencyList.get(calcName);
-        if (dependencies) { 
-            for (const dep of dependencies) { 
-                if (!required.has(dep)) { required.add(dep); queue.push(dep); } 
-            } 
-        } 
-    }
-    return required;
-}
-
-function detectCircularDependencies(manifestMap) {
-    let index      = 0;
-    const stack    = [];
-    const indices  = new Map();
-    const lowLinks = new Map();
-    const onStack  = new Set();
-    const cycles   = [];
-
-    function strongconnect(v) {
-        indices.set(v, index);
-        lowLinks.set(v, index);
-        index++;
-        stack.push(v);
-        onStack.add(v);
-
-        const entry = manifestMap.get(v);
-        if (entry && entry.dependencies) {
-            for (const w of entry.dependencies) {
-                if (!manifestMap.has(w)) continue; 
-                if (!indices.has(w)) {
-                    strongconnect(w);
-                    lowLinks.set(v, Math.min(lowLinks.get(v), lowLinks.get(w)));
-                } else if (onStack.has(w)) {
-                    lowLinks.set(v, Math.min(lowLinks.get(v), indices.get(w)));
-                }
-            }
-        }
-
-        if (lowLinks.get(v) === indices.get(v)) {
-            const scc = [];
-            let w;
-            do {
-                w = stack.pop();
-                onStack.delete(w);
-                scc.push(w);
-            } while (w !== v);
-
-            if (scc.length > 1) {
-                cycles.push(scc);
-            }
-        }
-    }
-
-    for (const name of manifestMap.keys()) {
-        if (!indices.has(name)) {
-            strongconnect(name);
-        }
-    }
-
-    if (cycles.length > 0) {
-        const cycle = cycles[0];
-        return cycle.join(' -> ') + ' -> ' + cycle[0];
-    }
-    return null;
-}
-
-function buildManifest(productLinesToRun = [], calculations) {
-    log.divider('Building Dynamic Manifest');
-    log.info(`[ManifestBuilder] Global System Epoch: ${SYSTEM_EPOCH}`);
-
-    const requestedLog = (!productLinesToRun || productLinesToRun.length === 0) 
-        ? "ALL (Wildcard/Empty)" 
-        : productLinesToRun.join(', ');
-    log.info(`[ManifestBuilder] 📥 Request received for: [${requestedLog}]`);
-    
-    const manifestMap = new Map();
-    const adjacency = new Map(); 
-    const reverseAdjacency = new Map(); 
-    const inDegree = new Map();
-    let hasFatalError = false;
-    
-    function processCalc(Class, name, folderName) {
-        if (!Class || typeof Class !== 'function') return;
-        const normalizedName = normalizeName(name);
-        
-        if (typeof Class.getMetadata     !== 'function') { log.fatal(`Calculation "${normalizedName}" missing static getMetadata().`);     hasFatalError = true; return; }
-        if (typeof Class.getDependencies !== 'function') { log.fatal(`Calculation "${normalizedName}" missing static getDependencies().`); hasFatalError = true; return; }
-        
-        const metadata     = Class.getMetadata();
-        const weight       = estimateComplexity(Class, metadata)
-        const dependencies = Class.getDependencies().map(normalizeName);
-        const codeStr      = Class.toString();
-        const selfCodeHash = generateCodeHash(codeStr);
-        
-        let compositeHashString = selfCodeHash + `|EPOCH:${SYSTEM_EPOCH}`; 
-        
-        const usedDeps = [];
-        const usedLayerHashes = {}; 
-
-        for (const [layerName, exportsMap] of Object.entries(LAYER_TRIGGERS)) {
-            const layerHashes = LAYER_HASHES[layerName];
-            for (const [exportName, triggers] of Object.entries(exportsMap)) {
-                if (triggers.some(trigger => codeStr.includes(trigger))) {
-                    const exportHash = layerHashes[exportName];
-                    if (exportHash) {
-                        compositeHashString += exportHash;
-                        usedDeps.push(`${layerName}.${exportName}`);
-                        
-                        if (!usedLayerHashes[layerName]) usedLayerHashes[layerName] = '';
-                        usedLayerHashes[layerName] += exportHash;
-                    }
-                }
-            }
-        }
-        
-        const layerComposition = {};
-        for(const [lName, lStr] of Object.entries(usedLayerHashes)) {
-            layerComposition[lName] = generateCodeHash(lStr);
-        }
-
-        let isSafeMode = false;
-        if (usedDeps.length === 0) {
-            isSafeMode = true;
-            Object.values(LAYER_HASHES).forEach(layerObj => { Object.values(layerObj).forEach(h => compositeHashString += h); }); 
-            layerComposition['ALL_SAFE_MODE'] = 'ALL';
-        }
-
-        const intrinsicHash = generateCodeHash(compositeHashString);
-
-        const manifestEntry = {
-            name:          normalizedName,
-            class:         Class,
-            category:      folderName,     
-            sourcePackage: folderName,   
-            type:                 metadata.type,
-            isPage:               metadata.isPage === true,
-            isHistorical:         metadata.isHistorical !== undefined ? metadata.isHistorical : false, 
-            isTest:               metadata.isTest === true, 
-            rootDataDependencies: metadata.rootDataDependencies || [],
-            rootDataSeries:       metadata.rootDataSeries || null,
-            dependencySeries:     metadata.dependencySeries || null,
-            mandatoryRoots:       metadata.mandatoryRoots || [],
-            canHaveMissingRoots:  metadata.canHaveMissingRoots || false,
-            canHaveMissingSeries: metadata.canHaveMissingSeries || false,
-            userType:             metadata.userType,
-            dependencies: dependencies,
-            schedule:             metadata.schedule || null,
-            ttlDays:              metadata.ttlDays, 
-            pass: 0,
-            hash: intrinsicHash,
-            weight: weight,
-            composition: {
-                epoch: SYSTEM_EPOCH,
-                code: selfCodeHash,
-                layers: layerComposition,
-                deps: {} 
-            },
-            debugUsedLayers: isSafeMode ? ['ALL (Safe Mode)'] : usedDeps
-        };
-        
-        manifestMap.set(normalizedName, manifestEntry);
-        adjacency.set(normalizedName, dependencies);
-        inDegree.set(normalizedName, dependencies.length);
-        dependencies.forEach(dep => { if (!reverseAdjacency.has(dep)) reverseAdjacency.set(dep, []); reverseAdjacency.get(dep).push(normalizedName); }); 
-    }
-    
-    for (const folderName in calculations) {
-        if (folderName === 'legacy') continue;
-        const calculationGroup = calculations[folderName];
-        for (const key in calculationGroup) {
-            const entry = calculationGroup[key];
-            if (typeof entry === 'function') { processCalc(entry, key, folderName); } 
-        } 
-    }
-
-    if (hasFatalError) throw new Error('Manifest build failed due to missing static methods.');
-    log.success(`Loaded ${manifestMap.size} calculations.`);
-
-    const allNames = new Set(manifestMap.keys());
-    for (const [name, entry] of manifestMap) {
-        for (const dep of entry.dependencies) {
-            if (!allNames.has(dep)) log.error(`${name} depends on unknown calculation "${dep}"`);
-        }
-    }
-
-    const productLineEndpoints = [];
-    const runAll = !productLinesToRun || productLinesToRun.length === 0 || productLinesToRun.includes('*');
-    for (const [name, entry] of manifestMap.entries()) {  
-        if (runAll || productLinesToRun.includes(entry.category)) { 
-            productLineEndpoints.push(name); 
-        }  
-    }
-
-    const requiredCalcs = getDependencySet(productLineEndpoints, adjacency);
-    log.info(`Filtered down to ${requiredCalcs.size} active calculations.`);
-
-    const activePackages = new Set();
-    requiredCalcs.forEach(name => {
-        const entry = manifestMap.get(name);
-        if (entry) activePackages.add(entry.sourcePackage);
-    });
-    
-    const activeList = Array.from(activePackages).sort().join(', ');
-    log.info(`[ManifestBuilder] ✅ FINAL ACTIVE PRODUCT LINES: [${activeList}]`);
-
-    const filteredManifestMap = new Map();
-    const filteredInDegree = new Map();
-    const filteredReverseAdjacency = new Map();
-
-    for (const name of requiredCalcs) { 
-        filteredManifestMap.set(name, manifestMap.get(name)); 
-        filteredInDegree.set(name, inDegree.get(name));
-        const consumers = (reverseAdjacency.get(name) || []).filter(consumer => requiredCalcs.has(consumer)); 
-        filteredReverseAdjacency.set(name, consumers); 
-    }
-
-    const sortedManifest = [];
-    const queue = [];
-    let maxPass = 0;
-
-    for (const [name, degree] of filteredInDegree) { 
-        if (degree === 0) { queue.push(name); filteredManifestMap.get(name).pass = 1; maxPass = 1; } 
-    }
-    queue.sort();
-    
-    while (queue.length) {
-        const currentName = queue.shift();
-        const currentEntry = filteredManifestMap.get(currentName);
-        sortedManifest.push(currentEntry);
-        
-        for (const neighborName of (filteredReverseAdjacency.get(currentName) || [])) { 
-            const newDegree = filteredInDegree.get(neighborName) - 1; 
-            filteredInDegree.set(neighborName, newDegree);
-            const neighborEntry = filteredManifestMap.get(neighborName);
-            if (neighborEntry.pass <= currentEntry.pass) { 
-                neighborEntry.pass = currentEntry.pass + 1; 
-                if (neighborEntry.pass > maxPass) maxPass = neighborEntry.pass; 
-            }
-            if (newDegree === 0) queue.push(neighborName); 
-        }
-        queue.sort(); 
-    }
-
-    if (sortedManifest.length !== filteredManifestMap.size) {
-        const cycle = detectCircularDependencies(filteredManifestMap);
-        if (cycle) {
-            throw new Error(`Circular dependency detected: ${cycle}`);
-        } else {
-            throw new Error('Circular dependency detected (Unknown topology error).');
-        }
-    }
-
-    for (const entry of sortedManifest) {
-        let dependencySignature = entry.hash; 
-        
-        if (entry.dependencies && entry.dependencies.length > 0) {
-            const depHashes = entry.dependencies.map(depName => { 
-                const depEntry = filteredManifestMap.get(depName);
-                if (depEntry) {
-                    entry.composition.deps[depName] = depEntry.hash;
-                    return depEntry.hash;
-                }
-                return ''; 
-            }).join('|'); 
-            dependencySignature += `|DEPS:${depHashes}`;
-        }
-        entry.hash = generateCodeHash(dependencySignature);
-    }
-
-    return sortedManifest;
-}
-
-function build(productLinesToRun, calculations) {
-    try { return buildManifest(productLinesToRun, calculations); } 
-    catch (error) { log.error(error.message); return null; }
-}
-
-module.exports = { build };

package/functions/computation-system/data/AvailabilityChecker.js

@@ -1,236 +0,0 @@
-/**
- * @fileoverview Checks availability of root data via the Root Data Index.
- * REFACTORED: Config-driven dependency checking and shared status normalization.
- */
-const { normalizeName } = require('../utils/utils');
-const { FieldPath } = require('@google-cloud/firestore');
-
-const INDEX_COLLECTION = process.env.ROOT_DATA_AVAILABILITY_COLLECTION || 'system_root_data_index';
-
-// =============================================================================
-// CONFIGURATION: Dependency Mappings
-// =============================================================================
-
-// Global dependencies that are NOT date-specific and are always considered available.
-// These are stored at fixed paths and don't rely on daily availability index.
-const GLOBAL_DEPS = new Set([
-    'piMasterList'  // Stored at /system_state/popular_investor_master_list (global CID->username mapping)
-]);
-
-// Dependencies that map directly to a single status flag (date-specific)
-const SIMPLE_DEP_MAP = {
-    rankings:     'piRankings',
-    verification: 'signedInUserVerification',
-    insights:     'hasInsights',
-    price:        'hasPrices',
-    ratings:      'piRatings',
-    pageViews:    'piPageViews',
-    watchlist:    'watchlistMembership',
-    alerts:       'piAlertHistory',
-    piMasterList: 'piMasterList' // Kept for normalizeStatus compatibility, but checked via GLOBAL_DEPS
-};
-
-// Dependencies that vary based on the userType of the calculation
-const COMPLEX_DEP_MAP = {
-    portfolio: {
-        speculator:       'speculatorPortfolio',
-        normal:           'normalPortfolio',
-        popular_investor: 'piPortfolios',
-        signed_in_user:   'signedInUserPortfolio',
-        _default:         'hasPortfolio'
-    },
-    history: {
-        speculator:       'speculatorHistory',
-        normal:           'normalHistory',
-        popular_investor: 'piHistory',
-        signed_in_user:   'signedInUserHistory',
-        _default:         'hasHistory'
-    },
-    social: {
-        popular_investor: 'hasPISocial',
-        signed_in_user:   'hasSignedInSocial',
-        _default:         'hasSocial'
-    }
-};
-
-// =============================================================================
-// LOGIC: Dependency Checking
-// =============================================================================
-
-/**
- * Checks if a specific calculation can run based on its dependencies and the current data status.
- */
-function checkRootDependencies(calcManifest, rootDataStatus) {
-    const missing = [];
-    const available = [];
-    const deps = calcManifest.rootDataDependencies || [];
-
-    if (!deps.length) return { canRun: true, missing, available };
-
-    const userType = (calcManifest.userType || 'all').toLowerCase();
-    const canHaveMissing = calcManifest.canHaveMissingRoots === true;
-
-    for (const dep of deps) {
-        let isAvailable = false;
-        let missingKey = dep;
-
-        // 0. Check for Global Dependencies (always available, not date-specific)
-        if (GLOBAL_DEPS.has(dep)) {
-            isAvailable = true;
-        }
-        // 1. Resolve Status Key for date-specific dependencies
-        else if (SIMPLE_DEP_MAP[dep]) {
-            const key = SIMPLE_DEP_MAP[dep];
-            if (rootDataStatus[key]) isAvailable = true;
-            else missingKey = key;
-        } 
-        else if (COMPLEX_DEP_MAP[dep]) {
-            const map = COMPLEX_DEP_MAP[dep];
-            const key = map[userType] || map._default;
-            if (rootDataStatus[key]) isAvailable = true;
-            else missingKey = key;
-        }
-
-        // 2. Check Availability
-        // [FIX] Removed Optimistic Series Fallback. 
-        // Just because we want history (series) doesn't mean the data type exists for this date.
-        if (isAvailable) {
-            available.push(dep);
-        } else {
-            missing.push(missingKey);
-        }
-    }
-
-    // 3. Enforce Mandatory Roots (Granular Override)
-    if (calcManifest.mandatoryRoots?.length) {
-        if (calcManifest.mandatoryRoots.some(r => !available.includes(r))) {
-            return { canRun: false, missing, available };
-        }
-    }
-
-    return { 
-        canRun: canHaveMissing ? available.length > 0 : missing.length === 0, 
-        missing, 
-        available 
-    };
-}
-
-/**
- * filters a list of calculations to those that can run given the current data.
- */
-function getViableCalculations(candidates, fullManifest, rootDataStatus, dailyStatus) {
-    const manifestMap = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
-    
-    return candidates.filter(calc => {
-        // 1. Check Root Data
-        const { canRun } = checkRootDependencies(calc, rootDataStatus);
-        if (!canRun) return false;
-
-        // 2. Check Computed Dependencies (Hashes must match)
-        if (calc.dependencies?.length) {
-            return calc.dependencies.every(depName => {
-                const norm = normalizeName(depName);
-                const stored = dailyStatus[norm];
-                const ref = manifestMap.get(norm);
-                // Dependency must exist, have run, and match the current manifest hash
-                return ref && stored && stored.hash === ref.hash;
-            });
-        }
-        return true;
-    });
-}
-
-// =============================================================================
-// DATA ACCESS: Status Normalization
-// =============================================================================
-
-/**
- * Standardizes the Firestore document into a flat status object.
- * Handles fallbacks and merges 'details' fields.
- */
-function normalizeStatus(data) {
-    // If data is null/undefined, return object with all false
-    const d = data || {};
-    const det = d.details || {};
-    
-    // Helper to check both root level and details object
-    const val = (key, rootFallback) => !!det[key] || (rootFallback ? !!d[rootFallback] : false);
-
-    return {
-        // Core Flags
-        hasPortfolio: !!d.hasPortfolio,
-        hasHistory:   !!d.hasHistory,
-        hasSocial:    !!d.hasSocial,
-        hasInsights:  !!d.hasInsights,
-        hasPrices:    !!d.hasPrices,
-        
-        // Granular Portfolio/History
-        speculatorPortfolio: !!det.speculatorPortfolio,
-        normalPortfolio:     !!det.normalPortfolio,
-        piPortfolios:        !!det.piPortfolios,
-        piDeepPortfolios:    !!det.piDeepPortfolios,
-        signedInUserPortfolio: !!det.signedInUserPortfolio,
-        
-        speculatorHistory:   !!det.speculatorHistory,
-        normalHistory:       !!det.normalHistory,
-        piHistory:           !!det.piHistory,
-        signedInUserHistory: !!det.signedInUserHistory,
-
-        // Meta Types
-        piRankings:               !!det.piRankings,
-        signedInUserVerification: !!det.signedInUserVerification,
-        hasPISocial:              val('hasPISocial', 'hasPISocial'),
-        hasSignedInSocial:        val('hasSignedInSocial', 'hasSignedInSocial'),
-        
-        // Extended Root Types
-        piRatings:           !!det.piRatings,
-        piPageViews:         !!det.piPageViews,
-        watchlistMembership: !!det.watchlistMembership,
-        piAlertHistory:      !!det.piAlertHistory,
-        piMasterList:        !!det.piMasterList
-    };
-}
-
-/**
- * Checks root data availability for a single date.
- */
-async function checkRootDataAvailability(dateStr, config, dependencies, earliestDates) {
-    const { logger, db } = dependencies;
-    try {
-        const doc = await db.collection(INDEX_COLLECTION).doc(dateStr).get();
-        if (!doc.exists) {
-            logger.log('WARN', `[Availability] Index not found for ${dateStr}.`);
-            return { status: normalizeStatus(null) };
-        }
-        return { 
-            status: normalizeStatus(doc.data()),
-            // Legacy placeholders preserved for compatibility
-            portfolioRefs: null, historyRefs: null, todayInsights: null, todaySocialPostInsights: null, yesterdayPortfolioRefs: null
-        };
-    } catch (err) {
-        logger.log('ERROR', `Error checking availability index: ${err.message}`);
-        return null;
-    }
-}
-
-/**
- * Fetches availability status for a range of dates.
- */
-async function getAvailabilityWindow(deps, startDateStr, endDateStr) {
-    const { db } = deps;
-    const snapshot = await db.collection(INDEX_COLLECTION)
-        .where(FieldPath.documentId(), '>=', startDateStr)
-        .where(FieldPath.documentId(), '<=', endDateStr)
-        .get();
-
-    const map = new Map();
-    snapshot.forEach(doc => map.set(doc.id, normalizeStatus(doc.data())));
-    return map;
-}
-
-module.exports = { 
-    checkRootDependencies, 
-    checkRootDataAvailability, 
-    getViableCalculations,
-    getAvailabilityWindow 
-};