bulltrackers-module 1.0.158 → 1.0.159

package/functions/computation-system/helpers/computation_manifest_builder.js

@@ -0,0 +1,264 @@
+/**
+ * @fileoverview
+ * Dynamic Manifest Builder (v3 - Refactored)
+ *
+ * This script builds the computation manifest dynamically by introspecting all
+ * calculation classes from the new product-line structure (core, gem, gauss, etc.).
+ * It validates metadata, resolves dependencies, and performs a topological sort
+ * to determine execution passes.
+ *
+ * It explicitly skips the 'legacy' package.
+ * It has removed all logic for deprecated folder structures (e.g., /historical/).
+ */
+
+const { calculations }   = require('aiden-shared-calculations-unified');
+const fs                 = require('fs');
+const path               = require('path');
+const Viz                = require('viz.js');
+const { Module, render } = require('viz.js/full.render.js');
+
+/* --------------------------------------------------
+ * Pretty Console Helpers
+ * -------------------------------------------------- */
+const log = {
+    info: (msg) => console.log('ℹ︎ ' + msg),
+    step: (msg) => console.log('› ' + msg),
+    warn: (msg) => console.warn('⚠︎ ' + msg),
+    success: (msg) => console.log('✔︎ ' + msg),
+    error: (msg) => console.error('✖ ' + msg),
+    fatal: (msg) => { console.error('✖ FATAL ✖ ' + msg); console.error('✖ FATAL ✖ Manifest build FAILED.'); },
+    divider: (label) => { const line = ''.padEnd(60, '─'); console.log(`\n${line}\n${label}\n${line}\n`); },
+};
+
+/* --------------------------------------------------
+ * Helper Utilities
+ * -------------------------------------------------- */
+
+const normalizeName = (name) => { if (typeof name !== 'string') return name; return name.trim().replace(/,$/, '').replace(/_/g, '-').toLowerCase(); };
+
+/**
+ * Finds the closest string matches for a typo.
+ */
+function suggestClosest(name, candidates, n = 3) {
+    const levenshtein = (a = '', b = '') => {
+        const m = a.length, n = b.length;
+        if (!m) return n; if (!n) return m;
+        const dp = Array.from({ length: m + 1 }, (_, i) => Array(n + 1).fill(i));
+        for (let j = 0; j <= n; j++) dp[0][j] = j;
+        for (let i = 1; i <= m; i++)
+        for (let j = 1; j <= n; j++)
+        dp[i][j] = a[i - 1] === b[j - 1] ? dp[i - 1][j - 1] : Math.min(dp[i - 1][j - 1], dp[i][j - 1], dp[i - 1][j]) + 1;
+        return dp[m][n];
+    };
+    const scores = candidates.map(c => [c, levenshtein(name, c)]);
+    scores.sort((a, b) => a[1] - b[1]);
+    return scores.slice(0, n).map(s => s[0]);
+}
+
+/**
+ * Checks for a cycle using Depth First Search.
+ */
+function findCycles(manifestMap, adjacencyList) {
+    const visited = new Set(), stack = new Set(), cycles = [];
+    const dfs = (node, path) => {
+        if (stack.has(node)) { const idx = path.indexOf(node); cycles.push([...path.slice(idx), node]); return; }
+        if (visited.has(node)) return;
+        visited.add(node); stack.add(node);
+        for (const nb of adjacencyList.get(node) || []) dfs(nb, [...path, nb]);
+        stack.delete(node); };
+    for (const name of manifestMap.keys()) dfs(name, [name]);
+    return cycles;
+}
+
+/**
+ * Recursively traces all dependencies for a given set of "endpoint" calcs.
+ */
+function getDependencySet(endpoints, adjacencyList) {
+    const required = new Set(endpoints);
+    const queue = [...endpoints];
+    while (queue.length > 0) { const calcName = queue.shift(); const dependencies = adjacencyList.get(calcName);
+    if (dependencies) { for (const dep of dependencies) { if (!required.has(dep)) { required.add(dep); queue.push(dep); } } } }
+    return required;
+}
+
+/* --------------------------------------------------
+ * Core Manifest Builder
+ * -------------------------------------------------- */
+
+/**
+ * Builds the full computation manifest.
+ * @param {string[]} productLinesToRun - Array of product line categories (folder names) to build for.
+ * @returns {object[]} The final sorted manifest array.
+ */
+function buildManifest(productLinesToRun = []) {
+    log.divider('Building Dynamic Manifest');
+    log.info(`Target Product Lines: [${productLinesToRun.join(', ')}]`);
+    const manifestMap      = new Map();
+    const adjacency        = new Map(); 
+    const reverseAdjacency = new Map(); 
+    const inDegree         = new Map();
+    let hasFatalError      = false;
+    /* ---------------- 1. Load All Calculations ---------------- */
+    log.step('Loading and validating all calculation classes…');
+    const allCalculationClasses = new Map();
+    /**
+     * Processes a single calculation class, validates it, and adds to maps.
+     * --- REFACTOR: Simplified signature, isHistorical is now read from metadata ---
+     */
+    function processCalc(Class, name, category) {
+        if (!Class || typeof Class !== 'function') return;
+        const normalizedName = normalizeName(name);
+        allCalculationClasses.set(normalizedName, Class);
+        // --- RULE 1: Check for static getMetadata() ---
+        if (typeof Class.getMetadata !== 'function') { log.fatal(`Calculation "${normalizedName}" is missing the static getMetadata() method. Build FAILED.`); hasFatalError = true; return; }
+        // --- RULE 2: Check for static getDependencies() ---
+        if (typeof Class.getDependencies !== 'function') { log.fatal(`Calculation "${normalizedName}" is missing the static getDependencies() method. Build FAILED.`); hasFatalError = true;return; }
+        // --- RULE 3: Check for static getSchema() ---
+        if (typeof Class.getSchema !== 'function') {log.warn(`Calculation "${normalizedName}" is missing the static getSchema() method. (Recommended)`); }
+        const metadata = Class.getMetadata();
+        const dependencies = Class.getDependencies().map(normalizeName);
+        // --- RULE 4: Check for isHistorical mismatch ---
+        if (metadata.isHistorical === true && !Class.toString().includes('yesterdayPortfolio')) { log.warn(`Calculation "${normalizedName}" is marked 'isHistorical: true' but does not seem to use 'yesterdayPortfolio'.`); }
+        const manifestEntry = {
+            name: normalizedName,
+            class: Class,
+            category: metadata.category || category, 
+            type: metadata.type,
+            isHistorical: metadata.isHistorical, 
+            rootDataDependencies: metadata.rootDataDependencies || [],
+            userType: metadata.userType,
+            dependencies: dependencies,
+            pass: 0,
+        };
+        manifestMap.set(normalizedName, manifestEntry);
+        adjacency.set(normalizedName, dependencies);
+        inDegree.set(normalizedName, dependencies.length);
+        dependencies.forEach(dep => { if (!reverseAdjacency.has(dep)) reverseAdjacency.set(dep, []); reverseAdjacency.get(dep).push(normalizedName); }); }
+    for (const category in calculations) {
+        if (category === 'legacy') { log.info('Skipping "legacy" calculations package.'); continue; }
+        const group = calculations[category];
+        for (const key in group) {
+            const entry = group[key];
+            if (typeof entry === 'function') { processCalc(entry, key, category);  } } }
+
+    if (hasFatalError) { throw new Error('Manifest build failed due to missing static methods in calculations.'); }
+    log.success(`Loaded and validated ${manifestMap.size} total calculations.`);
+    /* ---------------- 2. Validate Dependency Links ---------------- */
+    log.divider('Validating Dependency Links');
+    const allNames = new Set(manifestMap.keys());
+    let invalidLinks = false;
+    for (const [name, entry] of manifestMap) {
+        for (const dep of entry.dependencies) {
+            if (!allNames.has(dep)) {
+                invalidLinks = true;
+                const guesses = suggestClosest(dep, Array.from(allNames));
+                log.error(`${name} depends on unknown calculation "${dep}"`);
+                if (guesses.length) log.info(`Did you mean: ${guesses.join(', ')} ?`);
+            }
+            if (dep === name) {
+                invalidLinks = true;
+                log.error(`${name} has a circular dependency on *itself*!`);
+            }
+        }
+    }
+    if (invalidLinks) { throw new Error('Manifest validation failed. Fix missing or self-referencing dependencies.'); }
+    log.success('All dependency links are valid.');
+    /* ---------------- 3. Filter for Product Lines ---------------- */
+    log.divider('Filtering by Product Line');
+    // 1. Find all "endpoint" calculations (the final signals) in the target product lines.
+    const productLineEndpoints = [];
+    for (const [name, entry] of manifestMap.entries()) { if (productLinesToRun.includes(entry.category)) { productLineEndpoints.push(name); } }
+    // 2. Add 'core' calculations as they are always included.
+    for (const [name, entry] of manifestMap.entries()) { if (entry.category === 'core') { productLineEndpoints.push(name); } }
+    // 3. Trace all dependencies upwards from these endpoints.
+    const requiredCalcs = getDependencySet(productLineEndpoints, adjacency);
+    log.info(`Identified ${productLineEndpoints.length} endpoint/core calculations.`);
+    log.info(`Traced dependencies: ${requiredCalcs.size} total calculations are required.`);
+    // 4. Create the final, filtered maps for sorting.
+    const filteredManifestMap = new Map();
+    const filteredInDegree = new Map();
+    const filteredReverseAdjacency = new Map();
+    for (const name of requiredCalcs) { filteredManifestMap.set(name, manifestMap.get(name)); filteredInDegree.set(name, inDegree.get(name));
+    const consumers = (reverseAdjacency.get(name) || []).filter(consumer => requiredCalcs.has(consumer)); filteredReverseAdjacency.set(name, consumers); }
+    log.success(`Filtered manifest to ${filteredManifestMap.size} calculations.`);
+    /* ---------------- 4. Topological Sort (Kahn's Algorithm) ---------------- */
+    log.divider('Topological Sorting (Kahn\'s Algorithm)');
+    const sortedManifest = [];
+    const queue = [];
+    let maxPass = 0;
+    for (const [name, degree] of filteredInDegree) { if (degree === 0) { queue.push(name); filteredManifestMap.get(name).pass = 1; maxPass = 1; } }
+    queue.sort();
+    while (queue.length) {
+        const currentName = queue.shift();
+        const currentEntry = filteredManifestMap.get(currentName);
+        sortedManifest.push(currentEntry);
+        for (const neighborName of (filteredReverseAdjacency.get(currentName) || [])) { const newDegree = filteredInDegree.get(neighborName) - 1; filteredInDegree.set(neighborName, newDegree);
+            const neighborEntry = filteredManifestMap.get(neighborName);
+            if (neighborEntry.pass <= currentEntry.pass) { neighborEntry.pass = currentEntry.pass + 1; if (neighborEntry.pass > maxPass) maxPass = neighborEntry.pass; }
+            if (newDegree === 0) { queue.push(neighborName); } }
+        queue.sort(); }
+    if (sortedManifest.length !== filteredManifestMap.size) {
+        log.divider('Circular Dependency Detected');
+        const cycles = findCycles(filteredManifestMap, adjacency);
+        for (const c of cycles) log.error('Cycle: ' + c.join(' → '));
+        throw new Error('Circular dependency detected. Manifest build failed.'); }
+    /* ---------------- 5. Summary ---------------- */
+    log.divider('Manifest Summary');
+    log.success(`Total calculations in build: ${sortedManifest.length}`);
+    log.success(`Total passes required: ${maxPass}\n`);
+    const grouped = [...sortedManifest.reduce((m, e) => { if (!m.has(e.pass)) m.set(e.pass, []); m.get(e.pass).push(e); return m; }, new Map())].sort((a, b) => a[0] - b[0]);
+    for (const [pass, list] of grouped) { console.log(`Pass ${pass} (${list.length} calcs):`); console.log('  ' + list.map(l => l.name).join(', ')); }
+    log.divider('Build Complete');
+    return sortedManifest;
+}
+
+/**
+ * Generates an SVG dependency graph for a given manifest.
+ * @param {object[]} manifest - The sorted manifest array.
+ * @param {string} filename - The output filename (e.g., "full-dependency-tree.svg").
+ */
+async function generateSvgGraph(manifest, filename = 'dependency-tree.svg') {
+    log.divider(`Generating SVG Graph: ${filename}`);
+    const viz = new Viz({ Module, render });
+    const categories = [...new Set(manifest.map(e => e.category))];
+    const colors = ['#e6194b', '#3cb44b', '#ffe119', '#4363d8', '#f58231', '#911eb4', '#46f0f0', '#f032e6', '#bcf60c', '#fabebe', '#008080', '#e6beff', '#9a6324', '#fffac8', '#800000', '#aaffc3', '#808000', '#ffd8b1', '#000075', '#808080'];
+    const colorMap = new Map(categories.map((cat, i) => [cat, colors[i % colors.length]]));
+    colorMap.set('core', '#a9a9a9');
+    let dot = `digraph Manifest {\n  rankdir=LR;\n  node [shape=box, style=filled, fontname="Helvetica"];\n  layout=dot;\n  overlap=false;\n  splines=true;\n`;
+    for (const category of categories) {
+        if (category === 'core') continue;
+        dot += `  subgraph "cluster_${category}" {\n    label="${category.toUpperCase()} Product Line";\n    style=filled;\n    color="#f0f0f0";\n`;
+        for (const entry of manifest.filter(e => e.category === category)) { dot += `    "${entry.name}" [label="${entry.name}\\n(Pass ${entry.pass})", fillcolor="${colorMap.get(category)}"];\n`; }
+        dot += `  }\n`; }
+    dot += `  subgraph "cluster_core" {\n    label="CORE";\n    style=filled;\n    color="#e0e0e0";\n`;
+    for (const entry of manifest.filter(e => e.category === 'core')) { dot += `    "${entry.name}" [label="${entry.name}\\n(Pass ${entry.pass})", fillcolor="${colorMap.get('core')}"];\n`; }
+    dot += `  }\n`;
+    for (const entry of manifest) { for (const dep of entry.dependencies || []) { dot += `  "${dep}" -> "${entry.name}";\n`; } }
+    dot += '}\n';
+    try {
+        const svg = await viz.renderString(dot, { format: 'svg' });
+        const out = path.join(__dirname, '..', '..', '..', '..', 'config', filename);
+        fs.writeFileSync(out, svg);
+        log.success(`Dependency tree generated at ${out}`);
+    } catch (e) { log.error(`SVG generation failed: ${e.message}`); }
+}
+
+
+/**
+ * Main entry point for building and exporting the manifest.
+ * @param {string[]} productLinesToRun - Array of product line categories (folder names) to build for.
+ */
+async function build(productLinesToRun) {
+    try {
+        const manifest = buildManifest(productLinesToRun);
+        await generateSvgGraph(manifest, `dependency-tree-filtered.svg`);
+        const allProductLines = Object.keys(calculations).filter(c => c !== 'legacy');
+        const fullManifest = buildManifest(allProductLines);
+        await generateSvgGraph(fullManifest, 'dependency-tree-full.svg');
+        return manifest;
+    } catch (error) { log.error(error.message); return null; }
+}
+
+module.exports = { build };
+
+if (require.main === module) { (async () => { log.info('Running manifest builder in local debug mode...'); const productLines = Object.keys(calculations).filter(c => c !== 'legacy');await build(productLines); })(); }

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -55,17 +55,12 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
  */
 async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config, { db, logger }) {
     const manifestMap = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
-    
-    // --- FIX: Create a Set of *all* calcs we need to fetch ---
-    // This includes the calcs in this pass AND their dependencies.
     const calcsToFetch = new Set();
     
     for (const calc of calcsInPass) {
         const calcName = normalizeName(calc.name);
-        // Add the calc itself (for skipping)
         calcsToFetch.add(calcName);
         
-        // Add all its dependencies (for processing)
         if (calc.dependencies && calc.dependencies.length > 0) {
             for (const depName of calc.dependencies) {
                 calcsToFetch.add(normalizeName(depName));
@@ -327,25 +322,8 @@ async function runMetaComputationPass(date, calcs, passName, config, deps, fetch
     const name = normalizeName(mCalc.name), Cl = mCalc.class;
     if (typeof Cl !== 'function') { logger.log('ERROR', `Invalid class ${name}`); failedCalcs.push(name); continue; }
     const inst = new Cl();
-    
-    // --- FIX: This is the critical change for the "Structural Bug" ---
-    // The original code assumed a 'meta' calc *only* has a .process() method.
-    // We now check for .process() first, and if it's not there,
-    // we fall back to calling .getResult() and pass it the dependencies.
-    //
-    // This file, however, expects `crowd_sharpe_ratio_proxy` to be
-    // refactored to use `process()`. See the update to that file.
-    // This `runMetaComputationPass` function remains as-is,
-    // as the fix is in refactoring the calculation file itself.
-    
-    try { 
-        
-        // --- FIX: Refactored `crowd_sharpe_ratio_proxy` will now have this method ---
-        if (typeof inst.process !== 'function') {
-            logger.log('ERROR', `Meta-calc ${name} is missing a 'process' method.`);
-            failedCalcs.push(name);
-            continue;
-        }
+     
+    try { if (typeof inst.process !== 'function') { logger.log('ERROR', `Meta-calc ${name} is missing a 'process' method.`); failedCalcs.push(name); continue; }
 
         const result = await Promise.resolve(inst.process(dStr, { ...deps, rootData: fullRoot }, config, fetchedDeps));
         

package/index.js

@@ -29,10 +29,15 @@ const taskEngine =        { handleRequest            : require('./functions/task
                             handleVerify             : require('./functions/task-engine/helpers/verify_helpers')                    .handleVerify  ,
                             handleUpdate             : require('./functions/task-engine/helpers/update_helpers')                    .handleUpdate };
 
+// --- NEW IMPORT ---
+const { build: buildManifestFunc } = require('./functions/computation-system/helpers/computation_manifest_builder');
+
                             // Computation System
 const computationSystem = { runComputationPass       : require('./functions/computation-system/helpers/computation_pass_runner')    .runComputationPass,    
                             dataLoader               : require('./functions/computation-system/utils/data_loader'),
-                            computationUtils         : require('./functions/computation-system/utils/utils') };
+                            computationUtils         : require('./functions/computation-system/utils/utils'),
+                            buildManifest            : buildManifestFunc // <--- NEW PIPE EXPORT
+                        };
 
                             // API
 const api =               { createApiApp             : require('./functions/generic-api/index')                                       .createApiApp,
@@ -50,4 +55,4 @@ const maintenance =       { runSpeculatorCleanup     : require('./functions/spec
                             // Proxy
 const proxy =             { handlePost               : require('./functions/appscript-api/index')                                      .handlePost };
 
-module.exports =          { pipe: { core, orchestrator, dispatcher, taskEngine, computationSystem, api, maintenance, proxy } };
+module.exports =          { pipe: { core, orchestrator, dispatcher, taskEngine, computationSystem, api, maintenance, proxy } };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.158",
+  "version": "1.0.159",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [