bulltrackers-module 1.0.158 → 1.0.160

package/functions/computation-system/helpers/computation_manifest_builder.js

@@ -0,0 +1,264 @@
+/**
+ * @fileoverview
+ * Dynamic Manifest Builder (v3 - Refactored)
+ *
+ * This script builds the computation manifest dynamically by introspecting all
+ * calculation classes from the new product-line structure (core, gem, gauss, etc.).
+ * It validates metadata, resolves dependencies, and performs a topological sort
+ * to determine execution passes.
+ *
+ * It explicitly skips the 'legacy' package.
+ * It has removed all logic for deprecated folder structures (e.g., /historical/).
+ */
+
+const { calculations }   = require('aiden-shared-calculations-unified');
+const fs                 = require('fs');
+const path               = require('path');
+const Viz                = require('viz.js');
+const { Module, render } = require('viz.js/full.render.js');
+
+/* --------------------------------------------------
+ * Pretty Console Helpers
+ * -------------------------------------------------- */
+const log = {
+    info: (msg) => console.log('ℹ︎ ' + msg),
+    step: (msg) => console.log('› ' + msg),
+    warn: (msg) => console.warn('⚠︎ ' + msg),
+    success: (msg) => console.log('✔︎ ' + msg),
+    error: (msg) => console.error('✖ ' + msg),
+    fatal: (msg) => { console.error('✖ FATAL ✖ ' + msg); console.error('✖ FATAL ✖ Manifest build FAILED.'); },
+    divider: (label) => { const line = ''.padEnd(60, '─'); console.log(`\n${line}\n${label}\n${line}\n`); },
+};
+
+/* --------------------------------------------------
+ * Helper Utilities
+ * -------------------------------------------------- */
+
+const normalizeName = (name) => { if (typeof name !== 'string') return name; return name.trim().replace(/,$/, '').replace(/_/g, '-').toLowerCase(); };
+
+/**
+ * Finds the closest string matches for a typo.
+ */
+function suggestClosest(name, candidates, n = 3) {
+    const levenshtein = (a = '', b = '') => {
+        const m = a.length, n = b.length;
+        if (!m) return n; if (!n) return m;
+        const dp = Array.from({ length: m + 1 }, (_, i) => Array(n + 1).fill(i));
+        for (let j = 0; j <= n; j++) dp[0][j] = j;
+        for (let i = 1; i <= m; i++)
+        for (let j = 1; j <= n; j++)
+        dp[i][j] = a[i - 1] === b[j - 1] ? dp[i - 1][j - 1] : Math.min(dp[i - 1][j - 1], dp[i][j - 1], dp[i - 1][j]) + 1;
+        return dp[m][n];
+    };
+    const scores = candidates.map(c => [c, levenshtein(name, c)]);
+    scores.sort((a, b) => a[1] - b[1]);
+    return scores.slice(0, n).map(s => s[0]);
+}
+
+/**
+ * Checks for a cycle using Depth First Search.
+ */
+function findCycles(manifestMap, adjacencyList) {
+    const visited = new Set(), stack = new Set(), cycles = [];
+    const dfs = (node, path) => {
+        if (stack.has(node)) { const idx = path.indexOf(node); cycles.push([...path.slice(idx), node]); return; }
+        if (visited.has(node)) return;
+        visited.add(node); stack.add(node);
+        for (const nb of adjacencyList.get(node) || []) dfs(nb, [...path, nb]);
+        stack.delete(node); };
+    for (const name of manifestMap.keys()) dfs(name, [name]);
+    return cycles;
+}
+
+/**
+ * Recursively traces all dependencies for a given set of "endpoint" calcs.
+ */
+function getDependencySet(endpoints, adjacencyList) {
+    const required = new Set(endpoints);
+    const queue = [...endpoints];
+    while (queue.length > 0) { const calcName = queue.shift(); const dependencies = adjacencyList.get(calcName);
+    if (dependencies) { for (const dep of dependencies) { if (!required.has(dep)) { required.add(dep); queue.push(dep); } } } }
+    return required;
+}
+
+/* --------------------------------------------------
+ * Core Manifest Builder
+ * -------------------------------------------------- */
+
+/**
+ * Builds the full computation manifest.
+ * @param {string[]} productLinesToRun - Array of product line categories (folder names) to build for.
+ * @returns {object[]} The final sorted manifest array.
+ */
+function buildManifest(productLinesToRun = []) {
+    log.divider('Building Dynamic Manifest');
+    log.info(`Target Product Lines: [${productLinesToRun.join(', ')}]`);
+    const manifestMap      = new Map();
+    const adjacency        = new Map(); 
+    const reverseAdjacency = new Map(); 
+    const inDegree         = new Map();
+    let hasFatalError      = false;
+    /* ---------------- 1. Load All Calculations ---------------- */
+    log.step('Loading and validating all calculation classes…');
+    const allCalculationClasses = new Map();
+    /**
+     * Processes a single calculation class, validates it, and adds to maps.
+     * --- REFACTOR: Simplified signature, isHistorical is now read from metadata ---
+     */
+    function processCalc(Class, name, category) {
+        if (!Class || typeof Class !== 'function') return;
+        const normalizedName = normalizeName(name);
+        allCalculationClasses.set(normalizedName, Class);
+        // --- RULE 1: Check for static getMetadata() ---
+        if (typeof Class.getMetadata !== 'function') { log.fatal(`Calculation "${normalizedName}" is missing the static getMetadata() method. Build FAILED.`); hasFatalError = true; return; }
+        // --- RULE 2: Check for static getDependencies() ---
+        if (typeof Class.getDependencies !== 'function') { log.fatal(`Calculation "${normalizedName}" is missing the static getDependencies() method. Build FAILED.`); hasFatalError = true;return; }
+        // --- RULE 3: Check for static getSchema() ---
+        if (typeof Class.getSchema !== 'function') {log.warn(`Calculation "${normalizedName}" is missing the static getSchema() method. (Recommended)`); }
+        const metadata = Class.getMetadata();
+        const dependencies = Class.getDependencies().map(normalizeName);
+        // --- RULE 4: Check for isHistorical mismatch ---
+        if (metadata.isHistorical === true && !Class.toString().includes('yesterdayPortfolio')) { log.warn(`Calculation "${normalizedName}" is marked 'isHistorical: true' but does not seem to use 'yesterdayPortfolio'.`); }
+        const manifestEntry = {
+            name: normalizedName,
+            class: Class,
+            category: metadata.category || category, 
+            type: metadata.type,
+            isHistorical: metadata.isHistorical, 
+            rootDataDependencies: metadata.rootDataDependencies || [],
+            userType: metadata.userType,
+            dependencies: dependencies,
+            pass: 0,
+        };
+        manifestMap.set(normalizedName, manifestEntry);
+        adjacency.set(normalizedName, dependencies);
+        inDegree.set(normalizedName, dependencies.length);
+        dependencies.forEach(dep => { if (!reverseAdjacency.has(dep)) reverseAdjacency.set(dep, []); reverseAdjacency.get(dep).push(normalizedName); }); }
+    for (const category in calculations) {
+        if (category === 'legacy') { log.info('Skipping "legacy" calculations package.'); continue; }
+        const group = calculations[category];
+        for (const key in group) {
+            const entry = group[key];
+            if (typeof entry === 'function') { processCalc(entry, key, category);  } } }
+
+    if (hasFatalError) { throw new Error('Manifest build failed due to missing static methods in calculations.'); }
+    log.success(`Loaded and validated ${manifestMap.size} total calculations.`);
+    /* ---------------- 2. Validate Dependency Links ---------------- */
+    log.divider('Validating Dependency Links');
+    const allNames = new Set(manifestMap.keys());
+    let invalidLinks = false;
+    for (const [name, entry] of manifestMap) {
+        for (const dep of entry.dependencies) {
+            if (!allNames.has(dep)) {
+                invalidLinks = true;
+                const guesses = suggestClosest(dep, Array.from(allNames));
+                log.error(`${name} depends on unknown calculation "${dep}"`);
+                if (guesses.length) log.info(`Did you mean: ${guesses.join(', ')} ?`);
+            }
+            if (dep === name) {
+                invalidLinks = true;
+                log.error(`${name} has a circular dependency on *itself*!`);
+            }
+        }
+    }
+    if (invalidLinks) { throw new Error('Manifest validation failed. Fix missing or self-referencing dependencies.'); }
+    log.success('All dependency links are valid.');
+    /* ---------------- 3. Filter for Product Lines ---------------- */
+    log.divider('Filtering by Product Line');
+    // 1. Find all "endpoint" calculations (the final signals) in the target product lines.
+    const productLineEndpoints = [];
+    for (const [name, entry] of manifestMap.entries()) { if (productLinesToRun.includes(entry.category)) { productLineEndpoints.push(name); } }
+    // 2. Add 'core' calculations as they are always included.
+    for (const [name, entry] of manifestMap.entries()) { if (entry.category === 'core') { productLineEndpoints.push(name); } }
+    // 3. Trace all dependencies upwards from these endpoints.
+    const requiredCalcs = getDependencySet(productLineEndpoints, adjacency);
+    log.info(`Identified ${productLineEndpoints.length} endpoint/core calculations.`);
+    log.info(`Traced dependencies: ${requiredCalcs.size} total calculations are required.`);
+    // 4. Create the final, filtered maps for sorting.
+    const filteredManifestMap = new Map();
+    const filteredInDegree = new Map();
+    const filteredReverseAdjacency = new Map();
+    for (const name of requiredCalcs) { filteredManifestMap.set(name, manifestMap.get(name)); filteredInDegree.set(name, inDegree.get(name));
+    const consumers = (reverseAdjacency.get(name) || []).filter(consumer => requiredCalcs.has(consumer)); filteredReverseAdjacency.set(name, consumers); }
+    log.success(`Filtered manifest to ${filteredManifestMap.size} calculations.`);
+    /* ---------------- 4. Topological Sort (Kahn's Algorithm) ---------------- */
+    log.divider('Topological Sorting (Kahn\'s Algorithm)');
+    const sortedManifest = [];
+    const queue = [];
+    let maxPass = 0;
+    for (const [name, degree] of filteredInDegree) { if (degree === 0) { queue.push(name); filteredManifestMap.get(name).pass = 1; maxPass = 1; } }
+    queue.sort();
+    while (queue.length) {
+        const currentName = queue.shift();
+        const currentEntry = filteredManifestMap.get(currentName);
+        sortedManifest.push(currentEntry);
+        for (const neighborName of (filteredReverseAdjacency.get(currentName) || [])) { const newDegree = filteredInDegree.get(neighborName) - 1; filteredInDegree.set(neighborName, newDegree);
+            const neighborEntry = filteredManifestMap.get(neighborName);
+            if (neighborEntry.pass <= currentEntry.pass) { neighborEntry.pass = currentEntry.pass + 1; if (neighborEntry.pass > maxPass) maxPass = neighborEntry.pass; }
+            if (newDegree === 0) { queue.push(neighborName); } }
+        queue.sort(); }
+    if (sortedManifest.length !== filteredManifestMap.size) {
+        log.divider('Circular Dependency Detected');
+        const cycles = findCycles(filteredManifestMap, adjacency);
+        for (const c of cycles) log.error('Cycle: ' + c.join(' → '));
+        throw new Error('Circular dependency detected. Manifest build failed.'); }
+    /* ---------------- 5. Summary ---------------- */
+    log.divider('Manifest Summary');
+    log.success(`Total calculations in build: ${sortedManifest.length}`);
+    log.success(`Total passes required: ${maxPass}\n`);
+    const grouped = [...sortedManifest.reduce((m, e) => { if (!m.has(e.pass)) m.set(e.pass, []); m.get(e.pass).push(e); return m; }, new Map())].sort((a, b) => a[0] - b[0]);
+    for (const [pass, list] of grouped) { console.log(`Pass ${pass} (${list.length} calcs):`); console.log('  ' + list.map(l => l.name).join(', ')); }
+    log.divider('Build Complete');
+    return sortedManifest;
+}
+
+/**
+ * Generates an SVG dependency graph for a given manifest.
+ * @param {object[]} manifest - The sorted manifest array.
+ * @param {string} filename - The output filename (e.g., "full-dependency-tree.svg").
+ */
+async function generateSvgGraph(manifest, filename = 'dependency-tree.svg') {
+    log.divider(`Generating SVG Graph: ${filename}`);
+    const viz = new Viz({ Module, render });
+    const categories = [...new Set(manifest.map(e => e.category))];
+    const colors = ['#e6194b', '#3cb44b', '#ffe119', '#4363d8', '#f58231', '#911eb4', '#46f0f0', '#f032e6', '#bcf60c', '#fabebe', '#008080', '#e6beff', '#9a6324', '#fffac8', '#800000', '#aaffc3', '#808000', '#ffd8b1', '#000075', '#808080'];
+    const colorMap = new Map(categories.map((cat, i) => [cat, colors[i % colors.length]]));
+    colorMap.set('core', '#a9a9a9');
+    let dot = `digraph Manifest {\n  rankdir=LR;\n  node [shape=box, style=filled, fontname="Helvetica"];\n  layout=dot;\n  overlap=false;\n  splines=true;\n`;
+    for (const category of categories) {
+        if (category === 'core') continue;
+        dot += `  subgraph "cluster_${category}" {\n    label="${category.toUpperCase()} Product Line";\n    style=filled;\n    color="#f0f0f0";\n`;
+        for (const entry of manifest.filter(e => e.category === category)) { dot += `    "${entry.name}" [label="${entry.name}\\n(Pass ${entry.pass})", fillcolor="${colorMap.get(category)}"];\n`; }
+        dot += `  }\n`; }
+    dot += `  subgraph "cluster_core" {\n    label="CORE";\n    style=filled;\n    color="#e0e0e0";\n`;
+    for (const entry of manifest.filter(e => e.category === 'core')) { dot += `    "${entry.name}" [label="${entry.name}\\n(Pass ${entry.pass})", fillcolor="${colorMap.get('core')}"];\n`; }
+    dot += `  }\n`;
+    for (const entry of manifest) { for (const dep of entry.dependencies || []) { dot += `  "${dep}" -> "${entry.name}";\n`; } }
+    dot += '}\n';
+    try {
+        const svg = await viz.renderString(dot, { format: 'svg' });
+        const out = path.join(__dirname, '..', '..', '..', '..', 'config', filename);
+        fs.writeFileSync(out, svg);
+        log.success(`Dependency tree generated at ${out}`);
+    } catch (e) { log.error(`SVG generation failed: ${e.message}`); }
+}
+
+
+/**
+ * Main entry point for building and exporting the manifest.
+ * @param {string[]} productLinesToRun - Array of product line categories (folder names) to build for.
+ */
+async function build(productLinesToRun) {
+    try {
+        const manifest = buildManifest(productLinesToRun);
+        await generateSvgGraph(manifest, `dependency-tree-filtered.svg`);
+        const allProductLines = Object.keys(calculations).filter(c => c !== 'legacy');
+        const fullManifest = buildManifest(allProductLines);
+        await generateSvgGraph(fullManifest, 'dependency-tree-full.svg');
+        return manifest;
+    } catch (error) { log.error(error.message); return null; }
+}
+
+module.exports = { build };
+
+if (require.main === module) { (async () => { log.info('Running manifest builder in local debug mode...'); const productLines = Object.keys(calculations).filter(c => c !== 'legacy');await build(productLines); })(); }

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -55,17 +55,12 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
  */
 async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config, { db, logger }) {
     const manifestMap = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
-    
-    // --- FIX: Create a Set of *all* calcs we need to fetch ---
-    // This includes the calcs in this pass AND their dependencies.
     const calcsToFetch = new Set();
     
     for (const calc of calcsInPass) {
         const calcName = normalizeName(calc.name);
-        // Add the calc itself (for skipping)
         calcsToFetch.add(calcName);
         
-        // Add all its dependencies (for processing)
         if (calc.dependencies && calc.dependencies.length > 0) {
             for (const depName of calc.dependencies) {
                 calcsToFetch.add(normalizeName(depName));
@@ -104,7 +99,13 @@ async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config,
     const fetched = {};
     if (docRefs.length) {
         (await db.getAll(...docRefs)).forEach((doc, i) => {
-            fetched[depNames[i]] = doc.exists ? doc.data() : null;
+            // --- FIX [PROBLEM 8]: Add completion marker check ---
+            const data = doc.exists ? doc.data() : null;
+            if (data && data._completed === true) {
+                fetched[depNames[i]] = data;
+            } else {
+                fetched[depNames[i]] = null; // Treat as not existing if incomplete
+            }
         });
     }
     
@@ -144,19 +145,18 @@ function filterCalculations(standardCalcs, metaCalcs, rootDataStatus, existingRe
         return earliestRunDate;
     };
     const filterCalc = (calc) => {
-        if (existingResults[calc.name]) {logger.log('TRACE', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Result already exists.`); skipped.add(calc.name); return false;}
+        // --- FIX [PROBLEM 8]: The check for existingResults is now correct ---
+        if (existingResults[calc.name]) {logger.log('TRACE', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Result already exists (and is complete).`); skipped.add(calc.name); return false;}
+        
         const earliestRunDate = getTrueEarliestRunDate(calc);
         if (dateToProcess < earliestRunDate) {logger.log('TRACE', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Date is before true earliest run date (${earliestRunDate.toISOString().slice(0, 10)}).`); skipped.add(calc.name); return false; }
         const { canRun, missing: missingRoot } = checkRootDependencies(calc, rootDataStatus);
         if (!canRun) {logger.log('INFO', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Data missing for this date: [${missingRoot.join(', ')}]`);skipped.add(calc.name); return false;}
         
-        // --- FIX: This check is now robust ---
-        // 'existingResults' now contains all dependencies, so this check
-        // will correctly find 'pnl_distribution_per_stock' and *not* skip.
         if (calc.type === 'meta') { 
             const missingDeps = (calc.dependencies || [])
                 .map(normalizeName)
-                .filter(d => !existingResults[d]); 
+                .filter(d => !existingResults[d]); // This check is now robust
             if (missingDeps.length > 0) { 
                 logger.log('WARN', `[Pass ${passToRun} Meta] Skipping ${calc.name} for ${dateStr}. Missing computed deps: [${missingDeps.join(', ')}]`); 
                 skipped.add(calc.name); 
@@ -182,15 +182,20 @@ async function loadHistoricalData(date, calcs, config, deps, rootData) {
     const needsYesterdayInsights  = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('insights'));
     const needsYesterdaySocial    = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('social'));
     const needsYesterdayPortfolio = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('portfolio'));
+    
+    // --- FIX: Add T-1 COMPUTED dependency loading ---
+    const needsYesterdayDependencies = calcs.some(c => c.isHistorical && c.dependencies && c.dependencies.length > 0);
+    
     const prev = new Date(date);
     prev.setUTCDate(prev.getUTCDate() - 1);
     const prevStr = prev.toISOString().slice(0, 10);
+    
     if(needsYesterdayInsights) {
-        tasks.push((async()=>{ const prev=new Date(date); prev.setUTCDate(prev.getUTCDate()-1); const prevStr=prev.toISOString().slice(0,10); 
+        tasks.push((async()=>{ 
         logger.log('INFO', `[PassRunner] Loading YESTERDAY insights data for ${prevStr}`);
         updated.yesterdayInsights=await loadDailyInsights(config,deps,prevStr); })());}
     if(needsYesterdaySocial) {
-        tasks.push((async()=>{ const prev=new Date(date); prev.setUTCDate(prev.getUTCDate()-1); const prevStr=prev.toISOString().slice(0,10); 
+        tasks.push((async()=>{ 
         logger.log('INFO', `[PassRunner] Loading YESTERDAY social data for ${prevStr}`);
         updated.yesterdaySocialPostInsights=await loadDailySocialPostInsights(config,deps,prevStr); })());}
     
@@ -200,6 +205,17 @@ async function loadHistoricalData(date, calcs, config, deps, rootData) {
             updated.yesterdayPortfolioRefs = await getPortfolioPartRefs(config, deps, prevStr); 
         })());
     }
+
+    // --- FIX: Load T-1 COMPUTED dependencies ---
+    if(needsYesterdayDependencies) {
+        tasks.push((async()=>{ 
+            logger.log('INFO', `[PassRunner] Loading YESTERDAY computed dependencies for ${prevStr}`);
+            // This is a simplified fetch, assuming all calcs in this pass share the same T-1 deps
+            // A more robust solution would aggregate all unique T-1 deps.
+            updated.yesterdayDependencyData = await fetchExistingResults(prevStr, calcs, calcs.map(c => c.manifest), config, deps);
+        })());
+    }
+
     await Promise.all(tasks); 
     return updated; 
 }
@@ -209,9 +225,12 @@ async function loadHistoricalData(date, calcs, config, deps, rootData) {
  */
 async function streamAndProcess(dateStr, state, passName, config, deps, rootData, portfolioRefs, historyRefs) { 
     const { logger, calculationUtils } = deps;
-    const { todayInsights, yesterdayInsights, todaySocialPostInsights, yesterdaySocialPostInsights } = rootData;
+    const { todayInsights, yesterdayInsights, todaySocialPostInsights, yesterdaySocialPostInsights, yesterdayDependencyData } = rootData;
     const calcsThatStreamPortfolio = Object.values(state).filter(calc => calc && calc.manifest && (calc.manifest.rootDataDependencies.includes('portfolio') || calc.manifest.category === 'speculators'));
-    const context={instrumentMappings:(await calculationUtils.loadInstrumentMappings()).instrumentToTicker, sectorMapping:(await calculationUtils.loadInstrumentMappings()).instrumentToSector, todayDateStr:dateStr, dependencies:deps, config};
+    
+    // --- FIX: Add yesterday's computed data to context ---
+    const context={instrumentMappings:(await calculationUtils.loadInstrumentMappings()).instrumentToTicker, sectorMapping:(await calculationUtils.loadInstrumentMappings()).instrumentToSector, todayDateStr:dateStr, dependencies:deps, config, yesterdaysDependencyData: yesterdayDependencyData};
+    
     let firstUser=true; 
     for(const name in state){  const calc=state[name]; if(!calc||typeof calc.process!=='function') continue; 
     const cat=calc.manifest.category;
@@ -254,6 +273,11 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
     firstUser=false; 
     if (pY) { delete yesterdayPortfolios[uid]; }
     if (hT) { delete todayHistoryData[uid]; } }  }
+    
+    // --- FIX [PROBLEM 7]: Clear stale data to prevent memory leak ---
+    yesterdayPortfolios = {};
+    todayHistoryData = {};
+
     logger.log('INFO', `[${passName}] Finished streaming data for ${dateStr}.`);
 }
 
@@ -284,6 +308,8 @@ async function runStandardComputationPass(date, calcs, passName, config, deps, r
     } else { standardResult[key] = result[key]; }}
     if (Object.keys(standardResult).length > 0) {
     const docRef = deps.db.collection(config.resultsCollection).doc(dStr) .collection(config.resultsSubcollection).doc(calc.manifest.category) .collection(config.computationsSubcollection).doc(name);
+    // --- FIX [PROBLEM 8]: Add completion marker ---
+    standardResult._completed = true;
     standardWrites.push({ ref: docRef, data: standardResult });}
     const calcClass = calc.manifest.class;
     let staticSchema = null;
@@ -302,7 +328,10 @@ async function runStandardComputationPass(date, calcs, passName, config, deps, r
     if (docPath.includes('/')) { docRef = deps.db.doc(docPath); } else {
     const collection = (docPath.startsWith('user_profile_history'))  ? config.shardedUserProfileCollection : config.shardedProfitabilityCollection;
     docRef = deps.db.collection(collection).doc(docPath); }
-    if (docData && typeof docData === 'object' && !Array.isArray(docData)) {shardedDocWrites.push({ ref: docRef, data: docData });
+    if (docData && typeof docData === 'object' && !Array.isArray(docData)) {
+    // --- FIX [PROBLEM 8]: Add completion marker to sharded writes ---
+    docData._completed = true;
+    shardedDocWrites.push({ ref: docRef, data: docData });
     } else { logger.log('ERROR', `[${passName}] Invalid sharded document data for ${docPath}. Not an object.`, { data: docData }); }
     if (shardedDocWrites.length > 0) { await commitBatchInChunks(config, deps, shardedDocWrites, `${passName} Sharded ${docPath} ${dStr}`); } }
     const logMetadata = {};
@@ -327,25 +356,8 @@ async function runMetaComputationPass(date, calcs, passName, config, deps, fetch
     const name = normalizeName(mCalc.name), Cl = mCalc.class;
     if (typeof Cl !== 'function') { logger.log('ERROR', `Invalid class ${name}`); failedCalcs.push(name); continue; }
     const inst = new Cl();
-    
-    // --- FIX: This is the critical change for the "Structural Bug" ---
-    // The original code assumed a 'meta' calc *only* has a .process() method.
-    // We now check for .process() first, and if it's not there,
-    // we fall back to calling .getResult() and pass it the dependencies.
-    //
-    // This file, however, expects `crowd_sharpe_ratio_proxy` to be
-    // refactored to use `process()`. See the update to that file.
-    // This `runMetaComputationPass` function remains as-is,
-    // as the fix is in refactoring the calculation file itself.
-    
-    try { 
-        
-        // --- FIX: Refactored `crowd_sharpe_ratio_proxy` will now have this method ---
-        if (typeof inst.process !== 'function') {
-            logger.log('ERROR', `Meta-calc ${name} is missing a 'process' method.`);
-            failedCalcs.push(name);
-            continue;
-        }
+     
+    try { if (typeof inst.process !== 'function') { logger.log('ERROR', `Meta-calc ${name} is missing a 'process' method.`); failedCalcs.push(name); continue; }
 
         const result = await Promise.resolve(inst.process(dStr, { ...deps, rootData: fullRoot }, config, fetchedDeps));
         
@@ -358,6 +370,8 @@ async function runMetaComputationPass(date, calcs, passName, config, deps, fetch
             }
             if (Object.keys(standardResult).length > 0) {
                 const docRef = deps.db.collection(config.resultsCollection).doc(dStr) .collection(config.resultsSubcollection).doc(mCalc.category) .collection(config.computationsSubcollection).doc(name);
+                // --- FIX [PROBLEM 8]: Add completion marker ---
+                standardResult._completed = true;
                 standardWrites.push({ ref: docRef, data: standardResult }); 
             }
             const calcClass = mCalc.class;
@@ -375,7 +389,13 @@ async function runMetaComputationPass(date, calcs, passName, config, deps, fetch
     for (const collectionName in shardedWrites) {
     const docs = shardedWrites[collectionName];
     const shardedDocWrites = [];
-    for (const docId in docs) { const docRef = docId.includes('/') ? deps.db.doc(docId) : deps.db.collection(collectionName).doc(docId); shardedDocWrites.push({ ref: docRef, data: docs[docId] }); }
+    for (const docId in docs) { 
+        const docRef = docId.includes('/') ? deps.db.doc(docId) : deps.db.collection(collectionName).doc(docId); 
+        // --- FIX [PROBLEM 8]: Add completion marker to sharded writes ---
+        const docData = docs[docId];
+        docData._completed = true;
+        shardedDocWrites.push({ ref: docRef, data: docData }); 
+    }
     if (shardedDocWrites.length > 0) { await commitBatchInChunks(config, deps, shardedDocWrites, `${passName} Sharded ${collectionName} ${dStr}`); } }    
     const logMetadata = {};
     if (failedCalcs.length > 0) { logMetadata.failedComputations = failedCalcs; }

package/index.js

@@ -29,10 +29,15 @@ const taskEngine =        { handleRequest            : require('./functions/task
                             handleVerify             : require('./functions/task-engine/helpers/verify_helpers')                    .handleVerify  ,
                             handleUpdate             : require('./functions/task-engine/helpers/update_helpers')                    .handleUpdate };
 
+// --- NEW IMPORT ---
+const { build: buildManifestFunc } = require('./functions/computation-system/helpers/computation_manifest_builder');
+
                             // Computation System
 const computationSystem = { runComputationPass       : require('./functions/computation-system/helpers/computation_pass_runner')    .runComputationPass,    
                             dataLoader               : require('./functions/computation-system/utils/data_loader'),
-                            computationUtils         : require('./functions/computation-system/utils/utils') };
+                            computationUtils         : require('./functions/computation-system/utils/utils'),
+                            buildManifest            : buildManifestFunc // <--- NEW PIPE EXPORT
+                        };
 
                             // API
 const api =               { createApiApp             : require('./functions/generic-api/index')                                       .createApiApp,
@@ -50,4 +55,4 @@ const maintenance =       { runSpeculatorCleanup     : require('./functions/spec
                             // Proxy
 const proxy =             { handlePost               : require('./functions/appscript-api/index')                                      .handlePost };
 
-module.exports =          { pipe: { core, orchestrator, dispatcher, taskEngine, computationSystem, api, maintenance, proxy } };
+module.exports =          { pipe: { core, orchestrator, dispatcher, taskEngine, computationSystem, api, maintenance, proxy } };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.158",
+  "version": "1.0.160",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [