bulltrackers-module 1.0.252 → 1.0.254

package/functions/computation-system/data/AvailabilityChecker.js

@@ -1,6 +1,6 @@
 /**
  * @fileoverview Checks availability of root data via the Root Data Index.
- * REFACTORED: Now relies on the centralized 'system_root_data_index' map.
+ * REFACTORED: Now supports granular 'userType' checks (Speculator vs Normal).
  */
 const { normalizeName } = require('../utils/utils');
 
@@ -10,11 +10,34 @@ const INDEX_COLLECTION = process.env.ROOT_DATA_AVAILABILITY_COLLECTION || 'syste
 function checkRootDependencies(calcManifest, rootDataStatus) {
     const missing = [];
     if (!calcManifest.rootDataDependencies) return { canRun: true, missing };
+
+    const userType = calcManifest.userType || 'all';
+
     for (const dep of calcManifest.rootDataDependencies) {
-        if (dep === 'portfolio'     && !rootDataStatus.hasPortfolio) missing.push('portfolio');
+        if (dep === 'portfolio') {
+            if (userType === 'speculator') {
+                if (!rootDataStatus.speculatorPortfolio) missing.push('speculatorPortfolio');
+            } else if (userType === 'normal') {
+                if (!rootDataStatus.normalPortfolio) missing.push('normalPortfolio');
+            } else {
+                // 'all', 'aggregate', or 'n/a' -> Check if ANY portfolio data exists
+                // This satisfies the "either/or" requirement for generic calculations
+                if (!rootDataStatus.hasPortfolio) missing.push('portfolio');
+            }
+        }
+        else if (dep === 'history') {
+            if (userType === 'speculator') {
+                if (!rootDataStatus.speculatorHistory) missing.push('speculatorHistory');
+            } else if (userType === 'normal') {
+                if (!rootDataStatus.normalHistory) missing.push('normalHistory');
+            } else {
+                // 'all', 'aggregate', or 'n/a' -> Check if ANY history data exists
+                if (!rootDataStatus.hasHistory) missing.push('history');
+            }
+        }
+        // These data types are global and do not have user subtypes
         else if (dep === 'insights' && !rootDataStatus.hasInsights)  missing.push('insights');
         else if (dep === 'social'   && !rootDataStatus.hasSocial)    missing.push('social');
-        else if (dep === 'history'  && !rootDataStatus.hasHistory)   missing.push('history');
         else if (dep === 'price'    && !rootDataStatus.hasPrices)    missing.push('price');
     }
     return { canRun: missing.length === 0, missing };
@@ -49,27 +72,33 @@ function getViableCalculations(candidates, fullManifest, rootDataStatus, dailySt
 
 /**
  * Checks data availability by reading the centralized index.
- * Only falls back to raw checks if explicitly configured or index is missing.
+ * Extracts granular details to support Speculator-specific checks.
  */
 async function checkRootDataAvailability(dateStr, config, dependencies, earliestDates) {
     const { logger, db } = dependencies;
     
     try {
         // 1. Try reading the Index
-        const indexDoc = await db.collection(INDEX_COLLECTION).doc(dateStr).get();
+        const indexDoc = db.collection(INDEX_COLLECTION).doc(dateStr).get();
         
         if (indexDoc.exists) {
             const data = indexDoc.data();
-            // Return status based on the map
-            // Note: We return null references. The data loaders in streamPortfolioData
-            // have logic to fetch refs if providedRefs is null (which they are here).
+            const details = data.details || {}; // Extract granular details if present
+
             return {
                 status: {
+                    // Global flags (Aggregate - true if EITHER exists)
                     hasPortfolio: !!data.hasPortfolio,
                     hasHistory:   !!data.hasHistory,
                     hasSocial:    !!data.hasSocial,
                     hasInsights:  !!data.hasInsights,
-                    hasPrices:    !!data.hasPrices
+                    hasPrices:    !!data.hasPrices,
+
+                    // Granular flags (Specific)
+                    speculatorPortfolio: !!details.speculatorPortfolio,
+                    normalPortfolio:     !!details.normalPortfolio,
+                    speculatorHistory:   !!details.speculatorHistory,
+                    normalHistory:       !!details.normalHistory
                 },
                 portfolioRefs: null, 
                 historyRefs: null,
@@ -79,10 +108,12 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
             };
         } else {
             // Index missing: implies data hasn't been indexed yet or doesn't exist.
-            // For safety in this strict model, we assume MISSING.
             logger.log('WARN', `[Availability] Index not found for ${dateStr}. Assuming NO data.`);
             return {
-                status: { hasPortfolio: false, hasHistory: false, hasSocial: false, hasInsights: false, hasPrices: false }
+                status: { 
+                    hasPortfolio: false, hasHistory: false, hasSocial: false, hasInsights: false, hasPrices: false,
+                    speculatorPortfolio: false, normalPortfolio: false, speculatorHistory: false, normalHistory: false
+                }
             };
         }
 

package/functions/computation-system/helpers/computation_dispatcher.js

@@ -1,7 +1,7 @@
 /**
  * FILENAME: computation-system/helpers/computation_dispatcher.js
  * PURPOSE: "Smart Dispatcher" - Analyzes state and only dispatches valid, runnable tasks.
- * UPDATED: Implements pre-dispatch analysis to guarantee worker success.
+ * UPDATED: Implements Audit Ledger creation (PENDING state) before dispatch.
  */
 
 const { getExpectedDateStrings, normalizeName, DEFINITIVE_EARLIEST_DATES } = require('../utils/utils.js');
@@ -9,6 +9,7 @@ const { groupByPass, analyzeDateExecution }     = require('../WorkflowOrchestrat
 const { PubSubUtils }                           = require('../../core/utils/pubsub_utils');
 const { fetchComputationStatus, updateComputationStatus } = require('../persistence/StatusRepository');
 const { checkRootDataAvailability }             = require('../data/AvailabilityChecker');
+const { commitBatchInChunks }                   = require('../persistence/FirestoreUtils'); // [NEW IMPORT]
 const pLimit                                    = require('p-limit');
 
 const TOPIC_NAME = 'computation-tasks';
@@ -19,7 +20,7 @@ const STATUS_IMPOSSIBLE = 'IMPOSSIBLE';
  * Performs full pre-flight checks (Root Data, Dependencies, History) before emitting.
  */
 async function dispatchComputationPass(config, dependencies, computationManifest) {
-    const { logger }  = dependencies;
+    const { logger, db } = dependencies; // Added db destructuring
     const pubsubUtils = new PubSubUtils(dependencies);
     const passToRun   = String(config.COMPUTATION_PASS_TO_RUN);
     
@@ -86,7 +87,6 @@ async function dispatchComputationPass(config, dependencies, computationManifest
             });
 
             // Mark Blocked/Failed Deps (Temporary Failure)
-            // We write these so the status reflects reality, but we DO NOT dispatch them.
             [...report.blocked, ...report.failedDependency].forEach(item => {
                 statusUpdates[item.name] = { hash: false, category: 'unknown', reason: item.reason };
             });
@@ -103,6 +103,7 @@ async function dispatchComputationPass(config, dependencies, computationManifest
                     date: dateStr,
                     pass: passToRun,
                     computation: normalizeName(item.name),
+                    hash: item.hash || item.newHash, // [NEW] Ensure Hash is passed for Ledger
                     timestamp: Date.now()
                 });
             });
@@ -116,7 +117,30 @@ async function dispatchComputationPass(config, dependencies, computationManifest
 
     // 4. Batch Dispatch Valid Tasks
     if (tasksToDispatch.length > 0) {
-        logger.log('INFO', `[Dispatcher] ✅ Generated ${tasksToDispatch.length} VALID tasks. Dispatching...`);
+        // --- [NEW] STEP 4.1: CREATE AUDIT LEDGER ENTRIES ---
+        logger.log('INFO', `[Dispatcher] 📝 Creating Audit Ledger entries for ${tasksToDispatch.length} tasks...`);
+        
+        const ledgerWrites = [];
+        for (const task of tasksToDispatch) {
+            const ledgerRef = db.collection(`computation_audit_ledger/${task.date}/passes/${task.pass}/tasks`).doc(task.computation);
+            ledgerWrites.push({
+                ref: ledgerRef,
+                data: {
+                    status: 'PENDING',
+                    computation: task.computation,
+                    expectedHash: task.hash || 'unknown',
+                    createdAt: new Date(),
+                    retries: 0
+                },
+                options: { merge: true } // Merge allows updating retries/timestamps without wiping history
+            });
+        }
+
+        // Commit Ledger writes using chunked batch utility
+        await commitBatchInChunks(config, dependencies, ledgerWrites, 'AuditLedger Creation');
+        // ---------------------------------------------------
+
+        logger.log('INFO', `[Dispatcher] ✅ Generated ${tasksToDispatch.length} VALID tasks. Dispatching to Pub/Sub...`);
         
         await pubsubUtils.batchPublishTasks(dependencies, {
             topicName: TOPIC_NAME,

package/functions/computation-system/persistence/ResultCommitter.js

@@ -1,30 +1,36 @@
 /**
  * @fileoverview Handles saving computation results with observability and Smart Cleanup.
+ * UPDATED: Implements Audit Ledger completion logic ("Closing the Ledger").
  */
 const { commitBatchInChunks }     = require('./FirestoreUtils');
 const { updateComputationStatus } = require('./StatusRepository');
 const { batchStoreSchemas }       = require('../utils/schema_capture');
 const { generateProcessId, PROCESS_TYPES } = require('../logger/logger');
+// Note: normalizeName is typically needed for doc IDs, but keys in stateObj are usually already normalized.
+// If not, ensure it is imported. Based on StandardExecutor, keys are normalized.
 
 async function commitResults(stateObj, dStr, passName, config, deps, skipStatusWrite = false) {
     const successUpdates = {};
     const schemas        = [];
     const cleanupTasks   = []; // Tasks to delete old data
-    const { logger }     = deps;
+    const { logger, db } = deps;
     const pid            = generateProcessId(PROCESS_TYPES.STORAGE, passName, dStr);
+    
+    // [NEW] Extract numeric pass ID from string (e.g., "Pass 1" -> "1")
+    const passNum = passName.replace(/[^0-9]/g, '');
 
     for (const name in stateObj) {
         const calc = stateObj[name];
         try {
             const result = await calc.getResult();
             
-            // [UPDATE] Validate Result: Check for Null, Empty Object, or Zero
+            // Validate Result: Check for Null, Empty Object, or Zero
             const isEmpty = !result || 
                             (typeof result === 'object' && Object.keys(result).length === 0) ||
                             (typeof result === 'number' && result === 0);
 
             if (isEmpty) {
-                // [UPDATE] Mark status as FALSE (Failed/Empty) so it re-runs or is flagged
+                // Mark status as FALSE (Failed/Empty) so it re-runs or is flagged
                 if (calc.manifest.hash) {
                     successUpdates[name] = { 
                         hash: false, 
@@ -35,7 +41,7 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                 continue; 
             }
 
-            const mainDocRef = deps.db.collection(config.resultsCollection)
+            const mainDocRef = db.collection(config.resultsCollection)
                 .doc(dStr)
                 .collection(config.resultsSubcollection)
                 .doc(calc.manifest.category)
@@ -44,6 +50,22 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
 
             const updates = await prepareAutoShardedWrites(result, mainDocRef, logger);
 
+            // --- [NEW] ADD AUDIT LEDGER COMPLETION TO BATCH ---
+            if (passNum && calc.manifest) {
+                const ledgerRef = db.collection(`computation_audit_ledger/${dStr}/passes/${passNum}/tasks`).doc(name);
+                updates.push({
+                    ref: ledgerRef,
+                    data: {
+                        status: 'COMPLETED',
+                        completedAt: new Date(),
+                        actualHash: calc.manifest.hash,
+                        _verified: true
+                    },
+                    options: { merge: true }
+                });
+            }
+            // --------------------------------------------------
+
             // Capture Schema
             if (calc.manifest.class.getSchema) {
                 const { class: _cls, ...safeMetadata } = calc.manifest;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.252",
+  "version": "1.0.254",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [