bulltrackers-module 1.0.737 → 1.0.739

package/functions/computation-system-v2/handlers/adminTest.js

@@ -0,0 +1,327 @@
+/**
+ * @fileoverview Admin Test Endpoint for Computation System
+ * 
+ * SECURITY: This endpoint is protected by GCP IAM (requireAuth: true).
+ * Only service accounts and users with cloudfunctions.invoker can access it.
+ * 
+ * PURPOSE:
+ * - Test computations in production without waiting for schedule
+ * - Force re-runs of computations (bypass hash checks)
+ * - Test worker pool functionality
+ * - Run on specific entities for debugging
+ * 
+ * USAGE:
+ *   curl -X POST https://REGION-PROJECT.cloudfunctions.net/compute-admin-test \
+ *     -H "Authorization: Bearer $(gcloud auth print-identity-token)" \
+ *     -H "Content-Type: application/json" \
+ *     -d '{"action": "run", "computation": "UserPortfolioSummary", "date": "2026-01-25"}'
+ */
+
+const system = require('../index');
+
+/**
+ * Admin test handler.
+ */
+async function adminTestHandler(req, res) {
+    const startTime = Date.now();
+    
+    try {
+        const { 
+            action = 'status',
+            computation,
+            date = new Date().toISOString().split('T')[0],
+            entityIds,
+            limit = 10,
+            force = true,           // Default to force for testing
+            useWorkerPool,          // Override: true/false/undefined (use config)
+            dryRun = false
+        } = req.body || {};
+
+        console.log(`[AdminTest] Action: ${action}, Computation: ${computation}, Date: ${date}`);
+
+        switch (action) {
+            // =========================================================
+            // STATUS: Show system status and available computations
+            // =========================================================
+            case 'status': {
+                const manifest = await system.getManifest();
+                
+                return res.status(200).json({
+                    status: 'ok',
+                    action: 'status',
+                    systemInfo: {
+                        computationCount: manifest.length,
+                        computations: manifest.map(c => ({
+                            name: c.originalName || c.name,
+                            type: c.type,
+                            pass: c.pass,
+                            schedule: c.schedule
+                        })),
+                        workerPool: {
+                            enabled: process.env.WORKER_POOL_ENABLED === 'true',
+                            localMode: process.env.WORKER_LOCAL_MODE === 'true'
+                        }
+                    },
+                    timestamp: new Date().toISOString()
+                });
+            }
+
+            // =========================================================
+            // ANALYZE: Check what would run for a given date
+            // =========================================================
+            case 'analyze': {
+                const report = await system.analyze({ date });
+                
+                return res.status(200).json({
+                    status: 'ok',
+                    action: 'analyze',
+                    date,
+                    report: {
+                        runnable: report.runnable?.map(r => r.name || r) || [],
+                        skipped: report.skipped?.map(r => ({ name: r.name, reason: r.reason })) || [],
+                        blocked: report.blocked?.map(r => ({ name: r.name, reason: r.reason })) || [],
+                        impossible: report.impossible?.map(r => ({ name: r.name, reason: r.reason })) || []
+                    }
+                });
+            }
+
+            // =========================================================
+            // RUN: Execute a single computation
+            // =========================================================
+            case 'run': {
+                if (!computation) {
+                    return res.status(400).json({
+                        status: 'error',
+                        error: 'Missing "computation" field. Use action: "status" to list available computations.'
+                    });
+                }
+
+                // Log worker pool override if specified
+                if (useWorkerPool !== undefined) {
+                    console.log(`[AdminTest] Worker pool override: ${useWorkerPool ? 'ENABLED' : 'DISABLED'}`);
+                }
+
+                console.log(`[AdminTest] Running ${computation} for ${date}...`);
+                console.log(`[AdminTest] Options: force=${force}, dryRun=${dryRun}, entityIds=${entityIds?.join(',') || 'all'}`);
+
+                const result = await system.runComputation({
+                    date,
+                    computation,
+                    entityIds: entityIds || null,
+                    dryRun,
+                    force,
+                    // Pass worker pool override explicitly (avoids env var caching issues)
+                    useWorkerPool
+                });
+
+                const duration = Date.now() - startTime;
+
+                return res.status(200).json({
+                    status: 'ok',
+                    action: 'run',
+                    computation,
+                    date,
+                    result: {
+                        status: result.status,
+                        duration: result.duration,
+                        resultCount: result.resultCount,
+                        reason: result.reason,
+                        hash: result.hash
+                    },
+                    totalDuration: duration,
+                    workerPoolUsed: useWorkerPool ?? (process.env.WORKER_POOL_ENABLED === 'true')
+                });
+            }
+
+            // =========================================================
+            // RUN_LIMITED: Run on a limited number of entities (safer)
+            // =========================================================
+            case 'run_limited': {
+                if (!computation) {
+                    return res.status(400).json({
+                        status: 'error',
+                        error: 'Missing "computation" field'
+                    });
+                }
+
+                // Get a sample of entities from BigQuery
+                const sampleEntities = await getSampleEntities(computation, date, limit);
+                
+                if (!sampleEntities || sampleEntities.length === 0) {
+                    return res.status(404).json({
+                        status: 'error',
+                        error: `No entities found for ${computation} on ${date}`
+                    });
+                }
+
+                console.log(`[AdminTest] Running LIMITED test: ${sampleEntities.length} entities`);
+
+                const result = await system.runComputation({
+                    date,
+                    computation,
+                    entityIds: sampleEntities,
+                    dryRun,
+                    force,
+                    useWorkerPool  // Pass worker pool override
+                });
+
+                const duration = Date.now() - startTime;
+
+                return res.status(200).json({
+                    status: 'ok',
+                    action: 'run_limited',
+                    computation,
+                    date,
+                    entitiesTested: sampleEntities,
+                    result: {
+                        status: result.status,
+                        duration: result.duration,
+                        resultCount: result.resultCount
+                    },
+                    totalDuration: duration
+                });
+            }
+
+            // =========================================================
+            // TEST_WORKER: Direct test of worker function
+            // =========================================================
+            case 'test_worker': {
+                if (!computation || !entityIds || entityIds.length === 0) {
+                    return res.status(400).json({
+                        status: 'error',
+                        error: 'Requires "computation" and "entityIds" array'
+                    });
+                }
+
+                // Import worker's local execution function
+                const { executeLocal, loadComputation } = require('./worker');
+                
+                // Verify computation exists
+                const CompClass = loadComputation(computation);
+                if (!CompClass) {
+                    return res.status(400).json({
+                        status: 'error',
+                        error: `Unknown computation: ${computation}`
+                    });
+                }
+
+                // Fetch real data for one entity
+                const config = require('../config/bulltrackers.config');
+                const { DataFetcher } = require('../framework/data/DataFetcher');
+                const { QueryBuilder } = require('../framework/data/QueryBuilder');
+                const { SchemaRegistry } = require('../framework/data/SchemaRegistry');
+                
+                const schemaRegistry = new SchemaRegistry(config.bigquery, console);
+                const queryBuilder = new QueryBuilder(config.bigquery, schemaRegistry, console);
+                const dataFetcher = new DataFetcher(
+                    { ...config.bigquery, tables: config.tables }, 
+                    queryBuilder, 
+                    console
+                );
+
+                const compConfig = CompClass.getConfig();
+                const testEntityId = entityIds[0];
+                
+                console.log(`[AdminTest] Fetching data for entity ${testEntityId}...`);
+                const data = await dataFetcher.fetchForComputation(compConfig.requires, date, [testEntityId]);
+
+                // Execute worker logic locally
+                console.log(`[AdminTest] Executing worker logic...`);
+                const workerResult = await executeLocal({
+                    computationName: computation,
+                    entityId: testEntityId,
+                    date,
+                    contextPackage: {
+                        entityData: data,
+                        references: {},
+                        dependencies: {},
+                        config: {}
+                    }
+                });
+
+                const duration = Date.now() - startTime;
+
+                return res.status(200).json({
+                    status: 'ok',
+                    action: 'test_worker',
+                    computation,
+                    entityId: testEntityId,
+                    date,
+                    workerResult: workerResult.result,
+                    duration
+                });
+            }
+
+            default:
+                return res.status(400).json({
+                    status: 'error',
+                    error: `Unknown action: ${action}`,
+                    availableActions: ['status', 'analyze', 'run', 'run_limited', 'test_worker']
+                });
+        }
+
+    } catch (error) {
+        console.error('[AdminTest] Error:', error);
+        return res.status(500).json({
+            status: 'error',
+            error: error.message,
+            stack: process.env.NODE_ENV === 'development' ? error.stack : undefined
+        });
+    }
+}
+
+/**
+ * Get a sample of entity IDs for testing
+ */
+async function getSampleEntities(computation, date, limit) {
+    try {
+        const { BigQuery } = require('@google-cloud/bigquery');
+        const config = require('../config/bulltrackers.config');
+        
+        const bigquery = new BigQuery({
+            projectId: config.bigquery.projectId
+        });
+
+        // Load computation to get its config
+        const { loadComputation } = require('./worker');
+        const CompClass = loadComputation(computation);
+        
+        if (!CompClass) return null;
+        
+        const compConfig = CompClass.getConfig();
+        
+        // Find the driver table (first table with entityField)
+        let driverTable = null;
+        let entityField = null;
+        
+        for (const [tableName, tableSpec] of Object.entries(compConfig.requires || {})) {
+            const tableConfig = config.tables[tableName];
+            if (tableConfig?.entityField) {
+                driverTable = tableName;
+                entityField = tableConfig.entityField;
+                break;
+            }
+        }
+
+        if (!driverTable) return null;
+
+        const query = `
+            SELECT DISTINCT ${entityField} as entity_id
+            FROM \`${config.bigquery.projectId}.${config.bigquery.dataset}.${driverTable}\`
+            WHERE date = @date
+            LIMIT @limit
+        `;
+
+        const [rows] = await bigquery.query({
+            query,
+            params: { date, limit }
+        });
+
+        return rows.map(r => r.entity_id);
+    } catch (e) {
+        console.error('[AdminTest] Failed to get sample entities:', e);
+        return null;
+    }
+}
+
+module.exports = { adminTestHandler };

package/functions/computation-system-v2/handlers/index.js

@@ -12,6 +12,7 @@ const { schedulerHandler } = require('./scheduler');
 const { dispatcherHandler } = require('./dispatcher');
 const { onDemandHandler } = require('./onDemand');
 const { workerHandler, executeLocal } = require('./worker');
+const { adminTestHandler } = require('./adminTest');
 
 module.exports = {
     // Unified scheduler - triggered every minute by Cloud Scheduler
@@ -27,6 +28,9 @@ module.exports = {
     // Invoked by RemoteTaskRunner from Orchestrator
     computationWorker: workerHandler,
     
+    // Admin test endpoint - for testing computations in production
+    computeAdminTest: adminTestHandler,
+    
     // For local testing
     executeWorkerLocal: executeLocal
 };

package/functions/computation-system-v2/index.js

@@ -83,6 +83,16 @@ async function execute(options) {
 /**
  * WORKER ENTRY POINT: Run a single computation.
  * (Used by Cloud Functions / Dispatcher)
+ * 
+ * @param {Object} options
+ * @param {string} options.date - Target date (YYYY-MM-DD)
+ * @param {string} options.computation - Computation name
+ * @param {string[]} [options.entityIds] - Specific entities to run (null = all)
+ * @param {boolean} [options.dryRun] - If true, don't persist results
+ * @param {boolean} [options.force] - If true, bypass up-to-date checks
+ * @param {boolean} [options.useWorkerPool] - Override worker pool setting (undefined = use config)
+ * @param {Object} [options.config] - Override config
+ * @param {Object} [options.logger] - Custom logger
  */
 async function runComputation(options) {
     const {
@@ -90,6 +100,8 @@ async function runComputation(options) {
         computation,
         entityIds = null,
         dryRun = false,
+        force = false,
+        useWorkerPool,  // Runtime override for worker pool
         config: customConfig = null,
         logger = null
     } = options;
@@ -112,7 +124,9 @@ async function runComputation(options) {
     // This handles dependencies, data fetching, middleware, etc.
     return orch.runSingle(entry, date, { 
         entityIds, 
-        dryRun 
+        dryRun,
+        force,
+        useWorkerPool  // Pass override to Orchestrator
     });
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.737",
+  "version": "1.0.739",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [