bulltrackers-module 1.0.736 → 1.0.738

package/functions/computation-system-v2/config/bulltrackers.config.js

@@ -246,10 +246,12 @@ module.exports = {
     execution: {
         // Max concurrent entity processing (per-entity computations)
         // Higher = faster but more memory. Tune based on your Cloud Function memory.
-        entityConcurrency: 20,
+        entityConcurrency: 50,
         
         // Batch size for BigQuery inserts
         insertBatchSize: 500,
+
+        fetchBatchSize: 30000,
         
         // Memory safety: max entities to load for a dependency
         // If a dependency has more entities than this, use getDependency(name, entityId) instead
@@ -361,7 +363,7 @@ module.exports = {
         // Max concurrent worker invocations
         // Higher = faster but more network/GCS load
         // Recommended: 100-200 for production
-        concurrency: parseInt(process.env.WORKER_CONCURRENCY || '100', 10),
+        concurrency: 100,
         
         // Worker invocation timeout (ms)
         // Should be slightly less than worker function timeout
@@ -382,6 +384,8 @@ module.exports = {
         // Useful for testing specific computations
         forceOffloadComputations: process.env.WORKER_FORCE_COMPUTATIONS 
             ? process.env.WORKER_FORCE_COMPUTATIONS.split(',') 
-            : []
+            : [],
+        
+        minEntitiesForOffload: 100,
     }
 };

package/functions/computation-system-v2/docs/admin.md

@@ -0,0 +1,91 @@
+# Admin Test Endpoint
+
+## Deploy
+
+```bash
+node deploy.mjs ComputeAdminTest
+```
+
+## Usage Examples
+
+### 1. Check System Status
+
+```bash
+TOKEN=$(gcloud auth print-identity-token --audiences="https://europe-west1-stocks-12345.cloudfunctions.net/compute-admin-test")
+
+curl -X POST \
+    "https://europe-west1-stocks-12345.cloudfunctions.net/compute-admin-test" \
+    -H "Authorization: Bearer $TOKEN" \
+    -H "Content-Type: application/json" \
+    -d '{"action": "status"}'
+```
+
+### 2. Analyze What Would Run
+
+```bash
+curl -X POST \
+    "https://europe-west1-stocks-12345.cloudfunctions.net/compute-admin-test" \
+    -H "Authorization: Bearer $TOKEN" \
+    -H "Content-Type: application/json" \
+    -d '{"action": "analyze", "date": "2026-01-25"}'
+```
+
+### 3. Run Full Computation
+
+```bash
+curl -X POST \
+    "https://europe-west1-stocks-12345.cloudfunctions.net/compute-admin-test" \
+    -H "Authorization: Bearer $TOKEN" \
+    -H "Content-Type: application/json" \
+    -d '{"action": "run", "computation": "UserPortfolioSummary", "date": "2026-01-25", "force": true}'
+```
+
+### 4. Run Limited Test
+
+```bash
+curl -X POST \
+    "https://europe-west1-stocks-12345.cloudfunctions.net/compute-admin-test" \
+    -H "Authorization: Bearer $TOKEN" \
+    -H "Content-Type: application/json" \
+    -d '{"action": "run_limited", "computation": "UserPortfolioSummary", "date": "2026-01-25", "limit": 5}'
+```
+
+### 5. Test Specific Entities
+
+```bash
+curl -X POST \
+    "https://europe-west1-stocks-12345.cloudfunctions.net/compute-admin-test" \
+    -H "Authorization: Bearer $TOKEN" \
+    -H "Content-Type: application/json" \
+    -d '{"action": "run", "computation": "UserPortfolioSummary", "date": "2026-01-25", "entityIds": ["user-123", "user-456"], "force": true}'
+```
+
+### 6. Test Worker Directly
+
+```bash
+curl -X POST \
+    "https://europe-west1-stocks-12345.cloudfunctions.net/compute-admin-test" \
+    -H "Authorization: Bearer $TOKEN" \
+    -H "Content-Type: application/json" \
+    -d '{"action": "test_worker", "computation": "UserPortfolioSummary", "date": "2026-01-25", "entityIds": ["user-123"]}'
+```
+
+### 7. Test with Worker Pool Override
+
+```bash
+curl -X POST \
+    "https://europe-west1-stocks-12345.cloudfunctions.net/compute-admin-test" \
+    -H "Authorization: Bearer $TOKEN" \
+    -H "Content-Type: application/json" \
+    -d '{"action": "run", "computation": "UserPortfolioSummary", "date": "2026-01-25", "useWorkerPool": true, "force": true}'
+```
+
+## Available Actions
+
+| Action | Description |
+|--------|-------------|
+| `status` | List all computations and system status |
+| `analyze` | Check what would run for a given date |
+| `run` | Execute a full computation |
+| `run_limited` | Execute on N random entities (safer for testing) |
+| `test_worker` | Direct test of worker function logic |

package/functions/computation-system-v2/docs/architecture.md

@@ -0,0 +1,59 @@
+```mermaid
+graph TD
+    Root((System))
+
+    %% Subgraph: Scheduling & Control
+    subgraph Control_Plane [Control Plane]
+        Cron((Timer)) -->|Every Minute| Scheduler[Scheduler Handler]
+        Scheduler -->|Find Due & Zombies| StateRepo[(State DB)]
+        Scheduler -->|Dispatch Task| CloudTasks[Cloud Tasks Queue]
+        CloudTasks -->|HTTP POST w/ Backoff| Dispatcher[Dispatcher Handler]
+        Dispatcher -->|Run Computation| Orchestrator[Orchestrator]
+        Orchestrator -->|Return Status| Dispatcher
+        Dispatcher -.->|Blocked| Return503[503 Retry]
+        Return503 -.-> CloudTasks
+        Dispatcher -.->|Success / Skipped| Return200[200 OK]
+    end
+
+    %% Subgraph: Execution
+    subgraph Execution_Core [Execution Core]
+        Orchestrator --> Manifest[Manifest Builder]
+        Orchestrator -->|Check Hashes & Deps| StateRepo
+        Orchestrator -->|Fetch Data| BigQuery[(BigQuery)]
+        Orchestrator --> ExecMode{Mode?}
+
+        ExecMode -->|Global / Light| LocalExec[Local Execution]
+        LocalExec --> Logic[Computation Logic]
+        Logic --> LocalExec
+
+        ExecMode -->|Per-Entity / Heavy| RemoteRunner[Remote Task Runner]
+        RemoteRunner -->|Upload Context| GCS[(Cloud Storage)]
+        RemoteRunner --> Worker[Worker Handler]
+        Worker -->|Download Context| GCS
+        Worker -->|Execute| Logic
+        Worker -->|Return Result| RemoteRunner
+    end
+
+    %% Subgraph: Persistence
+    subgraph Persistence [Persistence Layer]
+        LocalExec -->|Commit Results| StateRepo
+        RemoteRunner -->|Commit Batch| StateRepo
+    end
+
+    %% Single-root anchoring (critical)
+    Root --> Cron
+    Root -.-> Orchestrator
+    Root -.-> LocalExec
+    Root -.-> RemoteRunner
+
+    %% Styling
+    classDef plain fill:#ffffff,stroke:#333,stroke-width:1px;
+    classDef db fill:#e1f5fe,stroke:#01579b,stroke-width:2px;
+    classDef logic fill:#e8f5e9,stroke:#2e7d32,stroke-width:2px;
+    classDef queue fill:#fff9c4,stroke:#fbc02d,stroke-width:2px;
+
+    class Cron,Scheduler,Dispatcher,Orchestrator,Manifest,LocalExec,RemoteRunner,Worker plain;
+    class StateRepo,BigQuery,GCS db;
+    class Logic logic;
+    class CloudTasks queue;
+```

package/functions/computation-system-v2/framework/execution/Orchestrator.js

@@ -362,8 +362,28 @@ class Orchestrator {
 
     /**
      * Determine if a computation should use remote workers
+     * 
+     * @param {Object} entry - Manifest entry
+     * @param {Object} options - Execution options
+     * @param {boolean} [options.useWorkerPool] - Runtime override (true/false/undefined)
+     * @param {boolean} [options.forceLocal] - Force local execution
      */
     _shouldUseRemoteWorkers(entry, options) {
+        // Runtime override takes precedence (for admin testing)
+        // useWorkerPool: true  -> force use worker pool
+        // useWorkerPool: false -> force local execution
+        // useWorkerPool: undefined -> use config
+        if (options.useWorkerPool === true) {
+            if (!this.remoteRunner) {
+                this._log('WARN', 'useWorkerPool=true but remoteRunner not initialized');
+                return false;
+            }
+            return true;
+        }
+        if (options.useWorkerPool === false) {
+            return false;
+        }
+        
         // No remote runner configured
         if (!this.remoteRunner) return false;
         

package/functions/computation-system-v2/handlers/adminTest.js

@@ -0,0 +1,327 @@
+/**
+ * @fileoverview Admin Test Endpoint for Computation System
+ * 
+ * SECURITY: This endpoint is protected by GCP IAM (requireAuth: true).
+ * Only service accounts and users with cloudfunctions.invoker can access it.
+ * 
+ * PURPOSE:
+ * - Test computations in production without waiting for schedule
+ * - Force re-runs of computations (bypass hash checks)
+ * - Test worker pool functionality
+ * - Run on specific entities for debugging
+ * 
+ * USAGE:
+ *   curl -X POST https://REGION-PROJECT.cloudfunctions.net/compute-admin-test \
+ *     -H "Authorization: Bearer $(gcloud auth print-identity-token)" \
+ *     -H "Content-Type: application/json" \
+ *     -d '{"action": "run", "computation": "UserPortfolioSummary", "date": "2026-01-25"}'
+ */
+
+const system = require('../index');
+
+/**
+ * Admin test handler.
+ */
+async function adminTestHandler(req, res) {
+    const startTime = Date.now();
+    
+    try {
+        const { 
+            action = 'status',
+            computation,
+            date = new Date().toISOString().split('T')[0],
+            entityIds,
+            limit = 10,
+            force = true,           // Default to force for testing
+            useWorkerPool,          // Override: true/false/undefined (use config)
+            dryRun = false
+        } = req.body || {};
+
+        console.log(`[AdminTest] Action: ${action}, Computation: ${computation}, Date: ${date}`);
+
+        switch (action) {
+            // =========================================================
+            // STATUS: Show system status and available computations
+            // =========================================================
+            case 'status': {
+                const manifest = await system.getManifest();
+                
+                return res.status(200).json({
+                    status: 'ok',
+                    action: 'status',
+                    systemInfo: {
+                        computationCount: manifest.length,
+                        computations: manifest.map(c => ({
+                            name: c.originalName || c.name,
+                            type: c.type,
+                            pass: c.pass,
+                            schedule: c.schedule
+                        })),
+                        workerPool: {
+                            enabled: process.env.WORKER_POOL_ENABLED === 'true',
+                            localMode: process.env.WORKER_LOCAL_MODE === 'true'
+                        }
+                    },
+                    timestamp: new Date().toISOString()
+                });
+            }
+
+            // =========================================================
+            // ANALYZE: Check what would run for a given date
+            // =========================================================
+            case 'analyze': {
+                const report = await system.analyze({ date });
+                
+                return res.status(200).json({
+                    status: 'ok',
+                    action: 'analyze',
+                    date,
+                    report: {
+                        runnable: report.runnable?.map(r => r.name || r) || [],
+                        skipped: report.skipped?.map(r => ({ name: r.name, reason: r.reason })) || [],
+                        blocked: report.blocked?.map(r => ({ name: r.name, reason: r.reason })) || [],
+                        impossible: report.impossible?.map(r => ({ name: r.name, reason: r.reason })) || []
+                    }
+                });
+            }
+
+            // =========================================================
+            // RUN: Execute a single computation
+            // =========================================================
+            case 'run': {
+                if (!computation) {
+                    return res.status(400).json({
+                        status: 'error',
+                        error: 'Missing "computation" field. Use action: "status" to list available computations.'
+                    });
+                }
+
+                // Log worker pool override if specified
+                if (useWorkerPool !== undefined) {
+                    console.log(`[AdminTest] Worker pool override: ${useWorkerPool ? 'ENABLED' : 'DISABLED'}`);
+                }
+
+                console.log(`[AdminTest] Running ${computation} for ${date}...`);
+                console.log(`[AdminTest] Options: force=${force}, dryRun=${dryRun}, entityIds=${entityIds?.join(',') || 'all'}`);
+
+                const result = await system.runComputation({
+                    date,
+                    computation,
+                    entityIds: entityIds || null,
+                    dryRun,
+                    force,
+                    // Pass worker pool override explicitly (avoids env var caching issues)
+                    useWorkerPool
+                });
+
+                const duration = Date.now() - startTime;
+
+                return res.status(200).json({
+                    status: 'ok',
+                    action: 'run',
+                    computation,
+                    date,
+                    result: {
+                        status: result.status,
+                        duration: result.duration,
+                        resultCount: result.resultCount,
+                        reason: result.reason,
+                        hash: result.hash
+                    },
+                    totalDuration: duration,
+                    workerPoolUsed: useWorkerPool ?? (process.env.WORKER_POOL_ENABLED === 'true')
+                });
+            }
+
+            // =========================================================
+            // RUN_LIMITED: Run on a limited number of entities (safer)
+            // =========================================================
+            case 'run_limited': {
+                if (!computation) {
+                    return res.status(400).json({
+                        status: 'error',
+                        error: 'Missing "computation" field'
+                    });
+                }
+
+                // Get a sample of entities from BigQuery
+                const sampleEntities = await getSampleEntities(computation, date, limit);
+                
+                if (!sampleEntities || sampleEntities.length === 0) {
+                    return res.status(404).json({
+                        status: 'error',
+                        error: `No entities found for ${computation} on ${date}`
+                    });
+                }
+
+                console.log(`[AdminTest] Running LIMITED test: ${sampleEntities.length} entities`);
+
+                const result = await system.runComputation({
+                    date,
+                    computation,
+                    entityIds: sampleEntities,
+                    dryRun,
+                    force,
+                    useWorkerPool  // Pass worker pool override
+                });
+
+                const duration = Date.now() - startTime;
+
+                return res.status(200).json({
+                    status: 'ok',
+                    action: 'run_limited',
+                    computation,
+                    date,
+                    entitiesTested: sampleEntities,
+                    result: {
+                        status: result.status,
+                        duration: result.duration,
+                        resultCount: result.resultCount
+                    },
+                    totalDuration: duration
+                });
+            }
+
+            // =========================================================
+            // TEST_WORKER: Direct test of worker function
+            // =========================================================
+            case 'test_worker': {
+                if (!computation || !entityIds || entityIds.length === 0) {
+                    return res.status(400).json({
+                        status: 'error',
+                        error: 'Requires "computation" and "entityIds" array'
+                    });
+                }
+
+                // Import worker's local execution function
+                const { executeLocal, loadComputation } = require('./worker');
+                
+                // Verify computation exists
+                const CompClass = loadComputation(computation);
+                if (!CompClass) {
+                    return res.status(400).json({
+                        status: 'error',
+                        error: `Unknown computation: ${computation}`
+                    });
+                }
+
+                // Fetch real data for one entity
+                const config = require('../config/bulltrackers.config');
+                const { DataFetcher } = require('../framework/data/DataFetcher');
+                const { QueryBuilder } = require('../framework/data/QueryBuilder');
+                const { SchemaRegistry } = require('../framework/data/SchemaRegistry');
+                
+                const schemaRegistry = new SchemaRegistry(config.bigquery, console);
+                const queryBuilder = new QueryBuilder(config.bigquery, schemaRegistry, console);
+                const dataFetcher = new DataFetcher(
+                    { ...config.bigquery, tables: config.tables }, 
+                    queryBuilder, 
+                    console
+                );
+
+                const compConfig = CompClass.getConfig();
+                const testEntityId = entityIds[0];
+                
+                console.log(`[AdminTest] Fetching data for entity ${testEntityId}...`);
+                const data = await dataFetcher.fetchForComputation(compConfig.requires, date, [testEntityId]);
+
+                // Execute worker logic locally
+                console.log(`[AdminTest] Executing worker logic...`);
+                const workerResult = await executeLocal({
+                    computationName: computation,
+                    entityId: testEntityId,
+                    date,
+                    contextPackage: {
+                        entityData: data,
+                        references: {},
+                        dependencies: {},
+                        config: {}
+                    }
+                });
+
+                const duration = Date.now() - startTime;
+
+                return res.status(200).json({
+                    status: 'ok',
+                    action: 'test_worker',
+                    computation,
+                    entityId: testEntityId,
+                    date,
+                    workerResult: workerResult.result,
+                    duration
+                });
+            }
+
+            default:
+                return res.status(400).json({
+                    status: 'error',
+                    error: `Unknown action: ${action}`,
+                    availableActions: ['status', 'analyze', 'run', 'run_limited', 'test_worker']
+                });
+        }
+
+    } catch (error) {
+        console.error('[AdminTest] Error:', error);
+        return res.status(500).json({
+            status: 'error',
+            error: error.message,
+            stack: process.env.NODE_ENV === 'development' ? error.stack : undefined
+        });
+    }
+}
+
+/**
+ * Get a sample of entity IDs for testing
+ */
+async function getSampleEntities(computation, date, limit) {
+    try {
+        const { BigQuery } = require('@google-cloud/bigquery');
+        const config = require('../config/bulltrackers.config');
+        
+        const bigquery = new BigQuery({
+            projectId: config.bigquery.projectId
+        });
+
+        // Load computation to get its config
+        const { loadComputation } = require('./worker');
+        const CompClass = loadComputation(computation);
+        
+        if (!CompClass) return null;
+        
+        const compConfig = CompClass.getConfig();
+        
+        // Find the driver table (first table with entityField)
+        let driverTable = null;
+        let entityField = null;
+        
+        for (const [tableName, tableSpec] of Object.entries(compConfig.requires || {})) {
+            const tableConfig = config.tables[tableName];
+            if (tableConfig?.entityField) {
+                driverTable = tableName;
+                entityField = tableConfig.entityField;
+                break;
+            }
+        }
+
+        if (!driverTable) return null;
+
+        const query = `
+            SELECT DISTINCT ${entityField} as entity_id
+            FROM \`${config.bigquery.projectId}.${config.bigquery.dataset}.${driverTable}\`
+            WHERE date = @date
+            LIMIT @limit
+        `;
+
+        const [rows] = await bigquery.query({
+            query,
+            params: { date, limit }
+        });
+
+        return rows.map(r => r.entity_id);
+    } catch (e) {
+        console.error('[AdminTest] Failed to get sample entities:', e);
+        return null;
+    }
+}
+
+module.exports = { adminTestHandler };

package/functions/computation-system-v2/handlers/index.js

@@ -12,6 +12,7 @@ const { schedulerHandler } = require('./scheduler');
 const { dispatcherHandler } = require('./dispatcher');
 const { onDemandHandler } = require('./onDemand');
 const { workerHandler, executeLocal } = require('./worker');
+const { adminTestHandler } = require('./adminTest');
 
 module.exports = {
     // Unified scheduler - triggered every minute by Cloud Scheduler
@@ -27,6 +28,9 @@ module.exports = {
     // Invoked by RemoteTaskRunner from Orchestrator
     computationWorker: workerHandler,
     
+    // Admin test endpoint - for testing computations in production
+    computeAdminTest: adminTestHandler,
+    
     // For local testing
     executeWorkerLocal: executeLocal
 };

package/functions/computation-system-v2/index.js

@@ -19,10 +19,12 @@ const { ManifestBuilder } = require('./framework/core/Manifest');
 const { Computation } = require('./framework/core/Computation');
 
 // Add computations to config
+// These are loaded from computation-system-v2/computations folder
 config.computations = [
     require('./computations/UserPortfolioSummary'),
     require('./computations/PopularInvestorProfileMetrics'),
     require('./computations/PopularInvestorRiskAssessment'),
+    require('./computations/PopularInvestorRiskMetrics'),
     // Add more computations here as they're migrated
 ];
 
@@ -81,6 +83,16 @@ async function execute(options) {
 /**
  * WORKER ENTRY POINT: Run a single computation.
  * (Used by Cloud Functions / Dispatcher)
+ * 
+ * @param {Object} options
+ * @param {string} options.date - Target date (YYYY-MM-DD)
+ * @param {string} options.computation - Computation name
+ * @param {string[]} [options.entityIds] - Specific entities to run (null = all)
+ * @param {boolean} [options.dryRun] - If true, don't persist results
+ * @param {boolean} [options.force] - If true, bypass up-to-date checks
+ * @param {boolean} [options.useWorkerPool] - Override worker pool setting (undefined = use config)
+ * @param {Object} [options.config] - Override config
+ * @param {Object} [options.logger] - Custom logger
  */
 async function runComputation(options) {
     const {
@@ -88,6 +100,8 @@ async function runComputation(options) {
         computation,
         entityIds = null,
         dryRun = false,
+        force = false,
+        useWorkerPool,  // Runtime override for worker pool
         config: customConfig = null,
         logger = null
     } = options;
@@ -110,7 +124,9 @@ async function runComputation(options) {
     // This handles dependencies, data fetching, middleware, etc.
     return orch.runSingle(entry, date, { 
         entityIds, 
-        dryRun 
+        dryRun,
+        force,
+        useWorkerPool  // Pass override to Orchestrator
     });
 }