bulltrackers-module 1.0.735 → 1.0.736

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (28) hide show
  1. package/functions/computation-system-v2/config/bulltrackers.config.js +75 -5
  2. package/functions/computation-system-v2/framework/data/DataFetcher.js +107 -105
  3. package/functions/computation-system-v2/framework/execution/Orchestrator.js +357 -150
  4. package/functions/computation-system-v2/framework/execution/RemoteTaskRunner.js +327 -0
  5. package/functions/computation-system-v2/framework/execution/middleware/LineageMiddleware.js +9 -4
  6. package/functions/computation-system-v2/framework/execution/middleware/ProfilerMiddleware.js +9 -21
  7. package/functions/computation-system-v2/framework/index.js +10 -3
  8. package/functions/computation-system-v2/framework/lineage/LineageTracker.js +53 -57
  9. package/functions/computation-system-v2/framework/monitoring/Profiler.js +54 -52
  10. package/functions/computation-system-v2/framework/resilience/Checkpointer.js +173 -27
  11. package/functions/computation-system-v2/framework/storage/StorageManager.js +419 -187
  12. package/functions/computation-system-v2/handlers/index.js +10 -1
  13. package/functions/computation-system-v2/handlers/scheduler.js +85 -193
  14. package/functions/computation-system-v2/handlers/worker.js +242 -0
  15. package/functions/computation-system-v2/test/analyze-results.js +238 -0
  16. package/functions/computation-system-v2/test/{test-dispatcher.js → other/test-dispatcher.js} +6 -6
  17. package/functions/computation-system-v2/test/{test-framework.js → other/test-framework.js} +14 -14
  18. package/functions/computation-system-v2/test/{test-real-execution.js → other/test-real-execution.js} +1 -1
  19. package/functions/computation-system-v2/test/{test-real-integration.js → other/test-real-integration.js} +3 -3
  20. package/functions/computation-system-v2/test/{test-refactor-e2e.js → other/test-refactor-e2e.js} +3 -3
  21. package/functions/computation-system-v2/test/{test-risk-metrics-computation.js → other/test-risk-metrics-computation.js} +4 -4
  22. package/functions/computation-system-v2/test/{test-scheduler.js → other/test-scheduler.js} +1 -1
  23. package/functions/computation-system-v2/test/{test-storage.js → other/test-storage.js} +2 -2
  24. package/functions/computation-system-v2/test/run-pipeline-test.js +554 -0
  25. package/functions/computation-system-v2/test/test-worker-pool.js +494 -0
  26. package/package.json +1 -1
  27. package/functions/computation-system-v2/computations/TestComputation.js +0 -46
  28. /package/functions/computation-system-v2/test/{test-results.json → other/test-results.json} +0 -0
@@ -0,0 +1,238 @@
1
+ /**
2
+ * @fileoverview Test Results Analyzer
3
+ *
4
+ * Analyzes test reports and compares them across runs.
5
+ *
6
+ * Usage:
7
+ * node test/analyze-results.js --report test-report-123.json
8
+ * node test/analyze-results.js --compare report1.json report2.json
9
+ */
10
+
11
+ const fs = require('fs');
12
+ const path = require('path');
13
+
14
+ class TestResultsAnalyzer {
15
+ constructor(reportPath) {
16
+ this.report = JSON.parse(fs.readFileSync(reportPath, 'utf8'));
17
+ }
18
+
19
+ analyze() {
20
+ console.log('╔════════════════════════════════════════════════════════════╗');
21
+ console.log('║ TEST RESULTS DETAILED ANALYSIS ║');
22
+ console.log('╚════════════════════════════════════════════════════════════╝\n');
23
+
24
+ this._printOverview();
25
+ this._analyzeLineage();
26
+ this._analyzeCosts();
27
+ this._analyzePerformance();
28
+ this._checkForIssues();
29
+ }
30
+
31
+ _printOverview() {
32
+ console.log('📋 OVERVIEW\n');
33
+ console.log(` Run ID: ${this.report.runId}`);
34
+ console.log(` Status: ${this.report.status}`);
35
+ console.log(` Start: ${this.report.startTime}`);
36
+ console.log(` End: ${this.report.endTime}`);
37
+
38
+ const duration = new Date(this.report.endTime) - new Date(this.report.startTime);
39
+ console.log(` Duration: ${Math.round(duration / 1000)}s\n`);
40
+ }
41
+
42
+ _analyzeLineage() {
43
+ console.log('🔗 LINEAGE ANALYSIS\n');
44
+
45
+ const { dependencyGraph, passSummary } = this.report.lineage;
46
+
47
+ // Check for correct topological ordering
48
+ console.log(' Pass Distribution:');
49
+ for (const [pass, comps] of Object.entries(passSummary)) {
50
+ console.log(` Pass ${pass}: ${comps.length} computations`);
51
+ }
52
+
53
+ // Verify dependencies executed before dependents
54
+ console.log('\n Dependency Verification:');
55
+ let violations = 0;
56
+
57
+ for (const comp of dependencyGraph) {
58
+ if (comp.dependencies.length === 0) continue;
59
+
60
+ for (const dep of comp.dependencies) {
61
+ const depData = dependencyGraph.find(c => c.computation === dep);
62
+ if (!depData) {
63
+ console.log(` ⚠️ ${comp.computation}: dependency ${dep} not found`);
64
+ violations++;
65
+ } else if (depData.pass >= comp.pass) {
66
+ console.log(` ❌ ${comp.computation} (Pass ${comp.pass}) runs before/same as dependency ${dep} (Pass ${depData.pass})`);
67
+ violations++;
68
+ }
69
+ }
70
+ }
71
+
72
+ if (violations === 0) {
73
+ console.log(' ✅ All dependencies correctly ordered');
74
+ }
75
+ console.log('');
76
+ }
77
+
78
+ _analyzeCosts() {
79
+ console.log('💰 COST ANALYSIS\n');
80
+
81
+ const { totalCostUSD, maxCostUSD, utilizationPct, byComputation } = this.report.cost;
82
+
83
+ console.log(` Total Cost: $${totalCostUSD.toFixed(6)}`);
84
+ console.log(` Budget: $${maxCostUSD}`);
85
+ console.log(` Utilization: ${utilizationPct.toFixed(1)}%`);
86
+
87
+ // Cost efficiency rating
88
+ let rating = '🟢 Excellent';
89
+ if (utilizationPct > 80) rating = '🔴 High (near limit)';
90
+ else if (utilizationPct > 50) rating = '🟡 Moderate';
91
+ console.log(` Rating: ${rating}\n`);
92
+
93
+ // Top cost drivers
94
+ console.log(' Top 5 Cost Drivers:');
95
+ byComputation.slice(0, 5).forEach((item, i) => {
96
+ const pct = (item.cost / totalCostUSD) * 100;
97
+ console.log(` ${i + 1}. ${item.name}: $${item.cost.toFixed(6)} (${pct.toFixed(1)}%)`);
98
+ });
99
+ console.log('');
100
+ }
101
+
102
+ _analyzePerformance() {
103
+ console.log('⚡ PERFORMANCE ANALYSIS\n');
104
+
105
+ const comps = this.report.computations;
106
+ const durations = comps.map(c => c.duration);
107
+ const avgDuration = durations.reduce((a, b) => a + b, 0) / durations.length;
108
+ const maxDuration = Math.max(...durations);
109
+ const totalDuration = durations.reduce((a, b) => a + b, 0);
110
+
111
+ console.log(` Total Execution Time: ${totalDuration}ms`);
112
+ console.log(` Average per Computation: ${Math.round(avgDuration)}ms`);
113
+ console.log(` Slowest: ${maxDuration}ms\n`);
114
+
115
+ // Slowest computations
116
+ const slowest = [...comps]
117
+ .sort((a, b) => b.duration - a.duration)
118
+ .slice(0, 5);
119
+
120
+ console.log(' Slowest Computations:');
121
+ slowest.forEach((comp, i) => {
122
+ const pct = (comp.duration / totalDuration) * 100;
123
+ console.log(` ${i + 1}. ${comp.name}: ${comp.duration}ms (${pct.toFixed(1)}%)`);
124
+ });
125
+ console.log('');
126
+ }
127
+
128
+ _checkForIssues() {
129
+ console.log('🔍 ISSUE DETECTION\n');
130
+
131
+ const issues = [];
132
+
133
+ // Check cost utilization
134
+ if (this.report.cost.utilizationPct > 90) {
135
+ issues.push({
136
+ severity: 'HIGH',
137
+ type: 'Cost',
138
+ message: `Cost utilization at ${this.report.cost.utilizationPct.toFixed(1)}% - near limit`
139
+ });
140
+ }
141
+
142
+ // Check for failed computations
143
+ const failed = this.report.computations.filter(c => c.status === 'failed');
144
+ if (failed.length > 0) {
145
+ issues.push({
146
+ severity: 'HIGH',
147
+ type: 'Execution',
148
+ message: `${failed.length} computations failed`
149
+ });
150
+ }
151
+
152
+ // Check for very slow computations (> 60s)
153
+ const verySlow = this.report.computations.filter(c => c.duration > 60000);
154
+ if (verySlow.length > 0) {
155
+ issues.push({
156
+ severity: 'MEDIUM',
157
+ type: 'Performance',
158
+ message: `${verySlow.length} computations took >60s`
159
+ });
160
+ }
161
+
162
+ if (issues.length === 0) {
163
+ console.log(' ✅ No issues detected\n');
164
+ } else {
165
+ for (const issue of issues) {
166
+ const icon = issue.severity === 'HIGH' ? '🔴' : '🟡';
167
+ console.log(` ${icon} [${issue.severity}] ${issue.type}: ${issue.message}`);
168
+ }
169
+ console.log('');
170
+ }
171
+ }
172
+
173
+ compare(otherReport) {
174
+ console.log('╔════════════════════════════════════════════════════════════╗');
175
+ console.log('║ COMPARISON ANALYSIS ║');
176
+ console.log('╚════════════════════════════════════════════════════════════╝\n');
177
+
178
+ const other = JSON.parse(fs.readFileSync(otherReport, 'utf8'));
179
+
180
+ // Cost comparison
181
+ console.log('💰 Cost Comparison:\n');
182
+ const costDiff = this.report.cost.totalCostUSD - other.cost.totalCostUSD;
183
+ const costDiffPct = (costDiff / other.cost.totalCostUSD) * 100;
184
+
185
+ console.log(` This run: $${this.report.cost.totalCostUSD.toFixed(6)}`);
186
+ console.log(` Previous run: $${other.cost.totalCostUSD.toFixed(6)}`);
187
+ console.log(` Difference: $${Math.abs(costDiff).toFixed(6)} (${costDiffPct > 0 ? '+' : ''}${costDiffPct.toFixed(1)}%)\n`);
188
+
189
+ // Performance comparison
190
+ console.log('⚡ Performance Comparison:\n');
191
+
192
+ const thisDuration = new Date(this.report.endTime) - new Date(this.report.startTime);
193
+ const otherDuration = new Date(other.endTime) - new Date(other.startTime);
194
+ const durationDiff = thisDuration - otherDuration;
195
+ const durationDiffPct = (durationDiff / otherDuration) * 100;
196
+
197
+ console.log(` This run: ${Math.round(thisDuration / 1000)}s`);
198
+ console.log(` Previous run: ${Math.round(otherDuration / 1000)}s`);
199
+ console.log(` Difference: ${Math.abs(Math.round(durationDiff / 1000))}s (${durationDiffPct > 0 ? '+' : ''}${durationDiffPct.toFixed(1)}%)\n`);
200
+
201
+ // Entity count comparison
202
+ console.log('📊 Entity Count Comparison:\n');
203
+ console.log(` This run: ${this.report.storage.totalEntities}`);
204
+ console.log(` Previous run: ${other.storage.totalEntities}\n`);
205
+ }
206
+ }
207
+
208
+ // ============================================================================
209
+ // CLI
210
+ // ============================================================================
211
+
212
+ function main() {
213
+ const args = process.argv.slice(2);
214
+
215
+ if (args.length === 0) {
216
+ console.log('Usage:');
217
+ console.log(' node test/analyze-results.js --report <path>');
218
+ console.log(' node test/analyze-results.js --compare <path1> <path2>');
219
+ process.exit(1);
220
+ }
221
+
222
+ if (args[0] === '--compare') {
223
+ const analyzer = new TestResultsAnalyzer(args[1]);
224
+ analyzer.compare(args[2]);
225
+ } else if (args[0] === '--report') {
226
+ const analyzer = new TestResultsAnalyzer(args[1]);
227
+ analyzer.analyze();
228
+ } else {
229
+ console.error('Unknown command:', args[0]);
230
+ process.exit(1);
231
+ }
232
+ }
233
+
234
+ if (require.main === module) {
235
+ main();
236
+ }
237
+
238
+ module.exports = { TestResultsAnalyzer };
@@ -11,18 +11,18 @@
11
11
  const path = require('path');
12
12
 
13
13
  // Use real config
14
- const config = require('../config/bulltrackers.config');
14
+ const config = require('../../config/bulltrackers.config');
15
15
 
16
16
  // Add test computations
17
17
  config.computations = [
18
- require('../computations/PopularInvestorProfileMetrics'),
19
- require('../computations/PopularInvestorRiskAssessment'),
18
+ require('../../computations/PopularInvestorProfileMetrics'),
19
+ require('../../computations/PopularInvestorRiskAssessment'),
20
20
  ];
21
21
 
22
22
  // Override config for dispatcher
23
- const dispatcherModule = require('../handlers/dispatcher');
24
- const onDemandModule = require('../handlers/onDemand');
25
- const { ManifestBuilder, ScheduleValidator } = require('../framework');
23
+ const dispatcherModule = require('../../handlers/dispatcher');
24
+ const onDemandModule = require('../../handlers/onDemand');
25
+ const { ManifestBuilder, ScheduleValidator } = require('../../framework');
26
26
 
27
27
  // Mock response object
28
28
  function createMockResponse() {
@@ -27,10 +27,10 @@ const logger = {
27
27
  async function testManifestBuilding() {
28
28
  console.log('\n=== TEST: Manifest Building with Dependencies ===\n');
29
29
 
30
- const { ManifestBuilder } = require('../framework/core/Manifest');
31
- const UserPortfolioSummary = require('../computations/UserPortfolioSummary');
32
- const PopularInvestorProfileMetrics = require('../computations/PopularInvestorProfileMetrics');
33
- const PopularInvestorRiskAssessment = require('../computations/PopularInvestorRiskAssessment');
30
+ const { ManifestBuilder } = require('../../framework/core/Manifest');
31
+ const UserPortfolioSummary = require('../../computations/UserPortfolioSummary');
32
+ const PopularInvestorProfileMetrics = require('../../computations/PopularInvestorProfileMetrics');
33
+ const PopularInvestorRiskAssessment = require('../../computations/PopularInvestorRiskAssessment');
34
34
 
35
35
  const config = { epoch: 'test-v1' };
36
36
  const builder = new ManifestBuilder(config, logger);
@@ -85,7 +85,7 @@ async function testManifestBuilding() {
85
85
  async function testSchemaRegistry() {
86
86
  console.log('\n=== TEST: Schema Registry ===\n');
87
87
 
88
- const { SchemaRegistry } = require('../framework/data/SchemaRegistry');
88
+ const { SchemaRegistry } = require('../../framework/data/SchemaRegistry');
89
89
 
90
90
  const config = {
91
91
  projectId: process.env.GCP_PROJECT_ID,
@@ -139,8 +139,8 @@ async function testSchemaRegistry() {
139
139
  async function testQueryBuilder() {
140
140
  console.log('\n=== TEST: Query Builder ===\n');
141
141
 
142
- const { SchemaRegistry } = require('../framework/data/SchemaRegistry');
143
- const { QueryBuilder } = require('../framework/data/QueryBuilder');
142
+ const { SchemaRegistry } = require('../../framework/data/SchemaRegistry');
143
+ const { QueryBuilder } = require('../../framework/data/QueryBuilder');
144
144
 
145
145
  const config = {
146
146
  projectId: process.env.GCP_PROJECT_ID,
@@ -179,10 +179,10 @@ async function testQueryBuilder() {
179
179
  async function testComputationValidation() {
180
180
  console.log('\n=== TEST: Computation Validation ===\n');
181
181
 
182
- const { Computation } = require('../framework/core/Computation');
183
- const UserPortfolioSummary = require('../computations/UserPortfolioSummary');
184
- const PopularInvestorProfileMetrics = require('../computations/PopularInvestorProfileMetrics');
185
- const PopularInvestorRiskAssessment = require('../computations/PopularInvestorRiskAssessment');
182
+ const { Computation } = require('../../framework/core/Computation');
183
+ const UserPortfolioSummary = require('../../computations/UserPortfolioSummary');
184
+ const PopularInvestorProfileMetrics = require('../../computations/PopularInvestorProfileMetrics');
185
+ const PopularInvestorRiskAssessment = require('../../computations/PopularInvestorRiskAssessment');
186
186
 
187
187
  // Test UserPortfolioSummary
188
188
  console.log('Testing UserPortfolioSummary...');
@@ -231,7 +231,7 @@ async function testDryRun() {
231
231
  console.log('\n=== TEST: Dry Run Analysis ===\n');
232
232
 
233
233
  try {
234
- const v2 = require('../index');
234
+ const v2 = require('../../index');
235
235
 
236
236
  const report = await v2.analyze({
237
237
  date: '2026-01-24',
@@ -280,7 +280,7 @@ async function testRealExecution() {
280
280
  console.log('⚠️ This test will execute computations and write to BigQuery!\n');
281
281
 
282
282
  try {
283
- const v2 = require('../index');
283
+ const v2 = require('../../index');
284
284
 
285
285
  // Find a date with data
286
286
  console.log('Looking for a date with available data...');
@@ -402,7 +402,7 @@ async function testDependentExecution(previousResult) {
402
402
  console.log(` Duration: ${depCompleted.duration}ms`);
403
403
 
404
404
  try {
405
- const v2 = require('../index');
405
+ const v2 = require('../../index');
406
406
 
407
407
  // Now try to run the dependent computation
408
408
  console.log(`\n--- Running Dependent Computation (Pass 2) ---`);
@@ -22,7 +22,7 @@ async function main() {
22
22
  console.log('║ Computation System v2 - Real Execution Test ║');
23
23
  console.log('╚════════════════════════════════════════════════════════════╝\n');
24
24
 
25
- const v2 = require('../index');
25
+ const v2 = require('../../index');
26
26
 
27
27
  // Step 1: Verify pass assignment
28
28
  console.log('=== STEP 1: Verify Pass Assignment ===\n');
@@ -9,9 +9,9 @@
9
9
  const fs = require('fs');
10
10
  const path = require('path');
11
11
  const { BigQuery } = require('@google-cloud/bigquery');
12
- const { Orchestrator } = require('../framework/execution/Orchestrator');
13
- const { Computation } = require('../framework/core/Computation');
14
- const realConfig = require('../config/bulltrackers.config');
12
+ const { Orchestrator } = require('../../framework/execution/Orchestrator');
13
+ const { Computation } = require('../../framework/core/Computation');
14
+ const realConfig = require('../../config/bulltrackers.config');
15
15
 
16
16
  // 1. DYNAMIC TEST COMPUTATION
17
17
  class LiveTestComputation extends Computation {
@@ -4,9 +4,9 @@
4
4
  */
5
5
 
6
6
  const assert = require('assert');
7
- const { Orchestrator } = require('../framework/execution/Orchestrator');
8
- const TestComputation = require('../computations/TestComputation');
9
- const { RulesRegistry } = require('../framework/core/Rules');
7
+ const { Orchestrator } = require('../../framework/execution/Orchestrator');
8
+ const TestComputation = require('../../computations/TestComputation');
9
+ const { RulesRegistry } = require('../../framework/core/Rules');
10
10
 
11
11
  // MOCK DATA
12
12
  const MOCK_USERS = {
@@ -7,10 +7,10 @@
7
7
  * - Firestore storage
8
8
  */
9
9
 
10
- const { Executor, ManifestBuilder } = require('../framework');
11
- const { StorageManager } = require('../framework/storage/StorageManager');
12
- const PopularInvestorRiskMetrics = require('../computations/PopularInvestorRiskMetrics');
13
- const config = require('../config/bulltrackers.config');
10
+ const { Executor, ManifestBuilder } = require('../../framework');
11
+ const { StorageManager } = require('../../framework/storage/StorageManager');
12
+ const PopularInvestorRiskMetrics = require('../../computations/PopularInvestorRiskMetrics');
13
+ const config = require('../../config/bulltrackers.config');
14
14
 
15
15
  console.log('╔════════════════════════════════════════════════════════════╗');
16
16
  console.log('║ PopularInvestorRiskMetrics - Full Integration Test ║');
@@ -7,7 +7,7 @@
7
7
  * 3. Finding due computations
8
8
  */
9
9
 
10
- const schedulerModule = require('../handlers/scheduler');
10
+ const schedulerModule = require('../../handlers/scheduler');
11
11
 
12
12
  // ============================================================================
13
13
  // Test Clock Drift Handling
@@ -4,8 +4,8 @@
4
4
  * Tests for the unified storage system (BigQuery + Firestore)
5
5
  */
6
6
 
7
- const { StorageManager } = require('../framework/storage/StorageManager');
8
- const config = require('../config/bulltrackers.config');
7
+ const { StorageManager } = require('../../framework/storage/StorageManager');
8
+ const config = require('../../config/bulltrackers.config');
9
9
 
10
10
  console.log('╔════════════════════════════════════════════════════════════╗');
11
11
  console.log('║ Computation System v2 - Storage Tests ║');