bulltrackers-module 1.0.294 → 1.0.296
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/functions/computation-system/executors/PriceBatchExecutor.js +0 -1
- package/functions/computation-system/executors/StandardExecutor.js +1 -1
- package/functions/computation-system/features.md +395 -0
- package/functions/computation-system/helpers/computation_dispatcher.js +9 -10
- package/functions/computation-system/layers/extractors.js +9 -9
- package/functions/computation-system/persistence/RunRecorder.js +9 -9
- package/functions/generic-api/admin-api/index.js +385 -0
- package/functions/generic-api/helpers/api_helpers.js +30 -4
- package/functions/generic-api/index.js +8 -1
- package/package.json +1 -1
|
@@ -13,7 +13,7 @@ const { ContextFactory } = require
|
|
|
13
13
|
const { commitResults } = require('../persistence/ResultCommitter');
|
|
14
14
|
const mathLayer = require('../layers/index');
|
|
15
15
|
const { performance } = require('perf_hooks');
|
|
16
|
-
const v8 = require('v8');
|
|
16
|
+
const v8 = require('v8');
|
|
17
17
|
|
|
18
18
|
class StandardExecutor {
|
|
19
19
|
static async run(date, calcs, passName, config, deps, rootData, fetchedDeps, previousFetchedDeps, skipStatusWrite = false) {
|
|
@@ -0,0 +1,395 @@
|
|
|
1
|
+
# Complete Feature Inventory of BullTrackers Computation System
|
|
2
|
+
|
|
3
|
+
## Core DAG Engine Features
|
|
4
|
+
|
|
5
|
+
### 1. **Topological Sorting (Kahn's Algorithm)**
|
|
6
|
+
- **Files**: `ManifestBuilder.js:187-205`
|
|
7
|
+
- **Implementation**: Builds execution passes by tracking in-degrees, queuing zero-dependency nodes
|
|
8
|
+
- **Niche aspect**: Dynamic pass assignment (line 201: `neighborEntry.pass = currentEntry.pass + 1`)
|
|
9
|
+
- **Common in**: Airflow, Prefect, Dagster (all use topological sort)
|
|
10
|
+
|
|
11
|
+
### 2. **Cycle Detection (Tarjan's SCC Algorithm)**
|
|
12
|
+
- **Files**: `ManifestBuilder.js:98-141`
|
|
13
|
+
- **Implementation**: Strongly Connected Components detection with stack-based traversal
|
|
14
|
+
- **Niche aspect**: Returns human-readable cycle chain (line 137: `cycle.join(' -> ') + ' -> ' + cycle[0]`)
|
|
15
|
+
- **Common in**: Academic graph libraries, rare in production DAG systems (most use simpler DFS)
|
|
16
|
+
|
|
17
|
+
### 3. **Auto-Discovery Manifest Building**
|
|
18
|
+
- **Files**: `ManifestBuilder.js:143-179`, `ManifestLoader.js:9-42`
|
|
19
|
+
- **Implementation**: Scans directories, instantiates classes, extracts metadata via `getMetadata()` static method
|
|
20
|
+
- **Niche aspect**: Singleton caching with multi-key support (ManifestLoader.js:9)
|
|
21
|
+
- **Common in**: Plugin systems (Airflow providers), less common for computation graphs
|
|
22
|
+
|
|
23
|
+
## Dependency Management & Optimization
|
|
24
|
+
|
|
25
|
+
### 4. **Multi-Layered Hash Composition**
|
|
26
|
+
- **Files**: `ManifestBuilder.js:56-95`, `HashManager.js:25-36`
|
|
27
|
+
- **Implementation**: Composite hash from code + epoch + infrastructure + layers + dependencies
|
|
28
|
+
- **Niche aspect**: Infrastructure hash (recursive file tree hashing, HashManager.js:38-79)
|
|
29
|
+
- **Common in**: Build systems (Bazel, Buck), **very rare** in data pipelines
|
|
30
|
+
|
|
31
|
+
### 5. **Content-Based Dependency Short-Circuiting**
|
|
32
|
+
- **Files**: `WorkflowOrchestrator.js:51-73`
|
|
33
|
+
- **Implementation**: Tracks `resultHash` (output data hash), skips re-run if output unchanged despite code change
|
|
34
|
+
- **Niche aspect**: `dependencyResultHashes` tracking (line 59-67)
|
|
35
|
+
- **Common in**: **Extremely rare** - only seen in specialized incremental computation systems
|
|
36
|
+
|
|
37
|
+
### 6. **Behavioral Stability Detection (SimHash)**
|
|
38
|
+
- **Files**: `BuildReporter.js:55-89`, `SimRunner.js:12-42`, `Fabricator.js:20-244`
|
|
39
|
+
- **Implementation**: Runs code against deterministic mock data, hashes output to detect "logic changes" vs "cosmetic changes"
|
|
40
|
+
- **Niche aspect**: Seeded random data generation (SeededRandom.js:1-38) for reproducible simulations
|
|
41
|
+
- **Common in**: **Unique** - haven't seen this elsewhere. Conceptually similar to property-based testing but for optimization
|
|
42
|
+
|
|
43
|
+
### 7. **System Epoch Forcing**
|
|
44
|
+
- **Files**: `system_epoch.js:1-2`, `ManifestBuilder.js:65`
|
|
45
|
+
- **Implementation**: Manual version bump to force global re-computation
|
|
46
|
+
- **Niche aspect**: Single-line file that invalidates all cached results
|
|
47
|
+
- **Common in**: Cache invalidation patterns, but unusual to have a dedicated module
|
|
48
|
+
|
|
49
|
+
## Execution & Resource Management
|
|
50
|
+
|
|
51
|
+
### 8. **Streaming Execution with Batch Flushing**
|
|
52
|
+
- **Files**: `StandardExecutor.js:86-158`
|
|
53
|
+
- **Implementation**: Async generators yield data chunks, flush to DB every N users
|
|
54
|
+
- **Niche aspect**: Adaptive flushing based on V8 heap pressure (line 128-145)
|
|
55
|
+
- **Common in**: ETL tools (Spark, Flink use micro-batching), **heap-aware flushing is rare**
|
|
56
|
+
|
|
57
|
+
### 9. **Memory Heartbeat (Flight Recorder)**
|
|
58
|
+
- **Files**: `computation_worker.js:30-53`
|
|
59
|
+
- **Implementation**: Background timer writes memory stats to Firestore every 2 seconds
|
|
60
|
+
- **Niche aspect**: Uses `.unref()` to prevent blocking process exit (line 50)
|
|
61
|
+
- **Common in**: APM tools (DataDog, New Relic), **embedding in workers is custom**
|
|
62
|
+
|
|
63
|
+
### 10. **Forensic Crash Analysis & Intelligent Routing**
|
|
64
|
+
- **Files**: `computation_dispatcher.js:31-68`
|
|
65
|
+
- **Implementation**: Reads last memory stats from failed runs, routes to high-mem queue if OOM suspected
|
|
66
|
+
- **Niche aspect**: Parses telemetry to distinguish crash types (line 44-50)
|
|
67
|
+
- **Common in**: Kubernetes autoscaling heuristics, **application-level routing is rare**
|
|
68
|
+
|
|
69
|
+
### 11. **Circuit Breaker Pattern**
|
|
70
|
+
- **Files**: `StandardExecutor.js:164-173`
|
|
71
|
+
- **Implementation**: Tracks error rate, fails fast if >10% failures after 100 items
|
|
72
|
+
- **Niche aspect**: Runs mid-stream (not just at job start)
|
|
73
|
+
- **Common in**: Microservices (Hystrix, Resilience4j), uncommon in data pipelines
|
|
74
|
+
|
|
75
|
+
### 12. **Incremental Auto-Sharding**
|
|
76
|
+
- **Files**: `ResultCommitter.js:234-302`
|
|
77
|
+
- **Implementation**: Dynamically splits results into Firestore subcollection shards, tracks shard index across flushes
|
|
78
|
+
- **Niche aspect**: `flushMode: INTERMEDIATE` flag (line 150) to avoid pointer updates mid-stream
|
|
79
|
+
- **Common in**: Database sharding, **dynamic document sharding is custom**
|
|
80
|
+
|
|
81
|
+
### 13. **GZIP Compression Strategy**
|
|
82
|
+
- **Files**: `ResultCommitter.js:128-157`
|
|
83
|
+
- **Implementation**: Compresses results >50KB, stores as binary blob if <900KB compressed
|
|
84
|
+
- **Niche aspect**: Falls back to sharding if compression fails or exceeds limit
|
|
85
|
+
- **Common in**: Storage layers, integration at application level is custom
|
|
86
|
+
|
|
87
|
+
## Data Quality & Validation
|
|
88
|
+
|
|
89
|
+
### 14. **Heuristic Validation (Grey Box)**
|
|
90
|
+
- **Files**: `ResultsValidator.js:8-96`
|
|
91
|
+
- **Implementation**: Statistical analysis (zero%, null%, flatline detection) without knowing schema
|
|
92
|
+
- **Niche aspect**: Weekend mode (line 57-64) - relaxes thresholds on Saturdays/Sundays
|
|
93
|
+
- **Common in**: Data quality tools (Great Expectations, Soda), **weekend-aware thresholds are domain-specific**
|
|
94
|
+
|
|
95
|
+
### 15. **Contract Discovery & Enforcement**
|
|
96
|
+
- **Files**: `ContractDiscoverer.js:11-120`, `ContractValidator.js:9-64`
|
|
97
|
+
- **Implementation**: Monte Carlo simulation learns behavioral bounds, enforces at runtime
|
|
98
|
+
- **Niche aspect**: Distinguishes "physics limits" (ratios 0-1) from "statistical envelopes" (6-sigma)
|
|
99
|
+
- **Common in**: **Unique** - closest analogue is schema inference (Pandas Profiling) but this is probabilistic + enforced
|
|
100
|
+
|
|
101
|
+
### 16. **Semantic Gates**
|
|
102
|
+
- **Files**: `ResultCommitter.js:118-127`
|
|
103
|
+
- **Implementation**: Blocks results that violate contracts before writing
|
|
104
|
+
- **Niche aspect**: Differentiated error handling - `SEMANTIC_GATE` errors are non-retryable (line 210-225)
|
|
105
|
+
- **Common in**: Type systems (TypeScript, Mypy), **runtime probabilistic checks are rare**
|
|
106
|
+
|
|
107
|
+
### 17. **Root Data Availability Tracking**
|
|
108
|
+
- **Files**: `AvailabilityChecker.js:49-87`, `utils.js:11-17`
|
|
109
|
+
- **Implementation**: Centralized index (`system_root_data_index`) tracks what data exists per day
|
|
110
|
+
- **Niche aspect**: Granular user-type checks (speculator vs normal portfolio, line 23-47)
|
|
111
|
+
- **Common in**: Data catalogs (Amundsen, DataHub), **day-level granularity is custom**
|
|
112
|
+
|
|
113
|
+
### 18. **Impossible State Propagation**
|
|
114
|
+
- **Files**: `WorkflowOrchestrator.js:94-96`, `logger.js:77-93`
|
|
115
|
+
- **Implementation**: Marks calculations as `IMPOSSIBLE` instead of failing them, allows graph to continue
|
|
116
|
+
- **Niche aspect**: Separate "impossible" category in analysis reports (logger.js:86-91)
|
|
117
|
+
- **Common in**: Workflow engines handle failures, **explicit impossible state is rare**
|
|
118
|
+
|
|
119
|
+
## Orchestration & Coordination
|
|
120
|
+
|
|
121
|
+
### 19. **Event-Driven Callback Pattern (Zero Polling)**
|
|
122
|
+
- **Files**: `bulltrackers_pipeline.yaml:49-76`, `computation_worker.js:82-104`
|
|
123
|
+
- **Implementation**: Workflow creates callback endpoint, worker POSTs on completion, workflow wakes
|
|
124
|
+
- **Niche aspect**: IAM authentication for callbacks (computation_worker.js:88-91)
|
|
125
|
+
- **Common in**: Cloud Workflows, AWS Step Functions (both support callbacks), **IAM-secured callbacks are best practice but not default**
|
|
126
|
+
|
|
127
|
+
### 20. **Run State Counter Pattern**
|
|
128
|
+
- **Files**: `computation_dispatcher.js:107-115`, `computation_worker.js:106-123`
|
|
129
|
+
- **Implementation**: Shared Firestore doc tracks `remainingTasks`, workers decrement on completion
|
|
130
|
+
- **Niche aspect**: Transaction-based decrement (computation_worker.js:109-119) ensures atomicity
|
|
131
|
+
- **Common in**: Distributed systems, **Firestore-specific implementation is custom**
|
|
132
|
+
|
|
133
|
+
### 21. **Audit Ledger (Ledger-DB Pattern)**
|
|
134
|
+
- **Files**: `computation_dispatcher.js:143-163`, `RunRecorder.js:26-99`
|
|
135
|
+
- **Implementation**: Write-once ledger per task (`computation_audit_ledger/{date}/passes/{pass}/tasks/{calc}`)
|
|
136
|
+
- **Niche aspect**: Stores granular timing breakdown (RunRecorder.js:64-70)
|
|
137
|
+
- **Common in**: Event sourcing systems, **granular profiling in ledger is uncommon**
|
|
138
|
+
|
|
139
|
+
### 22. **Poison Message Handling (DLQ)**
|
|
140
|
+
- **Files**: `computation_worker.js:36-60`
|
|
141
|
+
- **Implementation**: Max retries check via Pub/Sub `deliveryAttempt`, moves to dead letter queue
|
|
142
|
+
- **Niche aspect**: Differentiates deterministic errors (line 194-222) from transient failures
|
|
143
|
+
- **Common in**: Message queues (RabbitMQ, SQS), **logic-aware routing is custom**
|
|
144
|
+
|
|
145
|
+
### 23. **Catch-Up Logic (Historical Scan)**
|
|
146
|
+
- **Files**: `computation_dispatcher.js:65-81`
|
|
147
|
+
- **Implementation**: Scans full date range (earliest data → target date) instead of just target date
|
|
148
|
+
- **Niche aspect**: Parallel analysis with concurrency limit (line 85)
|
|
149
|
+
- **Common in**: Data pipelines (backfill mode), **integrated into dispatcher is convenient**
|
|
150
|
+
|
|
151
|
+
## Observability & Debugging
|
|
152
|
+
|
|
153
|
+
### 24. **Structured Logging System**
|
|
154
|
+
- **Files**: `logger.js:27-118`
|
|
155
|
+
- **Implementation**: Dual output (human-readable + JSON), process tracking, context inheritance
|
|
156
|
+
- **Niche aspect**: `ProcessLogger` class (line 120-148) for scoped logging with auto-stats
|
|
157
|
+
- **Common in**: Production apps (Winston, Bunyan), **process-scoped loggers are nice touch**
|
|
158
|
+
|
|
159
|
+
### 25. **Date Analysis Reports**
|
|
160
|
+
- **Files**: `logger.js:77-132`
|
|
161
|
+
- **Implementation**: Per-date breakdown of runnable/blocked/impossible/skipped calculations
|
|
162
|
+
- **Niche aspect**: Unicode symbols for visual parsing (line 103)
|
|
163
|
+
- **Common in**: DAG visualization tools, **inline CLI reports are developer-friendly**
|
|
164
|
+
|
|
165
|
+
### 26. **Build Report Generator**
|
|
166
|
+
- **Files**: `BuildReporter.js:138-248`
|
|
167
|
+
- **Implementation**: Pre-deployment impact analysis showing blast radius of code changes
|
|
168
|
+
- **Niche aspect**: Blast radius calculation (line 62-77) - finds all downstream dependents
|
|
169
|
+
- **Common in**: CI/CD tools (GitHub's "affected projects"), **calculation-level granularity is detailed**
|
|
170
|
+
|
|
171
|
+
### 27. **System Fingerprinting**
|
|
172
|
+
- **Files**: `BuildReporter.js:28-51`, `HashManager.js:80-111`
|
|
173
|
+
- **Implementation**: SHA-256 hash of entire codebase + manifest, triggers report on change
|
|
174
|
+
- **Niche aspect**: Recursive directory walk with ignore patterns (HashManager.js:44-60)
|
|
175
|
+
- **Common in**: Docker layer caching, **for change detection at deploy-time is creative**
|
|
176
|
+
|
|
177
|
+
### 28. **Execution Statistics Tracking**
|
|
178
|
+
- **Files**: `StandardExecutor.js:64-71`, `RunRecorder.js:57-70`
|
|
179
|
+
- **Implementation**: Tracks processed/skipped users, setup/stream/processing time breakdowns
|
|
180
|
+
- **Niche aspect**: Profiler-ready structure (RunRecorder.js:64-70) for BigQuery analysis
|
|
181
|
+
- **Common in**: Profilers (cProfile, pyflame), **baked into business logic is pragmatic**
|
|
182
|
+
|
|
183
|
+
## Data Access Patterns
|
|
184
|
+
|
|
185
|
+
### 29. **Smart Shard Indexing**
|
|
186
|
+
- **Files**: `data_loader.js:152-213`
|
|
187
|
+
- **Implementation**: Maintains `instrumentId → shardId` index to avoid scanning all shards
|
|
188
|
+
- **Niche aspect**: 24-hour TTL with rebuild logic (line 167-172)
|
|
189
|
+
- **Common in**: Database indexes, **application-level shard routing is custom**
|
|
190
|
+
|
|
191
|
+
### 30. **Async Generator Streaming**
|
|
192
|
+
- **Files**: `data_loader.js:130-150`
|
|
193
|
+
- **Implementation**: `async function*` yields data chunks, caller consumes with `for await`
|
|
194
|
+
- **Niche aspect**: Supports pre-provided refs (line 132) for dependency injection
|
|
195
|
+
- **Common in**: Node.js streams, **generator-based approach is modern/clean**
|
|
196
|
+
|
|
197
|
+
### 31. **Cached Data Loader**
|
|
198
|
+
- **Files**: `CachedDataLoader.js:14-73`
|
|
199
|
+
- **Implementation**: Execution-scoped cache for mappings/insights/social data
|
|
200
|
+
- **Niche aspect**: Decompression helper (line 24-32) for transparent GZIP handling
|
|
201
|
+
- **Common in**: Data layers (Apollo Client, React Query), **per-execution scope is appropriate**
|
|
202
|
+
|
|
203
|
+
### 32. **Deferred Hydration**
|
|
204
|
+
- **Files**: `DependencyFetcher.js:23-66`
|
|
205
|
+
- **Implementation**: Fetches metadata documents, hydrates sharded data on-demand
|
|
206
|
+
- **Niche aspect**: Parallel hydration promises (line 44-47)
|
|
207
|
+
- **Common in**: ORMs (lazy loading), **manual shard hydration is low-level**
|
|
208
|
+
|
|
209
|
+
## Domain-Specific Intelligence
|
|
210
|
+
|
|
211
|
+
### 33. **User Classification Engine**
|
|
212
|
+
- **Files**: `profiling.js:24-236`
|
|
213
|
+
- **Implementation**: "Smart Money" scoring with 18+ behavioral signals
|
|
214
|
+
- **Niche aspect**: Multi-factor scoring (portfolio allocation + trade history + execution timing)
|
|
215
|
+
- **Common in**: Fintech risk models, **granularity is impressive**
|
|
216
|
+
|
|
217
|
+
### 34. **Convex Hull Risk Geometry**
|
|
218
|
+
- **Files**: `profiling.js:338-365`
|
|
219
|
+
- **Implementation**: Monotone Chain algorithm for efficient frontier analysis
|
|
220
|
+
- **Niche aspect**: O(n log n) algorithm choice (profiling.js:345-363)
|
|
221
|
+
- **Common in**: Computational geometry libraries, **integration into user profiling is domain-specific**
|
|
222
|
+
|
|
223
|
+
### 35. **Kadane's Maximum Drawdown**
|
|
224
|
+
- **Files**: `extractors.js:27-52`
|
|
225
|
+
- **Implementation**: O(n) single-pass algorithm for peak-to-trough decline
|
|
226
|
+
- **Niche aspect**: Returns indices for visualization (line 47)
|
|
227
|
+
- **Common in**: Finance libraries (QuantLib), **clean implementation**
|
|
228
|
+
|
|
229
|
+
### 36. **Fast Fourier Transform (Cooley-Tukey)**
|
|
230
|
+
- **Files**: `mathematics.js:148-184`
|
|
231
|
+
- **Implementation**: O(n log n) frequency domain analysis with zero-padding
|
|
232
|
+
- **Niche aspect**: Recursive implementation (line 163-183)
|
|
233
|
+
- **Common in**: Signal processing (NumPy, SciPy), **JavaScript implementation is rare**
|
|
234
|
+
|
|
235
|
+
### 37. **Sliding Window Extrema (Monotonic Queue)**
|
|
236
|
+
- **Files**: `mathematics.js:227-259`
|
|
237
|
+
- **Implementation**: O(n) min/max calculation using deque
|
|
238
|
+
- **Niche aspect**: Dual deques (one for min, one for max, line 236-237)
|
|
239
|
+
- **Common in**: Competitive programming, **production usage is uncommon**
|
|
240
|
+
|
|
241
|
+
### 38. **Geometric Brownian Motion Simulator**
|
|
242
|
+
- **Files**: `mathematics.js:99-118`
|
|
243
|
+
- **Implementation**: Box-Muller transform for normal random variates, Monte Carlo simulation
|
|
244
|
+
- **Niche aspect**: Returns `Float32Array` for memory efficiency (line 106)
|
|
245
|
+
- **Common in**: Quant finance (Black-Scholes), **typed arrays are performance-conscious**
|
|
246
|
+
|
|
247
|
+
### 39. **Hit Probability Calculator**
|
|
248
|
+
- **Files**: `mathematics.js:75-97`
|
|
249
|
+
- **Implementation**: Closed-form barrier option pricing formula
|
|
250
|
+
- **Niche aspect**: Custom `normCDF` implementation (line 85-89) avoids external deps
|
|
251
|
+
- **Common in**: Options pricing libraries, **standalone implementation is self-contained**
|
|
252
|
+
|
|
253
|
+
### 40. **Kernel Density Estimation**
|
|
254
|
+
- **Files**: `mathematics.js:263-288`
|
|
255
|
+
- **Implementation**: Gaussian kernel with weighted samples
|
|
256
|
+
- **Niche aspect**: 3-bandwidth cutoff for performance (line 276)
|
|
257
|
+
- **Common in**: Stats packages (SciPy, R), **production KDE is uncommon**
|
|
258
|
+
|
|
259
|
+
## Schema & Type Management
|
|
260
|
+
|
|
261
|
+
### 41. **Schema Capture System**
|
|
262
|
+
- **Files**: `schema_capture.js:28-68`
|
|
263
|
+
- **Implementation**: Batch stores class-defined schemas to Firestore
|
|
264
|
+
- **Niche aspect**: Pre-commit validation (line 32-34) prevents batch failures
|
|
265
|
+
- **Common in**: Schema registries (Confluent), **lightweight alternative**
|
|
266
|
+
|
|
267
|
+
### 42. **Production Schema Validators**
|
|
268
|
+
- **Files**: `validators.js:14-137`
|
|
269
|
+
- **Implementation**: Structural validation matching schema.md definitions
|
|
270
|
+
- **Niche aspect**: Separate validators per data type (portfolio/history/social/insights/prices)
|
|
271
|
+
- **Common in**: Data quality frameworks, **schema.md alignment is discipline**
|
|
272
|
+
|
|
273
|
+
### 43. **Legacy Mapping System**
|
|
274
|
+
- **Files**: `HashManager.js:8-23`, `ContextFactory.js:12-17`
|
|
275
|
+
- **Implementation**: Alias mapping for backward compatibility (e.g., `extract` → `DataExtractor`)
|
|
276
|
+
- **Niche aspect**: Dual injection into context (line 14-16)
|
|
277
|
+
- **Common in**: API versioning, **maintaining during refactor is good practice**
|
|
278
|
+
|
|
279
|
+
## Infrastructure & Operations
|
|
280
|
+
|
|
281
|
+
### 44. **Self-Healing Sharding Strategy**
|
|
282
|
+
- **Files**: `ResultCommitter.js:234-302`
|
|
283
|
+
- **Implementation**: Progressively stricter sharding on failure (900KB → 450KB → 200KB → 100KB)
|
|
284
|
+
- **Niche aspect**: Strategy array iteration (line 241-246)
|
|
285
|
+
- **Common in**: Resilience patterns, **adaptive sharding is creative**
|
|
286
|
+
|
|
287
|
+
### 45. **Initial Write Cleanup Logic**
|
|
288
|
+
- **Files**: `ResultCommitter.js:111-127`, `StandardExecutor.js:122-124`
|
|
289
|
+
- **Implementation**: `isInitialWrite` flag triggers shard deletion before first write
|
|
290
|
+
- **Niche aspect**: Transition detection (line 115-121) from sharded → compressed
|
|
291
|
+
- **Common in**: Migration scripts, **baked into write path is convenient**
|
|
292
|
+
|
|
293
|
+
### 46. **Firestore Byte Calculator**
|
|
294
|
+
- **Files**: `ResultCommitter.js:319-324`
|
|
295
|
+
- **Implementation**: Estimates document size for batch limits
|
|
296
|
+
- **Niche aspect**: Handles `DocumentReference` paths (line 322)
|
|
297
|
+
- **Common in**: Firestore SDKs (internal), **custom implementation for control**
|
|
298
|
+
|
|
299
|
+
### 47. **Retry with Exponential Backoff**
|
|
300
|
+
- **Files**: `utils.js:65-79`
|
|
301
|
+
- **Implementation**: Async retry wrapper with configurable attempts and backoff
|
|
302
|
+
- **Niche aspect**: 1s → 2s → 4s progression (line 75)
|
|
303
|
+
- **Common in**: HTTP clients (axios, got), **standalone utility is reusable**
|
|
304
|
+
|
|
305
|
+
### 48. **Batch Commit Chunker**
|
|
306
|
+
- **Files**: `utils.js:86-128`
|
|
307
|
+
- **Implementation**: Splits writes into Firestore 500-op/10MB batches
|
|
308
|
+
- **Niche aspect**: Supports DELETE operations (line 103-108)
|
|
309
|
+
- **Common in**: ORMs (SQLAlchemy bulk), **DELETE support is complete**
|
|
310
|
+
|
|
311
|
+
### 49. **Date Range Generator**
|
|
312
|
+
- **Files**: `utils.js:131-139`
|
|
313
|
+
- **Implementation**: UTC-aware date string generation
|
|
314
|
+
- **Niche aspect**: Forces UTC via `Date.UTC()` constructor (line 133-134)
|
|
315
|
+
- **Common in**: Date libraries (date-fns, Luxon), **UTC enforcement is critical for finance**
|
|
316
|
+
|
|
317
|
+
### 50. **Earliest Date Discovery**
|
|
318
|
+
- **Files**: `utils.js:158-207`
|
|
319
|
+
- **Implementation**: Scans multiple collections to find first available data
|
|
320
|
+
- **Niche aspect**: Handles both flat and sharded collections (line 142-157, 160-174)
|
|
321
|
+
- **Common in**: Data discovery tools, **multi-source aggregation is thorough**
|
|
322
|
+
|
|
323
|
+
## Advanced Patterns
|
|
324
|
+
|
|
325
|
+
### 51. **Tarjan's Stack Management**
|
|
326
|
+
- **Files**: `ManifestBuilder.js:98-141`
|
|
327
|
+
- **Implementation**: Manual stack tracking for SCC detection
|
|
328
|
+
- **Niche aspect**: `onStack` Set for O(1) membership checks (line 106)
|
|
329
|
+
- **Common in**: Graph algorithm implementations, **production usage is advanced**
|
|
330
|
+
|
|
331
|
+
### 52. **Dependency-Injection Context Factory**
|
|
332
|
+
- **Files**: `ContextFactory.js:17-61`
|
|
333
|
+
- **Implementation**: Separate builders for per-user vs meta contexts
|
|
334
|
+
- **Niche aspect**: Math layer injection with legacy aliases (line 12-17)
|
|
335
|
+
- **Common in**: DI frameworks (Spring, Guice), **manual factory is lightweight**
|
|
336
|
+
|
|
337
|
+
### 53. **Price Batch Executor**
|
|
338
|
+
- **Files**: `PriceBatchExecutor.js:12-104`
|
|
339
|
+
- **Implementation**: Specialized executor for price-only calculations (optimization pass)
|
|
340
|
+
- **Niche aspect**: Outer concurrency (2) + shard batching (20) + write batching (50) nested limits
|
|
341
|
+
- **Common in**: MapReduce systems, **three-level batching is complex**
|
|
342
|
+
|
|
343
|
+
### 54. **Deterministic Mock Data Fabrication**
|
|
344
|
+
- **Files**: `Fabricator.js:20-244`, `SeededRandom.js:8-38`
|
|
345
|
+
- **Implementation**: LCG PRNG seeded by calculation name for reproducible fakes
|
|
346
|
+
- **Niche aspect**: Iteration-based seed rotation (Fabricator.js:29)
|
|
347
|
+
- **Common in**: Property-based testing (Hypothesis, QuickCheck), **for optimization is novel**
|
|
348
|
+
|
|
349
|
+
### 55. **Schema-Driven Fake Generation**
|
|
350
|
+
- **Files**: `Fabricator.js:48-71`
|
|
351
|
+
- **Implementation**: Recursively generates data matching JSON schema
|
|
352
|
+
- **Niche aspect**: Volume scaling flag (line 49) for aggregate vs per-item data
|
|
353
|
+
- **Common in**: Schema-based generators (JSF, json-schema-faker), **custom to domain**
|
|
354
|
+
|
|
355
|
+
### 56. **Migration Cleanup Hook**
|
|
356
|
+
- **Files**: `ResultCommitter.js:81-83`, `ResultCommitter.js:305-317`
|
|
357
|
+
- **Implementation**: Deletes old category data when calculation moves
|
|
358
|
+
- **Niche aspect**: `previousCategory` tracking in manifest (WorkflowOrchestrator.js:50-54)
|
|
359
|
+
- **Common in**: Schema migration tools (Alembic, Flyway), **inline cleanup is pragmatic**
|
|
360
|
+
|
|
361
|
+
### 57. **Non-Retryable Error Classification**
|
|
362
|
+
- **Files**: `ResultCommitter.js:18-21`, `computation_worker.js:194-225`
|
|
363
|
+
- **Implementation**: Distinguishes deterministic failures from transient errors
|
|
364
|
+
- **Niche aspect**: `error.stage` property for categorization (computation_worker.js:205-209)
|
|
365
|
+
- **Common in**: Error handling libraries (Sentry), **semantic error types are good practice**
|
|
366
|
+
|
|
367
|
+
### 58. **Reverse Adjacency Graph**
|
|
368
|
+
- **Files**: `BuildReporter.js:62-77`
|
|
369
|
+
- **Implementation**: Maintains child → parent edges for impact analysis
|
|
370
|
+
- **Niche aspect**: Used for blast radius calculation (line 66-74)
|
|
371
|
+
- **Common in**: Dependency analyzers (npm-why), **runtime maintenance is useful**
|
|
372
|
+
|
|
373
|
+
### 59. **Multi-Key Manifest Cache**
|
|
374
|
+
- **Files**: `ManifestLoader.js:9-14`
|
|
375
|
+
- **Implementation**: Cache key is JSON-stringified sorted product lines
|
|
376
|
+
- **Niche aspect**: Handles `['ALL']` vs `['crypto', 'stocks']` as different keys
|
|
377
|
+
- **Common in**: Memoization libraries (lodash.memoize), **cache key design is thoughtful**
|
|
378
|
+
|
|
379
|
+
### 60. **Workflow Variable Restoration**
|
|
380
|
+
- **Files**: `bulltrackers_pipeline.yaml:11-17`
|
|
381
|
+
- **Implementation**: Comment notes a bug fix restoring `passes` and `max_retries` variables
|
|
382
|
+
- **Niche aspect**: T-1 date logic (line 13-15) for "process yesterday" pattern
|
|
383
|
+
- **Common in**: Production YAML configs, **inline documentation is helpful**
|
|
384
|
+
|
|
385
|
+
---
|
|
386
|
+
|
|
387
|
+
## Summary Statistics
|
|
388
|
+
|
|
389
|
+
- **Total Features Identified**: 60
|
|
390
|
+
- **Unique/Rare Features**: ~15 (SimHash, content-based short-circuit, forensic routing, contract discovery, weekend validation, behavioral stability, heap-aware flushing, monotonic queue extrema, FFT, KDE, smart shard indexing, recursive infra hash, semantic gates, impossible propagation, blast radius)
|
|
391
|
+
- **Advanced CS Algorithms**: 8 (Kahn's, Tarjan's, Convex Hull, Kadane's, FFT, Box-Muller, Monotonic Queue, LCG)
|
|
392
|
+
- **Common Patterns (Elevated)**: ~25 (executed exceptionally well or with domain-specific twist)
|
|
393
|
+
- **Standard Infrastructure**: ~22 (logging, retries, batching, streaming, caching, validation, etc.)
|
|
394
|
+
|
|
395
|
+
**Verdict**: About 25% truly novel, 40% common patterns elevated to production-grade, 35% standard infrastructure executed well.
|
|
@@ -29,6 +29,7 @@ async function checkCrashForensics(db, date, pass, computationName) {
|
|
|
29
29
|
const ledgerPath = `computation_audit_ledger/${date}/passes/${pass}/tasks/${computationName}`;
|
|
30
30
|
const doc = await db.doc(ledgerPath).get();
|
|
31
31
|
|
|
32
|
+
// Default to standard
|
|
32
33
|
if (!doc.exists) return 'standard';
|
|
33
34
|
|
|
34
35
|
const data = doc.data();
|
|
@@ -64,8 +65,8 @@ async function checkCrashForensics(db, date, pass, computationName) {
|
|
|
64
65
|
*/
|
|
65
66
|
async function dispatchComputationPass(config, dependencies, computationManifest, reqBody = {}) {
|
|
66
67
|
const { logger, db } = dependencies;
|
|
67
|
-
const pubsubUtils
|
|
68
|
-
const passToRun
|
|
68
|
+
const pubsubUtils = new PubSubUtils(dependencies);
|
|
69
|
+
const passToRun = String(config.COMPUTATION_PASS_TO_RUN);
|
|
69
70
|
|
|
70
71
|
// Extract Date and Callback from request body (pushed by Workflow)
|
|
71
72
|
// NOTE: 'dateStr' acts as the "Target Date" (Ceiling), usually T-1.
|
|
@@ -75,9 +76,7 @@ async function dispatchComputationPass(config, dependencies, computationManifest
|
|
|
75
76
|
if (!passToRun) { return logger.log('ERROR', '[Dispatcher] No pass defined (COMPUTATION_PASS_TO_RUN). Aborting.'); }
|
|
76
77
|
if (!dateStr) { return logger.log('ERROR', '[Dispatcher] No date defined. Aborting.'); }
|
|
77
78
|
|
|
78
|
-
const currentManifestHash = generateCodeHash(
|
|
79
|
-
computationManifest.map(c => c.hash).sort().join('|')
|
|
80
|
-
);
|
|
79
|
+
const currentManifestHash = generateCodeHash( computationManifest.map(c => c.hash).sort().join('|') );
|
|
81
80
|
|
|
82
81
|
const passes = groupByPass(computationManifest);
|
|
83
82
|
const calcsInThisPass = passes[passToRun] || [];
|
|
@@ -129,9 +128,9 @@ async function dispatchComputationPass(config, dependencies, computationManifest
|
|
|
129
128
|
}
|
|
130
129
|
}
|
|
131
130
|
|
|
132
|
-
const results
|
|
133
|
-
const dailyStatus
|
|
134
|
-
const availability
|
|
131
|
+
const results = await Promise.all(fetchPromises);
|
|
132
|
+
const dailyStatus = results[0];
|
|
133
|
+
const availability = results[1];
|
|
135
134
|
const prevDailyStatus = (prevDateStr && results[2]) ? results[2] : (prevDateStr ? {} : null);
|
|
136
135
|
|
|
137
136
|
const rootDataStatus = availability ? availability.status : {
|
|
@@ -220,9 +219,9 @@ async function dispatchComputationPass(config, dependencies, computationManifest
|
|
|
220
219
|
|
|
221
220
|
// 3. Create Audit Ledger Entries
|
|
222
221
|
const finalDispatched = [];
|
|
223
|
-
const txnLimit
|
|
222
|
+
const txnLimit = pLimit(20);
|
|
224
223
|
|
|
225
|
-
const txnPromises
|
|
224
|
+
const txnPromises = tasksToDispatch.map(task => txnLimit(async () => {
|
|
226
225
|
const ledgerRef = db.collection(`computation_audit_ledger/${task.date}/passes/${task.pass}/tasks`).doc(task.computation);
|
|
227
226
|
|
|
228
227
|
try {
|
|
@@ -120,8 +120,8 @@ class DataExtractor {
|
|
|
120
120
|
return "Buy";
|
|
121
121
|
}
|
|
122
122
|
|
|
123
|
-
static getLeverage(position)
|
|
124
|
-
static getOpenRate(position)
|
|
123
|
+
static getLeverage(position) { return position ? (position.Leverage || 1) : 1; }
|
|
124
|
+
static getOpenRate(position) { return position ? (position.OpenRate || 0) : 0; }
|
|
125
125
|
static getCurrentRate(position) { return position ? (position.CurrentRate || 0) : 0; }
|
|
126
126
|
static getStopLossRate(position) {
|
|
127
127
|
const rate = position ? (position.StopLossRate || 0) : 0;
|
|
@@ -207,7 +207,7 @@ class HistoryExtractor {
|
|
|
207
207
|
});
|
|
208
208
|
}
|
|
209
209
|
const asset = assetsMap.get(instId);
|
|
210
|
-
const open
|
|
210
|
+
const open = new Date(t.OpenDateTime);
|
|
211
211
|
const close = new Date(t.CloseDateTime);
|
|
212
212
|
const durationMins = (close - open) / 60000;
|
|
213
213
|
if (durationMins > 0) {
|
|
@@ -287,25 +287,25 @@ class InsightsExtractor {
|
|
|
287
287
|
return insights.find(i => i.instrumentId === instrumentId) || null;
|
|
288
288
|
}
|
|
289
289
|
|
|
290
|
-
static getTotalOwners(insight)
|
|
291
|
-
static getLongPercent(insight)
|
|
292
|
-
static getShortPercent(insight)
|
|
290
|
+
static getTotalOwners(insight) { return insight ? (insight.total || 0) : 0; }
|
|
291
|
+
static getLongPercent(insight) { return insight ? (insight.buy || 0) : 0; }
|
|
292
|
+
static getShortPercent(insight) { return insight ? (insight.sell || 0) : 0; }
|
|
293
293
|
static getGrowthPercent(insight) { return insight ? (insight.growth || 0) : 0; }
|
|
294
294
|
|
|
295
295
|
static getLongCount(insight) {
|
|
296
|
-
const total
|
|
296
|
+
const total = this.getTotalOwners(insight);
|
|
297
297
|
const buyPct = this.getLongPercent(insight);
|
|
298
298
|
return Math.floor(total * (buyPct / 100));
|
|
299
299
|
}
|
|
300
300
|
|
|
301
301
|
static getShortCount(insight) {
|
|
302
|
-
const total
|
|
302
|
+
const total = this.getTotalOwners(insight);
|
|
303
303
|
const sellPct = this.getShortPercent(insight);
|
|
304
304
|
return Math.floor(total * (sellPct / 100));
|
|
305
305
|
}
|
|
306
306
|
|
|
307
307
|
static getNetOwnershipChange(insight) {
|
|
308
|
-
const total
|
|
308
|
+
const total = this.getTotalOwners(insight);
|
|
309
309
|
const growth = this.getGrowthPercent(insight);
|
|
310
310
|
if (total === 0) return 0;
|
|
311
311
|
const prevTotal = total / (1 + (growth / 100));
|
|
@@ -60,12 +60,12 @@ async function recordRunAttempt(db, context, status, error = null, detailedMetri
|
|
|
60
60
|
|
|
61
61
|
// [IDEA 2] Enhanced Execution Stats
|
|
62
62
|
executionStats: {
|
|
63
|
-
processedUsers: rawExecStats.processedUsers
|
|
64
|
-
skippedUsers: rawExecStats.skippedUsers
|
|
63
|
+
processedUsers: rawExecStats.processedUsers || 0,
|
|
64
|
+
skippedUsers: rawExecStats.skippedUsers || 0,
|
|
65
65
|
// Explicitly break out timings for BigQuery/Analysis
|
|
66
66
|
timings: {
|
|
67
|
-
setupMs: Math.round(timings.setup
|
|
68
|
-
streamMs: Math.round(timings.stream
|
|
67
|
+
setupMs: Math.round(timings.setup || 0),
|
|
68
|
+
streamMs: Math.round(timings.stream || 0),
|
|
69
69
|
processingMs: Math.round(timings.processing || 0)
|
|
70
70
|
}
|
|
71
71
|
},
|
|
@@ -73,8 +73,8 @@ async function recordRunAttempt(db, context, status, error = null, detailedMetri
|
|
|
73
73
|
outputStats: {
|
|
74
74
|
sizeMB: sizeMB,
|
|
75
75
|
isSharded: !!detailedMetrics.storage?.isSharded,
|
|
76
|
-
shardCount:
|
|
77
|
-
keysWritten: detailedMetrics.storage?.keys
|
|
76
|
+
shardCount: detailedMetrics.storage?.shardCount || 1,
|
|
77
|
+
keysWritten: detailedMetrics.storage?.keys || 0
|
|
78
78
|
},
|
|
79
79
|
|
|
80
80
|
anomalies: anomalies,
|
|
@@ -84,9 +84,9 @@ async function recordRunAttempt(db, context, status, error = null, detailedMetri
|
|
|
84
84
|
if (error) {
|
|
85
85
|
runEntry.error = {
|
|
86
86
|
message: error.message || 'Unknown Error',
|
|
87
|
-
stage:
|
|
88
|
-
stack:
|
|
89
|
-
code:
|
|
87
|
+
stage: error.stage || 'UNKNOWN',
|
|
88
|
+
stack: error.stack ? error.stack.substring(0, 1000) : null,
|
|
89
|
+
code: error.code || null
|
|
90
90
|
};
|
|
91
91
|
}
|
|
92
92
|
|
|
@@ -0,0 +1,385 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Admin API Router
|
|
3
|
+
* Sub-module for system observability, debugging, and visualization.
|
|
4
|
+
* Mounted at /admin within the Generic API.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
const express = require('express');
|
|
8
|
+
const pLimit = require('p-limit');
|
|
9
|
+
const { getManifest } = require('../../computation-system/topology/ManifestLoader');
|
|
10
|
+
const { normalizeName } = require('../../computation-system/utils/utils');
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Factory to create the Admin Router.
|
|
14
|
+
* @param {object} config - System configuration.
|
|
15
|
+
* @param {object} dependencies - { db, logger, ... }
|
|
16
|
+
* @param {object} unifiedCalculations - The injected calculations package.
|
|
17
|
+
*/
|
|
18
|
+
const createAdminRouter = (config, dependencies, unifiedCalculations) => {
|
|
19
|
+
const router = express.Router();
|
|
20
|
+
const { db, logger } = dependencies;
|
|
21
|
+
|
|
22
|
+
// Helper to get fresh manifest
|
|
23
|
+
const getFullManifest = () => getManifest([], unifiedCalculations, dependencies);
|
|
24
|
+
|
|
25
|
+
// --- 1. TOPOLOGY VISUALIZER ---
|
|
26
|
+
router.get('/topology', async (req, res) => {
|
|
27
|
+
try {
|
|
28
|
+
const manifest = getFullManifest();
|
|
29
|
+
const nodes = [];
|
|
30
|
+
const edges = [];
|
|
31
|
+
|
|
32
|
+
manifest.forEach(calc => {
|
|
33
|
+
nodes.push({
|
|
34
|
+
id: calc.name,
|
|
35
|
+
data: {
|
|
36
|
+
label: calc.name,
|
|
37
|
+
layer: calc.category,
|
|
38
|
+
pass: calc.pass,
|
|
39
|
+
isHistorical: calc.isHistorical,
|
|
40
|
+
type: calc.type
|
|
41
|
+
},
|
|
42
|
+
position: { x: 0, y: 0 }
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
if (calc.dependencies) {
|
|
46
|
+
calc.dependencies.forEach(dep => {
|
|
47
|
+
edges.push({
|
|
48
|
+
id: `e-${dep}-${calc.name}`,
|
|
49
|
+
source: normalizeName(dep),
|
|
50
|
+
target: calc.name,
|
|
51
|
+
type: 'smoothstep'
|
|
52
|
+
});
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
if (calc.rootDataDependencies) {
|
|
57
|
+
calc.rootDataDependencies.forEach(root => {
|
|
58
|
+
const rootId = `ROOT_${root.toUpperCase()}`;
|
|
59
|
+
if (!nodes.find(n => n.id === rootId)) {
|
|
60
|
+
nodes.push({
|
|
61
|
+
id: rootId,
|
|
62
|
+
type: 'input',
|
|
63
|
+
data: { label: `${root.toUpperCase()} DB` },
|
|
64
|
+
position: { x: 0, y: 0 },
|
|
65
|
+
style: { background: '#f0f0f0', border: '1px solid #777' }
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
edges.push({
|
|
69
|
+
id: `e-root-${root}-${calc.name}`,
|
|
70
|
+
source: rootId,
|
|
71
|
+
target: calc.name,
|
|
72
|
+
animated: true,
|
|
73
|
+
style: { stroke: '#ff0072' }
|
|
74
|
+
});
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
res.json({ summary: { totalNodes: nodes.length, totalEdges: edges.length }, nodes, edges });
|
|
80
|
+
} catch (e) {
|
|
81
|
+
logger.log('ERROR', '[AdminAPI] Topology build failed', e);
|
|
82
|
+
res.status(500).json({ error: e.message });
|
|
83
|
+
}
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
// --- 2. STATUS MATRIX (Calendar / State UI) ---
|
|
87
|
+
// Returns status of ALL computations across a date range.
|
|
88
|
+
// ENHANCED: Cross-references Manifest to detect "PENDING" (Not run yet) vs "MISSING".
|
|
89
|
+
router.get('/matrix', async (req, res) => {
|
|
90
|
+
const { start, end } = req.query;
|
|
91
|
+
if (!start || !end) return res.status(400).json({ error: "Start and End dates required." });
|
|
92
|
+
|
|
93
|
+
try {
|
|
94
|
+
const startDate = new Date(String(start));
|
|
95
|
+
const endDate = new Date(String(end));
|
|
96
|
+
const dates = [];
|
|
97
|
+
for (let d = new Date(startDate); d <= endDate; d.setDate(d.getDate() + 1)) {
|
|
98
|
+
dates.push(d.toISOString().slice(0, 10));
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
const manifest = getFullManifest();
|
|
102
|
+
const allCalcNames = new Set(manifest.map(c => c.name));
|
|
103
|
+
|
|
104
|
+
const limit = pLimit(20);
|
|
105
|
+
const matrix = {};
|
|
106
|
+
|
|
107
|
+
await Promise.all(dates.map(date => limit(async () => {
|
|
108
|
+
// Fetch Status and Root Data Availability
|
|
109
|
+
const [statusSnap, rootSnap] = await Promise.all([
|
|
110
|
+
db.collection('computation_status').doc(date).get(),
|
|
111
|
+
db.collection('system_root_data_index').doc(date).get()
|
|
112
|
+
]);
|
|
113
|
+
|
|
114
|
+
const statusData = statusSnap.exists ? statusSnap.data() : {};
|
|
115
|
+
const rootData = rootSnap.exists ? rootSnap.data() : { status: { hasPortfolio: false } };
|
|
116
|
+
|
|
117
|
+
const dateStatus = {};
|
|
118
|
+
|
|
119
|
+
// Check every calculation in the Manifest
|
|
120
|
+
allCalcNames.forEach(calcName => {
|
|
121
|
+
const entry = statusData[calcName];
|
|
122
|
+
|
|
123
|
+
if (!entry) {
|
|
124
|
+
// If root data exists but calc is missing -> PENDING
|
|
125
|
+
// If no root data -> WAITING_DATA
|
|
126
|
+
dateStatus[calcName] = rootData.status?.hasPortfolio ? 'PENDING' : 'WAITING_DATA';
|
|
127
|
+
} else if (typeof entry === 'object') {
|
|
128
|
+
if (entry.hash && typeof entry.hash === 'string' && entry.hash.startsWith('IMPOSSIBLE')) {
|
|
129
|
+
dateStatus[calcName] = 'IMPOSSIBLE';
|
|
130
|
+
} else if (entry.hash === false) {
|
|
131
|
+
dateStatus[calcName] = 'BLOCKED';
|
|
132
|
+
} else {
|
|
133
|
+
dateStatus[calcName] = 'COMPLETED';
|
|
134
|
+
}
|
|
135
|
+
} else if (entry === 'IMPOSSIBLE') {
|
|
136
|
+
dateStatus[calcName] = 'IMPOSSIBLE';
|
|
137
|
+
} else {
|
|
138
|
+
dateStatus[calcName] = 'COMPLETED';
|
|
139
|
+
}
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
matrix[date] = {
|
|
143
|
+
dataAvailable: rootData.status || {},
|
|
144
|
+
calculations: dateStatus
|
|
145
|
+
};
|
|
146
|
+
})));
|
|
147
|
+
|
|
148
|
+
res.json(matrix);
|
|
149
|
+
} catch (e) {
|
|
150
|
+
logger.log('ERROR', '[AdminAPI] Matrix fetch failed', e);
|
|
151
|
+
res.status(500).json({ error: e.message });
|
|
152
|
+
}
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
// --- 3. PIPELINE STATE (Progress Bar) ---
|
|
156
|
+
// Shows realtime status of the 5-pass system for a specific date
|
|
157
|
+
router.get('/pipeline/state', async (req, res) => {
|
|
158
|
+
const { date } = req.query;
|
|
159
|
+
if (!date) return res.status(400).json({ error: "Date required" });
|
|
160
|
+
|
|
161
|
+
try {
|
|
162
|
+
const passes = ['1', '2', '3', '4', '5'];
|
|
163
|
+
const state = await Promise.all(passes.map(async (pass) => {
|
|
164
|
+
// We use the Audit Ledger which is the source of truth for execution state
|
|
165
|
+
const tasksSnap = await db.collection(`computation_audit_ledger/${date}/passes/${pass}/tasks`).get();
|
|
166
|
+
|
|
167
|
+
const stats = {
|
|
168
|
+
pending: 0,
|
|
169
|
+
inProgress: 0,
|
|
170
|
+
completed: 0,
|
|
171
|
+
failed: 0,
|
|
172
|
+
totalMemoryMB: 0,
|
|
173
|
+
avgDurationMs: 0
|
|
174
|
+
};
|
|
175
|
+
|
|
176
|
+
const durations = [];
|
|
177
|
+
|
|
178
|
+
tasksSnap.forEach(doc => {
|
|
179
|
+
const data = doc.data();
|
|
180
|
+
const s = (data.status || 'UNKNOWN').toLowerCase();
|
|
181
|
+
if (stats[s] !== undefined) stats[s]++;
|
|
182
|
+
else stats[s] = 1;
|
|
183
|
+
|
|
184
|
+
if (data.telemetry?.lastMemory?.rssMB) {
|
|
185
|
+
stats.totalMemoryMB += data.telemetry.lastMemory.rssMB;
|
|
186
|
+
}
|
|
187
|
+
if (data.completedAt && data.startedAt) {
|
|
188
|
+
durations.push(new Date(data.completedAt).getTime() - new Date(data.startedAt).getTime());
|
|
189
|
+
}
|
|
190
|
+
});
|
|
191
|
+
|
|
192
|
+
stats.avgDurationMs = durations.length ?
|
|
193
|
+
Math.round(durations.reduce((a, b) => a + b, 0) / durations.length) : 0;
|
|
194
|
+
|
|
195
|
+
return { pass, stats, totalTasks: tasksSnap.size };
|
|
196
|
+
}));
|
|
197
|
+
|
|
198
|
+
res.json({ date, passes: state });
|
|
199
|
+
} catch (e) {
|
|
200
|
+
res.status(500).json({ error: e.message });
|
|
201
|
+
}
|
|
202
|
+
});
|
|
203
|
+
|
|
204
|
+
// --- 4. DEPENDENCY TRACER (Blast Radius) ---
|
|
205
|
+
router.get('/trace/:calcName', async (req, res) => {
|
|
206
|
+
const { calcName } = req.params;
|
|
207
|
+
const mode = req.query.mode || 'downstream'; // 'upstream' or 'downstream'
|
|
208
|
+
|
|
209
|
+
try {
|
|
210
|
+
const manifest = getFullManifest();
|
|
211
|
+
const manifestMap = new Map(manifest.map(c => [c.name, c]));
|
|
212
|
+
|
|
213
|
+
if (!manifestMap.has(calcName)) return res.status(404).json({ error: 'Calculation not found' });
|
|
214
|
+
|
|
215
|
+
const trace = { root: calcName, chain: [] };
|
|
216
|
+
|
|
217
|
+
if (mode === 'upstream') {
|
|
218
|
+
// What does X depend on?
|
|
219
|
+
const visited = new Set();
|
|
220
|
+
const walk = (name, depth = 0) => {
|
|
221
|
+
if (visited.has(name) || depth > 10) return;
|
|
222
|
+
visited.add(name);
|
|
223
|
+
const calc = manifestMap.get(name);
|
|
224
|
+
if (!calc) return;
|
|
225
|
+
|
|
226
|
+
trace.chain.push({
|
|
227
|
+
name, depth, pass: calc.pass, type: calc.type
|
|
228
|
+
});
|
|
229
|
+
|
|
230
|
+
calc.dependencies?.forEach(dep => walk(dep, depth + 1));
|
|
231
|
+
};
|
|
232
|
+
walk(calcName);
|
|
233
|
+
} else {
|
|
234
|
+
// What depends on X? (Downstream / Impact)
|
|
235
|
+
const reverseGraph = new Map();
|
|
236
|
+
manifest.forEach(c => {
|
|
237
|
+
c.dependencies?.forEach(dep => {
|
|
238
|
+
const normDep = normalizeName(dep);
|
|
239
|
+
if (!reverseGraph.has(normDep)) reverseGraph.set(normDep, []);
|
|
240
|
+
reverseGraph.get(normDep).push(c.name);
|
|
241
|
+
});
|
|
242
|
+
});
|
|
243
|
+
|
|
244
|
+
const visited = new Set();
|
|
245
|
+
const walk = (name, depth = 0) => {
|
|
246
|
+
if (visited.has(name) || depth > 10) return;
|
|
247
|
+
visited.add(name);
|
|
248
|
+
|
|
249
|
+
const calc = manifestMap.get(name);
|
|
250
|
+
trace.chain.push({
|
|
251
|
+
name, depth, pass: calc?.pass
|
|
252
|
+
});
|
|
253
|
+
|
|
254
|
+
reverseGraph.get(name)?.forEach(child => walk(child, depth + 1));
|
|
255
|
+
};
|
|
256
|
+
walk(calcName);
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
res.json(trace);
|
|
260
|
+
} catch (e) {
|
|
261
|
+
res.status(500).json({ error: e.message });
|
|
262
|
+
}
|
|
263
|
+
});
|
|
264
|
+
|
|
265
|
+
// --- 5. CONTRACT VIOLATIONS (Quality Gate) ---
|
|
266
|
+
router.get('/violations', async (req, res) => {
|
|
267
|
+
const days = parseInt(String(req.query.days)) || 7;
|
|
268
|
+
const cutoff = new Date();
|
|
269
|
+
cutoff.setDate(cutoff.getDate() - days);
|
|
270
|
+
|
|
271
|
+
try {
|
|
272
|
+
// Check DLQ for Semantic Failures (Hard Violations)
|
|
273
|
+
const dlqSnap = await db.collection('computation_dead_letter_queue')
|
|
274
|
+
.where('finalAttemptAt', '>', cutoff)
|
|
275
|
+
.where('error.stage', '==', 'SEMANTIC_GATE')
|
|
276
|
+
.limit(50)
|
|
277
|
+
.get();
|
|
278
|
+
|
|
279
|
+
const violations = [];
|
|
280
|
+
dlqSnap.forEach(doc => {
|
|
281
|
+
const data = doc.data();
|
|
282
|
+
violations.push({
|
|
283
|
+
id: doc.id,
|
|
284
|
+
computation: data.originalData.computation,
|
|
285
|
+
date: data.originalData.date,
|
|
286
|
+
reason: data.error.message,
|
|
287
|
+
type: 'HARD_VIOLATION',
|
|
288
|
+
timestamp: data.finalAttemptAt
|
|
289
|
+
});
|
|
290
|
+
});
|
|
291
|
+
|
|
292
|
+
// Check Audit Logs for Soft Anomalies (Statistical warnings)
|
|
293
|
+
const anomalySnap = await db.collectionGroup('history')
|
|
294
|
+
.where('triggerTime', '>', cutoff.toISOString())
|
|
295
|
+
.where('anomalies', '!=', []) // Firestore != operator
|
|
296
|
+
.limit(50)
|
|
297
|
+
.get();
|
|
298
|
+
|
|
299
|
+
anomalySnap.forEach(doc => {
|
|
300
|
+
const data = doc.data();
|
|
301
|
+
data.anomalies?.forEach(anomaly => {
|
|
302
|
+
violations.push({
|
|
303
|
+
id: doc.id,
|
|
304
|
+
computation: data.computationName,
|
|
305
|
+
date: data.targetDate,
|
|
306
|
+
reason: anomaly,
|
|
307
|
+
type: 'SOFT_ANOMALY',
|
|
308
|
+
timestamp: data.triggerTime
|
|
309
|
+
});
|
|
310
|
+
});
|
|
311
|
+
});
|
|
312
|
+
|
|
313
|
+
// Sort by time desc
|
|
314
|
+
violations.sort((a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime());
|
|
315
|
+
|
|
316
|
+
res.json({ count: violations.length, violations });
|
|
317
|
+
} catch (e) {
|
|
318
|
+
res.status(500).json({ error: e.message });
|
|
319
|
+
}
|
|
320
|
+
});
|
|
321
|
+
|
|
322
|
+
// --- 6. MEMORY HOTSPOTS (Forensics) ---
|
|
323
|
+
router.get('/memory/hotspots', async (req, res) => {
|
|
324
|
+
const thresholdMB = parseInt(String(req.query.threshold)) || 1000; // 1GB default
|
|
325
|
+
|
|
326
|
+
try {
|
|
327
|
+
// Ledger tasks maintain 'telemetry.lastMemory'
|
|
328
|
+
// We use collectionGroup to search across all dates/passes
|
|
329
|
+
const snapshot = await db.collectionGroup('tasks')
|
|
330
|
+
.where('telemetry.lastMemory.rssMB', '>', thresholdMB)
|
|
331
|
+
.orderBy('telemetry.lastMemory.rssMB', 'desc')
|
|
332
|
+
.limit(20)
|
|
333
|
+
.get();
|
|
334
|
+
|
|
335
|
+
const hotspots = [];
|
|
336
|
+
snapshot.forEach(doc => {
|
|
337
|
+
const data = doc.data();
|
|
338
|
+
hotspots.push({
|
|
339
|
+
computation: data.computation,
|
|
340
|
+
rssMB: data.telemetry.lastMemory.rssMB,
|
|
341
|
+
heapMB: data.telemetry.lastMemory.heapUsedMB,
|
|
342
|
+
status: data.status,
|
|
343
|
+
worker: data.workerId,
|
|
344
|
+
date: doc.ref.parent.parent.parent.parent.id // traversing path to get date
|
|
345
|
+
});
|
|
346
|
+
});
|
|
347
|
+
|
|
348
|
+
res.json({ count: hotspots.length, hotspots });
|
|
349
|
+
} catch (e) {
|
|
350
|
+
res.status(500).json({ error: e.message });
|
|
351
|
+
}
|
|
352
|
+
});
|
|
353
|
+
|
|
354
|
+
// --- 7. FLIGHT RECORDER (Inspection) ---
|
|
355
|
+
// Existing inspection endpoint kept for drill-down
|
|
356
|
+
router.get('/inspect/:date/:calcName', async (req, res) => {
|
|
357
|
+
const { date, calcName } = req.params;
|
|
358
|
+
try {
|
|
359
|
+
const passes = ['1', '2', '3', '4', '5'];
|
|
360
|
+
let executionRecord = null;
|
|
361
|
+
|
|
362
|
+
await Promise.all(passes.map(async (pass) => {
|
|
363
|
+
if (executionRecord) return;
|
|
364
|
+
const ref = db.doc(`computation_audit_ledger/${date}/passes/${pass}/tasks/${calcName}`);
|
|
365
|
+
const snap = await ref.get();
|
|
366
|
+
if (snap.exists) executionRecord = { pass, ...snap.data() };
|
|
367
|
+
}));
|
|
368
|
+
|
|
369
|
+
if (!executionRecord) return res.status(404).json({ status: 'NOT_FOUND' });
|
|
370
|
+
|
|
371
|
+
const contractSnap = await db.collection('system_contracts').doc(calcName).get();
|
|
372
|
+
|
|
373
|
+
res.json({
|
|
374
|
+
execution: executionRecord,
|
|
375
|
+
contract: contractSnap.exists ? contractSnap.data() : null
|
|
376
|
+
});
|
|
377
|
+
} catch (e) {
|
|
378
|
+
res.status(500).json({ error: e.message });
|
|
379
|
+
}
|
|
380
|
+
});
|
|
381
|
+
|
|
382
|
+
return router;
|
|
383
|
+
};
|
|
384
|
+
|
|
385
|
+
module.exports = createAdminRouter;
|
|
@@ -1,10 +1,31 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @fileoverview API sub-pipes.
|
|
3
|
-
* REFACTORED:
|
|
4
|
-
*
|
|
3
|
+
* REFACTORED: API V3 - Status-Aware Data Fetching.
|
|
4
|
+
* UPDATED: Added GZIP Decompression support for fetching compressed results.
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
7
|
const { FieldPath } = require('@google-cloud/firestore');
|
|
8
|
+
const zlib = require('zlib'); // [NEW] Required for decompression
|
|
9
|
+
|
|
10
|
+
// --- HELPER: DECOMPRESSION ---
|
|
11
|
+
/**
|
|
12
|
+
* Checks if data is compressed and inflates it if necessary.
|
|
13
|
+
* @param {object} data - The raw Firestore document data.
|
|
14
|
+
* @returns {object} The original (decompressed) JSON object.
|
|
15
|
+
*/
|
|
16
|
+
function tryDecompress(data) {
|
|
17
|
+
if (data && data._compressed === true && data.payload) {
|
|
18
|
+
try {
|
|
19
|
+
// Firestore returns Buffers automatically for Blob types
|
|
20
|
+
return JSON.parse(zlib.gunzipSync(data.payload).toString());
|
|
21
|
+
} catch (e) {
|
|
22
|
+
console.error('[API] Decompression failed:', e);
|
|
23
|
+
// Return empty object or original data on failure to avoid crashing response
|
|
24
|
+
return {};
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
return data;
|
|
28
|
+
}
|
|
8
29
|
|
|
9
30
|
// --- AVAILABILITY CACHE ---
|
|
10
31
|
class AvailabilityCache {
|
|
@@ -147,6 +168,7 @@ const buildCalculationMap = (unifiedCalculations) => {
|
|
|
147
168
|
|
|
148
169
|
/**
|
|
149
170
|
* Sub-pipe: pipe.api.helpers.fetchUnifiedData
|
|
171
|
+
* UPDATED: Uses tryDecompress to handle compressed payloads.
|
|
150
172
|
*/
|
|
151
173
|
const fetchUnifiedData = async (config, dependencies, calcKeys, dateStrings, calcMap) => {
|
|
152
174
|
const { db, logger } = dependencies;
|
|
@@ -191,7 +213,8 @@ const fetchUnifiedData = async (config, dependencies, calcKeys, dateStrings, cal
|
|
|
191
213
|
snapshots.forEach((doc, idx) => {
|
|
192
214
|
const { date, key } = chunk[idx];
|
|
193
215
|
if (doc.exists) {
|
|
194
|
-
|
|
216
|
+
// [UPDATED] Decompress data if needed
|
|
217
|
+
response[date][key] = tryDecompress(doc.data());
|
|
195
218
|
} else {
|
|
196
219
|
response[date][key] = null;
|
|
197
220
|
}
|
|
@@ -321,8 +344,11 @@ async function getComputationStructure(computationName, calcMap, config, depende
|
|
|
321
344
|
.collection(compsSub).doc(computationName);
|
|
322
345
|
const doc = await docRef.get();
|
|
323
346
|
if (!doc.exists) { return { status: 'error', computation: computationName, message: `Summary flag was present for ${latestStoredDate} but doc is missing.` }; }
|
|
324
|
-
|
|
347
|
+
|
|
348
|
+
// [UPDATED] Decompress data for structure inspection
|
|
349
|
+
const fullData = tryDecompress(doc.data());
|
|
325
350
|
const structureSnippet = createStructureSnippet(fullData);
|
|
351
|
+
|
|
326
352
|
return { status: 'success', computation: computationName, category: category, latestStoredDate: latestStoredDate, structureSnippet: structureSnippet, };
|
|
327
353
|
} catch (error) {
|
|
328
354
|
logger.log('ERROR', `API /structure/${computationName} helper failed.`, { errorMessage: error.message });
|
|
@@ -2,12 +2,16 @@
|
|
|
2
2
|
* @fileoverview Main entry point for the Generic API module.
|
|
3
3
|
* Export the 'createApiApp' main pipe function.
|
|
4
4
|
* REFACTORED: API V3 - Status-Aware Data Fetching.
|
|
5
|
+
* UPDATED: Added Admin API Mount.
|
|
5
6
|
*/
|
|
6
7
|
|
|
7
8
|
const express = require('express');
|
|
8
9
|
const cors = require('cors');
|
|
9
10
|
const { buildCalculationMap, createApiHandler, getComputationStructure, createManifestHandler, getDynamicSchema } = require('./helpers/api_helpers.js');
|
|
10
11
|
|
|
12
|
+
// [NEW] Import Admin Router
|
|
13
|
+
const createAdminRouter = require('./admin-api/index');
|
|
14
|
+
|
|
11
15
|
/**
|
|
12
16
|
* In-Memory Cache Handler
|
|
13
17
|
* Wrapper that adds TTL cache to GET requests.
|
|
@@ -71,8 +75,11 @@ function createApiApp(config, dependencies, unifiedCalculations) {
|
|
|
71
75
|
app.use(cors({ origin: true }));
|
|
72
76
|
app.use(express.json());
|
|
73
77
|
|
|
78
|
+
// --- [NEW] MOUNT ADMIN API ---
|
|
79
|
+
// This injects the dependencies and the calculations package into the admin router
|
|
80
|
+
app.use('/admin', createAdminRouter(config, dependencies, unifiedCalculations));
|
|
81
|
+
|
|
74
82
|
// --- Main API V3 Endpoint ---
|
|
75
|
-
// createApiHandler now initializes the AvailabilityCache internally
|
|
76
83
|
const originalApiHandler = createApiHandler(config, dependencies, calcMap);
|
|
77
84
|
const cachedApiHandler = createCacheHandler(originalApiHandler, dependencies);
|
|
78
85
|
|