bulltrackers-module 1.0.294 → 1.0.296

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -70,7 +70,6 @@ async function runBatchPriceComputation(config, deps, dateStrings, calcs, target
70
70
  .collection(config.computationsSubcollection)
71
71
  .doc(normalizeName(calcManifest.name));
72
72
 
73
- // [UPDATE] Add _lastUpdated timestamp
74
73
  writes.push({
75
74
  ref: docRef,
76
75
  data: {
@@ -13,7 +13,7 @@ const { ContextFactory } = require
13
13
  const { commitResults } = require('../persistence/ResultCommitter');
14
14
  const mathLayer = require('../layers/index');
15
15
  const { performance } = require('perf_hooks');
16
- const v8 = require('v8'); // [NEW] For Memory introspection
16
+ const v8 = require('v8');
17
17
 
18
18
  class StandardExecutor {
19
19
  static async run(date, calcs, passName, config, deps, rootData, fetchedDeps, previousFetchedDeps, skipStatusWrite = false) {
@@ -0,0 +1,395 @@
1
+ # Complete Feature Inventory of BullTrackers Computation System
2
+
3
+ ## Core DAG Engine Features
4
+
5
+ ### 1. **Topological Sorting (Kahn's Algorithm)**
6
+ - **Files**: `ManifestBuilder.js:187-205`
7
+ - **Implementation**: Builds execution passes by tracking in-degrees, queuing zero-dependency nodes
8
+ - **Niche aspect**: Dynamic pass assignment (line 201: `neighborEntry.pass = currentEntry.pass + 1`)
9
+ - **Common in**: Airflow, Prefect, Dagster (all use topological sort)
10
+
11
+ ### 2. **Cycle Detection (Tarjan's SCC Algorithm)**
12
+ - **Files**: `ManifestBuilder.js:98-141`
13
+ - **Implementation**: Strongly Connected Components detection with stack-based traversal
14
+ - **Niche aspect**: Returns human-readable cycle chain (line 137: `cycle.join(' -> ') + ' -> ' + cycle[0]`)
15
+ - **Common in**: Academic graph libraries, rare in production DAG systems (most use simpler DFS)
16
+
17
+ ### 3. **Auto-Discovery Manifest Building**
18
+ - **Files**: `ManifestBuilder.js:143-179`, `ManifestLoader.js:9-42`
19
+ - **Implementation**: Scans directories, instantiates classes, extracts metadata via `getMetadata()` static method
20
+ - **Niche aspect**: Singleton caching with multi-key support (ManifestLoader.js:9)
21
+ - **Common in**: Plugin systems (Airflow providers), less common for computation graphs
22
+
23
+ ## Dependency Management & Optimization
24
+
25
+ ### 4. **Multi-Layered Hash Composition**
26
+ - **Files**: `ManifestBuilder.js:56-95`, `HashManager.js:25-36`
27
+ - **Implementation**: Composite hash from code + epoch + infrastructure + layers + dependencies
28
+ - **Niche aspect**: Infrastructure hash (recursive file tree hashing, HashManager.js:38-79)
29
+ - **Common in**: Build systems (Bazel, Buck), **very rare** in data pipelines
30
+
31
+ ### 5. **Content-Based Dependency Short-Circuiting**
32
+ - **Files**: `WorkflowOrchestrator.js:51-73`
33
+ - **Implementation**: Tracks `resultHash` (output data hash), skips re-run if output unchanged despite code change
34
+ - **Niche aspect**: `dependencyResultHashes` tracking (line 59-67)
35
+ - **Common in**: **Extremely rare** - only seen in specialized incremental computation systems
36
+
37
+ ### 6. **Behavioral Stability Detection (SimHash)**
38
+ - **Files**: `BuildReporter.js:55-89`, `SimRunner.js:12-42`, `Fabricator.js:20-244`
39
+ - **Implementation**: Runs code against deterministic mock data, hashes output to detect "logic changes" vs "cosmetic changes"
40
+ - **Niche aspect**: Seeded random data generation (SeededRandom.js:1-38) for reproducible simulations
41
+ - **Common in**: **Unique** - haven't seen this elsewhere. Conceptually similar to property-based testing but for optimization
42
+
43
+ ### 7. **System Epoch Forcing**
44
+ - **Files**: `system_epoch.js:1-2`, `ManifestBuilder.js:65`
45
+ - **Implementation**: Manual version bump to force global re-computation
46
+ - **Niche aspect**: Single-line file that invalidates all cached results
47
+ - **Common in**: Cache invalidation patterns, but unusual to have a dedicated module
48
+
49
+ ## Execution & Resource Management
50
+
51
+ ### 8. **Streaming Execution with Batch Flushing**
52
+ - **Files**: `StandardExecutor.js:86-158`
53
+ - **Implementation**: Async generators yield data chunks, flush to DB every N users
54
+ - **Niche aspect**: Adaptive flushing based on V8 heap pressure (line 128-145)
55
+ - **Common in**: ETL tools (Spark, Flink use micro-batching), **heap-aware flushing is rare**
56
+
57
+ ### 9. **Memory Heartbeat (Flight Recorder)**
58
+ - **Files**: `computation_worker.js:30-53`
59
+ - **Implementation**: Background timer writes memory stats to Firestore every 2 seconds
60
+ - **Niche aspect**: Uses `.unref()` to prevent blocking process exit (line 50)
61
+ - **Common in**: APM tools (DataDog, New Relic), **embedding in workers is custom**
62
+
63
+ ### 10. **Forensic Crash Analysis & Intelligent Routing**
64
+ - **Files**: `computation_dispatcher.js:31-68`
65
+ - **Implementation**: Reads last memory stats from failed runs, routes to high-mem queue if OOM suspected
66
+ - **Niche aspect**: Parses telemetry to distinguish crash types (line 44-50)
67
+ - **Common in**: Kubernetes autoscaling heuristics, **application-level routing is rare**
68
+
69
+ ### 11. **Circuit Breaker Pattern**
70
+ - **Files**: `StandardExecutor.js:164-173`
71
+ - **Implementation**: Tracks error rate, fails fast if >10% failures after 100 items
72
+ - **Niche aspect**: Runs mid-stream (not just at job start)
73
+ - **Common in**: Microservices (Hystrix, Resilience4j), uncommon in data pipelines
74
+
75
+ ### 12. **Incremental Auto-Sharding**
76
+ - **Files**: `ResultCommitter.js:234-302`
77
+ - **Implementation**: Dynamically splits results into Firestore subcollection shards, tracks shard index across flushes
78
+ - **Niche aspect**: `flushMode: INTERMEDIATE` flag (line 150) to avoid pointer updates mid-stream
79
+ - **Common in**: Database sharding, **dynamic document sharding is custom**
80
+
81
+ ### 13. **GZIP Compression Strategy**
82
+ - **Files**: `ResultCommitter.js:128-157`
83
+ - **Implementation**: Compresses results >50KB, stores as binary blob if <900KB compressed
84
+ - **Niche aspect**: Falls back to sharding if compression fails or exceeds limit
85
+ - **Common in**: Storage layers, integration at application level is custom
86
+
87
+ ## Data Quality & Validation
88
+
89
+ ### 14. **Heuristic Validation (Grey Box)**
90
+ - **Files**: `ResultsValidator.js:8-96`
91
+ - **Implementation**: Statistical analysis (zero%, null%, flatline detection) without knowing schema
92
+ - **Niche aspect**: Weekend mode (line 57-64) - relaxes thresholds on Saturdays/Sundays
93
+ - **Common in**: Data quality tools (Great Expectations, Soda), **weekend-aware thresholds are domain-specific**
94
+
95
+ ### 15. **Contract Discovery & Enforcement**
96
+ - **Files**: `ContractDiscoverer.js:11-120`, `ContractValidator.js:9-64`
97
+ - **Implementation**: Monte Carlo simulation learns behavioral bounds, enforces at runtime
98
+ - **Niche aspect**: Distinguishes "physics limits" (ratios 0-1) from "statistical envelopes" (6-sigma)
99
+ - **Common in**: **Unique** - closest analogue is schema inference (Pandas Profiling) but this is probabilistic + enforced
100
+
101
+ ### 16. **Semantic Gates**
102
+ - **Files**: `ResultCommitter.js:118-127`
103
+ - **Implementation**: Blocks results that violate contracts before writing
104
+ - **Niche aspect**: Differentiated error handling - `SEMANTIC_GATE` errors are non-retryable (line 210-225)
105
+ - **Common in**: Type systems (TypeScript, Mypy), **runtime probabilistic checks are rare**
106
+
107
+ ### 17. **Root Data Availability Tracking**
108
+ - **Files**: `AvailabilityChecker.js:49-87`, `utils.js:11-17`
109
+ - **Implementation**: Centralized index (`system_root_data_index`) tracks what data exists per day
110
+ - **Niche aspect**: Granular user-type checks (speculator vs normal portfolio, line 23-47)
111
+ - **Common in**: Data catalogs (Amundsen, DataHub), **day-level granularity is custom**
112
+
113
+ ### 18. **Impossible State Propagation**
114
+ - **Files**: `WorkflowOrchestrator.js:94-96`, `logger.js:77-93`
115
+ - **Implementation**: Marks calculations as `IMPOSSIBLE` instead of failing them, allows graph to continue
116
+ - **Niche aspect**: Separate "impossible" category in analysis reports (logger.js:86-91)
117
+ - **Common in**: Workflow engines handle failures, **explicit impossible state is rare**
118
+
119
+ ## Orchestration & Coordination
120
+
121
+ ### 19. **Event-Driven Callback Pattern (Zero Polling)**
122
+ - **Files**: `bulltrackers_pipeline.yaml:49-76`, `computation_worker.js:82-104`
123
+ - **Implementation**: Workflow creates callback endpoint, worker POSTs on completion, workflow wakes
124
+ - **Niche aspect**: IAM authentication for callbacks (computation_worker.js:88-91)
125
+ - **Common in**: Cloud Workflows, AWS Step Functions (both support callbacks), **IAM-secured callbacks are best practice but not default**
126
+
127
+ ### 20. **Run State Counter Pattern**
128
+ - **Files**: `computation_dispatcher.js:107-115`, `computation_worker.js:106-123`
129
+ - **Implementation**: Shared Firestore doc tracks `remainingTasks`, workers decrement on completion
130
+ - **Niche aspect**: Transaction-based decrement (computation_worker.js:109-119) ensures atomicity
131
+ - **Common in**: Distributed systems, **Firestore-specific implementation is custom**
132
+
133
+ ### 21. **Audit Ledger (Ledger-DB Pattern)**
134
+ - **Files**: `computation_dispatcher.js:143-163`, `RunRecorder.js:26-99`
135
+ - **Implementation**: Write-once ledger per task (`computation_audit_ledger/{date}/passes/{pass}/tasks/{calc}`)
136
+ - **Niche aspect**: Stores granular timing breakdown (RunRecorder.js:64-70)
137
+ - **Common in**: Event sourcing systems, **granular profiling in ledger is uncommon**
138
+
139
+ ### 22. **Poison Message Handling (DLQ)**
140
+ - **Files**: `computation_worker.js:36-60`
141
+ - **Implementation**: Max retries check via Pub/Sub `deliveryAttempt`, moves to dead letter queue
142
+ - **Niche aspect**: Differentiates deterministic errors (line 194-222) from transient failures
143
+ - **Common in**: Message queues (RabbitMQ, SQS), **logic-aware routing is custom**
144
+
145
+ ### 23. **Catch-Up Logic (Historical Scan)**
146
+ - **Files**: `computation_dispatcher.js:65-81`
147
+ - **Implementation**: Scans full date range (earliest data → target date) instead of just target date
148
+ - **Niche aspect**: Parallel analysis with concurrency limit (line 85)
149
+ - **Common in**: Data pipelines (backfill mode), **integrated into dispatcher is convenient**
150
+
151
+ ## Observability & Debugging
152
+
153
+ ### 24. **Structured Logging System**
154
+ - **Files**: `logger.js:27-118`
155
+ - **Implementation**: Dual output (human-readable + JSON), process tracking, context inheritance
156
+ - **Niche aspect**: `ProcessLogger` class (line 120-148) for scoped logging with auto-stats
157
+ - **Common in**: Production apps (Winston, Bunyan), **process-scoped loggers are nice touch**
158
+
159
+ ### 25. **Date Analysis Reports**
160
+ - **Files**: `logger.js:77-132`
161
+ - **Implementation**: Per-date breakdown of runnable/blocked/impossible/skipped calculations
162
+ - **Niche aspect**: Unicode symbols for visual parsing (line 103)
163
+ - **Common in**: DAG visualization tools, **inline CLI reports are developer-friendly**
164
+
165
+ ### 26. **Build Report Generator**
166
+ - **Files**: `BuildReporter.js:138-248`
167
+ - **Implementation**: Pre-deployment impact analysis showing blast radius of code changes
168
+ - **Niche aspect**: Blast radius calculation (line 62-77) - finds all downstream dependents
169
+ - **Common in**: CI/CD tools (GitHub's "affected projects"), **calculation-level granularity is detailed**
170
+
171
+ ### 27. **System Fingerprinting**
172
+ - **Files**: `BuildReporter.js:28-51`, `HashManager.js:80-111`
173
+ - **Implementation**: SHA-256 hash of entire codebase + manifest, triggers report on change
174
+ - **Niche aspect**: Recursive directory walk with ignore patterns (HashManager.js:44-60)
175
+ - **Common in**: Docker layer caching, **for change detection at deploy-time is creative**
176
+
177
+ ### 28. **Execution Statistics Tracking**
178
+ - **Files**: `StandardExecutor.js:64-71`, `RunRecorder.js:57-70`
179
+ - **Implementation**: Tracks processed/skipped users, setup/stream/processing time breakdowns
180
+ - **Niche aspect**: Profiler-ready structure (RunRecorder.js:64-70) for BigQuery analysis
181
+ - **Common in**: Profilers (cProfile, pyflame), **baked into business logic is pragmatic**
182
+
183
+ ## Data Access Patterns
184
+
185
+ ### 29. **Smart Shard Indexing**
186
+ - **Files**: `data_loader.js:152-213`
187
+ - **Implementation**: Maintains `instrumentId → shardId` index to avoid scanning all shards
188
+ - **Niche aspect**: 24-hour TTL with rebuild logic (line 167-172)
189
+ - **Common in**: Database indexes, **application-level shard routing is custom**
190
+
191
+ ### 30. **Async Generator Streaming**
192
+ - **Files**: `data_loader.js:130-150`
193
+ - **Implementation**: `async function*` yields data chunks, caller consumes with `for await`
194
+ - **Niche aspect**: Supports pre-provided refs (line 132) for dependency injection
195
+ - **Common in**: Node.js streams, **generator-based approach is modern/clean**
196
+
197
+ ### 31. **Cached Data Loader**
198
+ - **Files**: `CachedDataLoader.js:14-73`
199
+ - **Implementation**: Execution-scoped cache for mappings/insights/social data
200
+ - **Niche aspect**: Decompression helper (line 24-32) for transparent GZIP handling
201
+ - **Common in**: Data layers (Apollo Client, React Query), **per-execution scope is appropriate**
202
+
203
+ ### 32. **Deferred Hydration**
204
+ - **Files**: `DependencyFetcher.js:23-66`
205
+ - **Implementation**: Fetches metadata documents, hydrates sharded data on-demand
206
+ - **Niche aspect**: Parallel hydration promises (line 44-47)
207
+ - **Common in**: ORMs (lazy loading), **manual shard hydration is low-level**
208
+
209
+ ## Domain-Specific Intelligence
210
+
211
+ ### 33. **User Classification Engine**
212
+ - **Files**: `profiling.js:24-236`
213
+ - **Implementation**: "Smart Money" scoring with 18+ behavioral signals
214
+ - **Niche aspect**: Multi-factor scoring (portfolio allocation + trade history + execution timing)
215
+ - **Common in**: Fintech risk models, **granularity is impressive**
216
+
217
+ ### 34. **Convex Hull Risk Geometry**
218
+ - **Files**: `profiling.js:338-365`
219
+ - **Implementation**: Monotone Chain algorithm for efficient frontier analysis
220
+ - **Niche aspect**: O(n log n) algorithm choice (profiling.js:345-363)
221
+ - **Common in**: Computational geometry libraries, **integration into user profiling is domain-specific**
222
+
223
+ ### 35. **Kadane's Maximum Drawdown**
224
+ - **Files**: `extractors.js:27-52`
225
+ - **Implementation**: O(n) single-pass algorithm for peak-to-trough decline
226
+ - **Niche aspect**: Returns indices for visualization (line 47)
227
+ - **Common in**: Finance libraries (QuantLib), **clean implementation**
228
+
229
+ ### 36. **Fast Fourier Transform (Cooley-Tukey)**
230
+ - **Files**: `mathematics.js:148-184`
231
+ - **Implementation**: O(n log n) frequency domain analysis with zero-padding
232
+ - **Niche aspect**: Recursive implementation (line 163-183)
233
+ - **Common in**: Signal processing (NumPy, SciPy), **JavaScript implementation is rare**
234
+
235
+ ### 37. **Sliding Window Extrema (Monotonic Queue)**
236
+ - **Files**: `mathematics.js:227-259`
237
+ - **Implementation**: O(n) min/max calculation using deque
238
+ - **Niche aspect**: Dual deques (one for min, one for max, line 236-237)
239
+ - **Common in**: Competitive programming, **production usage is uncommon**
240
+
241
+ ### 38. **Geometric Brownian Motion Simulator**
242
+ - **Files**: `mathematics.js:99-118`
243
+ - **Implementation**: Box-Muller transform for normal random variates, Monte Carlo simulation
244
+ - **Niche aspect**: Returns `Float32Array` for memory efficiency (line 106)
245
+ - **Common in**: Quant finance (Black-Scholes), **typed arrays are performance-conscious**
246
+
247
+ ### 39. **Hit Probability Calculator**
248
+ - **Files**: `mathematics.js:75-97`
249
+ - **Implementation**: Closed-form barrier option pricing formula
250
+ - **Niche aspect**: Custom `normCDF` implementation (line 85-89) avoids external deps
251
+ - **Common in**: Options pricing libraries, **standalone implementation is self-contained**
252
+
253
+ ### 40. **Kernel Density Estimation**
254
+ - **Files**: `mathematics.js:263-288`
255
+ - **Implementation**: Gaussian kernel with weighted samples
256
+ - **Niche aspect**: 3-bandwidth cutoff for performance (line 276)
257
+ - **Common in**: Stats packages (SciPy, R), **production KDE is uncommon**
258
+
259
+ ## Schema & Type Management
260
+
261
+ ### 41. **Schema Capture System**
262
+ - **Files**: `schema_capture.js:28-68`
263
+ - **Implementation**: Batch stores class-defined schemas to Firestore
264
+ - **Niche aspect**: Pre-commit validation (line 32-34) prevents batch failures
265
+ - **Common in**: Schema registries (Confluent), **lightweight alternative**
266
+
267
+ ### 42. **Production Schema Validators**
268
+ - **Files**: `validators.js:14-137`
269
+ - **Implementation**: Structural validation matching schema.md definitions
270
+ - **Niche aspect**: Separate validators per data type (portfolio/history/social/insights/prices)
271
+ - **Common in**: Data quality frameworks, **schema.md alignment is discipline**
272
+
273
+ ### 43. **Legacy Mapping System**
274
+ - **Files**: `HashManager.js:8-23`, `ContextFactory.js:12-17`
275
+ - **Implementation**: Alias mapping for backward compatibility (e.g., `extract` → `DataExtractor`)
276
+ - **Niche aspect**: Dual injection into context (line 14-16)
277
+ - **Common in**: API versioning, **maintaining during refactor is good practice**
278
+
279
+ ## Infrastructure & Operations
280
+
281
+ ### 44. **Self-Healing Sharding Strategy**
282
+ - **Files**: `ResultCommitter.js:234-302`
283
+ - **Implementation**: Progressively stricter sharding on failure (900KB → 450KB → 200KB → 100KB)
284
+ - **Niche aspect**: Strategy array iteration (line 241-246)
285
+ - **Common in**: Resilience patterns, **adaptive sharding is creative**
286
+
287
+ ### 45. **Initial Write Cleanup Logic**
288
+ - **Files**: `ResultCommitter.js:111-127`, `StandardExecutor.js:122-124`
289
+ - **Implementation**: `isInitialWrite` flag triggers shard deletion before first write
290
+ - **Niche aspect**: Transition detection (line 115-121) from sharded → compressed
291
+ - **Common in**: Migration scripts, **baked into write path is convenient**
292
+
293
+ ### 46. **Firestore Byte Calculator**
294
+ - **Files**: `ResultCommitter.js:319-324`
295
+ - **Implementation**: Estimates document size for batch limits
296
+ - **Niche aspect**: Handles `DocumentReference` paths (line 322)
297
+ - **Common in**: Firestore SDKs (internal), **custom implementation for control**
298
+
299
+ ### 47. **Retry with Exponential Backoff**
300
+ - **Files**: `utils.js:65-79`
301
+ - **Implementation**: Async retry wrapper with configurable attempts and backoff
302
+ - **Niche aspect**: 1s → 2s → 4s progression (line 75)
303
+ - **Common in**: HTTP clients (axios, got), **standalone utility is reusable**
304
+
305
+ ### 48. **Batch Commit Chunker**
306
+ - **Files**: `utils.js:86-128`
307
+ - **Implementation**: Splits writes into Firestore 500-op/10MB batches
308
+ - **Niche aspect**: Supports DELETE operations (line 103-108)
309
+ - **Common in**: ORMs (SQLAlchemy bulk), **DELETE support is complete**
310
+
311
+ ### 49. **Date Range Generator**
312
+ - **Files**: `utils.js:131-139`
313
+ - **Implementation**: UTC-aware date string generation
314
+ - **Niche aspect**: Forces UTC via `Date.UTC()` constructor (line 133-134)
315
+ - **Common in**: Date libraries (date-fns, Luxon), **UTC enforcement is critical for finance**
316
+
317
+ ### 50. **Earliest Date Discovery**
318
+ - **Files**: `utils.js:158-207`
319
+ - **Implementation**: Scans multiple collections to find first available data
320
+ - **Niche aspect**: Handles both flat and sharded collections (line 142-157, 160-174)
321
+ - **Common in**: Data discovery tools, **multi-source aggregation is thorough**
322
+
323
+ ## Advanced Patterns
324
+
325
+ ### 51. **Tarjan's Stack Management**
326
+ - **Files**: `ManifestBuilder.js:98-141`
327
+ - **Implementation**: Manual stack tracking for SCC detection
328
+ - **Niche aspect**: `onStack` Set for O(1) membership checks (line 106)
329
+ - **Common in**: Graph algorithm implementations, **production usage is advanced**
330
+
331
+ ### 52. **Dependency-Injection Context Factory**
332
+ - **Files**: `ContextFactory.js:17-61`
333
+ - **Implementation**: Separate builders for per-user vs meta contexts
334
+ - **Niche aspect**: Math layer injection with legacy aliases (line 12-17)
335
+ - **Common in**: DI frameworks (Spring, Guice), **manual factory is lightweight**
336
+
337
+ ### 53. **Price Batch Executor**
338
+ - **Files**: `PriceBatchExecutor.js:12-104`
339
+ - **Implementation**: Specialized executor for price-only calculations (optimization pass)
340
+ - **Niche aspect**: Outer concurrency (2) + shard batching (20) + write batching (50) nested limits
341
+ - **Common in**: MapReduce systems, **three-level batching is complex**
342
+
343
+ ### 54. **Deterministic Mock Data Fabrication**
344
+ - **Files**: `Fabricator.js:20-244`, `SeededRandom.js:8-38`
345
+ - **Implementation**: LCG PRNG seeded by calculation name for reproducible fakes
346
+ - **Niche aspect**: Iteration-based seed rotation (Fabricator.js:29)
347
+ - **Common in**: Property-based testing (Hypothesis, QuickCheck), **for optimization is novel**
348
+
349
+ ### 55. **Schema-Driven Fake Generation**
350
+ - **Files**: `Fabricator.js:48-71`
351
+ - **Implementation**: Recursively generates data matching JSON schema
352
+ - **Niche aspect**: Volume scaling flag (line 49) for aggregate vs per-item data
353
+ - **Common in**: Schema-based generators (JSF, json-schema-faker), **custom to domain**
354
+
355
+ ### 56. **Migration Cleanup Hook**
356
+ - **Files**: `ResultCommitter.js:81-83`, `ResultCommitter.js:305-317`
357
+ - **Implementation**: Deletes old category data when calculation moves
358
+ - **Niche aspect**: `previousCategory` tracking in manifest (WorkflowOrchestrator.js:50-54)
359
+ - **Common in**: Schema migration tools (Alembic, Flyway), **inline cleanup is pragmatic**
360
+
361
+ ### 57. **Non-Retryable Error Classification**
362
+ - **Files**: `ResultCommitter.js:18-21`, `computation_worker.js:194-225`
363
+ - **Implementation**: Distinguishes deterministic failures from transient errors
364
+ - **Niche aspect**: `error.stage` property for categorization (computation_worker.js:205-209)
365
+ - **Common in**: Error handling libraries (Sentry), **semantic error types are good practice**
366
+
367
+ ### 58. **Reverse Adjacency Graph**
368
+ - **Files**: `BuildReporter.js:62-77`
369
+ - **Implementation**: Maintains child → parent edges for impact analysis
370
+ - **Niche aspect**: Used for blast radius calculation (line 66-74)
371
+ - **Common in**: Dependency analyzers (npm-why), **runtime maintenance is useful**
372
+
373
+ ### 59. **Multi-Key Manifest Cache**
374
+ - **Files**: `ManifestLoader.js:9-14`
375
+ - **Implementation**: Cache key is JSON-stringified sorted product lines
376
+ - **Niche aspect**: Handles `['ALL']` vs `['crypto', 'stocks']` as different keys
377
+ - **Common in**: Memoization libraries (lodash.memoize), **cache key design is thoughtful**
378
+
379
+ ### 60. **Workflow Variable Restoration**
380
+ - **Files**: `bulltrackers_pipeline.yaml:11-17`
381
+ - **Implementation**: Comment notes a bug fix restoring `passes` and `max_retries` variables
382
+ - **Niche aspect**: T-1 date logic (line 13-15) for "process yesterday" pattern
383
+ - **Common in**: Production YAML configs, **inline documentation is helpful**
384
+
385
+ ---
386
+
387
+ ## Summary Statistics
388
+
389
+ - **Total Features Identified**: 60
390
+ - **Unique/Rare Features**: ~15 (SimHash, content-based short-circuit, forensic routing, contract discovery, weekend validation, behavioral stability, heap-aware flushing, monotonic queue extrema, FFT, KDE, smart shard indexing, recursive infra hash, semantic gates, impossible propagation, blast radius)
391
+ - **Advanced CS Algorithms**: 8 (Kahn's, Tarjan's, Convex Hull, Kadane's, FFT, Box-Muller, Monotonic Queue, LCG)
392
+ - **Common Patterns (Elevated)**: ~25 (executed exceptionally well or with domain-specific twist)
393
+ - **Standard Infrastructure**: ~22 (logging, retries, batching, streaming, caching, validation, etc.)
394
+
395
+ **Verdict**: About 25% truly novel, 40% common patterns elevated to production-grade, 35% standard infrastructure executed well.
@@ -29,6 +29,7 @@ async function checkCrashForensics(db, date, pass, computationName) {
29
29
  const ledgerPath = `computation_audit_ledger/${date}/passes/${pass}/tasks/${computationName}`;
30
30
  const doc = await db.doc(ledgerPath).get();
31
31
 
32
+ // Default to standard
32
33
  if (!doc.exists) return 'standard';
33
34
 
34
35
  const data = doc.data();
@@ -64,8 +65,8 @@ async function checkCrashForensics(db, date, pass, computationName) {
64
65
  */
65
66
  async function dispatchComputationPass(config, dependencies, computationManifest, reqBody = {}) {
66
67
  const { logger, db } = dependencies;
67
- const pubsubUtils = new PubSubUtils(dependencies);
68
- const passToRun = String(config.COMPUTATION_PASS_TO_RUN);
68
+ const pubsubUtils = new PubSubUtils(dependencies);
69
+ const passToRun = String(config.COMPUTATION_PASS_TO_RUN);
69
70
 
70
71
  // Extract Date and Callback from request body (pushed by Workflow)
71
72
  // NOTE: 'dateStr' acts as the "Target Date" (Ceiling), usually T-1.
@@ -75,9 +76,7 @@ async function dispatchComputationPass(config, dependencies, computationManifest
75
76
  if (!passToRun) { return logger.log('ERROR', '[Dispatcher] No pass defined (COMPUTATION_PASS_TO_RUN). Aborting.'); }
76
77
  if (!dateStr) { return logger.log('ERROR', '[Dispatcher] No date defined. Aborting.'); }
77
78
 
78
- const currentManifestHash = generateCodeHash(
79
- computationManifest.map(c => c.hash).sort().join('|')
80
- );
79
+ const currentManifestHash = generateCodeHash( computationManifest.map(c => c.hash).sort().join('|') );
81
80
 
82
81
  const passes = groupByPass(computationManifest);
83
82
  const calcsInThisPass = passes[passToRun] || [];
@@ -129,9 +128,9 @@ async function dispatchComputationPass(config, dependencies, computationManifest
129
128
  }
130
129
  }
131
130
 
132
- const results = await Promise.all(fetchPromises);
133
- const dailyStatus = results[0];
134
- const availability = results[1];
131
+ const results = await Promise.all(fetchPromises);
132
+ const dailyStatus = results[0];
133
+ const availability = results[1];
135
134
  const prevDailyStatus = (prevDateStr && results[2]) ? results[2] : (prevDateStr ? {} : null);
136
135
 
137
136
  const rootDataStatus = availability ? availability.status : {
@@ -220,9 +219,9 @@ async function dispatchComputationPass(config, dependencies, computationManifest
220
219
 
221
220
  // 3. Create Audit Ledger Entries
222
221
  const finalDispatched = [];
223
- const txnLimit = pLimit(20);
222
+ const txnLimit = pLimit(20);
224
223
 
225
- const txnPromises = tasksToDispatch.map(task => txnLimit(async () => {
224
+ const txnPromises = tasksToDispatch.map(task => txnLimit(async () => {
226
225
  const ledgerRef = db.collection(`computation_audit_ledger/${task.date}/passes/${task.pass}/tasks`).doc(task.computation);
227
226
 
228
227
  try {
@@ -120,8 +120,8 @@ class DataExtractor {
120
120
  return "Buy";
121
121
  }
122
122
 
123
- static getLeverage(position) { return position ? (position.Leverage || 1) : 1; }
124
- static getOpenRate(position) { return position ? (position.OpenRate || 0) : 0; }
123
+ static getLeverage(position) { return position ? (position.Leverage || 1) : 1; }
124
+ static getOpenRate(position) { return position ? (position.OpenRate || 0) : 0; }
125
125
  static getCurrentRate(position) { return position ? (position.CurrentRate || 0) : 0; }
126
126
  static getStopLossRate(position) {
127
127
  const rate = position ? (position.StopLossRate || 0) : 0;
@@ -207,7 +207,7 @@ class HistoryExtractor {
207
207
  });
208
208
  }
209
209
  const asset = assetsMap.get(instId);
210
- const open = new Date(t.OpenDateTime);
210
+ const open = new Date(t.OpenDateTime);
211
211
  const close = new Date(t.CloseDateTime);
212
212
  const durationMins = (close - open) / 60000;
213
213
  if (durationMins > 0) {
@@ -287,25 +287,25 @@ class InsightsExtractor {
287
287
  return insights.find(i => i.instrumentId === instrumentId) || null;
288
288
  }
289
289
 
290
- static getTotalOwners(insight) { return insight ? (insight.total || 0) : 0; }
291
- static getLongPercent(insight) { return insight ? (insight.buy || 0) : 0; }
292
- static getShortPercent(insight) { return insight ? (insight.sell || 0) : 0; }
290
+ static getTotalOwners(insight) { return insight ? (insight.total || 0) : 0; }
291
+ static getLongPercent(insight) { return insight ? (insight.buy || 0) : 0; }
292
+ static getShortPercent(insight) { return insight ? (insight.sell || 0) : 0; }
293
293
  static getGrowthPercent(insight) { return insight ? (insight.growth || 0) : 0; }
294
294
 
295
295
  static getLongCount(insight) {
296
- const total = this.getTotalOwners(insight);
296
+ const total = this.getTotalOwners(insight);
297
297
  const buyPct = this.getLongPercent(insight);
298
298
  return Math.floor(total * (buyPct / 100));
299
299
  }
300
300
 
301
301
  static getShortCount(insight) {
302
- const total = this.getTotalOwners(insight);
302
+ const total = this.getTotalOwners(insight);
303
303
  const sellPct = this.getShortPercent(insight);
304
304
  return Math.floor(total * (sellPct / 100));
305
305
  }
306
306
 
307
307
  static getNetOwnershipChange(insight) {
308
- const total = this.getTotalOwners(insight);
308
+ const total = this.getTotalOwners(insight);
309
309
  const growth = this.getGrowthPercent(insight);
310
310
  if (total === 0) return 0;
311
311
  const prevTotal = total / (1 + (growth / 100));
@@ -60,12 +60,12 @@ async function recordRunAttempt(db, context, status, error = null, detailedMetri
60
60
 
61
61
  // [IDEA 2] Enhanced Execution Stats
62
62
  executionStats: {
63
- processedUsers: rawExecStats.processedUsers || 0,
64
- skippedUsers: rawExecStats.skippedUsers || 0,
63
+ processedUsers: rawExecStats.processedUsers || 0,
64
+ skippedUsers: rawExecStats.skippedUsers || 0,
65
65
  // Explicitly break out timings for BigQuery/Analysis
66
66
  timings: {
67
- setupMs: Math.round(timings.setup || 0),
68
- streamMs: Math.round(timings.stream || 0),
67
+ setupMs: Math.round(timings.setup || 0),
68
+ streamMs: Math.round(timings.stream || 0),
69
69
  processingMs: Math.round(timings.processing || 0)
70
70
  }
71
71
  },
@@ -73,8 +73,8 @@ async function recordRunAttempt(db, context, status, error = null, detailedMetri
73
73
  outputStats: {
74
74
  sizeMB: sizeMB,
75
75
  isSharded: !!detailedMetrics.storage?.isSharded,
76
- shardCount: detailedMetrics.storage?.shardCount || 1,
77
- keysWritten: detailedMetrics.storage?.keys || 0
76
+ shardCount: detailedMetrics.storage?.shardCount || 1,
77
+ keysWritten: detailedMetrics.storage?.keys || 0
78
78
  },
79
79
 
80
80
  anomalies: anomalies,
@@ -84,9 +84,9 @@ async function recordRunAttempt(db, context, status, error = null, detailedMetri
84
84
  if (error) {
85
85
  runEntry.error = {
86
86
  message: error.message || 'Unknown Error',
87
- stage: error.stage || 'UNKNOWN',
88
- stack: error.stack ? error.stack.substring(0, 1000) : null,
89
- code: error.code || null
87
+ stage: error.stage || 'UNKNOWN',
88
+ stack: error.stack ? error.stack.substring(0, 1000) : null,
89
+ code: error.code || null
90
90
  };
91
91
  }
92
92
 
@@ -0,0 +1,385 @@
1
+ /**
2
+ * @fileoverview Admin API Router
3
+ * Sub-module for system observability, debugging, and visualization.
4
+ * Mounted at /admin within the Generic API.
5
+ */
6
+
7
+ const express = require('express');
8
+ const pLimit = require('p-limit');
9
+ const { getManifest } = require('../../computation-system/topology/ManifestLoader');
10
+ const { normalizeName } = require('../../computation-system/utils/utils');
11
+
12
+ /**
13
+ * Factory to create the Admin Router.
14
+ * @param {object} config - System configuration.
15
+ * @param {object} dependencies - { db, logger, ... }
16
+ * @param {object} unifiedCalculations - The injected calculations package.
17
+ */
18
+ const createAdminRouter = (config, dependencies, unifiedCalculations) => {
19
+ const router = express.Router();
20
+ const { db, logger } = dependencies;
21
+
22
+ // Helper to get fresh manifest
23
+ const getFullManifest = () => getManifest([], unifiedCalculations, dependencies);
24
+
25
+ // --- 1. TOPOLOGY VISUALIZER ---
26
+ router.get('/topology', async (req, res) => {
27
+ try {
28
+ const manifest = getFullManifest();
29
+ const nodes = [];
30
+ const edges = [];
31
+
32
+ manifest.forEach(calc => {
33
+ nodes.push({
34
+ id: calc.name,
35
+ data: {
36
+ label: calc.name,
37
+ layer: calc.category,
38
+ pass: calc.pass,
39
+ isHistorical: calc.isHistorical,
40
+ type: calc.type
41
+ },
42
+ position: { x: 0, y: 0 }
43
+ });
44
+
45
+ if (calc.dependencies) {
46
+ calc.dependencies.forEach(dep => {
47
+ edges.push({
48
+ id: `e-${dep}-${calc.name}`,
49
+ source: normalizeName(dep),
50
+ target: calc.name,
51
+ type: 'smoothstep'
52
+ });
53
+ });
54
+ }
55
+
56
+ if (calc.rootDataDependencies) {
57
+ calc.rootDataDependencies.forEach(root => {
58
+ const rootId = `ROOT_${root.toUpperCase()}`;
59
+ if (!nodes.find(n => n.id === rootId)) {
60
+ nodes.push({
61
+ id: rootId,
62
+ type: 'input',
63
+ data: { label: `${root.toUpperCase()} DB` },
64
+ position: { x: 0, y: 0 },
65
+ style: { background: '#f0f0f0', border: '1px solid #777' }
66
+ });
67
+ }
68
+ edges.push({
69
+ id: `e-root-${root}-${calc.name}`,
70
+ source: rootId,
71
+ target: calc.name,
72
+ animated: true,
73
+ style: { stroke: '#ff0072' }
74
+ });
75
+ });
76
+ }
77
+ });
78
+
79
+ res.json({ summary: { totalNodes: nodes.length, totalEdges: edges.length }, nodes, edges });
80
+ } catch (e) {
81
+ logger.log('ERROR', '[AdminAPI] Topology build failed', e);
82
+ res.status(500).json({ error: e.message });
83
+ }
84
+ });
85
+
86
+ // --- 2. STATUS MATRIX (Calendar / State UI) ---
87
+ // Returns status of ALL computations across a date range.
88
+ // ENHANCED: Cross-references Manifest to detect "PENDING" (Not run yet) vs "MISSING".
89
+ router.get('/matrix', async (req, res) => {
90
+ const { start, end } = req.query;
91
+ if (!start || !end) return res.status(400).json({ error: "Start and End dates required." });
92
+
93
+ try {
94
+ const startDate = new Date(String(start));
95
+ const endDate = new Date(String(end));
96
+ const dates = [];
97
+ for (let d = new Date(startDate); d <= endDate; d.setDate(d.getDate() + 1)) {
98
+ dates.push(d.toISOString().slice(0, 10));
99
+ }
100
+
101
+ const manifest = getFullManifest();
102
+ const allCalcNames = new Set(manifest.map(c => c.name));
103
+
104
+ const limit = pLimit(20);
105
+ const matrix = {};
106
+
107
+ await Promise.all(dates.map(date => limit(async () => {
108
+ // Fetch Status and Root Data Availability
109
+ const [statusSnap, rootSnap] = await Promise.all([
110
+ db.collection('computation_status').doc(date).get(),
111
+ db.collection('system_root_data_index').doc(date).get()
112
+ ]);
113
+
114
+ const statusData = statusSnap.exists ? statusSnap.data() : {};
115
+ const rootData = rootSnap.exists ? rootSnap.data() : { status: { hasPortfolio: false } };
116
+
117
+ const dateStatus = {};
118
+
119
+ // Check every calculation in the Manifest
120
+ allCalcNames.forEach(calcName => {
121
+ const entry = statusData[calcName];
122
+
123
+ if (!entry) {
124
+ // If root data exists but calc is missing -> PENDING
125
+ // If no root data -> WAITING_DATA
126
+ dateStatus[calcName] = rootData.status?.hasPortfolio ? 'PENDING' : 'WAITING_DATA';
127
+ } else if (typeof entry === 'object') {
128
+ if (entry.hash && typeof entry.hash === 'string' && entry.hash.startsWith('IMPOSSIBLE')) {
129
+ dateStatus[calcName] = 'IMPOSSIBLE';
130
+ } else if (entry.hash === false) {
131
+ dateStatus[calcName] = 'BLOCKED';
132
+ } else {
133
+ dateStatus[calcName] = 'COMPLETED';
134
+ }
135
+ } else if (entry === 'IMPOSSIBLE') {
136
+ dateStatus[calcName] = 'IMPOSSIBLE';
137
+ } else {
138
+ dateStatus[calcName] = 'COMPLETED';
139
+ }
140
+ });
141
+
142
+ matrix[date] = {
143
+ dataAvailable: rootData.status || {},
144
+ calculations: dateStatus
145
+ };
146
+ })));
147
+
148
+ res.json(matrix);
149
+ } catch (e) {
150
+ logger.log('ERROR', '[AdminAPI] Matrix fetch failed', e);
151
+ res.status(500).json({ error: e.message });
152
+ }
153
+ });
154
+
155
+ // --- 3. PIPELINE STATE (Progress Bar) ---
156
+ // Shows realtime status of the 5-pass system for a specific date
157
+ router.get('/pipeline/state', async (req, res) => {
158
+ const { date } = req.query;
159
+ if (!date) return res.status(400).json({ error: "Date required" });
160
+
161
+ try {
162
+ const passes = ['1', '2', '3', '4', '5'];
163
+ const state = await Promise.all(passes.map(async (pass) => {
164
+ // We use the Audit Ledger which is the source of truth for execution state
165
+ const tasksSnap = await db.collection(`computation_audit_ledger/${date}/passes/${pass}/tasks`).get();
166
+
167
+ const stats = {
168
+ pending: 0,
169
+ inProgress: 0,
170
+ completed: 0,
171
+ failed: 0,
172
+ totalMemoryMB: 0,
173
+ avgDurationMs: 0
174
+ };
175
+
176
+ const durations = [];
177
+
178
+ tasksSnap.forEach(doc => {
179
+ const data = doc.data();
180
+ const s = (data.status || 'UNKNOWN').toLowerCase();
181
+ if (stats[s] !== undefined) stats[s]++;
182
+ else stats[s] = 1;
183
+
184
+ if (data.telemetry?.lastMemory?.rssMB) {
185
+ stats.totalMemoryMB += data.telemetry.lastMemory.rssMB;
186
+ }
187
+ if (data.completedAt && data.startedAt) {
188
+ durations.push(new Date(data.completedAt).getTime() - new Date(data.startedAt).getTime());
189
+ }
190
+ });
191
+
192
+ stats.avgDurationMs = durations.length ?
193
+ Math.round(durations.reduce((a, b) => a + b, 0) / durations.length) : 0;
194
+
195
+ return { pass, stats, totalTasks: tasksSnap.size };
196
+ }));
197
+
198
+ res.json({ date, passes: state });
199
+ } catch (e) {
200
+ res.status(500).json({ error: e.message });
201
+ }
202
+ });
203
+
204
+ // --- 4. DEPENDENCY TRACER (Blast Radius) ---
205
+ router.get('/trace/:calcName', async (req, res) => {
206
+ const { calcName } = req.params;
207
+ const mode = req.query.mode || 'downstream'; // 'upstream' or 'downstream'
208
+
209
+ try {
210
+ const manifest = getFullManifest();
211
+ const manifestMap = new Map(manifest.map(c => [c.name, c]));
212
+
213
+ if (!manifestMap.has(calcName)) return res.status(404).json({ error: 'Calculation not found' });
214
+
215
+ const trace = { root: calcName, chain: [] };
216
+
217
+ if (mode === 'upstream') {
218
+ // What does X depend on?
219
+ const visited = new Set();
220
+ const walk = (name, depth = 0) => {
221
+ if (visited.has(name) || depth > 10) return;
222
+ visited.add(name);
223
+ const calc = manifestMap.get(name);
224
+ if (!calc) return;
225
+
226
+ trace.chain.push({
227
+ name, depth, pass: calc.pass, type: calc.type
228
+ });
229
+
230
+ calc.dependencies?.forEach(dep => walk(dep, depth + 1));
231
+ };
232
+ walk(calcName);
233
+ } else {
234
+ // What depends on X? (Downstream / Impact)
235
+ const reverseGraph = new Map();
236
+ manifest.forEach(c => {
237
+ c.dependencies?.forEach(dep => {
238
+ const normDep = normalizeName(dep);
239
+ if (!reverseGraph.has(normDep)) reverseGraph.set(normDep, []);
240
+ reverseGraph.get(normDep).push(c.name);
241
+ });
242
+ });
243
+
244
+ const visited = new Set();
245
+ const walk = (name, depth = 0) => {
246
+ if (visited.has(name) || depth > 10) return;
247
+ visited.add(name);
248
+
249
+ const calc = manifestMap.get(name);
250
+ trace.chain.push({
251
+ name, depth, pass: calc?.pass
252
+ });
253
+
254
+ reverseGraph.get(name)?.forEach(child => walk(child, depth + 1));
255
+ };
256
+ walk(calcName);
257
+ }
258
+
259
+ res.json(trace);
260
+ } catch (e) {
261
+ res.status(500).json({ error: e.message });
262
+ }
263
+ });
264
+
265
+ // --- 5. CONTRACT VIOLATIONS (Quality Gate) ---
266
+ router.get('/violations', async (req, res) => {
267
+ const days = parseInt(String(req.query.days)) || 7;
268
+ const cutoff = new Date();
269
+ cutoff.setDate(cutoff.getDate() - days);
270
+
271
+ try {
272
+ // Check DLQ for Semantic Failures (Hard Violations)
273
+ const dlqSnap = await db.collection('computation_dead_letter_queue')
274
+ .where('finalAttemptAt', '>', cutoff)
275
+ .where('error.stage', '==', 'SEMANTIC_GATE')
276
+ .limit(50)
277
+ .get();
278
+
279
+ const violations = [];
280
+ dlqSnap.forEach(doc => {
281
+ const data = doc.data();
282
+ violations.push({
283
+ id: doc.id,
284
+ computation: data.originalData.computation,
285
+ date: data.originalData.date,
286
+ reason: data.error.message,
287
+ type: 'HARD_VIOLATION',
288
+ timestamp: data.finalAttemptAt
289
+ });
290
+ });
291
+
292
+ // Check Audit Logs for Soft Anomalies (Statistical warnings)
293
+ const anomalySnap = await db.collectionGroup('history')
294
+ .where('triggerTime', '>', cutoff.toISOString())
295
+ .where('anomalies', '!=', []) // Firestore != operator
296
+ .limit(50)
297
+ .get();
298
+
299
+ anomalySnap.forEach(doc => {
300
+ const data = doc.data();
301
+ data.anomalies?.forEach(anomaly => {
302
+ violations.push({
303
+ id: doc.id,
304
+ computation: data.computationName,
305
+ date: data.targetDate,
306
+ reason: anomaly,
307
+ type: 'SOFT_ANOMALY',
308
+ timestamp: data.triggerTime
309
+ });
310
+ });
311
+ });
312
+
313
+ // Sort by time desc
314
+ violations.sort((a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime());
315
+
316
+ res.json({ count: violations.length, violations });
317
+ } catch (e) {
318
+ res.status(500).json({ error: e.message });
319
+ }
320
+ });
321
+
322
+ // --- 6. MEMORY HOTSPOTS (Forensics) ---
323
+ router.get('/memory/hotspots', async (req, res) => {
324
+ const thresholdMB = parseInt(String(req.query.threshold)) || 1000; // 1GB default
325
+
326
+ try {
327
+ // Ledger tasks maintain 'telemetry.lastMemory'
328
+ // We use collectionGroup to search across all dates/passes
329
+ const snapshot = await db.collectionGroup('tasks')
330
+ .where('telemetry.lastMemory.rssMB', '>', thresholdMB)
331
+ .orderBy('telemetry.lastMemory.rssMB', 'desc')
332
+ .limit(20)
333
+ .get();
334
+
335
+ const hotspots = [];
336
+ snapshot.forEach(doc => {
337
+ const data = doc.data();
338
+ hotspots.push({
339
+ computation: data.computation,
340
+ rssMB: data.telemetry.lastMemory.rssMB,
341
+ heapMB: data.telemetry.lastMemory.heapUsedMB,
342
+ status: data.status,
343
+ worker: data.workerId,
344
+ date: doc.ref.parent.parent.parent.parent.id // traversing path to get date
345
+ });
346
+ });
347
+
348
+ res.json({ count: hotspots.length, hotspots });
349
+ } catch (e) {
350
+ res.status(500).json({ error: e.message });
351
+ }
352
+ });
353
+
354
+ // --- 7. FLIGHT RECORDER (Inspection) ---
355
+ // Existing inspection endpoint kept for drill-down
356
+ router.get('/inspect/:date/:calcName', async (req, res) => {
357
+ const { date, calcName } = req.params;
358
+ try {
359
+ const passes = ['1', '2', '3', '4', '5'];
360
+ let executionRecord = null;
361
+
362
+ await Promise.all(passes.map(async (pass) => {
363
+ if (executionRecord) return;
364
+ const ref = db.doc(`computation_audit_ledger/${date}/passes/${pass}/tasks/${calcName}`);
365
+ const snap = await ref.get();
366
+ if (snap.exists) executionRecord = { pass, ...snap.data() };
367
+ }));
368
+
369
+ if (!executionRecord) return res.status(404).json({ status: 'NOT_FOUND' });
370
+
371
+ const contractSnap = await db.collection('system_contracts').doc(calcName).get();
372
+
373
+ res.json({
374
+ execution: executionRecord,
375
+ contract: contractSnap.exists ? contractSnap.data() : null
376
+ });
377
+ } catch (e) {
378
+ res.status(500).json({ error: e.message });
379
+ }
380
+ });
381
+
382
+ return router;
383
+ };
384
+
385
+ module.exports = createAdminRouter;
@@ -1,10 +1,31 @@
1
1
  /**
2
2
  * @fileoverview API sub-pipes.
3
- * REFACTORED: Fixed Category Resolution to match ManifestBuilder logic.
4
- * Implements Status-Based Availability Caching and Smart Date Resolution.
3
+ * REFACTORED: API V3 - Status-Aware Data Fetching.
4
+ * UPDATED: Added GZIP Decompression support for fetching compressed results.
5
5
  */
6
6
 
7
7
  const { FieldPath } = require('@google-cloud/firestore');
8
+ const zlib = require('zlib'); // [NEW] Required for decompression
9
+
10
+ // --- HELPER: DECOMPRESSION ---
11
+ /**
12
+ * Checks if data is compressed and inflates it if necessary.
13
+ * @param {object} data - The raw Firestore document data.
14
+ * @returns {object} The original (decompressed) JSON object.
15
+ */
16
+ function tryDecompress(data) {
17
+ if (data && data._compressed === true && data.payload) {
18
+ try {
19
+ // Firestore returns Buffers automatically for Blob types
20
+ return JSON.parse(zlib.gunzipSync(data.payload).toString());
21
+ } catch (e) {
22
+ console.error('[API] Decompression failed:', e);
23
+ // Return empty object or original data on failure to avoid crashing response
24
+ return {};
25
+ }
26
+ }
27
+ return data;
28
+ }
8
29
 
9
30
  // --- AVAILABILITY CACHE ---
10
31
  class AvailabilityCache {
@@ -147,6 +168,7 @@ const buildCalculationMap = (unifiedCalculations) => {
147
168
 
148
169
  /**
149
170
  * Sub-pipe: pipe.api.helpers.fetchUnifiedData
171
+ * UPDATED: Uses tryDecompress to handle compressed payloads.
150
172
  */
151
173
  const fetchUnifiedData = async (config, dependencies, calcKeys, dateStrings, calcMap) => {
152
174
  const { db, logger } = dependencies;
@@ -191,7 +213,8 @@ const fetchUnifiedData = async (config, dependencies, calcKeys, dateStrings, cal
191
213
  snapshots.forEach((doc, idx) => {
192
214
  const { date, key } = chunk[idx];
193
215
  if (doc.exists) {
194
- response[date][key] = doc.data();
216
+ // [UPDATED] Decompress data if needed
217
+ response[date][key] = tryDecompress(doc.data());
195
218
  } else {
196
219
  response[date][key] = null;
197
220
  }
@@ -321,8 +344,11 @@ async function getComputationStructure(computationName, calcMap, config, depende
321
344
  .collection(compsSub).doc(computationName);
322
345
  const doc = await docRef.get();
323
346
  if (!doc.exists) { return { status: 'error', computation: computationName, message: `Summary flag was present for ${latestStoredDate} but doc is missing.` }; }
324
- const fullData = doc.data();
347
+
348
+ // [UPDATED] Decompress data for structure inspection
349
+ const fullData = tryDecompress(doc.data());
325
350
  const structureSnippet = createStructureSnippet(fullData);
351
+
326
352
  return { status: 'success', computation: computationName, category: category, latestStoredDate: latestStoredDate, structureSnippet: structureSnippet, };
327
353
  } catch (error) {
328
354
  logger.log('ERROR', `API /structure/${computationName} helper failed.`, { errorMessage: error.message });
@@ -2,12 +2,16 @@
2
2
  * @fileoverview Main entry point for the Generic API module.
3
3
  * Export the 'createApiApp' main pipe function.
4
4
  * REFACTORED: API V3 - Status-Aware Data Fetching.
5
+ * UPDATED: Added Admin API Mount.
5
6
  */
6
7
 
7
8
  const express = require('express');
8
9
  const cors = require('cors');
9
10
  const { buildCalculationMap, createApiHandler, getComputationStructure, createManifestHandler, getDynamicSchema } = require('./helpers/api_helpers.js');
10
11
 
12
+ // [NEW] Import Admin Router
13
+ const createAdminRouter = require('./admin-api/index');
14
+
11
15
  /**
12
16
  * In-Memory Cache Handler
13
17
  * Wrapper that adds TTL cache to GET requests.
@@ -71,8 +75,11 @@ function createApiApp(config, dependencies, unifiedCalculations) {
71
75
  app.use(cors({ origin: true }));
72
76
  app.use(express.json());
73
77
 
78
+ // --- [NEW] MOUNT ADMIN API ---
79
+ // This injects the dependencies and the calculations package into the admin router
80
+ app.use('/admin', createAdminRouter(config, dependencies, unifiedCalculations));
81
+
74
82
  // --- Main API V3 Endpoint ---
75
- // createApiHandler now initializes the AvailabilityCache internally
76
83
  const originalApiHandler = createApiHandler(config, dependencies, calcMap);
77
84
  const cachedApiHandler = createCacheHandler(originalApiHandler, dependencies);
78
85
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "bulltrackers-module",
3
- "version": "1.0.294",
3
+ "version": "1.0.296",
4
4
  "description": "Helper Functions for Bulltrackers.",
5
5
  "main": "index.js",
6
6
  "files": [